diff --git a/.github/workflows/mesh-integration.yml b/.github/workflows/mesh-integration.yml index 8b49078e..7343069f 100644 --- a/.github/workflows/mesh-integration.yml +++ b/.github/workflows/mesh-integration.yml @@ -1,6 +1,6 @@ name: Mesh integration -# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. rejoin removes the bootstrap Docker server's etcd member and checks the server idles on its tombstone, refuses to start a second cluster from its moved-aside datastore with no server to join, joins the existing cluster from a join URL in its role directory with the same cluster ID, and idles as a server whose role file names the agent. role_controller adds a standby agent, runs the role controller's probe and controller from the image built from this checkout, makes a voter slow with tc netem and pauses another, and checks every probe reports the link medium the cluster role labelled its node with, the controller reports the voter unfit with the swap named, reports that it would not act with a voter paused, and changes nothing. role_actuator acts as the role controller by hand on three Docker servers and an eligible native standby: the native host's role actuator promotes it to a voter with writes succeeding throughout, steps it down to etcd-only with no pod's shim left, and once its member is removed through k3s's annotation finishes it as an agent with the same node password and Tailscale address and its datastore moved aside, the two k3s units never running together; then a Docker server's role-actuator service demotes it the same way, its agent keeping the server's node password and Tailscale address as one Node with the datastore moved aside, and promotes it again into the same etcd cluster. role_swap, role_swap_paused, role_swap_blocked_peer and role_swap_refused run the role controller's own voter swap by command, with the controller's image built from this checkout: a native standby in and the Docker bootstrap host out with writes throughout and the bootstrap host back in a second swap; the outgoing voter paused (the removal still completes, and the tombstone keeps it out on its return); the candidate's peer port dropped (the promotion is rolled back); and a swap refused while a voter is paused. role_controller_swap runs the controller at the ARC role's default period, makes a voter slow, and turns the dry run off, checking the controller swaps the slow voter for the Docker standby on its own decision; it is a scenario of its own because filling a window of samples at that period takes as long as the controller says. Only on changes that can affect how nodes join the mesh. +# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. rejoin removes the bootstrap Docker server's etcd member and checks the server idles on its tombstone, refuses to start a second cluster from its moved-aside datastore with no server to join, joins the existing cluster from a join URL in its role directory with the same cluster ID, and idles as a server whose role file names the agent. role_controller adds a standby agent, runs the role controller's probe and controller from the image built from this checkout, makes a voter slow with tc netem and pauses another, and checks every probe reports the link medium the cluster role labelled its node with, the controller reports the voter unfit with the swap named, reports that it would not act with a voter paused, and changes nothing. role_actuator acts as the role controller by hand on three Docker servers and an eligible native standby: the native host's role actuator promotes it to a voter with writes succeeding throughout, steps it down to etcd-only with no pod's shim left, and once its member is removed through k3s's annotation finishes it as an agent with the same node password and Tailscale address and its datastore moved aside, the two k3s units never running together; then a Docker server's role-actuator service demotes it the same way, its agent keeping the server's node password and Tailscale address as one Node with the datastore moved aside, and promotes it again into the same etcd cluster. role_swap, role_swap_paused, role_swap_blocked_peer and role_swap_refused run the role controller's own voter swap by command, with the controller's image built from this checkout: a native standby in and the Docker bootstrap host out with writes throughout and the bootstrap host back in a second swap; the outgoing voter paused (the removal still completes, and the tombstone keeps it out on its return); the candidate's peer port dropped (the promotion is rolled back); and a swap refused while a voter is paused. role_controller_swap runs the controller at the ARC role's default period, makes a voter slow, and turns the dry run off, checking the controller swaps the slow voter for the Docker standby on its own decision; it is a scenario of its own because filling a window of samples at that period takes as long as the controller says. etcd_lan runs three servers on the host network of daemons that share the test network, cuts one off the mesh to show its member leaves etcd, then moves every member onto its LAN address through the role, one per run with writes throughout, shows a server cut off the mesh keeps its member, and moves them back. Only on changes that can affect how nodes join the mesh. # # Runs on GitHub-hosted ubuntu-latest, whose Docker daemon runs the privileged containers the test needs (k3s, and Tailscale inside it). The work directory sits under runner.temp, which the job always has write access to. @@ -57,6 +57,7 @@ jobs: - {scenario: in_cluster, compose: v5.5.1, pin: true} - {scenario: context_checks, compose: v5.5.1} - {scenario: host_network, compose: v5.5.1} + - {scenario: etcd_lan, compose: v5.5.1} steps: - uses: actions/checkout@v7 diff --git a/docs/architecture.md b/docs/architecture.md index d4df6ab0..0cccbf7d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -8,7 +8,7 @@ ARC's `gha-runner-scale-set` solves both by design: it runs one fresh pod per jo The control plane itself runs genuine embedded-etcd HA across three `server`-role hosts, not a single node with worker capacity bolted on: one host bootstraps with `--cluster-init` (the one supported migration path from k3s's default single-node SQLite datastore to embedded etcd), and two further hosts join as full `k3s server` members - a real vote each in etcd's own raft quorum, not just schedulable capacity. 3 is the practical minimum: etcd tolerates the loss of any *minority* of its members without losing quorum, and with 2 servers that minority is zero - a second server can only ever make quorum strictly worse than staying at one, never better, so "2-node HA" isn't a real intermediate step, it's a trap. Getting there over Tailscale rather than a shared LAN needed real, non-obvious work - see the [Non-obvious behaviour](non-obvious-behaviour.md#non-obvious-behaviour) section's etcd-specific entries for what actually broke and why. -Hosts on different networks can't reach each other's LAN addresses, and even hosts that share a LAN can't from inside their Docker VMs by default: each node's k3s container sits behind its VM's NAT, so its own Tailscale client has no address to offer its peers and every pair of nodes is relayed. So cross-node cluster traffic (a node joining, cross-node pod-to-pod traffic, the API server's proxy to a remote kubelet, and etcd's own peer traffic between `server` hosts) runs over a real overlay with mesh addresses - see [Tailscale](#tailscale) below for how that overlay is built and the DNS pitfall it introduces. Where hosts do share a LAN, `github_runner_cluster_network_mode: host`, on a Docker VM that has an address of its own on that LAN, lets their Tailscale clients reach each other directly over it, with the relay as the fallback when a host leaves; nothing k3s uses changes (see `roles/github_runner_cluster/README.md`, Networking). +Hosts on different networks can't reach each other's LAN addresses, and even hosts that share a LAN can't from inside their Docker VMs by default: each node's k3s container sits behind its VM's NAT, so its own Tailscale client has no address to offer its peers and every pair of nodes is relayed. So cross-node cluster traffic (a node joining, cross-node pod-to-pod traffic, the API server's proxy to a remote kubelet, and etcd's own peer traffic between `server` hosts) runs over a real overlay with mesh addresses - see [Tailscale](#tailscale) below for how that overlay is built and the DNS pitfall it introduces. Where hosts do share a LAN, `github_runner_cluster_network_mode: host`, on a Docker VM that has an address of its own on that LAN, lets their Tailscale clients reach each other directly over it, with the relay as the fallback when a host leaves; nothing k3s uses changes (see `roles/github_runner_cluster/README.md`, Networking). On such hosts etcd's own peer traffic can also move onto the LAN addresses, so that a server whose mesh device stops working keeps its etcd quorum; k3s holds one etcd address per member, so the fallback to the tailnet is decided per run, not per connection (see the role README, etcd peers over the LAN). The custom runner toolchain (GCC 11/G++ for C++20, Bun, the `gh` CLI — see `Dockerfile`) carries over from the previous design, rebased onto ARC's own [`actions/actions-runner`](https://github.com/actions/runner/pkgs/container/actions-runner) base image instead of `catthehacker/ubuntu:act-latest`. ARC's runner pods use their own registration flow, so the previous design's vendored [myoung34/docker-github-actions-runner](https://github.com/myoung34/docker-github-actions-runner) scripts (registration, GitHub App JWT signing, ephemeral re-registration) are not needed. diff --git a/docs/non-obvious-behaviour.md b/docs/non-obvious-behaviour.md index 92f7785b..22653768 100644 --- a/docs/non-obvious-behaviour.md +++ b/docs/non-obvious-behaviour.md @@ -13,7 +13,7 @@ - **The metrics API's pod list and `kubectl top nodes` require metrics-server.** k3s bundles it by default (the cluster role's `docker-compose.yml` only disables `traefik`), but confirm it is actually running during the first real autoscaler dry run — `scripts/autoscaler.sh` fails safe to the static floor if either is unreachable. A very short-lived job (seconds, not minutes) can complete and its pod be garbage-collected before metrics-server's own scrape interval ever captures its usage - confirmed live against `test-arc-runner.yml`'s ~15-20s jobs, where `kubectl top pod` genuinely never returns data. This correctly triggers the fail-safe rather than a wrong reading, and doesn't matter in practice: a job that short never accumulates meaningful memory pressure anyway. `test-autoscaler.yml`'s own probe jobs hold for 120s specifically so metrics-server has time to observe them. - **A replaced node can stay permanently `NotReady` with "Node password rejected" in its logs.** k3s keeps a node's password in `/etc/rancher/node/password` and the cluster holds a Secret for it (`kube-system/.node-password.k3s`). A recreated container keeps its password, because `/etc/rancher/node` is a named volume that the cluster role seeds from the running container before the first recreate that mounts it (see the role README's "Node passwords and recreating servers"), so this only happens to a node that is genuinely new under an old name: a replaced host, or a node whose volume was removed. Its container then generates a password the bootstrap host's Secret does not match, and the bootstrap host rejects the rejoin. Fix: on the bootstrap host, `kubectl delete secret .node-password.k3s -n kube-system` (and, for an agent, `kubectl delete node `), then restart the container so it registers fresh. Never delete a server's Node object: k3s then removes that server's etcd member, and a removed member's etcd refuses to start until its local datastore (`/var/lib/rancher/k3s/server/db`) is moved aside, after which it rejoins as a new member. `roles/github_runner_cluster/tasks/main.yml` self-heals the password mismatch automatically, only for hosts whose node is missing or not Ready, and deletes Node objects only for agents. - **etcd's own peer traffic does not follow vpn-auth's `--node-ip`/`--advertise-address` the way every other kind of cluster traffic does - it stays on the container's own internal, cross-machine-unroutable Docker-bridge IP by default.** k3s logs this outright ("Etcd IP (PrivateIP) remains the local IP. Running etcd traffic over VPN is not recommended due to performance issues") and deliberately avoids the VPN interface for etcd specifically, assuming a real local network is the better alternative - an assumption that doesn't hold here, since a container's own bridge IP is never reachable from another machine at all. Confirmed live: a second server's join failed outright with `MemberAdd request timed out`, because it was trying to reach the first server's own unroutable bridge IP. An `--etcd-arg="initial-advertise-peer-urls=..."` passthrough looks like the fix but isn't: it only overrides etcd's own raw flag, while k3s independently builds its own `--initial-cluster` string from `config.PrivateIP`, computed separately and earlier - the two then disagree and etcd's bootstrap validation rejects the mismatch outright (`"--initial-cluster has ...= but missing from --initial-advertise-peer-urls="`). The actual fix, traced through k3s's own source (`pkg/cli/server/server.go`, `pkg/executor/embed/embed.go`): `config.PrivateIP` is populated from `--node-ip`'s own value if set, and this happens *before* `--vpn-auth`'s own executor-layer code later rebuilds `NodeIP` from its own Tailscale detection - so passing `--node-ip` explicitly ourselves, to the same real Tailscale IP vpn-auth would itself have detected, is what actually makes `PrivateIP` (and therefore etcd's peer-advertise address) correct, with no second, disagreeing source of truth for the same value. `docker-compose.yml`'s `k3s` service entrypoint brings Tailscale up itself and reads `tailscale ip -4` before exec'ing k3s specifically to learn this IP first, since `exec` replaces the script's own process and nothing can react afterward to what `--vpn-auth`'s own internal call would have produced. -- **A global `--etcd-arg` override is not scoped to the "real" server's own etcd instance - it also applies to k3s's separate, internal "temporary etcd", spun up on every *restart* (never on first boot) to reconcile with an already-initialized on-disk datastore before the real instance starts.** An earlier `--etcd-arg="listen-peer-urls=https://0.0.0.0:2380"` (kept as a "harmless" belt-and-braces bind override after fixing the peer-URL issue above) forced that temporary instance onto the same TLS `0.0.0.0:2380` bind it has no certificates for by design (it normally listens on a local, ephemeral, non-TLS port). Confirmed live: this only broke on a restart against an already-initialized datastore, never on a fresh `--cluster-init`, which is exactly why it wasn't caught until well after the first working deploy - fatal: `"cannot listen on TLS for [::]:2380: KeyFile and CertFile are not presented"`. `--node-ip` alone (above) is sufficient; etcd's own default `listen-peer-urls` already derives from the same `PrivateIP` correctly, so there's nothing left for an explicit override to add. +- **A global `--etcd-arg` override is not scoped to the "real" server's own etcd instance - it also applies to k3s's separate, internal "temporary etcd", spun up on every *restart* (never on first boot) to reconcile with an already-initialized on-disk datastore before the real instance starts.** An earlier `--etcd-arg="listen-peer-urls=https://0.0.0.0:2380"` (kept as a "harmless" belt-and-braces bind override after fixing the peer-URL issue above) forced that temporary instance onto the same TLS `0.0.0.0:2380` bind it has no certificates for by design (it normally listens on a local, ephemeral, non-TLS port). Confirmed live: this only broke on a restart against an already-initialized datastore, never on a fresh `--cluster-init`, which is exactly why it wasn't caught until well after the first working deploy - fatal: `"cannot listen on TLS for [::]:2380: KeyFile and CertFile are not presented"`. `--node-ip` alone (above) is sufficient; etcd's own default `listen-peer-urls` already derives from the same `PrivateIP` correctly, so there's nothing left for an explicit override to add. The same holds for moving etcd onto the servers' LAN addresses (`github_runner_cluster_etcd_peers_over_lan`, the role README's etcd peers over the LAN): the role sets `--node-ip` to the LAN address and passes no `--etcd-arg` for it, so the temporary instance is untouched. - **Calling `tailscale up --authkey=...` unconditionally on every container restart re-registers a brand new tailnet device each time, instead of reconnecting the persisted one - even with the join key's own state volume intact.** A reusable Tailscale auth key is designed to authorize provisioning *new* devices, and a naive `tailscale up` call on every restart (needed here to learn this node's own Tailscale IP before `exec`ing k3s - see above) doesn't check whether the daemon is already authenticated first, so it keeps minting new devices under names like `node1`, `node1-1`, `node1-2`... k3s's own `--vpn-auth` integration avoids exactly this from v1.35.0 on, by checking the Tailscale backend state first and skipping its own `tailscale up` call entirely once already `"Running"` (`StartVPN()` in `pkg/vpn/vpn.go`; the pinned v1.34 releases run `tailscale up --authkey ... --reset` on every start whatever the state) - the manual pre-empt call needs the identical guard: check `tailscale ip -4` succeeds *before* calling `tailscale up` at all, only calling it when not already authenticated. - **Tailscale assigns an IP and wires up its own MagicDNS resolver as two separate steps, not atomically - a node having a real IP is not proof DNS resolution against another tailnet member is working yet.** A joining server needs to resolve the bootstrap host's own MagicDNS hostname (`K3S_JOIN_SERVER_URL`) before it can bootstrap against it at all. Confirmed live: k3s failed outright with `"dial tcp: lookup k3s-server.example.ts.net: no such host"` immediately after `tailscale ip -4` had already returned a real address for the *local* node. Fix: wait for a real `nslookup` success against the join server's own hostname specifically, not just local Tailscale readiness, before invoking k3s (the `k3s` service's entrypoint only does this when it has a server to join, `K3S_JOIN_SERVER_URL` or the role directory's `join-url` - the bootstrap host has nothing else to resolve). - **A server that memory-pressure regrading has stepped to `etcd-only` is `NotReady` by design, and must stay registered.** With `github_runner_cluster_regrade` on, a server under sustained memory pressure steps down (`full`, `control-plane`, `etcd-only`) while it keeps its etcd member and vote; at `etcd-only` k3s runs with `--disable-apiserver --disable-controller-manager --disable-scheduler --disable-agent`, so there is no kubelet to keep its Node Ready, and the Node goes `NotReady` with `Unknown` conditions and stays so. That is not a failure to clear. Its Node object must not be deleted (k3s removes that server's etcd member with it), so anything that culls long-`NotReady` nodes must leave a Node carrying the `github-runner.exadev.com/regrade-level` label alone, and the role's wait for every node to be Ready already does. `kubectl` on such a server's own container is unreliable (its API server is off); the regrade service always reaches the cluster through another server. See the cluster role's README, Memory-pressure regrading. Confirmed against a three-server k3s cluster: the same etcd member ids before, during and after, the cluster taking writes with the stepped-down server as one of the two live voters, and the same Node and node password after stepping back up. diff --git a/docs/role-controller.md b/docs/role-controller.md index 557a32b8..b8745ad3 100644 --- a/docs/role-controller.md +++ b/docs/role-controller.md @@ -4,7 +4,7 @@ `fleet-role-probe` and `fleet-role` (an optional DaemonSet and Deployment in the platform namespace, see `roles/github_runner_arc/templates/fleet-role.yaml.j2`, `scripts/fleet-role-probe.sh` and `scripts/fleet-role.sh`; `github_runner_cluster_role_controller`, off by default, which `github_runner_arc_role_controller_enabled` takes as its default) measure which hosts are fit to hold an etcd vote, report the swap of an unfit voter for a fit standby that the controller would make and, when asked or when `github_runner_arc_role_controller_dry_run` is false, carry the swap out ([Voter swap](#voter-swap)). The probe only reports: its ClusterRole can `get` and `list` nodes and nothing else, and the only object it writes is its own ConfigMap. The controller runs unchanged when no host is marked eligible, when it reports the voters' fitness alone. -The probe runs on every Linux node that is an etcd voter (k3s's `node-role.kubernetes.io/etcd` label) or is marked eligible and has a kubelet (a server regraded to etcd-only has none, and its side is read from its peers), on the node's own network, through a node affinity on those two labels, so a node that is neither (a Unraid host, an image-flashed agent) has no pod and nothing is asked of its kernel. A pod stopped on a node that has lost both labels, or is gone, removes the node's entry as it ends, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the control plane and not with the fleet. Every `github_runner_arc_role_controller_sample_seconds` (15) it appends one sample to its node's key of the `fleet-role-probe` ConfigMap, which keeps the last window of them so that a replaced pod loses nothing. A sample holds the TCP connect time to every voter's etcd peer port (a connect that does not finish within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what its etcd metrics (the `127.0.0.1:2381` listener) gained since the last sample: the WAL fsync and backend commit histograms, the round trip to each peer, leader changes, and whether it has a leader, is a learner and knows as many members as there are voters (a learner or a member that has not started shows as an extra known member); on a node that is not a voter, an fdatasync reading from fio with the write size etcd's own disk check uses, once per window; the memory pressure (`/proc/pressure/memory` of the pod's own procfs, which serves the whole node's, so nothing is mounted) and `MemAvailable` the regrade core reads, with the core's own functions, where a node on a kernel without pressure stall information (built or booted without it, which has no `/proc/pressure` or, on older kernels, has the files and refuses to read them) gives `MemAvailable` and `MemTotal` and no pressure figure; and the medium of the node's link, `wired`, `wifi` or `unknown`: the value of the node's `github-runner.exadev.com/link` label (`github_runner_arc_role_controller_link_label`), which the cluster role reads on the host itself (see the cluster role's README) since a node in a container or a VM sees only its own virtual interface, and otherwise the medium of the interface the pod's default route leaves by (an interface that is virtual, such as the k3s container's veth, hides the link beneath it), each sample saying which (`link_source`, `label` or `interface`). The pod's own reading is taken from the interface's `/sys/class/net` entry and never by walking `/sys`, whose symlinks loop. `scripts/fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of these readings alone (the link as the pod's own interface gives it), with no cluster access, to compare with a reading taken by hand on the host. +The probe runs on every Linux node that is an etcd voter (k3s's `node-role.kubernetes.io/etcd` label) or is marked eligible and has a kubelet (a server regraded to etcd-only has none, and its side is read from its peers), on the node's own network, through a node affinity on those two labels, so a node that is neither (a Unraid host, an image-flashed agent) has no pod and nothing is asked of its kernel. A pod stopped on a node that has lost both labels, or is gone, removes the node's entry as it ends, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the control plane and not with the fleet. Every `github_runner_arc_role_controller_sample_seconds` (15) it appends one sample to its node's key of the `fleet-role-probe` ConfigMap, which keeps the last window of them so that a replaced pod loses nothing. A sample holds the TCP connect time to every voter's etcd peer port, at the address k3s records for the voter's etcd member (the Node's `etcd.k3s.cattle.io/node-address` annotation, its mesh address or, once moved, its LAN address) (a connect that does not finish within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what its etcd metrics (the `127.0.0.1:2381` listener) gained since the last sample: the WAL fsync and backend commit histograms, the round trip to each peer, leader changes, and whether it has a leader, is a learner and knows as many members as there are voters (a learner or a member that has not started shows as an extra known member); on a node that is not a voter, an fdatasync reading from fio with the write size etcd's own disk check uses, once per window; the memory pressure (`/proc/pressure/memory` of the pod's own procfs, which serves the whole node's, so nothing is mounted) and `MemAvailable` the regrade core reads, with the core's own functions, where a node on a kernel without pressure stall information (built or booted without it, which has no `/proc/pressure` or, on older kernels, has the files and refuses to read them) gives `MemAvailable` and `MemTotal` and no pressure figure; and the medium of the node's link, `wired`, `wifi` or `unknown`: the value of the node's `github-runner.exadev.com/link` label (`github_runner_arc_role_controller_link_label`), which the cluster role reads on the host itself (see the cluster role's README) since a node in a container or a VM sees only its own virtual interface, and otherwise the medium of the interface the pod's default route leaves by (an interface that is virtual, such as the k3s container's veth, hides the link beneath it), each sample saying which (`link_source`, `label` or `interface`). The pod's own reading is taken from the interface's `/sys/class/net` entry and never by walking `/sys`, whose symlinks loop. `scripts/fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of these readings alone (the link as the pod's own interface gives it), with no cluster access, to compare with a reading taken by hand on the host. The controller reads the Nodes and the probes' samples every sample period and publishes `fleet-role-status`, which the heartbeat republishes in its gist beside the autoscaler's status. For every voter and every eligible host it gives a verdict, `fit`, `unfit` or `unknown`, with each reason and the numbers behind it. Every limit derives from etcd's guidance and the cluster's own timing, not chosen numbers: diff --git a/plugins/filter/cluster_etcd_lan.py b/plugins/filter/cluster_etcd_lan.py new file mode 100644 index 00000000..fa137a7e --- /dev/null +++ b/plugins/filter/cluster_etcd_lan.py @@ -0,0 +1,139 @@ +"""Filters the github_runner_cluster role uses to move etcd's peer traffic between k3s servers onto their LAN addresses (github_runner_cluster_etcd_lan_address and github_runner_cluster_etcd_peers_over_lan), and back: the checks on the inventory, the reading of etcd's member list and of the other servers' certificates, and the decision of which server changes its address in a run.""" + +from __future__ import annotations + +import ipaddress +import json +import re +from collections.abc import Mapping, Sequence +from typing import Any + +from ansible.errors import AnsibleFilterError + +# k3s names a server's etcd member after its node, a hyphen and the first eight characters of a random UUID (setName in pkg/etcd/etcd.go at v1.34.1+k3s1). +MEMBER_SUFFIX = r"-[0-9a-f]{8}" + + +def _text(value: Any) -> str: + return "" if value is None else str(value).strip() + + +def cluster_etcd_lan_problems(hosts: Any, peers_over_lan: Any) -> list[str]: + """Return what in the inventory keeps etcd's peer traffic from moving onto the servers' LAN addresses, or would break it once moved. + + :param hosts: one mapping per host of the cluster group, with ``name``, ``node_name``, ``role`` (the topology's server or agent), ``eligible`` (github_runner_cluster_voter_eligible), ``node_mode``, ``network_mode``, ``lan_cidr``, ``mesh``, ``address`` (github_runner_cluster_etcd_lan_address) and ``peers_over_lan`` (that host's github_runner_cluster_etcd_peers_over_lan). :param peers_over_lan: this host's github_runner_cluster_etcd_peers_over_lan. :returns: one sentence per problem, empty when there is none. :raises AnsibleFilterError: if ``hosts`` is not a list of mappings. + + An address is a server's own: an IPv4 address in the LAN the nodes share, on a Docker node on the host network, whose k3s then holds the address in the namespace etcd listens in. k3s registers one peer URL per etcd member and joins a new member with only the first URL of each existing one, and etcd refuses a join whose URLs differ from the members' (ValidateClusterAndAssignIDs in server/etcdserver/api/membership/cluster.go), so a member cannot carry the LAN address and the mesh address together: with peers_over_lan every server and every host that may become one (an eligible host) needs an address of its own, and every host of the group must agree on the setting. The off-mesh start of a headscale_in_cluster bootstrap server passes no node address to k3s, so its etcd address would change between its two starts. + """ + if not isinstance(hosts, Sequence) or isinstance(hosts, str) or not all(isinstance(host, Mapping) for host in hosts): + raise AnsibleFilterError("The hosts must be a list of mappings.") + problems: list[str] = [] + seen: dict[str, str] = {} + wanted = bool(peers_over_lan) + for host in hosts: + name = _text(host.get("name")) + address = _text(host.get("address")) + takes_vote = host.get("role") == "server" or bool(host.get("eligible")) + if bool(host.get("peers_over_lan")) != wanted: + problems.append(f"{name} sets github_runner_cluster_etcd_peers_over_lan to {bool(host.get('peers_over_lan'))}, and this host to {wanted}; set it once for the whole group.") + if not address: + if wanted and takes_vote: + problems.append(f"{name} is {'a server' if host.get('role') == 'server' else 'eligible for a vote'} but sets no github_runner_cluster_etcd_lan_address, and with github_runner_cluster_etcd_peers_over_lan every etcd member must be reached on its LAN address (k3s registers one peer URL per member).") + continue + try: + parsed = ipaddress.IPv4Address(address) + except ValueError: + problems.append(f"{name}'s github_runner_cluster_etcd_lan_address {address!r} is not an IPv4 address.") + continue + if not takes_vote: + problems.append(f"{name} sets github_runner_cluster_etcd_lan_address but is neither a server nor eligible for a vote, so it runs no etcd member.") + if host.get("node_mode") == "native": + problems.append(f"{name} is a native node, which the LAN address is not supported on: its node address comes from files/native/write-node-ip.sh, and its etcd member cannot be updated with the role's etcdctl container. Leave the address unset there, and unmark it eligible before turning github_runner_cluster_etcd_peers_over_lan on.") + if host.get("network_mode") != "host": + problems.append(f"{name} sets github_runner_cluster_etcd_lan_address with github_runner_cluster_network_mode {host.get('network_mode')}. On the bridge network the node's container has only its Docker bridge address: a LAN address published through the Docker VM's NAT reaches etcd from the VM's address instead of the peer's, which fails etcd's check of the peer certificate against the connection's source, and macOS drops a user-mode VM's traffic to other Macs. Move the node to the host network of a VM with an address on the LAN first (the role README's Networking section).") + cidr = _text(host.get("lan_cidr")) + try: + in_lan = bool(cidr) and parsed in ipaddress.IPv4Network(cidr, strict=False) + except ValueError: + in_lan = False + if not in_lan: + problems.append(f"{name}'s github_runner_cluster_etcd_lan_address {address} is not in github_runner_cluster_lan_cidr ({cidr or 'unset'}).") + if address in seen: + problems.append(f"{name} and {seen[address]} both set github_runner_cluster_etcd_lan_address {address}; each etcd member needs an address of its own.") + seen.setdefault(address, name) + if wanted and host.get("mesh") == "headscale_in_cluster": + problems.append(f"{name} runs the headscale_in_cluster mesh, whose bootstrap server first starts k3s off the mesh with no node address, so its etcd address would change between its two starts; github_runner_cluster_etcd_peers_over_lan is not supported with it.") + return list(dict.fromkeys(problems)) + + +def cluster_etcd_members(member_list: Any) -> list[dict[str, Any]]: + """Return the members of ``etcdctl member list -w json`` output. + + :param member_list: the command's standard output. :returns: one mapping per member: ``id`` (hexadecimal, as ``etcdctl member update`` takes it), ``name``, ``peer_hosts`` (the host of each peer URL, in order) and ``learner``. :raises AnsibleFilterError: if the output is not etcdctl's JSON member list. + """ + try: + data = json.loads(_text(member_list)) + except ValueError as error: + raise AnsibleFilterError(f"etcdctl's member list is not JSON: {error}") from error + members = data.get("members") if isinstance(data, Mapping) else None + if not isinstance(members, list): + raise AnsibleFilterError("etcdctl's member list has no members array.") + found: list[dict[str, Any]] = [] + for member in members: + if not isinstance(member, Mapping) or not isinstance(member.get("ID"), int): + raise AnsibleFilterError(f"etcdctl's member list holds an entry without a numeric ID: {member!r}") + hosts = [] + for url in member.get("peerURLs") or []: + match = re.fullmatch(r"https?://\[?([^\]/]+?)\]?:[0-9]+/?", str(url)) + if match is None: + raise AnsibleFilterError(f"etcd member {member.get('name')!r} has a peer URL etcdctl printed as {url!r}, which is not scheme://host:port.") + hosts.append(match.group(1)) + found.append({"id": format(member["ID"], "x"), "name": _text(member.get("name")), "peer_hosts": hosts, "learner": bool(member.get("isLearner"))}) + return found + + +def cluster_etcd_member_of(members: Any, node_name: Any) -> dict[str, Any] | None: + """Return the member k3s registered for a node, or None when it has none. + + :param members: the output of cluster_etcd_members. :param node_name: the node's name. :raises AnsibleFilterError: if more than one member carries the node's name, which k3s itself refuses to join with. + """ + pattern = re.compile(re.escape(_text(node_name)) + MEMBER_SUFFIX) + matches = [member for member in members if pattern.fullmatch(str(member.get("name", "")))] + if len(matches) > 1: + raise AnsibleFilterError(f"etcd lists {len(matches)} members for {node_name}: {', '.join(member['name'] for member in matches)}. Remove the stale one before changing its address.") + return matches[0] if matches else None + + +def cluster_etcd_lan_switches(hosts: Any) -> list[str]: + """Return the hosts that change their etcd address in this run: the first of those that would, in play order. + + :param hosts: one mapping per host of the play, in play order, with ``name`` and ``switch`` (empty, ``to_lan`` or ``to_mesh``). :returns: a list of at most one host name. Changing a member's address stops it until its container is recreated, so two in one run could leave the cluster without a quorum; the others change in later runs. + """ + return [_text(host.get("name")) for host in hosts if _text(host.get("switch"))][:1] + + +def cluster_etcd_lan_readiness(certificates: Any, reaches: Any, addresses: Any) -> dict[str, list[str]]: + """Return the other servers that keep a member from moving onto its LAN address. + + :param certificates: the looped results of reading each other server's etcd peer certificate's IP addresses, one per line (``item`` the host, ``rc`` and ``stdout_lines``; an unreachable host has no ``rc``). + :param reaches: the looped results of each other server pinging the member's LAN address (``item`` and ``rc``). + :param addresses: each other server's own LAN address, by host. + :returns: ``unnamed``, the servers whose certificate could not be read or does not name their own LAN address, and ``unreached``, the servers that did not reach the member's address. + """ + unnamed = [str(result["item"]) for result in certificates if result.get("rc") != 0 or _text(addresses.get(result["item"])) not in [_text(line) for line in result.get("stdout_lines") or []]] + unreached = [str(result["item"]) for result in reaches if result.get("rc") != 0] + return {"unnamed": unnamed, "unreached": unreached} + + +class FilterModule: + """Registers the etcd LAN filters with Ansible.""" + + def filters(self) -> dict[str, Any]: + """Return the filters this plugin provides.""" + return { + "cluster_etcd_lan_problems": cluster_etcd_lan_problems, + "cluster_etcd_members": cluster_etcd_members, + "cluster_etcd_member_of": cluster_etcd_member_of, + "cluster_etcd_lan_switches": cluster_etcd_lan_switches, + "cluster_etcd_lan_readiness": cluster_etcd_lan_readiness, + } diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 7172f3bc..b2e351c8 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -378,11 +378,11 @@ The identity moves through `files/node-identity.sh`, the same pipe that seeds a ## Networking -Every node's k3s uses its mesh address for everything: its node IP, etcd's peer URLs, the API server and pod traffic. Which physical path the mesh takes between two nodes is Tailscale's choice, among the endpoints each node's Tailscale client can offer its peers. On Compose's default bridge network (`github_runner_cluster_network_mode: bridge`, the default) the client sits behind the Docker host's NAT and, inside a Docker VM, behind the VM's too, so it has no address its peers can reach and every pair of nodes is relayed, even on one LAN. +Every node's k3s uses its mesh address for everything: its node IP, etcd's peer URLs, the API server and pod traffic, unless [etcd peers over the LAN](#etcd-peers-over-the-lan) moves etcd's onto the servers' LAN addresses. Which physical path the mesh takes between two nodes is Tailscale's choice, among the endpoints each node's Tailscale client can offer its peers. On Compose's default bridge network (`github_runner_cluster_network_mode: bridge`, the default) the client sits behind the Docker host's NAT and, inside a Docker VM, behind the VM's too, so it has no address its peers can reach and every pair of nodes is relayed, even on one LAN. `github_runner_cluster_network_mode: host` runs the node's container on the Docker host's own network instead (`docker-compose.host-network.yml`, merged last). Its client then offers the host's own addresses, so nodes that share a network connect directly and fall back to a relay or a hole-punched path when they do not. The container publishes no port, since its ports are the host's; with `github_runner_cluster_wait_for_nodes` or `github_runner_cluster_record_deployed_version` on, the role checks that the API server answers on the host's loopback before it uses the kubeconfig there. k3s's ports, routes and firewall rules, Tailscale's, and the entrypoint's unreachable route for the mesh ranges then live in the host's namespace, shared with any other container on the host network there. Nothing k3s uses changes, so a node moves between the two modes with a container recreate. -On the host network the node's ports are the Docker host's. The API server (6443) and kubelet (10250) then answer on every address the host has, so on a bridged VM they are reachable from the LAN on the VM's address; etcd stays on loopback and the mesh address. In bridge mode only the API server's port is published, on the Mac's own addresses. Any other container on the same Docker engine would gain a route into the mesh through the node's Tailscale routing and its kube-proxy rules, so the host network needs a Docker engine dedicated to the node, and the role enforces it: before changing anything it refuses (under `--check` too) while the engine has a running container that is not part of the node's own Compose project (`github_runner_cluster_compose_project`: the k3s container, the pruner and the regrade service), naming each. A bridged Colima profile of its own per node meets it; a node that shares its engine with other containers, a second cluster or a Headscale server included, stays on the bridge network. +On the host network the node's ports are the Docker host's. The API server (6443) and kubelet (10250) then answer on every address the host has, so on a bridged VM they are reachable from the LAN on the VM's address; etcd stays on loopback and the mesh address, or the LAN address once [etcd peers over the LAN](#etcd-peers-over-the-lan) has moved it, where its peer and client ports answer only a peer or client with a certificate from the cluster's etcd CA (k3s starts etcd with client certificate authentication on both). In bridge mode only the API server's port is published, on the Mac's own addresses. Any other container on the same Docker engine would gain a route into the mesh through the node's Tailscale routing and its kube-proxy rules, so the host network needs a Docker engine dedicated to the node, and the role enforces it: before changing anything it refuses (under `--check` too) while the engine has a running container that is not part of the node's own Compose project (`github_runner_cluster_compose_project`: the k3s container, the pruner and the regrade service), naming each. A bridged Colima profile of its own per node meets it; a node that shares its engine with other containers, a second cluster or a Headscale server included, stays on the bridge network. On a Linux machine running Docker without a VM, the host network is the machine's own. The node's Tailscale client would then create `tailscale0`, Tailscale's routing table and rules and its firewall chains beside the machine's own Tailscale client, and a second node on the same Docker host would share all of that and the kubelet's port. Neither works, so the role refuses host mode when the Docker host's network already has a Tailscale device that is not the node's own (as it is once the node runs on the host network), and a node that should go direct on such a machine needs a VM of its own. @@ -439,6 +439,22 @@ Move the engine first and the network after, one change at a time, so that the s Tailscale settings a node needs go on the `tailscaled` command line in `k3s/node-entrypoint.sh` or through `--vpn-auth`'s `extraArgs`: k3s runs `tailscale up --reset` on every start, which drops any other preference. +## etcd peers over the LAN + +On the mesh, a server whose mesh device stops working (an expired node key, a control server it cannot reach) loses its etcd peers with it, even when every server sits on one LAN: etcd's peer and client URLs are mesh addresses, so the member logs reads that could not reach a quorum (`timed out waiting for read index response`) and the cluster is a voter short. `github_runner_cluster_etcd_lan_address` (per server) and `github_runner_cluster_etcd_peers_over_lan` (for the group) put etcd's peer and client URLs on the servers' LAN addresses instead, so etcd keeps its quorum through a mesh outage on one server. Only etcd moves: the node's IP, flannel, the API server's advertise address and every other kind of cluster traffic stay on the mesh, because with `--vpn-auth` k3s takes only etcd's address from `--node-ip` and overrides the rest with the mesh address ("VPN provider overrides configured node-ip parameter" in `pkg/agent/config/config.go`, and the advertise address in `pkg/cli/server/server.go`, at [v1.34.1+k3s1](https://github.com/k3s-io/k3s/tree/v1.34.1%2Bk3s1)). The move goes through `--node-ip`, never an `--etcd-arg` override of `listen-peer-urls`: k3s passes every `--etcd-arg` to the temporary etcd it starts on each restart as well (`StartEmbeddedTemporary` in `pkg/etcd/etcd.go`), which has no certificates for such a listener (see the collection README's known issues), while `--node-ip` sets only the address the real instance's URLs are built from. The addresses are set explicitly rather than detected, because a host on a LAN can carry several addresses (a dock's adapter and Wi-Fi), the one that matters is the one the other servers reach, and the role checks that and the address's place in `github_runner_cluster_lan_cidr` before it uses it. + +**Why there is no automatic fallback per connection.** etcd can hold several peer URLs per member and fails over between them, but k3s registers exactly one (`peerURL` in `pkg/etcd/etcd.go`), checks at start that a member carries it (`membershipError`), and joins a new server with only the first URL of each member; etcd then refuses the join unless each member's URLs match exactly (`ValidateClusterAndAssignIDs` in `server/etcdserver/api/membership/cluster.go` at [etcd v3.6.4](https://github.com/etcd-io/etcd/blob/v3.6.4/server/etcdserver/api/membership/cluster.go)), so a member with a LAN and a mesh URL would break every later join and voter swap. Each member therefore has one address, and the fallback to the mesh is taken when the role decides: a server another server cannot reach on its LAN address keeps its mesh address, and the run says so. Once moved, a member that leaves the LAN (a laptop taken elsewhere) loses its peers as a mesh member loses them in a mesh outage, so the setting is for servers that stay on one LAN; move it back first. + +**What it needs.** Every server, and every host eligible for a vote (`github_runner_cluster_voter_eligible`), needs an address of its own, since a member on the mesh cannot reach one on the LAN and a host the role controller promotes joins with the group's addresses. Each must be a Docker node on the host network (`github_runner_cluster_network_mode: host`) of a Docker host with that address on its own network, which on macOS is a bridged VM (see [Moving a node to a bridged Colima VM](#moving-a-node-to-a-bridged-colima-vm)). On the bridge network it cannot work, which the role refuses: the node's container has only its Docker bridge address, a LAN port published through the Docker VM's NAT reaches etcd from the VM's address instead of the peer's, which fails etcd's check of a peer's certificate against the address its connection comes from (`checkCertSAN` in etcd's `client/pkg/transport/listener_tls.go`), and macOS drops a user-mode VM's traffic to other Macs. Native nodes are refused too, since their node address comes from `files/native/write-node-ip.sh` and the role's etcdctl runs in a container; and so is the `headscale_in_cluster` mesh, whose bootstrap server starts k3s off the mesh with no node address. The checks run before anything changes, under `--check` too, along with `openssl` on each server's host to read its certificate. + +**Rollout.** Nothing changes for a host with no address, and its `.env`, Compose files and k3s arguments are the same as before the settings existed. The rollout then takes one run per stage, each server's change in its turn of starting the servers, so no two are ever away together: + +1. Set `github_runner_cluster_etcd_lan_address` for every server and eligible host and run the role. Each server's `.env` gains `K3S_ETCD_LAN_ADDRESS` and its Compose files gain `docker-compose.etcd-lan.yml`, so it is recreated in its turn (through the same quorum guard and wait for Ready as every recreate) with the address added to its certificates' names (`--tls-san`). This has to come first for every server: once a member listens on its LAN address the others dial it from theirs, and its etcd checks each one's certificate names that address. No member moves. +2. Set `github_runner_cluster_etcd_peers_over_lan: true` for the group and run the role once per server, with `--check --diff` first if you like. Each run moves the first server of the play whose member is still on the mesh and reports the others as waiting. Before it changes anything it checks that every other server's live etcd peer certificate names that server's own LAN address, that every other server reaches the moving server's address (three pings from its host network; a server that does not reach it keeps the member on the mesh and the run says so), that every server is Ready, that every etcd member reports healthy, and that no server logged a read that could not reach a quorum in the last `github_runner_cluster_etcd_lan_quiet_minutes` (10). It then updates the member's peer URL (`etcdctl member update`, from `github_runner_cluster_etcdctl_image` in a throwaway container that shares the k3s container's network and mounts its data volume read-only for the client certificate) and recreates the server with `K3S_ETCD_PEER_ADDRESS` set, which starts k3s with `--node-ip` at the LAN address, so etcd listens there. Between the update and the restart the other members dial the new address, where nothing listens yet, so the member is away for as long as its recreate, as any recreated server is. Afterwards the run waits for the node to be Ready on a heartbeat from the new container, for etcd to list the member at its new address and every member to report healthy, and for `github_runner_cluster_etcd_lan_settle_seconds` (60) with no server logging such a read. If anything fails, it updates the member back to its old URL (through this server or, when it does not answer, through every other), writes `.env` back, recreates the server on its old address, waits for it, and fails the run saying so. +3. To move back, set `github_runner_cluster_etcd_peers_over_lan: false` and run once per server: each run moves one member back to its mesh address the same way. Keep the addresses set until every member is back: the role refuses a host whose `.env` still starts etcd on a LAN address once its address is removed, since k3s started without it would look for its member under an address etcd no longer lists. + +The `etcd_lan` scenario of `tests/mesh/run.sh` runs the whole of it on three servers on the host network of Docker daemons that share one network: the same cut of one server off the mesh takes its member out of etcd while etcd is on the mesh, and leaves it healthy, and the server writable, once etcd is on the LAN. + ## Multiple sites Each site is its own cluster: its own inventory group, bootstrap server and mesh provider, run by its own play with `github_runner_cluster_group` set to that group (see `examples/two-site/`). One cluster spanning two meshes is not supported: flannel's tailscale backend carries pod traffic over the single mesh every node belongs to, so every node of a cluster must be on the same tailnet or Headscale server. An inventory host belongs to at most one cluster group; to run a second cluster on the same machines, give each machine a second inventory host (see [Several clusters on one host](#several-clusters-on-one-host)). @@ -489,6 +505,7 @@ The role controller holds the regrade Lease (`kube-system/fleet-regrade`, the Le - `github_runner_cluster_regrade` (default false): run the `regrade` service, or on a native server the `fleet-regrade` unit, which steps this server down under memory pressure while it keeps its etcd vote (see [Memory-pressure regrading](#memory-pressure-regrading)). Servers only, and a Docker host marked eligible, whose service runs only while its recorded role is server (see [A Docker host's recorded role](#a-docker-hosts-recorded-role)). `github_runner_cluster_regrade_pressure_threshold` (percent, default 10) and `github_runner_cluster_regrade_available_floor_percent` (default 15) are the limit; `github_runner_cluster_regrade_step_up` (`automatic`, the default, or `manual`), `github_runner_cluster_regrade_sample_seconds`, `github_runner_cluster_regrade_sustain_seconds`, `github_runner_cluster_regrade_cooldown_seconds`, `github_runner_cluster_regrade_operation_timeout_seconds` and `github_runner_cluster_regrade_lease_duration_seconds` are the timing, all derived from `github_runner_cluster_regrade_pressure_window_seconds` (the kernel's `avg300`, 300 seconds, not a setting) unless set. `github_runner_cluster_regrade_level_label` and `github_runner_cluster_regrade_taint` are derived, never set directly. - Per-host overrides: `github_runner_cluster_node_role` (`server` or `agent`), `github_runner_cluster_bootstrap` (true picks the bootstrap host, false excludes a host), `github_runner_cluster_server_url` (join this server instead, for a cluster whose servers are outside the inventory), `github_runner_cluster_tls_sans`, `github_runner_cluster_node_address`, and `github_runner_cluster_voter_eligible` (true stages a native host as a standby server, its server configuration written and its name on every server's certificate, without making it a server; see [Native servers](#native-servers)). The run fails if the result has an even number of servers, no servers, or more than one bootstrap host. - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)). +- `github_runner_cluster_etcd_lan_address` (per server, default empty) and `github_runner_cluster_etcd_peers_over_lan` (for the group, default false): a server's LAN address for etcd, and whether etcd's members use them, one member moved per run; `github_runner_cluster_etcd_lan_quiet_minutes` (default 10) and `github_runner_cluster_etcd_lan_settle_seconds` (default 60) bound the checks before and after each move (see [etcd peers over the LAN](#etcd-peers-over-the-lan)). - `github_runner_cluster_network_mode` (default `bridge`, or `host`): the network the node's container runs on, and with `host`, `github_runner_cluster_lan_cidr` (required: the LAN the nodes share) and `github_runner_cluster_network_probe_image` (default `busybox:1.37`, for reading the host's addresses) (see [Networking](#networking)). - `github_runner_cluster_runtime_autostart_at` (default `auto`; `login` or `boot`) and `github_runner_cluster_runtime_boot_profile` (default `node`): whether the Colima profile is started by a launchd agent at login or by a launchd daemon at boot, and which profile `auto` starts at boot (see [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). - `github_runner_cluster_runtime_force_stop_wedged` (default true), `github_runner_cluster_runtime_probe_attempts` (default 5), `github_runner_cluster_runtime_probe_delay` (default 10), `github_runner_cluster_runtime_probe_timeout` (default 20), `github_runner_cluster_runtime_boot_grace` (default 600), `github_runner_cluster_runtime_kill_grace` (default 5) and `github_runner_cluster_runtime_clean_stop_timeout` (default 120): when a Colima profile listed as running with a silent runtime is stopped, see the Colima paragraph above. diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index b52c62ab..a86fb0db 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -28,6 +28,42 @@ github_runner_cluster_node_taints: [] github_runner_cluster_etcd_heartbeat_interval: "" github_runner_cluster_etcd_election_timeout: "" +# This server's own IPv4 address on the LAN the servers share (in github_runner_cluster_lan_cidr), for etcd's peer traffic: empty, the default, leaves the host exactly as it was. Set, the role adds it to the names in the server's certificates (a container recreate, one server at a time as every recreate is) and checks it: the host must be a Docker node on the host network (github_runner_cluster_network_mode host, which puts the address in the namespace etcd listens in), a server or a host eligible for a vote, and the address must be in the LAN, on the Docker host's own network, and nobody else's. The address alone moves no traffic; github_runner_cluster_etcd_peers_over_lan does. See "etcd peers over the LAN" in the role README. +github_runner_cluster_etcd_lan_address: "" +# Whether the servers' etcd members use their LAN addresses (github_runner_cluster_etcd_lan_address) instead of their mesh addresses, for their peer and client URLs: false, the default, keeps etcd on the mesh. Set it once for the whole group. Turned on, each run moves at most one running server's member onto its LAN address (an `etcdctl member update`, then a recreate of that server's container with k3s's --node-ip at the LAN address), and only once every other server's certificate names its own LAN address, every server is Ready, etcd's members are all healthy, no server logged a read that could not reach a quorum in the last github_runner_cluster_etcd_lan_quiet_minutes, and every other server reaches the address; a server that does not reach it keeps the mesh address, which the run reports. A move that does not come back healthy is moved back to the mesh address and the run fails. Turned off again, each run moves one member back the same way. The other servers wait for later runs, so a cluster of three moves over three runs. Every host that is a server or eligible for a vote needs an address, since k3s registers one peer URL per member. +github_runner_cluster_etcd_peers_over_lan: false +# How far back, in minutes, the role reads every server's k3s log for etcd reads that could not reach a quorum ("timed out waiting for read index response" and its kin) before it moves a member's address. A cluster that has just had one is not changed. +github_runner_cluster_etcd_lan_quiet_minutes: 10 +# Not a setting: the messages etcd logs when a linearizable read cannot get its read index from the leader in time (linearizableReadLoop and sendReadIndex in server/etcdserver/v3_server.go at etcd v3.6.4), which mean the member could not reach a quorum for it, the symptom of a voter cut off from its peers. tasks/etcd_lan_quiet.yml looks for them in every server's log. +github_runner_cluster_etcd_read_index_messages: + - timed out waiting for read index response + - waiting for ReadIndex response took too long + - failed to get read index from Raft +# Not a setting: the etcd image whose etcdctl reads the member list and updates a member's peer URL, from a throwaway container that shares the k3s container's network and reads its client certificate from the k3s data volume (as the role README's runbook does by hand). The release of etcd that k3s v1.34.1+k3s1 embeds (its go.mod's go.etcd.io/etcd replacement, v3.6.4-k3s3). +github_runner_cluster_etcdctl_image: "gcr.io/etcd-development/etcd:v3.6.4" +# Derived, never set directly: whether this host's k3s starts etcd on its LAN address (K3S_ETCD_PEER_ADDRESS in .env), and the move of its etcd member this run makes (empty, to_lan or to_mesh), both as tasks/etcd_lan.yml decides them; false and empty for a host that sets no address. +github_runner_cluster_etcd_peer_on_lan: false +github_runner_cluster_etcd_lan_switch: "" +# How long, in seconds, the role watches every server's log after a moved member is healthy again, for etcd reads that could not reach a quorum, before it takes the move as done. +github_runner_cluster_etcd_lan_settle_seconds: 60 +# Derived, never set directly: the start of the command that runs this host's etcdctl, in a throwaway container from github_runner_cluster_etcdctl_image that shares the k3s container's network and mounts its data volume read-only for the client certificate, as tasks/etcd_members_read.yml runs it through each server. +github_runner_cluster_etcdctl_argv: + - docker + - run + - --rm + - --network + - "container:{{ github_runner_cluster_container_name }}" + - --volume + - "{{ github_runner_cluster_compose_project }}_k3s-data:/var/lib/rancher/k3s:ro" + - --entrypoint + - etcdctl + - "{{ github_runner_cluster_etcdctl_image }}" + - --endpoints=https://127.0.0.1:2379 + - --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt + - --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt + - --key=/var/lib/rancher/k3s/server/tls/etcd/client.key + - "--command-timeout={{ github_runner_cluster_cluster_read_timeout | int }}s" + # Memory-pressure regrading (see the role README): a server steps down through three levels when memory runs short, and back up, without ever leaving etcd. full is the server as it runs; control-plane is still a server and an etcd voter, cordoned and drained; etcd-only has its API server, controller manager, scheduler and agent switched off, and holds the node's etcd seat and nothing else. Giving up the seat is never automatic. false (the default) installs and changes nothing. true runs the `regrade` Compose service beside this server's k3s container, a server host only, with the Docker socket mounted (the same permission cost as the docker-prune service). The run refuses it where a step down could not work: an agent, a cluster with no other server in the inventory to keep the API up, and the bootstrap server of a headscale_in_cluster mesh, whose Headscale server runs on the agent that etcd-only switches off. github_runner_cluster_regrade: false # Whether the service also steps up by itself. automatic (the default) goes back up one level at a time once the server has been calm, that is the pressure under the threshold and MemAvailable at or above the floor, for the sustain period, and the cooldown since the last step has passed; it takes the same Lease and checks the same peers as a step down. manual leaves stepping up to `fleet-regrade up`. Stepping down is always automatic, and giving up the etcd vote never is. @@ -168,7 +204,7 @@ github_runner_cluster_headscale_standby_host: "" # headscale_in_cluster keeps Headscale's state on the bootstrap server's k3s data volume, which a static pod cannot replace with a replicated persistent volume (kubelet runs it with no API server, so it cannot bind a claim), so the provider requires Litestream. Set this to true to run without it and accept that losing that volume loses every registration. github_runner_cluster_headscale_accept_node_local_state: false -# The network the node's k3s container runs on. bridge (the default) is Compose's own bridge network, behind the Docker host's NAT, with the API server's port published. host puts the container on the Docker host's own network (on macOS, the Docker VM's, which then needs an address of its own on the LAN; see the role README's Networking section): its Tailscale client then offers the host's own addresses to its peers, so nodes that share a network connect directly instead of through a relay, and fall back to a relay or a hole-punched path when they do not. Nothing k3s uses changes (node IP, etcd peer URLs and certificate names stay on the mesh), so a host can move between the two with a container recreate. With host, k3s's ports, routes and firewall rules (kube-proxy, flannel, Tailscale's own and the entrypoint's unreachable route for the mesh ranges) live in the Docker host's namespace, shared with every other container on the host network there, and the API server (6443) and kubelet (10250) answer on every address the host has, its LAN address included. One node per Docker host: the role refuses host mode where the host network already has a Tailscale device that is not the node's own, as on a Linux machine running its own Tailscale client with no VM between, or beside a second node on the same Docker host (see the role README's Networking section). It also refuses host mode while the Docker engine runs any container that is not part of the node's own Compose project, since each would get a route into the mesh: the engine must be dedicated to the node. Per host, in its host_vars. +# The network the node's k3s container runs on. bridge (the default) is Compose's own bridge network, behind the Docker host's NAT, with the API server's port published. host puts the container on the Docker host's own network (on macOS, the Docker VM's, which then needs an address of its own on the LAN; see the role README's Networking section): its Tailscale client then offers the host's own addresses to its peers, so nodes that share a network connect directly instead of through a relay, and fall back to a relay or a hole-punched path when they do not. Nothing k3s uses changes (node IP, etcd peer URLs and certificate names stay on the mesh; only github_runner_cluster_etcd_peers_over_lan moves etcd's onto the LAN, and it needs host), so a host can move between the two with a container recreate. With host, k3s's ports, routes and firewall rules (kube-proxy, flannel, Tailscale's own and the entrypoint's unreachable route for the mesh ranges) live in the Docker host's namespace, shared with every other container on the host network there, and the API server (6443) and kubelet (10250) answer on every address the host has, its LAN address included. One node per Docker host: the role refuses host mode where the host network already has a Tailscale device that is not the node's own, as on a Linux machine running its own Tailscale client with no VM between, or beside a second node on the same Docker host (see the role README's Networking section). It also refuses host mode while the Docker engine runs any container that is not part of the node's own Compose project, since each would get a route into the mesh: the engine must be dedicated to the node. Per host, in its host_vars. github_runner_cluster_network_mode: bridge # Required with github_runner_cluster_network_mode host: the LAN the nodes share, in CIDR notation (for example 192.0.2.0/24). Before starting the node the role checks that the Docker host's own network (on macOS, the Docker VM's) has an address in it, and fails without changing anything when it has none, since a node on the host network of a VM without a LAN address gains nothing and still loses its published port. The address is read with `ip` from a throwaway container on the host network, run from github_runner_cluster_network_probe_image. github_runner_cluster_lan_cidr: "" @@ -176,13 +212,14 @@ github_runner_cluster_network_probe_image: "busybox:1.37" # Extra Compose files merged after docker-compose.yml for this host's k3s project, as paths relative to github_runner_cluster_dir or absolute. Nothing needs them in normal use; the mesh integration tests use one to put several nodes on one Docker host. github_runner_cluster_compose_files: [] -# Derived, never set directly: every Compose file for this host's k3s project, in merge order: docker-compose.yml, then the override the headscale_in_cluster provider writes for the bootstrap server, then docker-compose.api-port.yml when github_runner_cluster_api_port is not the default, then docker-compose.node-labels.yml when the host has node labels, then docker-compose.fleet-role.yml when the host takes part in the role controller (github_runner_cluster_publishes_node_labels, the same hosts), then github_runner_cluster_compose_files, then, with github_runner_cluster_network_mode host, docker-compose.host-network.yml, last so that its emptied port list wins over any file that publishes one. With no files beyond docker-compose.yml the role passes none at all and Compose reads it by itself, exactly as a plain `docker compose up` in github_runner_cluster_dir would. +# Derived, never set directly: every Compose file for this host's k3s project, in merge order: docker-compose.yml, then the override the headscale_in_cluster provider writes for the bootstrap server, then docker-compose.api-port.yml when github_runner_cluster_api_port is not the default, then docker-compose.node-labels.yml when the host has node labels, then docker-compose.fleet-role.yml when the host takes part in the role controller (github_runner_cluster_publishes_node_labels, the same hosts), then docker-compose.etcd-lan.yml when the host sets github_runner_cluster_etcd_lan_address, then github_runner_cluster_compose_files, then, with github_runner_cluster_network_mode host, docker-compose.host-network.yml, last so that its emptied port list wins over any file that publishes one. With no files beyond docker-compose.yml the role passes none at all and Compose reads it by itself, exactly as a plain `docker compose up` in github_runner_cluster_dir would. github_runner_cluster_compose_file_list: >- {{ ['docker-compose.yml'] + (['docker-compose.mesh.yml'] if github_runner_cluster_mesh == 'headscale_in_cluster' and github_runner_cluster_effective_bootstrap | default(false) | bool else []) + (['docker-compose.api-port.yml'] if github_runner_cluster_api_port | int != github_runner_cluster_k3s_default_api_port | int else []) + (['docker-compose.node-labels.yml'] if github_runner_cluster_node_labels | length > 0 else []) + (['docker-compose.fleet-role.yml'] if github_runner_cluster_publishes_node_labels | bool else []) + + (['docker-compose.etcd-lan.yml'] if github_runner_cluster_etcd_lan_address | trim | length > 0 else []) + github_runner_cluster_compose_files + (['docker-compose.host-network.yml'] if github_runner_cluster_network_mode == 'host' else []) }} # Derived, never set directly: the Compose files this role writes into github_runner_cluster_dir itself, by the name the list above gives them. A user's own entry in github_runner_cluster_compose_files is not among them, and a missing one of those is an error in a check run as in a real one. @@ -191,6 +228,7 @@ github_runner_cluster_compose_files_written: - docker-compose.api-port.yml - docker-compose.node-labels.yml - docker-compose.fleet-role.yml + - docker-compose.etcd-lan.yml - docker-compose.host-network.yml - docker-compose.mesh.yml # Derived, never set directly: the Compose files a Compose command run now can name. Every file in the list above in a real run, since the run has written the ones it owns before any Compose command; under --check, which writes nothing, tasks/compose_files_in_use.yml leaves out the ones this role would write and the host does not have yet. Compose refuses a command that names a file that is not there, and a check run that fails on one fails the README's promise that --check runs before every deploy. diff --git a/roles/github_runner_cluster/files/docker-compose.etcd-lan.yml b/roles/github_runner_cluster/files/docker-compose.etcd-lan.yml new file mode 100644 index 00000000..e24e4f56 --- /dev/null +++ b/roles/github_runner_cluster/files/docker-compose.etcd-lan.yml @@ -0,0 +1,7 @@ +# Merged after docker-compose.yml by the github_runner_cluster role when github_runner_cluster_etcd_lan_address is set for the host (see github_runner_cluster_compose_file_list in defaults/main.yml): tells the server's entrypoint the host's address on the LAN, which it adds to the certificates' names, and, once the host's etcd member has moved onto that address, the address k3s is to start etcd on (see "etcd peers over the LAN" in the role README). A separate file rather than entries in docker-compose.yml, because any entry there, even an empty one, changes the configuration hash Compose compares and so would recreate every running server when the collection is upgraded. The agent runs no etcd and takes neither. +services: + k3s: + environment: + - K3S_ETCD_LAN_ADDRESS=${K3S_ETCD_LAN_ADDRESS:?K3S_ETCD_LAN_ADDRESS must be set in .env - this host's own address on the LAN the servers share} + # Empty while the host's etcd member is on the mesh address. + - K3S_ETCD_PEER_ADDRESS=${K3S_ETCD_PEER_ADDRESS:-} diff --git a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh index 3893ad9a..5d961acb 100755 --- a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh +++ b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/sh # Entry point for the k3s and k3s-agent services in docker-compose.yml: starts Tailscale inside the node's own container, then k3s with its --vpn-auth integration, so each node joins the mesh as its own device. Bind-mounted from the Compose project directory the role manages (github_runner_cluster_dir) rather than baked into the k3s image, so a host picks up a change to it with the Compose file that uses it, without rebuilding an image Compose would not otherwise rebuild. # -# Usage: node-entrypoint.sh server|agent. Reads, from the container's environment (docker-compose.yml, from .env): K3S_VPN_AUTH_JOIN_KEY, K3S_VPN_AUTH_CONTROL_SERVER_URL (empty for Tailscale's own control plane, a Headscale URL otherwise), K3S_JOIN_SERVER_URL, K3S_TLS_SAN_LIST, K3S_DATASTORE, K3S_API_PORT, K3S_ETCD_HEARTBEAT_INTERVAL and K3S_ETCD_ELECTION_TIMEOUT (server), and K3S_MESH_SELF_HOSTED_HEALTH_URL (the bootstrap server of a cluster that runs its own Headscale, see supervise_self_hosted below); and, when docker-compose.fleet-role.yml mounts it, the role directory's `role` and `join-url` (fleet_role_dir below). +# Usage: node-entrypoint.sh server|agent. Reads, from the container's environment (docker-compose.yml, from .env): K3S_VPN_AUTH_JOIN_KEY, K3S_VPN_AUTH_CONTROL_SERVER_URL (empty for Tailscale's own control plane, a Headscale URL otherwise), K3S_JOIN_SERVER_URL, K3S_TLS_SAN_LIST, K3S_DATASTORE, K3S_API_PORT, K3S_ETCD_HEARTBEAT_INTERVAL and K3S_ETCD_ELECTION_TIMEOUT (server), K3S_ETCD_LAN_ADDRESS and K3S_ETCD_PEER_ADDRESS (server, both unset unless docker-compose.etcd-lan.yml passes them), and K3S_MESH_SELF_HOSTED_HEALTH_URL (the bootstrap server of a cluster that runs its own Headscale, see supervise_self_hosted below); and, when docker-compose.fleet-role.yml mounts it, the role directory's `role` and `join-url` (fleet_role_dir below). # No set -e: a failed `tailscale up` falls through to waiting for an address, as it always has, rather than exiting and restarting the container. set -u @@ -210,6 +210,10 @@ install_tailscale_up_wait() { # # --disable-network-policy: this fleet has no use for Kubernetes NetworkPolicy, and kube-router's enforcement chains sit directly in the FORWARD path every cross-node pod packet takes. # +# --node-ip is K3S_ETCD_PEER_ADDRESS when set (docker-compose.etcd-lan.yml passes it, for a server whose etcd moved onto its LAN address; see "etcd peers over the LAN" in the role README), and otherwise the mesh IP. With --vpn-auth, k3s takes only etcd's address and the certificates' extra names from it: the kubelet's node IP is replaced by the mesh address ("VPN provider overrides configured node-ip parameter", pkg/agent/config/config.go) and so is the API server's advertise address (pkg/cli/server/server.go), which k3s also adds to every certificate's names, so the node, flannel and the API stay on the mesh. +# +# --tls-san K3S_ETCD_LAN_ADDRESS, when set, puts the LAN address into the certificates k3s writes at start, the etcd peer certificate among them (genETCDCerts and fieldsChanged in pkg/daemons/control/deps/deps.go, which regenerate a certificate missing a name). etcd checks a peer's certificate against the address its connection comes from (checkCertSAN in client/pkg/transport/listener_tls.go), and a server that dials a peer's LAN address does so from its own LAN address, so every server needs its own in its certificate before any server's etcd moves onto the LAN. +# # --node-ip forces etcd's peer-advertise address onto this node's mesh IP. k3s builds etcd's --initial-cluster from config.PrivateIP, which is populated from --node-ip before --vpn-auth's own executor code rebuilds NodeIP from its Tailscale detection (traced through pkg/cli/server/server.go and pkg/executor/embed/embed.go). Without it PrivateIP stays the container's bridge IP, which no other machine can reach, and joining a second server failed with "MemberAdd request timed out". An --etcd-arg override is not a substitute: it also applies to the temporary etcd k3s starts on every restart to reconcile with its datastore, which has no TLS certificates for a 0.0.0.0:2380 listener and failed with "cannot listen on TLS for [::]:2380: KeyFile and CertFile are not presented". # # --cluster-init (only when join_url, set by set_join_url above, is empty, and K3S_DATASTORE is not sqlite) bootstraps embedded etcd, or converts an existing single-node SQLite datastore to it, the one migration k3s documents; once initialised, k3s ignores it on restart, so it stays in place permanently. refuse_second_cluster above has already stopped a former member with no join URL, so here it is a first bootstrap or a datastore k3s reuses. A host with a join URL passes --server instead and joins as another control-plane/etcd member. @@ -225,14 +229,14 @@ mesh_server_args() { set -- --disable=traefik --disable-network-policy --write-kubeconfig-mode=644 \ --vpn-auth="$vpn_auth" \ --resolv-conf=/etc/k3s-resolv.conf \ - --node-ip="$mesh_ip" + --node-ip="${K3S_ETCD_PEER_ADDRESS:-$mesh_ip}" for flag in $api_port_flags; do set -- "$@" "$flag"; done if [ -n "$join_url" ]; then set -- "$@" --server "$join_url" elif [ "${K3S_DATASTORE:-etcd}" != sqlite ]; then set -- "$@" --cluster-init fi - for san in ${K3S_TLS_SAN_LIST:-}; do set -- "$@" --tls-san "$san"; done + for san in ${K3S_TLS_SAN_LIST:-} ${K3S_ETCD_LAN_ADDRESS:-}; do set -- "$@" --tls-san "$san"; done for flag in $etcd_timing_flags; do set -- "$@" "$flag"; done for flag in $regrade_flags; do set -- "$@" "$flag"; done server_args="$*" diff --git a/roles/github_runner_cluster/tasks/etcd_lan.yml b/roles/github_runner_cluster/tasks/etcd_lan.yml new file mode 100644 index 00000000..c4d1bdf4 --- /dev/null +++ b/roles/github_runner_cluster/tasks/etcd_lan.yml @@ -0,0 +1,169 @@ +--- +# Decides, before the role changes anything on the host, where this host's etcd member is to be reached: its mesh address, as always, or its LAN address (github_runner_cluster_etcd_lan_address, moved onto by github_runner_cluster_etcd_peers_over_lan; see "etcd peers over the LAN" in the role README). Sets github_runner_cluster_etcd_peer_on_lan, which env.j2 reads, and github_runner_cluster_etcd_lan_switch, which a server's turn (tasks/start_server.yml) acts on: empty, or to_lan or to_mesh for the one server of the play whose running member moves in this run. With no address set it reads nothing but whether the host's .env already starts etcd on a LAN address, so a host that never sets one runs exactly as before. + +- name: Note what this host asks of etcd's addresses + ansible.builtin.set_fact: + github_runner_cluster_etcd_lan_wanted: "{{ github_runner_cluster_etcd_lan_address | trim | length > 0 }}" + github_runner_cluster_etcd_peer_on_lan: false + github_runner_cluster_etcd_lan_switch: "" + github_runner_cluster_etcd_lan_candidate: "" + +- name: Read whether this host's .env starts etcd on a LAN address + # Only when no address is set: the address is what the role needs to move a member back, so removing it while the member is on the LAN would start k3s on its mesh address, which etcd no longer lists for it ("membershipError" in k3s's log), and the server would not come back. grep -c prints a count and nothing of the file, which holds secrets. + ansible.builtin.shell: | + set -eu + if [ -e "$env_file" ]; then grep -c '^K3S_ETCD_PEER_ADDRESS=.' "$env_file" || true; else echo 0; fi + environment: + env_file: "{{ github_runner_cluster_dir }}/.env" + register: github_runner_cluster_etcd_lan_left + changed_when: false + check_mode: false + when: not github_runner_cluster_etcd_lan_wanted | bool + +- name: Fail if the LAN address was removed while etcd still uses it + ansible.builtin.fail: + msg: >- + {{ inventory_hostname }}'s .env starts etcd on its LAN address, but github_runner_cluster_etcd_lan_address is no longer set. + Started without it, k3s would look for its member under the mesh address, which etcd no longer lists, and the server would not + come back. Set the address again with github_runner_cluster_etcd_peers_over_lan false, and run once per server to move each + member back to its mesh address; then remove the address. Nothing was changed. + when: + - not github_runner_cluster_etcd_lan_wanted | bool + - (github_runner_cluster_etcd_lan_left.stdout | trim | int) > 0 + +- name: Check etcd's LAN addresses and decide this host's + when: github_runner_cluster_etcd_lan_wanted | bool or github_runner_cluster_etcd_peers_over_lan | bool + block: + - name: Fail if the inventory's LAN addresses cannot carry etcd's peer traffic + vars: + github_runner_cluster_etcd_lan_hosts: >- + {% set rows = [] %}{% for name, host in github_runner_cluster_topology.hosts.items() %}{% set settings = hostvars[name] %}{% set _ = rows.append({ + 'name': name, 'node_name': host.node_name, 'role': host.role, + 'eligible': settings.github_runner_cluster_voter_eligible | default(false) | bool, + 'node_mode': settings.github_runner_cluster_node_mode | default('compose'), + 'network_mode': settings.github_runner_cluster_network_mode | default('bridge'), + 'lan_cidr': settings.github_runner_cluster_lan_cidr | default(''), + 'mesh': settings.github_runner_cluster_mesh | default(github_runner_cluster_mesh), + 'address': settings.github_runner_cluster_etcd_lan_address | default(''), + 'peers_over_lan': settings.github_runner_cluster_etcd_peers_over_lan | default(false) | bool}) %}{% endfor %}{{ rows }} + github_runner_cluster_etcd_lan_problems: >- + {{ github_runner_cluster_etcd_lan_hosts | exadev.github_runner.cluster_etcd_lan_problems(github_runner_cluster_etcd_peers_over_lan | bool) }} + ansible.builtin.fail: + msg: "Not changing etcd's addresses: {{ github_runner_cluster_etcd_lan_problems | join(' ') }} Nothing was changed." + when: github_runner_cluster_etcd_lan_problems | length > 0 + + - name: Fail if the LAN address is not one of the Docker host's own + # host_network.yml read the Docker host's addresses from a throwaway container on its network, which is the namespace etcd listens in on the host network; they are parsed here as its own LAN check parses them. + vars: + github_runner_cluster_etcd_lan_own: "{{ github_runner_cluster_host_network.stdout.split('\n--\n') | first | split('\n') | exadev.github_runner.addresses_in_network(github_runner_cluster_lan_cidr) }}" + ansible.builtin.fail: + msg: >- + github_runner_cluster_etcd_lan_address is {{ github_runner_cluster_etcd_lan_address | trim }} for {{ inventory_hostname }}, which is not + an address of the Docker host its node runs on (its addresses in {{ github_runner_cluster_lan_cidr }}: {{ github_runner_cluster_etcd_lan_own | join(', ') or 'none' }}), + so etcd could not listen on it. Set it to one of those. Nothing was changed. + when: + - github_runner_cluster_etcd_lan_wanted | bool + - (github_runner_cluster_etcd_lan_address | trim) not in github_runner_cluster_etcd_lan_own + + - name: Read etcd's members, through this server's container and then every other server's + ansible.builtin.include_tasks: etcd_members_read.yml + when: github_runner_cluster_etcd_lan_wanted | bool + + - name: Work out where this host's member is and where it is to be + vars: + github_runner_cluster_etcd_lan_found: >- + {{ (github_runner_cluster_etcd_members | exadev.github_runner.cluster_etcd_member_of(github_runner_cluster_effective_node_name)) + if github_runner_cluster_etcd_members is not none else none }} + ansible.builtin.set_fact: + github_runner_cluster_etcd_lan_member: "{{ github_runner_cluster_etcd_lan_found }}" + github_runner_cluster_etcd_lan_live_on_lan: >- + {{ github_runner_cluster_etcd_lan_found is not none + and (github_runner_cluster_etcd_lan_found.peer_hosts | first | default('')) == (github_runner_cluster_etcd_lan_address | trim) }} + when: github_runner_cluster_etcd_lan_wanted | bool + + - name: Fail if etcd cannot be read while it may have a member for this host + # A host none of whose servers answers may still have a member, on either address, and starting it on the wrong one would keep it out of etcd. A host that has never run a server (no datastore on its data volume) has none to find. + ansible.builtin.fail: + msg: >- + Not deciding {{ inventory_hostname }}'s etcd address: etcd's member list could not be read through any server + ({{ github_runner_cluster_etcd_members_problem }}), so whether {{ github_runner_cluster_effective_node_name }} already has a member, + and on which address, cannot be told. Bring a server back and run again. Nothing was changed. + when: + - github_runner_cluster_etcd_lan_wanted | bool + - github_runner_cluster_etcd_members is none + - github_runner_cluster_etcd_members_needed | bool + + - name: Fail if this host's member is stopped where its address would change + # Moving a member's address is an update in etcd and a restart of the member on the new one; a stopped container would start on the address .env gives it while etcd still lists the other. + ansible.builtin.fail: + msg: >- + {{ inventory_hostname }}'s etcd member is on its {{ 'LAN' if github_runner_cluster_etcd_lan_live_on_lan | bool else 'mesh' }} address and + github_runner_cluster_etcd_peers_over_lan asks for the {{ 'LAN' if github_runner_cluster_etcd_peers_over_lan | bool else 'mesh' }} one, + but its k3s container is not running. Start it as it is (`docker compose up -d` in {{ github_runner_cluster_dir }}, with no change to + .env) and run again. Nothing was changed. + when: + - github_runner_cluster_etcd_lan_wanted | bool + - github_runner_cluster_etcd_lan_member is not none + - github_runner_cluster_etcd_lan_live_on_lan | bool != github_runner_cluster_etcd_peers_over_lan | bool + - github_runner_cluster_docker_role == 'server' + - not github_runner_cluster_network_running | bool + + - name: Note the move this host's member would make + ansible.builtin.set_fact: + github_runner_cluster_etcd_lan_candidate: >- + {{ '' if not (github_runner_cluster_etcd_lan_wanted | bool) or github_runner_cluster_etcd_lan_member is none + or github_runner_cluster_etcd_lan_live_on_lan | bool == github_runner_cluster_etcd_peers_over_lan | bool + else ('to_lan' if github_runner_cluster_etcd_peers_over_lan | bool else 'to_mesh') }} + + - name: Read this host's mesh address, which a member moving back to the mesh returns to + # The address k3s's own --node-ip took before the move (node-entrypoint.sh's mesh_ip), from the container's own Tailscale client. + ansible.builtin.command: + argv: [docker, exec, "{{ github_runner_cluster_container_name }}", tailscale, ip, "-4"] + environment: "{{ github_runner_cluster_docker_cli_environment }}" + register: github_runner_cluster_etcd_lan_mesh_ip + changed_when: false + check_mode: false + when: github_runner_cluster_etcd_lan_candidate == 'to_mesh' + + - name: Note the address this host's member would move to + ansible.builtin.set_fact: + github_runner_cluster_etcd_lan_target: >- + {{ github_runner_cluster_etcd_lan_address | trim if github_runner_cluster_etcd_lan_candidate == 'to_lan' + else github_runner_cluster_etcd_lan_mesh_ip.stdout | trim }} + when: github_runner_cluster_etcd_lan_candidate | length > 0 + + - name: Check every other server is ready for this member to move onto its LAN address + ansible.builtin.include_tasks: etcd_lan_ready.yml + vars: + github_runner_cluster_etcd_lan_others: >- + {{ github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') + | map(attribute='key') | reject('equalto', inventory_hostname) | list }} + when: github_runner_cluster_etcd_lan_candidate == 'to_lan' + +- name: Choose the one member of the play that moves in this run + # Each host has noted its own move above, which every host's facts now hold; the first in play order moves, and the others are reported and wait for a later run. + vars: + github_runner_cluster_etcd_lan_movers: >- + {% set rows = [] %}{% for host in ansible_play_hosts %}{% set _ = rows.append({'name': host, 'switch': hostvars[host].github_runner_cluster_etcd_lan_candidate | default('')}) %}{% endfor %}{{ rows }} + ansible.builtin.set_fact: + github_runner_cluster_etcd_lan_switch: >- + {{ github_runner_cluster_etcd_lan_candidate + if inventory_hostname in (github_runner_cluster_etcd_lan_movers | exadev.github_runner.cluster_etcd_lan_switches) else '' }} + +- name: Report a member that waits for a later run + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }}'s etcd member would move to its {{ 'LAN' if github_runner_cluster_etcd_lan_candidate == 'to_lan' else 'mesh' }} address, + but another server's member moves in this run, and a member's move stops it until its container is back. Run again to move this one. + when: + - github_runner_cluster_etcd_lan_candidate | length > 0 + - github_runner_cluster_etcd_lan_switch | length == 0 + +- name: Note whether this host's k3s starts etcd on its LAN address + # A running member keeps the address etcd lists for it unless it moves in this run. A host with no member (an agent eligible for a vote, a server that has not joined yet) starts on the address the group asks for, since it joins with it. + ansible.builtin.set_fact: + github_runner_cluster_etcd_peer_on_lan: >- + {{ github_runner_cluster_etcd_lan_wanted | bool + and ((github_runner_cluster_etcd_lan_switch == 'to_lan') + or (github_runner_cluster_etcd_lan_switch == '' and github_runner_cluster_etcd_lan_member is not none and github_runner_cluster_etcd_lan_live_on_lan | bool) + or (github_runner_cluster_etcd_lan_member is none and github_runner_cluster_etcd_peers_over_lan | bool)) }} diff --git a/roles/github_runner_cluster/tasks/etcd_lan_move.yml b/roles/github_runner_cluster/tasks/etcd_lan_move.yml new file mode 100644 index 00000000..7a292e95 --- /dev/null +++ b/roles/github_runner_cluster/tasks/etcd_lan_move.yml @@ -0,0 +1,134 @@ +--- +# Moves this server's etcd member to its LAN address (github_runner_cluster_etcd_lan_switch to_lan) or back to its mesh address (to_mesh), in its turn of starting the servers (tasks/start_server.yml), which has already refused unless every other server is Ready. etcd keeps each member's peer URL in its membership, and k3s rewrites it only for a cluster of one member (pkg/etcd/etcd.go, "failed to update peer url: etcd still has more than one member"), so the move is the one etcd documents for a member of a running cluster: `etcdctl member update` to the new URL, then the member restarted listening on it, which k3s does when it starts with --node-ip at the new address (.env's K3S_ETCD_PEER_ADDRESS, which this run has already written). Between the two the other members dial the new URL, where nothing listens yet, so the member is away for as long as its container's recreate, as any recreated server is. +# +# Before the update: every member reports healthy, and no server logged an etcd read that could not reach a quorum in the last github_runner_cluster_etcd_lan_quiet_minutes. After the recreate: the node is Ready on a heartbeat from the new container (tasks/server_ready.yml), etcd lists the member at the new address, every member reports healthy, and no server logs such a read over the next github_runner_cluster_etcd_lan_settle_seconds. Anything that fails moves the member back to the address it had, starts the server there again, waits for it, and fails the run. + +- name: Report the move under --check + ansible.builtin.debug: + msg: >- + Would move {{ inventory_hostname }}'s etcd member {{ github_runner_cluster_etcd_lan_member.name }} from + https://{{ github_runner_cluster_etcd_lan_member.peer_hosts | first }}:2380 to https://{{ github_runner_cluster_etcd_lan_target }}:2380 and recreate its + container to listen there. + changed_when: true + when: ansible_check_mode + +- name: Move the member, start the server on its new address and check the cluster + when: not ansible_check_mode + vars: + github_runner_cluster_etcd_lan_poll_seconds: 10 + github_runner_cluster_etcd_lan_poll_retries: "{{ ((github_runner_cluster_server_ready_timeout | int) / 10) | round(0, 'ceil') | int }}" + block: + - name: Read the health of every etcd member before the move + ansible.builtin.command: + argv: "{{ github_runner_cluster_etcdctl_argv + ['endpoint', 'health', '--cluster', '--write-out=json'] }}" + environment: "{{ github_runner_cluster_docker_cli_environment }}" + register: github_runner_cluster_etcd_lan_health_before + changed_when: false + failed_when: >- + github_runner_cluster_etcd_lan_health_before.rc != 0 + or (github_runner_cluster_etcd_lan_health_before.stdout | from_json | rejectattr('health') | list | length > 0) + + - name: Read every server's log for etcd reads that could not reach a quorum + ansible.builtin.include_tasks: etcd_lan_quiet.yml + vars: + github_runner_cluster_etcd_lan_since: "{{ github_runner_cluster_etcd_lan_quiet_minutes | int }}m" + + - name: Update the member's peer URL in etcd + ansible.builtin.command: + argv: "{{ github_runner_cluster_etcdctl_argv + ['member', 'update', github_runner_cluster_etcd_lan_member.id, '--peer-urls=https://' ~ github_runner_cluster_etcd_lan_target ~ ':2380'] }}" + environment: "{{ github_runner_cluster_docker_cli_environment }}" + changed_when: true + + - name: Note that etcd lists the member at its new address + ansible.builtin.set_fact: + github_runner_cluster_etcd_lan_updated: true + + - name: Recreate the server with k3s on the new address + ansible.builtin.include_tasks: server_compose_up.yml + + - name: Wait for the server's node to report Ready from the new container + ansible.builtin.include_tasks: server_ready.yml + + - name: Read etcd's member list until it lists this member at its new address + ansible.builtin.command: + argv: "{{ github_runner_cluster_etcdctl_argv + ['member', 'list', '--write-out=json'] }}" + environment: "{{ github_runner_cluster_docker_cli_environment }}" + register: github_runner_cluster_etcd_lan_members_after + changed_when: false + until: >- + github_runner_cluster_etcd_lan_members_after.rc == 0 + and (github_runner_cluster_etcd_lan_members_after.stdout | exadev.github_runner.cluster_etcd_members + | exadev.github_runner.cluster_etcd_member_of(github_runner_cluster_effective_node_name)).peer_hosts == [github_runner_cluster_etcd_lan_target] + retries: "{{ github_runner_cluster_etcd_lan_poll_retries }}" + delay: "{{ github_runner_cluster_etcd_lan_poll_seconds }}" + + - name: Read the health of every etcd member until all are healthy + ansible.builtin.command: + argv: "{{ github_runner_cluster_etcdctl_argv + ['endpoint', 'health', '--cluster', '--write-out=json'] }}" + environment: "{{ github_runner_cluster_docker_cli_environment }}" + register: github_runner_cluster_etcd_lan_health_after + changed_when: false + failed_when: false + until: >- + github_runner_cluster_etcd_lan_health_after.rc == 0 + and (github_runner_cluster_etcd_lan_health_after.stdout | from_json | rejectattr('health') | list | length == 0) + retries: "{{ github_runner_cluster_etcd_lan_poll_retries }}" + delay: "{{ github_runner_cluster_etcd_lan_poll_seconds }}" + + - name: Fail if a member is not healthy after the move + ansible.builtin.fail: + msg: "etcd's members are not all healthy after the move: {{ github_runner_cluster_etcd_lan_health_after.stdout | default('') }} {{ github_runner_cluster_etcd_lan_health_after.stderr | default('') }}" + when: >- + github_runner_cluster_etcd_lan_health_after.rc != 0 + or (github_runner_cluster_etcd_lan_health_after.stdout | from_json | rejectattr('health') | list | length > 0) + + - name: Read the time on this host, from which the logs are watched + ansible.builtin.command: date -u +%Y-%m-%dT%H:%M:%SZ + register: github_runner_cluster_etcd_lan_settled_from + changed_when: false + + - name: Watch the cluster settle with the member on its new address + ansible.builtin.wait_for: + timeout: "{{ github_runner_cluster_etcd_lan_settle_seconds | int }}" + + - name: Read every server's log since then for etcd reads that could not reach a quorum + ansible.builtin.include_tasks: etcd_lan_quiet.yml + vars: + github_runner_cluster_etcd_lan_since: "{{ github_runner_cluster_etcd_lan_settled_from.stdout | trim }}" + + rescue: + - name: Keep what failed, for the message below + ansible.builtin.set_fact: + github_runner_cluster_etcd_lan_failure: "{{ ansible_failed_task.name }}: {{ ansible_failed_result.msg | default(ansible_failed_result.stderr | default('')) }}" + + - name: Write .env as it was before the move + # This run wrote .env for the new address before the server's turn; the container still runs on the old one unless the move got as far as recreating it. + ansible.builtin.template: + src: env.j2 + dest: "{{ github_runner_cluster_dir }}/.env" + mode: "0600" + vars: + github_runner_cluster_etcd_peer_on_lan: "{{ github_runner_cluster_etcd_lan_switch == 'to_mesh' }}" + no_log: true + + - name: Move the member back to the address it had, and start the server there again + when: github_runner_cluster_etcd_lan_updated | default(false) | bool + block: + - name: Update the member's peer URL back, through this server or any other + ansible.builtin.include_tasks: etcd_lan_update_back.yml + vars: + github_runner_cluster_etcd_lan_back: "--peer-urls=https://{{ github_runner_cluster_etcd_lan_member.peer_hosts | first }}:2380" + + - name: Recreate the server on the address it had + ansible.builtin.include_tasks: server_compose_up.yml + + - name: Wait for the server's node to report Ready again + ansible.builtin.include_tasks: server_ready.yml + + - name: Fail the run, naming what went wrong + ansible.builtin.fail: + msg: >- + Moving {{ inventory_hostname }}'s etcd member to https://{{ github_runner_cluster_etcd_lan_target }}:2380 failed at + {{ github_runner_cluster_etcd_lan_failure }}. + {% if github_runner_cluster_etcd_lan_updated | default(false) | bool %}The member was moved back to + https://{{ github_runner_cluster_etcd_lan_member.peer_hosts | first }}:2380 and the server started there again.{% else %}etcd was not changed, and .env was written back as it was.{% endif %} diff --git a/roles/github_runner_cluster/tasks/etcd_lan_quiet.yml b/roles/github_runner_cluster/tasks/etcd_lan_quiet.yml new file mode 100644 index 00000000..d923aaec --- /dev/null +++ b/roles/github_runner_cluster/tasks/etcd_lan_quiet.yml @@ -0,0 +1,39 @@ +--- +# Fails unless no server of the inventory logged, since github_runner_cluster_etcd_lan_since (a duration such as 10m, or a UTC timestamp), an etcd read that could not reach a quorum: one of github_runner_cluster_etcd_read_index_messages, which etcd logs when a member cannot get a read index from its leader in time, the symptom of a voter cut off from its peers. Each server's k3s container log is read on its own host, with its pinned Docker context, and only the matching lines leave it. A server whose log cannot be read fails the check too, since its etcd cannot be vouched for. + +- name: Read each server's log for etcd reads that could not reach a quorum + ansible.builtin.shell: | + set -euo pipefail + docker logs --since "$since" "$container" > /dev/null + docker logs --since "$since" "$container" 2>&1 | grep -F {% for message in github_runner_cluster_etcd_read_index_messages %}-e {{ message | quote }} {% endfor %}|| true + args: + executable: /bin/bash + environment: >- + {{ ({'DOCKER_CONTEXT': github_runner_cluster_etcd_quiet_context} if github_runner_cluster_etcd_quiet_context | length > 0 else {}) + | combine({'container': github_runner_cluster_etcd_quiet_container, 'since': github_runner_cluster_etcd_lan_since}) }} + delegate_to: "{{ item }}" + loop: >- + {{ [inventory_hostname] + (github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') + | map(attribute='key') | reject('equalto', inventory_hostname) | list) }} + vars: + github_runner_cluster_etcd_quiet_container: >- + {{ github_runner_cluster_container_name if item == inventory_hostname + else hostvars[item].github_runner_cluster_container_name | default('github-runner-k3s') }} + github_runner_cluster_etcd_quiet_context: >- + {{ (github_runner_cluster_docker_context | default('') if item == inventory_hostname + else hostvars[item].github_runner_cluster_docker_context | default('')) | trim }} + register: github_runner_cluster_etcd_quiet_reads + changed_when: false + failed_when: false + ignore_unreachable: true + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + +- name: Fail if a server logged such a read, or its log could not be read + vars: + github_runner_cluster_etcd_quiet_problems: >- + {% set rows = [] %}{% for read in github_runner_cluster_etcd_quiet_reads.results %}{% if read.rc is not defined or read.rc != 0 %}{% set _ = rows.append(read.item ~ "'s log could not be read") %}{% elif read.stdout | trim | length > 0 %}{% set _ = rows.append(read.item ~ ' logged ' ~ (read.stdout_lines | length) ~ ' (last: ' ~ (read.stdout_lines | last | truncate(300)) ~ ')') %}{% endif %}{% endfor %}{{ rows }} + ansible.builtin.fail: + msg: >- + etcd reads could not reach a quorum since {{ github_runner_cluster_etcd_lan_since }}: {{ github_runner_cluster_etcd_quiet_problems | join('; ') }}. + when: github_runner_cluster_etcd_quiet_problems | length > 0 diff --git a/roles/github_runner_cluster/tasks/etcd_lan_ready.yml b/roles/github_runner_cluster/tasks/etcd_lan_ready.yml new file mode 100644 index 00000000..86ce4dfd --- /dev/null +++ b/roles/github_runner_cluster/tasks/etcd_lan_ready.yml @@ -0,0 +1,65 @@ +--- +# Before this host's member may move onto its LAN address (tasks/etcd_lan.yml, github_runner_cluster_etcd_lan_candidate to_lan), on every other server of the inventory, on its own host: its live etcd peer certificate must name that server's own LAN address, since once this member listens on the LAN the others dial it from their LAN addresses and etcd checks each peer's certificate against where its connection comes from (checkCertSAN in etcd's client/pkg/transport/listener_tls.go); and it must reach this host's LAN address, from a throwaway container on its host network, which is the namespace its etcd dials from. A certificate that lacks its name leaves this member where it is until a later run (the run that sets an address recreates that server with the name); a server that does not reach the address leaves this member on the mesh, the fallback, and the run says so. The certificate is public; openssl on that host prints its names. + +- name: Read the IP addresses each other server's etcd peer certificate names + ansible.builtin.shell: | + set -euo pipefail + docker exec "$container" cat /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt | openssl x509 -noout -text | grep -Eo 'IP Address:[0-9.]+' | sed 's/^IP Address://' + args: + executable: /bin/bash + environment: >- + {{ ({'DOCKER_CONTEXT': hostvars[item].github_runner_cluster_docker_context} if hostvars[item].github_runner_cluster_docker_context | default('') | length > 0 else {}) + | combine({'container': hostvars[item].github_runner_cluster_container_name | default('github-runner-k3s')}) }} + delegate_to: "{{ item }}" + loop: "{{ github_runner_cluster_etcd_lan_others }}" + register: github_runner_cluster_etcd_lan_certificates + changed_when: false + failed_when: false + ignore_unreachable: true + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + +- name: Ask each other server whether it reaches this host's LAN address + ansible.builtin.command: + argv: [docker, run, --rm, --network, host, "{{ github_runner_cluster_network_probe_image }}", ping, -c, "3", -W, "2", "{{ github_runner_cluster_etcd_lan_address | trim }}"] + environment: >- + {{ {'DOCKER_CONTEXT': hostvars[item].github_runner_cluster_docker_context} if hostvars[item].github_runner_cluster_docker_context | default('') | length > 0 else {} }} + delegate_to: "{{ item }}" + loop: "{{ github_runner_cluster_etcd_lan_others }}" + register: github_runner_cluster_etcd_lan_reach + changed_when: false + failed_when: false + ignore_unreachable: true + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + +- name: Work out what keeps this member on the mesh + vars: + github_runner_cluster_etcd_lan_addresses: >- + {{ dict(github_runner_cluster_etcd_lan_others | zip(github_runner_cluster_etcd_lan_others | map('extract', hostvars, 'github_runner_cluster_etcd_lan_address') | map('trim'))) }} + ansible.builtin.set_fact: + github_runner_cluster_etcd_lan_readiness: >- + {{ github_runner_cluster_etcd_lan_certificates.results + | exadev.github_runner.cluster_etcd_lan_readiness(github_runner_cluster_etcd_lan_reach.results, github_runner_cluster_etcd_lan_addresses) }} + +- name: Report a member that waits for the other servers' certificates + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }}'s etcd member stays on its mesh address in this run: {{ github_runner_cluster_etcd_lan_readiness.unnamed | join(', ') }} + {{ 'does' if github_runner_cluster_etcd_lan_readiness.unnamed | length == 1 else 'do' }} not yet name {{ 'its' if github_runner_cluster_etcd_lan_readiness.unnamed | length == 1 else 'their' }} + own LAN address in the etcd peer certificate (or could not be read), and would be refused by this member's etcd once it listens on the LAN. + The run that sets a server's address recreates it with the name; run again once every server has been. + when: github_runner_cluster_etcd_lan_readiness.unnamed | length > 0 + +- name: Report a member that stays on the mesh because the LAN does not reach it + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }}'s etcd member stays on its mesh address: {{ github_runner_cluster_etcd_lan_readiness.unreached | join(', ') }} + could not reach {{ github_runner_cluster_etcd_lan_address | trim }} (three pings from {{ 'its' if github_runner_cluster_etcd_lan_readiness.unreached | length == 1 else 'their' }} + host network), so etcd's peer traffic keeps to the mesh, which reaches every server. Check the LAN and run again. + when: github_runner_cluster_etcd_lan_readiness.unreached | length > 0 + +- name: Keep this member on the mesh when another server is not ready for it + ansible.builtin.set_fact: + github_runner_cluster_etcd_lan_candidate: "" + when: (github_runner_cluster_etcd_lan_readiness.unnamed + github_runner_cluster_etcd_lan_readiness.unreached) | length > 0 diff --git a/roles/github_runner_cluster/tasks/etcd_lan_update_back.yml b/roles/github_runner_cluster/tasks/etcd_lan_update_back.yml new file mode 100644 index 00000000..1baf8c03 --- /dev/null +++ b/roles/github_runner_cluster/tasks/etcd_lan_update_back.yml @@ -0,0 +1,55 @@ +--- +# Moves this server's etcd member back to the peer URL it had before tasks/etcd_lan_move.yml moved it: through this server's own etcd, and, when that does not answer (the server did not come back), through every other server of the inventory on its own host. An update to the URL a member already has changes nothing, so asking every other server is safe. Fails unless one of them made it. + +- name: Update the member's peer URL back through this server + ansible.builtin.command: + argv: "{{ github_runner_cluster_etcdctl_argv + ['member', 'update', github_runner_cluster_etcd_lan_member.id, github_runner_cluster_etcd_lan_back] }}" + environment: "{{ github_runner_cluster_docker_cli_environment }}" + register: github_runner_cluster_etcd_lan_back_here + changed_when: github_runner_cluster_etcd_lan_back_here.rc == 0 + failed_when: false + +- name: Update it back through the other servers when this one did not answer + ansible.builtin.command: + argv: + - docker + - run + - --rm + - --network + - "container:{{ hostvars[item].github_runner_cluster_container_name | default('github-runner-k3s') }}" + - --volume + - "{{ hostvars[item].github_runner_cluster_compose_project | default('github-runner') }}_k3s-data:/var/lib/rancher/k3s:ro" + - --entrypoint + - etcdctl + - "{{ github_runner_cluster_etcdctl_image }}" + - --endpoints=https://127.0.0.1:2379 + - --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt + - --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt + - --key=/var/lib/rancher/k3s/server/tls/etcd/client.key + - "--command-timeout={{ github_runner_cluster_cluster_read_timeout | int }}s" + - member + - update + - "{{ github_runner_cluster_etcd_lan_member.id }}" + - "{{ github_runner_cluster_etcd_lan_back }}" + environment: >- + {{ {'DOCKER_CONTEXT': hostvars[item].github_runner_cluster_docker_context} if hostvars[item].github_runner_cluster_docker_context | default('') | length > 0 else {} }} + delegate_to: "{{ item }}" + loop: >- + {{ github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') + | map(attribute='key') | reject('equalto', inventory_hostname) | list }} + register: github_runner_cluster_etcd_lan_back_there + changed_when: github_runner_cluster_etcd_lan_back_there.rc | default(1) == 0 + failed_when: false + ignore_unreachable: true + when: github_runner_cluster_etcd_lan_back_here.rc != 0 + +- name: Fail if no server could move the member back + ansible.builtin.fail: + msg: >- + etcd still lists {{ inventory_hostname }}'s member {{ github_runner_cluster_etcd_lan_member.name }} at its new address: no server could update it + back to {{ github_runner_cluster_etcd_lan_back }} ({{ github_runner_cluster_etcd_lan_back_here.stderr | default('') }}). Update it by hand with + `etcdctl member update {{ github_runner_cluster_etcd_lan_member.id }} {{ github_runner_cluster_etcd_lan_back }}` through a healthy server + (the role README's etcdctl runbook) before anything else. + when: + - github_runner_cluster_etcd_lan_back_here.rc != 0 + - github_runner_cluster_etcd_lan_back_there.results | default([]) | selectattr('rc', 'defined') | selectattr('rc', 'equalto', 0) | list | length == 0 diff --git a/roles/github_runner_cluster/tasks/etcd_members_read.yml b/roles/github_runner_cluster/tasks/etcd_members_read.yml new file mode 100644 index 00000000..4fd5f57b --- /dev/null +++ b/roles/github_runner_cluster/tasks/etcd_members_read.yml @@ -0,0 +1,61 @@ +--- +# Reads etcd's member list with a throwaway etcdctl (github_runner_cluster_etcdctl_image) that shares a server's k3s container's network and reads the client certificate from its data volume, mounted read-only: through this host's container when it runs as a server, then through every other server of the inventory on its own host, and keeps the first answer. Sets github_runner_cluster_etcd_members (cluster_etcd_members' list, or none when no server answered), github_runner_cluster_etcd_members_problem (why not) and github_runner_cluster_etcd_members_needed (whether this host's k3s container exists, so that it may have a member etcd could not be asked about). Each read is bounded by github_runner_cluster_cluster_read_timeout, as a whole and as etcdctl's own command timeout. + +- name: Read etcd's member list through each server + ansible.builtin.command: + argv: + - docker + - run + - --rm + - --network + - "container:{{ github_runner_cluster_etcd_read_container }}" + - --volume + - "{{ github_runner_cluster_etcd_read_project }}_k3s-data:/var/lib/rancher/k3s:ro" + - --entrypoint + - etcdctl + - "{{ github_runner_cluster_etcdctl_image }}" + - --endpoints=https://127.0.0.1:2379 + - --cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt + - --cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt + - --key=/var/lib/rancher/k3s/server/tls/etcd/client.key + - "--command-timeout={{ github_runner_cluster_cluster_read_timeout | int }}s" + - member + - list + - --write-out=json + environment: >- + {{ {'DOCKER_CONTEXT': github_runner_cluster_etcd_read_context} if github_runner_cluster_etcd_read_context | length > 0 else {} }} + delegate_to: "{{ item }}" + loop: >- + {{ ([inventory_hostname] if github_runner_cluster_docker_role == 'server' and github_runner_cluster_network_running | bool else []) + + (github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') + | map(attribute='key') | reject('equalto', inventory_hostname) | list) }} + vars: + github_runner_cluster_etcd_read_container: >- + {{ github_runner_cluster_container_name if item == inventory_hostname + else hostvars[item].github_runner_cluster_container_name | default('github-runner-k3s') }} + github_runner_cluster_etcd_read_project: >- + {{ github_runner_cluster_compose_project if item == inventory_hostname + else hostvars[item].github_runner_cluster_compose_project | default('github-runner') }} + github_runner_cluster_etcd_read_context: >- + {{ (github_runner_cluster_docker_context | default('') if item == inventory_hostname + else hostvars[item].github_runner_cluster_docker_context | default('')) | trim }} + register: github_runner_cluster_etcd_member_reads + changed_when: false + failed_when: false + ignore_unreachable: true + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + +- name: Keep the first member list a server gave + vars: + github_runner_cluster_etcd_member_answers: >- + {{ github_runner_cluster_etcd_member_reads.results | selectattr('rc', 'defined') | selectattr('rc', 'equalto', 0) | list }} + ansible.builtin.set_fact: + github_runner_cluster_etcd_members: >- + {{ (github_runner_cluster_etcd_member_answers | first).stdout | exadev.github_runner.cluster_etcd_members + if github_runner_cluster_etcd_member_answers | length > 0 else none }} + github_runner_cluster_etcd_members_problem: >- + {{ github_runner_cluster_etcd_member_reads.results + | map(attribute='item') | zip(github_runner_cluster_etcd_member_reads.results | map(attribute='stderr', default='unreachable')) + | map('join', ': ') | join('; ') if github_runner_cluster_etcd_member_answers | length == 0 else '' }} + github_runner_cluster_etcd_members_needed: "{{ github_runner_cluster_network_node_container.stdout | trim | length > 0 }}" diff --git a/roles/github_runner_cluster/tasks/main.yml b/roles/github_runner_cluster/tasks/main.yml index 5f1de626..2c0ecc9d 100644 --- a/roles/github_runner_cluster/tasks/main.yml +++ b/roles/github_runner_cluster/tasks/main.yml @@ -90,6 +90,10 @@ # Before anything changes on the host, and so before the mesh step, which starts the bootstrap server's Compose project itself with the headscale_in_cluster provider. ansible.builtin.include_tasks: host_network.yml +- name: Decide whether this host's etcd member uses its LAN address, and which member of the play moves in this run + # After host_network.yml, whose reads of the node's container and the Docker host's addresses it uses, and before anything changes on the host or .env is templated. See tasks/etcd_lan.yml. + ansible.builtin.include_tasks: etcd_lan.yml + - name: Size this host's Colima VM from the inventory, one node at a time through the quorum guard # After the runtime is up and the topology decided, both of which the resize reads. See tasks/colima_size.yml. ansible.builtin.include_tasks: colima_size.yml @@ -130,6 +134,7 @@ - {src: docker-compose.api-port.yml, mode: "0644"} - {src: docker-compose.node-labels.yml, mode: "0644"} - {src: docker-compose.fleet-role.yml, mode: "0644"} + - {src: docker-compose.etcd-lan.yml, mode: "0644"} - {src: docker-compose.host-network.yml, mode: "0644"} - {src: k3s/Dockerfile, mode: "0644"} - {src: k3s/node-entrypoint.sh, mode: "0755"} diff --git a/roles/github_runner_cluster/tasks/server_compose_up.yml b/roles/github_runner_cluster/tasks/server_compose_up.yml new file mode 100644 index 00000000..aca29629 --- /dev/null +++ b/roles/github_runner_cluster/tasks/server_compose_up.yml @@ -0,0 +1,11 @@ +--- +# Starts this server's k3s with Docker Compose, from the files and .env on the host: recreating its container when its configuration changed, as Compose decides. tasks/start_server.yml checks first whether that is safe; tasks/etcd_lan_move.yml starts a server here after moving its etcd member, and again to move it back. + +- name: Start k3s via docker compose (bootstrapping or joining, per this host's own .env) + community.docker.docker_compose_v2: + project_src: "{{ github_runner_cluster_dir }}" + project_name: "{{ github_runner_cluster_compose_project }}" + profiles: ["server"] + cli_context: "{{ github_runner_cluster_docker_cli_context or omit }}" + files: "{{ github_runner_cluster_compose_files_in_use if github_runner_cluster_compose_files_in_use | length > 1 else omit }}" + state: present diff --git a/roles/github_runner_cluster/tasks/start_server.yml b/roles/github_runner_cluster/tasks/start_server.yml index da79db36..436e1442 100644 --- a/roles/github_runner_cluster/tasks/start_server.yml +++ b/roles/github_runner_cluster/tasks/start_server.yml @@ -22,14 +22,10 @@ - not github_runner_cluster_dir_state.stat.exists - name: Start k3s via docker compose (bootstrapping or joining, per this host's own .env) - when: not (ansible_check_mode and not github_runner_cluster_dir_state.stat.exists) - community.docker.docker_compose_v2: - project_src: "{{ github_runner_cluster_dir }}" - project_name: "{{ github_runner_cluster_compose_project }}" - profiles: ["server"] - cli_context: "{{ github_runner_cluster_docker_cli_context or omit }}" - files: "{{ github_runner_cluster_compose_files_in_use if github_runner_cluster_compose_files_in_use | length > 1 else omit }}" - state: present + ansible.builtin.include_tasks: server_compose_up.yml + when: + - not (ansible_check_mode and not github_runner_cluster_dir_state.stat.exists) + - github_runner_cluster_etcd_lan_switch == '' - name: Wait for this server's node to report Ready again after its container was recreated # Only after a recreate of a running server that other servers are registered with: a first start waits for nothing here, a bootstrap server restarted before anything has joined it has no quorum to hand back and may not be Ready until the mesh steps after it have run, and a server Compose only started was not part of the quorum. @@ -39,3 +35,9 @@ - github_runner_cluster_recreate_running | bool - github_runner_cluster_recreate_held | bool - not ansible_check_mode + - github_runner_cluster_etcd_lan_switch == '' + +- name: Move this server's etcd member to its other address, start it there and check the cluster, or move it back + # Only for the one server whose running member this run moves (tasks/etcd_lan.yml). The check above has already refused unless every other server is Ready; see tasks/etcd_lan_move.yml for the rest. + ansible.builtin.include_tasks: etcd_lan_move.yml + when: github_runner_cluster_etcd_lan_switch | length > 0 diff --git a/roles/github_runner_cluster/templates/env.j2 b/roles/github_runner_cluster/templates/env.j2 index 55b0eefa..cfcb2648 100644 --- a/roles/github_runner_cluster/templates/env.j2 +++ b/roles/github_runner_cluster/templates/env.j2 @@ -31,3 +31,9 @@ K3S_NODE_TAINTS={{ github_runner_cluster_node_taints | join(' ') }} K3S_ETCD_HEARTBEAT_INTERVAL={{ github_runner_cluster_etcd_heartbeat_interval }} K3S_ETCD_ELECTION_TIMEOUT={{ github_runner_cluster_etcd_election_timeout }} {% endif %} +{% if github_runner_cluster_etcd_lan_address | trim | length > 0 %} +K3S_ETCD_LAN_ADDRESS={{ github_runner_cluster_etcd_lan_address | trim }} +{% if github_runner_cluster_etcd_peer_on_lan | default(false) | bool %} +K3S_ETCD_PEER_ADDRESS={{ github_runner_cluster_etcd_lan_address | trim }} +{% endif %} +{% endif %} diff --git a/scripts/fleet-role-probe.sh b/scripts/fleet-role-probe.sh index 67bf6142..d4cb8b8f 100755 --- a/scripts/fleet-role-probe.sh +++ b/scripts/fleet-role-probe.sh @@ -80,9 +80,9 @@ fsync_ms() { | jq -r --arg percentile "${tail_percentile}.000000" '.jobs[0].sync.lat_ns.percentile[$percentile] / 1000000' } -# The nodes JSON's voters, as a JSON array of name and InternalIP. +# The nodes JSON's voters, as a JSON array of name and the address their etcd member listens on for peers: the etcd.k3s.cattle.io/node-address annotation, which k3s's etcd metadata controller keeps on every etcd server's Node at the address it builds the member's peer URL from (pkg/etcd/metadata_controller.go at v1.34.1+k3s1). That is the node's InternalIP while etcd uses the mesh address, and its LAN address once github_runner_cluster_etcd_peers_over_lan has moved it, when nothing listens for peers on the InternalIP. voters_of() { - jq -c --arg label "${VOTER_LABEL%%=*}" --arg value "${VOTER_LABEL#*=}" '[.items[] | select((.metadata.labels // {})[$label] == $value) | {name: .metadata.name, ip: ([.status.addresses[]? | select(.type == "InternalIP") | .address] | .[0])}]' <<< "$1" + jq -c --arg label "${VOTER_LABEL%%=*}" --arg value "${VOTER_LABEL#*=}" '[.items[] | select((.metadata.labels // {})[$label] == $value) | {name: .metadata.name, ip: .metadata.annotations["etcd.k3s.cattle.io/node-address"]}]' <<< "$1" } # Whether this node is among the voters (a JSON array from voters_of). diff --git a/tests/etcd_lan/play.yml b/tests/etcd_lan/play.yml new file mode 100644 index 00000000..ababefd1 --- /dev/null +++ b/tests/etcd_lan/play.yml @@ -0,0 +1,24 @@ +--- +# Runs the cluster role's tasks/etcd_lan.yml on its own, for tests/unit/test_cluster_etcd_lan_role.py, against a stand-in docker command the test puts first on PATH, on the hosts of an inventory the test writes (all on a local connection, so a task delegated to another server runs here too). The role's defaults come from the inventory's group variables, which the test copies from defaults/main.yml so that each host's own settings override them as host variables do. Sets what tasks/host_network.yml would have read about each host's container and addresses from the inventory's stand_in_* variables, and writes the decision each host reaches to /.json. +- name: Decide etcd's addresses + hosts: all + connection: local + gather_facts: false + vars: + github_runner_cluster_docker_cli_environment: "{{ {'DOCKER_CONTEXT': github_runner_cluster_docker_context} if github_runner_cluster_docker_context | length > 0 else {} }}" + github_runner_cluster_docker_role: server + github_runner_cluster_effective_node_name: "{{ inventory_hostname }}" + github_runner_cluster_network_running: "{{ stand_in_running | default(true) }}" + github_runner_cluster_network_node_container: {stdout: "{{ 'running host' if stand_in_running | default(true) else stand_in_container_state | default('') }}"} + github_runner_cluster_host_network: {stdout: "{{ stand_in_addresses | default('') }}\n--\n1: lo: "} + tasks: + - name: Include the role's etcd address decision + ansible.builtin.include_tasks: ../../roles/github_runner_cluster/tasks/etcd_lan.yml + + - name: Write the decision this host reached + ansible.builtin.copy: + content: >- + {{ {'peer_on_lan': github_runner_cluster_etcd_peer_on_lan | bool, 'switch': github_runner_cluster_etcd_lan_switch, + 'candidate': github_runner_cluster_etcd_lan_candidate, 'target': github_runner_cluster_etcd_lan_target | default('')} | to_json }} + dest: "{{ output_dir }}/{{ inventory_hostname }}.json" + mode: "0644" diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 8085a34f..30467238 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). native_server three Docker servers and a native host (a privileged systemd container, tests/mesh/native/Dockerfile, managed over Ansible's docker connection) that joins as an agent, is then made a server by the inventory and promoted in place: it must join etcd as a learner, be promoted to a voter with its mesh address as its peer URL, and keep the cluster writable throughout, and a rerun must not restart it. role_controller four hosts (three servers and a standby agent, with etcd's data on memory so the machine's disk does not decide): the role controller's probe and controller run in the cluster from the image built from this checkout, with the manifests the ARC role renders; every host is reported fit from a full window of real samples, with the link the cluster role labelled its node with, a voter made slow with tc netem on its mesh interface is reported unfit with the swap named and every unfit verdict attributed to it, a paused voter makes the controller report that it would not act, and the controller changes nothing (not in the default set; needs jq, and takes a window of 100 samples before it can judge). role_actuator three Docker servers and a native standby, every host eligible and the role controller on, with the test acting as the controller (holding the regrade Lease and writing the Node's desired-role label and swap annotations by hand): the native host's role actuator promotes it (a learner, then a voter, with writes succeeding throughout), steps it down (its etcd serving at etcd-only with no pod's shim left), and, once its member is removed through k3s's annotation, finishes it as an agent with the same node password and Tailscale address and its datastore moved aside, the two k3s units never running together. role_swap three Docker servers and an eligible native standby, with the role controller's probe and controller running from this checkout's image and none pinned: the controller's own `swap` command takes the Docker bootstrap host out and the native host in as one swap under the regrade Lease, with writes succeeding throughout, three voters at the end, the demoted node's labels and annotations gone, its datastore moved aside and its identity kept, and a second swap brings the former bootstrap host back into the same etcd cluster. role_swap_paused the outgoing voter's k3s is paused once the new voter votes: the removal still completes with three voters, and on unpause the tombstone keeps the removed server out of etcd for a learner stall time while its actuator finishes it as an agent. role_swap_blocked_peer the candidate's etcd peer port is dropped, so its learner never catches up: the controller rolls the promotion back and leaves the members, the voters and the host as they were. role_swap_refused a voter is paused before the swap is asked for: the command exits 1 naming it and nothing is set. The role_controller scenario also turns the dry run off after the slow voter is reported unfit, and the controller swaps it for the Docker standby. rejoin three Docker servers whose hosts the inventory marks eligible, so the role mounts each one's role directory: the bootstrap server's etcd member is removed through another server, and restarted on its tombstone it idles instead of joining again; with its datastore moved aside and no server to join it refuses to start rather than bootstrap a second cluster; given a join URL naming another server in its role directory it joins the existing cluster as a learner and is promoted, with the same cluster ID before and after; and a role file naming the agent leaves its server container idle with no k3s. compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. +# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). native_server three Docker servers and a native host (a privileged systemd container, tests/mesh/native/Dockerfile, managed over Ansible's docker connection) that joins as an agent, is then made a server by the inventory and promoted in place: it must join etcd as a learner, be promoted to a voter with its mesh address as its peer URL, and keep the cluster writable throughout, and a rerun must not restart it. role_controller four hosts (three servers and a standby agent, with etcd's data on memory so the machine's disk does not decide): the role controller's probe and controller run in the cluster from the image built from this checkout, with the manifests the ARC role renders; every host is reported fit from a full window of real samples, with the link the cluster role labelled its node with, a voter made slow with tc netem on its mesh interface is reported unfit with the swap named and every unfit verdict attributed to it, a paused voter makes the controller report that it would not act, and the controller changes nothing (not in the default set; needs jq, and takes a window of 100 samples before it can judge). role_actuator three Docker servers and a native standby, every host eligible and the role controller on, with the test acting as the controller (holding the regrade Lease and writing the Node's desired-role label and swap annotations by hand): the native host's role actuator promotes it (a learner, then a voter, with writes succeeding throughout), steps it down (its etcd serving at etcd-only with no pod's shim left), and, once its member is removed through k3s's annotation, finishes it as an agent with the same node password and Tailscale address and its datastore moved aside, the two k3s units never running together. role_swap three Docker servers and an eligible native standby, with the role controller's probe and controller running from this checkout's image and none pinned: the controller's own `swap` command takes the Docker bootstrap host out and the native host in as one swap under the regrade Lease, with writes succeeding throughout, three voters at the end, the demoted node's labels and annotations gone, its datastore moved aside and its identity kept, and a second swap brings the former bootstrap host back into the same etcd cluster. role_swap_paused the outgoing voter's k3s is paused once the new voter votes: the removal still completes with three voters, and on unpause the tombstone keeps the removed server out of etcd for a learner stall time while its actuator finishes it as an agent. role_swap_blocked_peer the candidate's etcd peer port is dropped, so its learner never catches up: the controller rolls the promotion back and leaves the members, the voters and the host as they were. role_swap_refused a voter is paused before the swap is asked for: the command exits 1 naming it and nothing is set. The role_controller scenario also turns the dry run off after the slow voter is reported unfit, and the controller swaps it for the Docker standby. rejoin three Docker servers whose hosts the inventory marks eligible, so the role mounts each one's role directory: the bootstrap server's etcd member is removed through another server, and restarted on its tombstone it idles instead of joining again; with its datastore moved aside and no server to join it refuses to start rather than bootstrap a second cluster; given a join URL naming another server in its role directory it joins the existing cluster as a learner and is promoted, with the same cluster ID before and after; and a role file naming the agent leaves its server container idle with no k3s. compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. etcd_lan three servers as in host_network: cutting one off the mesh (its Tailscale interface's traffic dropped) takes its etcd member out; then setting each server's github_runner_cluster_etcd_lan_address moves no member, github_runner_cluster_etcd_peers_over_lan moves exactly one member onto its LAN address per run with writes succeeding throughout, a rerun restarts nothing, the same cut then leaves every member healthy and the cut server writable, and turning the setting off moves one member back per run. Linux only, as host_network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. # # With GRTEST_PIN_CONTEXT=1, the cluster scenarios run the role, and the in_cluster recovery playbook, with github_runner_cluster_docker_context pinned while the current Docker context they see points at nothing (tests/lib/docker_context.sh), so any docker call that ignored the pin fails the run; afterwards the current context must be unchanged. context_checks the role refuses a pinned context that does not exist, and a pinned context alongside DOCKER_HOST, before writing anything (not in the default set). # @@ -22,6 +22,9 @@ etcd_image=gcr.io/etcd-development/etcd:v3.5.21 dind_image=docker:27.5.1-dind # Set by scenario_host_network: every node's containers are then on that node's own daemon, reached through a Docker context per node. host_network=0 +# Set by scenario_etcd_lan: whether write_host_network_inventory gives every node its daemon's address as github_runner_cluster_etcd_lan_address, and whether assert_cluster expects etcd's members on those addresses rather than on the mesh. +etcd_lan_addresses=0 +etcd_peers_on_lan=0 busybox_image=busybox:1.37 # The native_server scenario's native host: its container, image, address and data volume. Unset (empty native_role) in every other scenario, which then has no native host. native_host=grtest-native @@ -358,8 +361,10 @@ assert_cluster() { echo "$members" expected_members=$(echo "$members" | grep -c ', started, ') [ "$expected_members" = "$node_count" ] || fail "expected ${node_count} started etcd members, found ${expected_members}" - # Every member's peer address must be its mesh (100.64.0.0/10) address, not a container bridge address. - if echo "$members" | grep -v 'https://100\.' | grep -q ', started, '; then fail "an etcd member advertises a peer address outside the mesh"; fi + # Every member's peer address must be its mesh (100.64.0.0/10) address, not a container bridge address; or, once the etcd_lan scenario has moved them, its daemon's address on the test network, which stands in for the LAN. + if [ "$etcd_peers_on_lan" = 1 ]; then + if echo "$members" | grep -v "https://${subnet_prefix}\\." | grep -q ', started, '; then fail "an etcd member advertises a peer address outside the LAN"; fi + elif echo "$members" | grep -v 'https://100\.' | grep -q ', started, '; then fail "an etcd member advertises a peer address outside the mesh"; fi log "Checking pods on different nodes reach each other over the mesh" kubectl_on 1 apply -f - < "${work}/inventory.yml" } @@ -706,6 +712,115 @@ move_between_networks() { [ "$(docker_on 3 exec "$(container 3)" tailscale ip -4 /dev/null | jq '[.[] | select(.health | not)] | length' 2>/dev/null)" = 0 ]; } + +# Whether node $2's member (at its LAN address or the mesh address $3) does not answer when asked through node $1. etcdctl exits non-zero whenever a member is unhealthy, which is the case looked for, so its output is read whatever its status. +member_unhealthy_from() { + local health + health=$(etcd_on "$1" endpoint health --cluster -w json 2>/dev/null || true) + jq -e --arg ip "$(node_ip "$2")" --arg mesh "$3" '[.[] | select(.health | not) | .endpoint | select(contains($mesh) or contains($ip))] | length > 0' <<< "$health" >/dev/null 2>&1 +} + +# Cuts node $1 off the mesh, as an expired node key does, and only the mesh: every packet in or out of its Tailscale interface is dropped in its VM's namespace, which on the host network is the node's, while the test network (its LAN) stays up. +cut_mesh() { docker exec "$(vm "$1")" sh -c 'iptables -I INPUT 1 -i tailscale0 -j DROP && iptables -I OUTPUT 1 -o tailscale0 -j DROP'; } +restore_mesh() { docker exec "$(vm "$1")" sh -c 'iptables -D INPUT -i tailscale0 -j DROP && iptables -D OUTPUT -o tailscale0 -j DROP'; } + +# Whether node $1's live etcd peer certificate names its daemon's address. +peer_certificate_names_lan() { + docker_on "$1" exec "$(container "$1")" cat /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt &1) || { echo "$output"; fail "the role failed to move member ${expected} onto the LAN"; } + echo "$output" | grep -E 'etcd member|^(PLAY RECAP|grtest)' || true + [ "$(members_on_lan)" = "$expected" ] || fail "after run ${expected}, $(members_on_lan) members are on the LAN, not ${expected}" + if [ "$expected" -lt "$node_count" ]; then grep -F "another server's member moves in this run" <<< "$output" >/dev/null || fail "run ${expected} did not report the members that wait"; fi + touch "$stop" + wait + assert_writes_recovered "$writes" || fail "writes through $(node_name "$api_node") did not recover within a leader election while $(node_name "$expected")'s member moved" + done + api_node=1 + etcd_peers_on_lan=1 + for index in $(seq 1 "$node_count"); do replace_echo_pod "$index"; done + assert_cluster + + log "Rerunning the role, which must not restart any node" + before=$(started_at) + run_role -e github_runner_cluster_etcd_peers_over_lan=true || fail "a rerun with every member on the LAN failed" + after=$(started_at) + [ "$before" = "$after" ] || fail "rerunning the role restarted a node container: ${before} -> ${after}" + + log "With every member on the LAN, cutting $(node_name 3) off the mesh leaves its member in etcd" + cut_mesh 3 + # Longer than etcd's election timeout several times over, so a member that had lost its peers would have been reported. + for _ in $(seq 1 12); do + sleep 5 + all_members_healthy_from 1 || { restore_mesh 3; fail "a member stopped answering with $(node_name 3) off the mesh"; } + all_members_healthy_from 3 || { restore_mesh 3; fail "$(node_name 3) lost its peers when cut off the mesh"; } + done + kubectl_on 3 create configmap "grtest-etcd-lan-${RANDOM}" --from-literal=a=1 /dev/null || { restore_mesh 3; fail "a write through $(node_name 3)'s API server failed with it off the mesh"; } + restore_mesh 3 + log "$(node_name 3)'s member kept its peers over the LAN while it was off the mesh" + + log "Moving the members back to the mesh, one per run" + for expected in 2 1 0; do + sleep 65 + output=$(run_role -e github_runner_cluster_etcd_lan_quiet_minutes=1 2>&1) || { echo "$output"; fail "the role failed to move a member back to the mesh"; } + [ "$(members_on_lan)" = "$expected" ] || fail "$(members_on_lan) members are on the LAN, not ${expected}" + done + etcd_peers_on_lan=0 + for index in $(seq 1 "$node_count"); do replace_echo_pod "$index"; done + assert_cluster +} + # One node and only the settings the role reads before its first check fails, for the scenarios that expect it to stop before touching the host. write_minimal_inventory() { local node="$1" @@ -1081,14 +1196,14 @@ node_is_voter() { [ "$(kubectl_on "$api_node" get node "${1:-$native_host}" -o 'jsonpath={.status.conditions[?(@.type=="EtcdIsVoter")].status}' 2>/dev/null)" = True ] } -# Writes to the cluster through node 1 once a second until the stop file appears, recording each write as ok, or as failed with the time and what kubectl said. Each write is a ConfigMap named by its sequence number, so no two writes share a name: write_marker's random suffix repeats often enough over a promotion's worth of writes that a refusal for a name already taken would read as a lost write. +# Writes to the cluster through node $api_node once a second until the stop file appears (with $3, when given, before each name, so that writers started one after another do not collide), recording each write as ok, or as failed with the time and what kubectl said. Each write is a ConfigMap named by its sequence number, so no two writes share a name: write_marker's random suffix repeats often enough over a promotion's worth of writes that a refusal for a name already taken would read as a lost write. write_throughout() { - local log="$1" stop="$2" sequence=0 error + local log="$1" stop="$2" prefix="${3:-}" sequence=0 error : > "$log" : > "${log}.times" until [ -e "$stop" ]; do sequence=$((sequence + 1)) - if error=$(kubectl_on "$api_node" create configmap "grtest-native-write-${sequence}" --from-literal=a=1 2>&1 >/dev/null &1 >/dev/null > "$log" echo "$(date +%s) ok" >> "${log}.times" else @@ -2187,6 +2302,7 @@ for scenario in "${scenarios[@]}"; do role_swap_blocked_peer) scenario_role_swap_blocked_peer ;; role_swap_refused) scenario_role_swap_refused ;; host_network) scenario_host_network ;; + etcd_lan) scenario_etcd_lan ;; compose_faulty) scenario_compose_faulty ;; context_checks) scenario_context_checks ;; # Re-checks a cluster a previous run kept with GRTEST_KEEP=1 and the same GRTEST_WORK, without rebuilding it. diff --git a/tests/unit/fleet_role_support.py b/tests/unit/fleet_role_support.py index 7df64045..68068497 100644 --- a/tests/unit/fleet_role_support.py +++ b/tests/unit/fleet_role_support.py @@ -369,6 +369,7 @@ def node( ready_for: float = 24 * 3600, level: str | None = None, address: str | None = None, + etcd_address: str | None = None, link: str | None = None, ) -> dict[str, Any]: """A Node as `kubectl get nodes -o json` lists it; a voter carries the etcd label and the EtcdIsVoter condition, a learner the label alone (k3s labels a member when it joins and sets the condition when it votes), ready_for is how long ago its Ready condition last changed, and link the medium the cluster role labelled it with, none when it has no such label.""" @@ -387,7 +388,8 @@ def node( if voter: conditions.append({"type": VOTER_CONDITION, "status": "True", "lastTransitionTime": transition_time(ready_for)}) return { - "metadata": {"name": name, "labels": labels}, + # k3s keeps the address an etcd server's member listens on for peers in this annotation, which the probe dials: the InternalIP while etcd uses the mesh address, unless etcd_address says the member moved. + "metadata": {"name": name, "labels": labels, "annotations": {"etcd.k3s.cattle.io/node-address": etcd_address or address or "127.0.0.1"} if voter or learner else {}}, "spec": {"unschedulable": True} if cordoned else {}, "status": { "conditions": conditions, diff --git a/tests/unit/test_cluster_etcd_lan.py b/tests/unit/test_cluster_etcd_lan.py new file mode 100644 index 00000000..d51da14d --- /dev/null +++ b/tests/unit/test_cluster_etcd_lan.py @@ -0,0 +1,290 @@ +"""Tests for moving etcd's peer traffic between k3s servers onto their LAN addresses (github_runner_cluster_etcd_lan_address and github_runner_cluster_etcd_peers_over_lan): the filters that check the inventory, read etcd's member list and decide which member moves (plugins/filter/cluster_etcd_lan.py), the flags node-entrypoint.sh passes to k3s, the server's .env and Compose file list (templates/env.j2 and defaults/main.yml, rendered by tests/cluster_versions/render_env.yml), the Compose override that passes the two variables, and where the move sits in a server's turn. A host that sets neither keeps exactly the arguments, .env and Compose files it had.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import re +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ROLE = ROOT / "roles" / "github_runner_cluster" +ENTRYPOINT = ROLE / "files" / "k3s" / "node-entrypoint.sh" +OVERRIDE = ROLE / "files" / "docker-compose.etcd-lan.yml" +ANSIBLE_PLAYBOOK = os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook") +DEFAULTS: dict[str, Any] = yaml.safe_load((ROLE / "defaults" / "main.yml").read_text()) + +MESH_IP = "100.64.0.1" +LAN_IP = "192.0.2.11" +# The arguments mesh_server_args built before the LAN address existed, for a bootstrap server and for a joining server with two TLS names: a host that sets no address must keep them exactly. +BOOTSTRAP_ARGS = ["--disable=traefik", "--disable-network-policy", "--write-kubeconfig-mode=644", "--vpn-auth=name=tailscale,joinKey=example", "--resolv-conf=/etc/k3s-resolv.conf", f"--node-ip={MESH_IP}", "--cluster-init"] +JOIN_ARGS = [*BOOTSTRAP_ARGS[:-1], "--server", "https://node1.example.internal:6443", "--tls-san", "node1.example.internal", "--tls-san", "node2.example.internal"] + + +def _filters() -> Any: + spec = importlib.util.spec_from_file_location("cluster_etcd_lan", ROOT / "plugins" / "filter" / "cluster_etcd_lan.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +FILTERS = _filters() + + +def host(name: str, **overrides: Any) -> dict[str, Any]: + """A server of the group as tasks/etcd_lan.yml describes it to cluster_etcd_lan_problems: a Docker node on the host network with an address in the LAN.""" + index = int(name[-1]) + return {"name": name, "node_name": name, "role": "server", "eligible": False, "node_mode": "compose", "network_mode": "host", "lan_cidr": "192.0.2.0/24", "mesh": "tailscale", "address": f"192.0.2.1{index}", "peers_over_lan": True} | overrides + + +GROUP = [host("node1"), host("node2"), host("node3")] + + +def member_list(*members: tuple[int, str, list[str]]) -> str: + """etcdctl member list -w json output, whose IDs are 64-bit integers.""" + return json.dumps({"header": {"cluster_id": 1}, "members": [{"ID": ident, "name": name, "peerURLs": urls, "clientURLs": []} for ident, name, urls in members]}) + + +class ProblemsTest(unittest.TestCase): + def problems(self, hosts: list[dict[str, Any]], peers_over_lan: bool = True) -> list[str]: + result: list[str] = FILTERS.cluster_etcd_lan_problems(hosts, peers_over_lan) + return result + + def test_three_servers_on_the_host_network_with_addresses_of_their_own_pass(self) -> None: + self.assertEqual(self.problems(GROUP), []) + + def test_addresses_without_the_switch_pass_and_an_agent_without_one_is_not_asked_for_one(self) -> None: + group = [host(name, peers_over_lan=False) for name in ("node1", "node2", "node3")] + [host("node4", role="agent", address="", peers_over_lan=False)] + self.assertEqual(self.problems(group, peers_over_lan=False), []) + + def test_a_server_or_an_eligible_host_without_an_address_is_named(self) -> None: + problems = self.problems([host("node1"), host("node2", address=""), host("node3"), host("node4", role="agent", eligible=True, address="")]) + self.assertEqual(len(problems), 2) + self.assertIn("node2 is a server but sets no", problems[0]) + self.assertIn("node4 is eligible for a vote but sets no", problems[1]) + + def test_the_bridge_network_is_refused_with_why(self) -> None: + (problem,) = self.problems([host("node1"), host("node2", network_mode="bridge"), host("node3")]) + self.assertIn("node2", problem) + self.assertIn("etcd's check of the peer certificate against the connection's source", problem) + + def test_a_native_node_is_refused(self) -> None: + (problem,) = self.problems([host("node1"), host("node2", node_mode="native"), host("node3")]) + self.assertIn("node2 is a native node", problem) + + def test_an_address_outside_the_lan_not_ipv4_or_taken_twice_is_named(self) -> None: + self.assertIn("is not in github_runner_cluster_lan_cidr", self.problems([host("node1", address="198.51.100.1"), host("node2"), host("node3")])[0]) + self.assertIn("is not an IPv4 address", self.problems([host("node1", address="node1.lan"), host("node2"), host("node3")])[0]) + self.assertIn("node2 and node1 both set", self.problems([host("node1"), host("node2", address="192.0.2.11"), host("node3")])[0]) + + def test_an_agent_that_cannot_take_a_vote_may_not_set_one(self) -> None: + (problem,) = self.problems([*GROUP, host("node4", role="agent")]) + self.assertIn("node4 sets github_runner_cluster_etcd_lan_address but is neither a server nor eligible", problem) + + def test_the_switch_must_be_the_same_on_every_host(self) -> None: + (problem,) = self.problems([host("node1"), host("node2", peers_over_lan=False), host("node3")]) + self.assertIn("node2 sets github_runner_cluster_etcd_peers_over_lan to False", problem) + + def test_headscale_in_cluster_is_refused_with_the_switch_on_and_allowed_with_it_off(self) -> None: + group = [host(name, mesh="headscale_in_cluster") for name in ("node1", "node2", "node3")] + self.assertTrue(all("headscale_in_cluster" in problem for problem in self.problems(group))) + self.assertEqual(self.problems([entry | {"peers_over_lan": False} for entry in group], peers_over_lan=False), []) + + +class MembersTest(unittest.TestCase): + def test_ids_come_back_in_hexadecimal_and_peer_urls_as_hosts(self) -> None: + (member,) = FILTERS.cluster_etcd_members(member_list((12345678901234567890, "node1-0a1b2c3d", [f"https://{MESH_IP}:2380"]))) + self.assertEqual(member, {"id": "ab54a98ceb1f0ad2", "name": "node1-0a1b2c3d", "peer_hosts": [MESH_IP], "learner": False}) + + def test_a_node_s_member_is_found_by_k3s_s_naming(self) -> None: + members = FILTERS.cluster_etcd_members(member_list((1, "node1-0a1b2c3d", [f"https://{MESH_IP}:2380"]), (2, "node10-0a1b2c3d", ["https://100.64.0.10:2380"]))) + self.assertEqual(FILTERS.cluster_etcd_member_of(members, "node1")["id"], "1") + self.assertIsNone(FILTERS.cluster_etcd_member_of(members, "node2")) + + def test_two_members_for_one_node_are_refused(self) -> None: + members = FILTERS.cluster_etcd_members(member_list((1, "node1-0a1b2c3d", []), (2, "node1-ffffffff", []))) + with self.assertRaisesRegex(Exception, "lists 2 members for node1"): + FILTERS.cluster_etcd_member_of(members, "node1") + + def test_output_that_is_not_a_member_list_is_refused(self) -> None: + for output in ("", "{}", '{"members": [{"name": "x"}]}', member_list((1, "a-00000000", ["unix://socket"]))): + with self.subTest(output=output), self.assertRaises(Exception): + FILTERS.cluster_etcd_members(output) + + +class DecisionTest(unittest.TestCase): + def test_only_the_first_member_that_would_move_moves(self) -> None: + hosts = [{"name": "node1", "switch": ""}, {"name": "node2", "switch": "to_lan"}, {"name": "node3", "switch": "to_lan"}] + self.assertEqual(FILTERS.cluster_etcd_lan_switches(hosts), ["node2"]) + self.assertEqual(FILTERS.cluster_etcd_lan_switches([{"name": "node1", "switch": ""}]), []) + + def test_a_certificate_without_its_own_address_or_an_unread_one_holds_the_move(self) -> None: + certificates = [{"item": "node2", "rc": 0, "stdout_lines": ["100.64.0.2", "192.0.2.12"]}, {"item": "node3", "rc": 0, "stdout_lines": ["100.64.0.3"]}, {"item": "node4", "unreachable": True}] + readiness = FILTERS.cluster_etcd_lan_readiness(certificates, [], {"node2": "192.0.2.12", "node3": "192.0.2.13", "node4": "192.0.2.14"}) + self.assertEqual(readiness, {"unnamed": ["node3", "node4"], "unreached": []}) + + def test_a_server_that_does_not_reach_the_address_keeps_the_member_on_the_mesh(self) -> None: + certificates = [{"item": "node2", "rc": 0, "stdout_lines": ["192.0.2.12"]}] + readiness = FILTERS.cluster_etcd_lan_readiness(certificates, [{"item": "node2", "rc": 1}], {"node2": "192.0.2.12"}) + self.assertEqual(readiness, {"unnamed": [], "unreached": ["node2"]}) + + +def function_source(name: str) -> str: + match = re.search(rf"^{name}\(\) \{{\n.*?^\}}\n", ENTRYPOINT.read_text(), re.MULTILINE | re.DOTALL) + assert match is not None, f"{name} is not defined in node-entrypoint.sh" + return match.group(0) + + +class EntrypointTest(unittest.TestCase): + """mesh_server_args, run in a shell with the other flag sets empty, as test_cluster_etcd_timing.py runs it.""" + + def server_args(self, **env: str) -> list[str]: + script = "\n".join( + [ + "set -u", + 'vpn_auth="name=tailscale,joinKey=example"', + f'mesh_ip="{MESH_IP}"', + 'regrade_flags=""', + 'api_port_flags=""', + 'etcd_timing_flags=""', + 'join_url="${K3S_JOIN_SERVER_URL:-}"', + function_source("mesh_server_args"), + "mesh_server_args", + 'printf "%s\\n" "$server_args"', + ] + ) + clean = {key: value for key, value in os.environ.items() if not key.startswith("K3S_")} + result = subprocess.run(["/bin/sh", "-c", script], capture_output=True, text=True, check=False, env=clean | env) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout.split() + + def test_without_the_variables_the_arguments_are_what_they_were(self) -> None: + self.assertEqual(self.server_args(), BOOTSTRAP_ARGS) + self.assertEqual(self.server_args(K3S_ETCD_LAN_ADDRESS="", K3S_ETCD_PEER_ADDRESS=""), BOOTSTRAP_ARGS) + joining = {"K3S_JOIN_SERVER_URL": "https://node1.example.internal:6443", "K3S_TLS_SAN_LIST": "node1.example.internal node2.example.internal"} + self.assertEqual(self.server_args(**joining), JOIN_ARGS) + + def test_the_lan_address_alone_names_it_in_the_certificates_and_keeps_etcd_on_the_mesh(self) -> None: + args = self.server_args(K3S_ETCD_LAN_ADDRESS=LAN_IP, K3S_TLS_SAN_LIST="node1.example.internal") + self.assertIn(f"--node-ip={MESH_IP}", args) + self.assertEqual(args[-4:], ["--tls-san", "node1.example.internal", "--tls-san", LAN_IP]) + self.assertIn("--vpn-auth=name=tailscale,joinKey=example", args) + + def test_a_member_on_the_lan_starts_etcd_there_with_the_mesh_still_carrying_the_node(self) -> None: + args = self.server_args(K3S_ETCD_LAN_ADDRESS=LAN_IP, K3S_ETCD_PEER_ADDRESS=LAN_IP) + self.assertIn(f"--node-ip={LAN_IP}", args) + self.assertNotIn(f"--node-ip={MESH_IP}", args) + self.assertEqual(args[-2:], ["--tls-san", LAN_IP]) + # The node, flannel and the API stay on the mesh through --vpn-auth, which overrides the kubelet's node IP and the advertise address. + self.assertIn("--vpn-auth=name=tailscale,joinKey=example", args) + + def test_the_agent_command_line_takes_neither(self) -> None: + agent_line = next(line for line in ENTRYPOINT.read_text().splitlines() if "k3s agent" in line) + self.assertNotIn("ETCD", agent_line) + self.assertNotIn("node-ip", agent_line) + + +class EnvironmentTest(unittest.TestCase): + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.work) + + def render(self, *extra: str) -> tuple[list[str], list[str]]: + env = self.work / "env" + files = self.work / "files.json" + args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"compose_files_output={files}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra] + environment = {**os.environ, "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", "ANSIBLE_RETRY_FILES_ENABLED": "false"} + result = subprocess.run(args, capture_output=True, text=True, check=False, env=environment, cwd=self.work) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + compose_files: list[str] = json.loads(files.read_text()) + return env.read_text().splitlines(), compose_files + + @unittest.skipIf(shutil.which(ANSIBLE_PLAYBOOK) is None, "ansible-playbook is not installed") + def test_no_address_writes_nothing_new_and_adds_no_compose_file(self) -> None: + lines, files = self.render() + self.assertEqual([line for line in lines if "ETCD_LAN" in line or "ETCD_PEER" in line], []) + self.assertEqual(files, ["docker-compose.yml"]) + self.assertEqual(self.render("-e", "github_runner_cluster_etcd_peers_over_lan=true"), (lines, files)) + + @unittest.skipIf(shutil.which(ANSIBLE_PLAYBOOK) is None, "ansible-playbook is not installed") + def test_the_address_is_written_and_its_override_merged_before_the_host_network(self) -> None: + lines, files = self.render("-e", f"github_runner_cluster_etcd_lan_address={LAN_IP}", "-e", "github_runner_cluster_network_mode=host") + self.assertIn(f"K3S_ETCD_LAN_ADDRESS={LAN_IP}", lines) + self.assertEqual([line for line in lines if line.startswith("K3S_ETCD_PEER_ADDRESS")], []) + self.assertEqual(files, ["docker-compose.yml", "docker-compose.etcd-lan.yml", "docker-compose.host-network.yml"]) + + @unittest.skipIf(shutil.which(ANSIBLE_PLAYBOOK) is None, "ansible-playbook is not installed") + def test_a_member_on_the_lan_gets_the_peer_address_too(self) -> None: + lines, _ = self.render("-e", f"github_runner_cluster_etcd_lan_address={LAN_IP}", "-e", '{"github_runner_cluster_etcd_peer_on_lan": true}') + self.assertIn(f"K3S_ETCD_LAN_ADDRESS={LAN_IP}", lines) + self.assertIn(f"K3S_ETCD_PEER_ADDRESS={LAN_IP}", lines) + + +class ComposeOverrideTest(unittest.TestCase): + def test_only_the_server_service_takes_the_two_variables_and_only_from_the_override(self) -> None: + services = yaml.safe_load(OVERRIDE.read_text())["services"] + self.assertEqual(list(services), ["k3s"]) + names = [entry.split("=", 1)[0] for entry in services["k3s"]["environment"]] + self.assertEqual(names, ["K3S_ETCD_LAN_ADDRESS", "K3S_ETCD_PEER_ADDRESS"]) + self.assertNotIn("ETCD_LAN", (ROLE / "files" / "docker-compose.yml").read_text()) + self.assertNotIn("ETCD_PEER", (ROLE / "files" / "docker-compose.yml").read_text()) + + def test_the_role_writes_the_override_and_counts_it_as_its_own(self) -> None: + self.assertIn("docker-compose.etcd-lan.yml", DEFAULTS["github_runner_cluster_compose_files_written"]) + copied = next(task for task in yaml.safe_load((ROLE / "tasks" / "main.yml").read_text()) if task.get("name") == "Place the collection's Compose file and k3s build context in it") + self.assertIn({"src": "docker-compose.etcd-lan.yml", "mode": "0644"}, copied["loop"]) + + +class DefaultsTest(unittest.TestCase): + def test_nothing_moves_by_default(self) -> None: + self.assertEqual(DEFAULTS["github_runner_cluster_etcd_lan_address"], "") + self.assertIs(DEFAULTS["github_runner_cluster_etcd_peers_over_lan"], False) + self.assertIs(DEFAULTS["github_runner_cluster_etcd_peer_on_lan"], False) + self.assertEqual(DEFAULTS["github_runner_cluster_etcd_lan_switch"], "") + + def test_the_etcdctl_image_is_pinned_to_a_release(self) -> None: + self.assertRegex(DEFAULTS["github_runner_cluster_etcdctl_image"], r"^gcr\.io/etcd-development/etcd:v[0-9]+\.[0-9]+\.[0-9]+$") + + +class TurnTest(unittest.TestCase): + def test_the_move_takes_the_place_of_the_plain_start_in_the_moving_server_s_turn(self) -> None: + tasks = yaml.safe_load((ROLE / "tasks" / "start_server.yml").read_text()) + includes = [task.get("ansible.builtin.include_tasks") for task in tasks] + self.assertEqual(includes, ["server_recreate_check.yml", None, None, "server_compose_up.yml", "server_ready.yml", "etcd_lan_move.yml"]) + for task in tasks[3:5]: + self.assertIn("github_runner_cluster_etcd_lan_switch == ''", task["when"]) + self.assertEqual(tasks[5]["when"], "github_runner_cluster_etcd_lan_switch | length > 0") + + def test_the_move_updates_etcd_only_after_checking_and_moves_back_on_any_failure(self) -> None: + (report, move) = yaml.safe_load((ROLE / "tasks" / "etcd_lan_move.yml").read_text()) + self.assertEqual(report["when"], "ansible_check_mode") + names = [task["name"] for task in move["block"]] + update = names.index("Update the member's peer URL in etcd") + self.assertLess(names.index("Read the health of every etcd member before the move"), update) + self.assertLess(names.index("Read every server's log for etcd reads that could not reach a quorum"), update) + self.assertEqual(move["block"][update + 1]["ansible.builtin.set_fact"], {"github_runner_cluster_etcd_lan_updated": True}) + # .env is written back first whatever failed, since the run wrote it for the new address before the server's turn. + self.assertEqual(move["rescue"][1]["ansible.builtin.template"]["src"], "env.j2") + self.assertEqual(move["rescue"][1]["vars"], {"github_runner_cluster_etcd_peer_on_lan": "{{ github_runner_cluster_etcd_lan_switch == 'to_mesh' }}"}) + self.assertNotIn("when", move["rescue"][1]) + rescue_includes = [task.get("ansible.builtin.include_tasks") for task in move["rescue"][2]["block"]] + self.assertEqual(rescue_includes, ["etcd_lan_update_back.yml", "server_compose_up.yml", "server_ready.yml"]) + self.assertEqual(move["rescue"][2]["when"], "github_runner_cluster_etcd_lan_updated | default(false) | bool") + self.assertIn("ansible.builtin.fail", move["rescue"][-1]) + + def test_the_decision_comes_before_anything_changes_on_the_host(self) -> None: + names = [task.get("ansible.builtin.include_tasks") for task in yaml.safe_load((ROLE / "tasks" / "main.yml").read_text())] + self.assertEqual(names.index("etcd_lan.yml"), names.index("host_network.yml") + 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_cluster_etcd_lan_role.py b/tests/unit/test_cluster_etcd_lan_role.py new file mode 100644 index 00000000..5db8c0a1 --- /dev/null +++ b/tests/unit/test_cluster_etcd_lan_role.py @@ -0,0 +1,203 @@ +"""Tests for the cluster role's tasks/etcd_lan.yml, run with ansible-playbook (tests/etcd_lan/play.yml) on three servers against a stand-in docker command that records what it is asked and answers from files the test writes: etcd's member list, each server's etcd peer certificate (real certificates, read by the real openssl), whether a server's ping reaches an address, and a server's mesh address. A host with no address reads nothing and keeps its .env; with the addresses alone no member moves; with github_runner_cluster_etcd_peers_over_lan exactly one running member moves per run, and only once every other server's certificate names its own LAN address and every other server reaches the mover's, the mesh being kept otherwise; a member on the LAN stays there and moves back when the setting is off; and the address cannot be removed while .env still starts etcd on it.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +PLAY = ROOT / "tests" / "etcd_lan" / "play.yml" +ANSIBLE_PLAYBOOK = os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook") +NAMES = ("node1", "node2", "node3") +LAN = {name: f"192.0.2.1{index}" for index, name in enumerate(NAMES, start=1)} +MESH = {name: f"100.64.0.{index}" for index, name in enumerate(NAMES, start=1)} +# The role's defaults, as the inventory's group variables: below every host's own settings, as role defaults are. +DEFAULTS: dict[str, Any] = yaml.safe_load((ROOT / "roles" / "github_runner_cluster" / "defaults" / "main.yml").read_text()) +TOPOLOGY = {"hosts": {name: {"role": "server", "bootstrap": name == "node1", "node_name": name} for name in NAMES}} + +# The stand-in docker logs each call with its DOCKER_CONTEXT, and answers: `run` of etcdctl with members.json (or fails while etcd-down exists), `run` of ping with failure when unreachable--
exists, `exec ... cat` of the peer certificate with cert-.pem, and `exec ... tailscale ip -4` with mesh-. +STUB = """\ + #!/bin/sh + echo "docker ${DOCKER_CONTEXT:--} $*" >> "$STUB_DIR/calls.log" + case "$1" in + run) + case "$*" in + *etcdctl*) [ -e "$STUB_DIR/etcd-down" ] && { echo "context deadline exceeded" >&2; exit 1; }; cat "$STUB_DIR/members.json" ;; + *ping*) for last; do :; done; [ -e "$STUB_DIR/unreachable-${DOCKER_CONTEXT:--}-$last" ] && exit 1; exit 0 ;; + esac ;; + exec) + case "$*" in + *peer-server-client.crt*) cat "$STUB_DIR/cert-$2.pem" ;; + *tailscale*) cat "$STUB_DIR/mesh-$2" ;; + esac ;; + esac + """ + + +def member(index: int, name: str, host: str) -> dict[str, Any]: + return {"ID": 1000 + index, "name": f"{name}-0a1b2c3{index}", "peerURLs": [f"https://{host}:2380"], "clientURLs": [f"https://{host}:2379"]} + + +@unittest.skipIf(shutil.which(ANSIBLE_PLAYBOOK) is None or shutil.which("openssl") is None, "ansible-playbook or openssl is not installed") +class EtcdLanRoleTest(unittest.TestCase): + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.work) + self.bin = self.work / "bin" + self.bin.mkdir() + docker = self.bin / "docker" + docker.write_text(textwrap.dedent(STUB)) + docker.chmod(0o755) + (self.work / "out").mkdir() + self.members(on_lan=()) + for name in NAMES: + self.certificate(name, named=True) + (self.work / f"mesh-{name}-k3s").write_text(MESH[name] + "\n") + + def certificate(self, name: str, named: bool) -> None: + """Writes the peer certificate node ``name``'s container holds: its mesh address always, and its LAN address when ``named``.""" + sans = f"IP:{MESH[name]}" + (f",IP:{LAN[name]}" if named else "") + key = self.work / f"{name}.key" + subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", str(key), "-out", str(self.work / f"cert-{name}-k3s.pem"), "-days", "1", "-subj", "/CN=etcd-peer", "-addext", f"subjectAltName={sans}"], check=True, capture_output=True) + + def members(self, on_lan: tuple[str, ...]) -> None: + listing = [member(index, name, LAN[name] if name in on_lan else MESH[name]) for index, name in enumerate(NAMES, start=1)] + (self.work / "members.json").write_text(json.dumps({"header": {}, "members": listing})) + + def unreachable(self, from_host: str, address: str) -> None: + (self.work / f"unreachable-ctx-{from_host}-{address}").touch() + + def run_tasks(self, addresses: bool = True, limit: str = "all", **extra: Any) -> subprocess.CompletedProcess[str]: + hosts: dict[str, dict[str, Any]] = {} + for name in NAMES: + hosts[name] = { + "github_runner_cluster_container_name": f"{name}-k3s", + "github_runner_cluster_docker_context": f"ctx-{name}", + "github_runner_cluster_dir": str(self.work / name), + "github_runner_cluster_network_mode": "host", + "github_runner_cluster_lan_cidr": "192.0.2.0/24", + "stand_in_addresses": f"3: col0 inet {LAN[name]}/24 brd 192.0.2.255 scope global col0", + } + if addresses: + hosts[name]["github_runner_cluster_etcd_lan_address"] = LAN[name] + (self.work / "inventory.json").write_text(json.dumps({"all": {"vars": DEFAULTS, "hosts": hosts}})) + extra_file = self.work / "extra.json" + extra_file.write_text(json.dumps({"github_runner_cluster_topology": TOPOLOGY, "output_dir": str(self.work / "out"), **extra})) + env = { + **os.environ, + "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}", + "STUB_DIR": str(self.work), + "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), + "ANSIBLE_LOCALHOST_WARNING": "false", + "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", + "ANSIBLE_RETRY_FILES_ENABLED": "false", + } + return subprocess.run([ANSIBLE_PLAYBOOK, "-i", str(self.work / "inventory.json"), str(PLAY), "--limit", limit, "-e", f"@{extra_file}"], capture_output=True, text=True, check=False, env=env, cwd=self.work, timeout=300) + + def decision(self, name: str) -> dict[str, Any]: + result: dict[str, Any] = json.loads((self.work / "out" / f"{name}.json").read_text()) + return result + + def calls(self) -> list[str]: + log = self.work / "calls.log" + return log.read_text().splitlines() if log.exists() else [] + + def passed(self, result: subprocess.CompletedProcess[str]) -> None: + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_no_address_reads_nothing_and_changes_nothing(self) -> None: + result = self.run_tasks(addresses=False) + self.passed(result) + self.assertEqual(self.calls(), []) + for name in NAMES: + self.assertEqual(self.decision(name), {"peer_on_lan": False, "switch": "", "candidate": "", "target": ""}) + + def test_the_addresses_alone_read_the_members_and_move_none(self) -> None: + self.passed(self.run_tasks()) + for name in NAMES: + self.assertEqual(self.decision(name)["switch"], "") + self.assertFalse(self.decision(name)["peer_on_lan"]) + self.assertTrue(all("etcdctl" in call for call in self.calls()), self.calls()) + + def test_with_the_setting_on_only_the_first_member_moves_and_the_others_wait(self) -> None: + result = self.run_tasks(github_runner_cluster_etcd_peers_over_lan=True) + self.passed(result) + self.assertEqual(self.decision("node1"), {"peer_on_lan": True, "switch": "to_lan", "candidate": "to_lan", "target": LAN["node1"]}) + for name in ("node2", "node3"): + self.assertEqual(self.decision(name)["switch"], "") + self.assertFalse(self.decision(name)["peer_on_lan"]) + self.assertIn("another server's member moves in this run", result.stdout) + # The other servers were asked to reach node1's address, each through its own Docker context. + self.assertIn(f"docker ctx-node2 run --rm --network host busybox:1.37 ping -c 3 -W 2 {LAN['node1']}", self.calls()) + self.assertIn(f"docker ctx-node3 run --rm --network host busybox:1.37 ping -c 3 -W 2 {LAN['node1']}", self.calls()) + + def test_a_certificate_without_its_server_s_lan_address_holds_the_move(self) -> None: + self.certificate("node2", named=False) + result = self.run_tasks(github_runner_cluster_etcd_peers_over_lan=True) + self.passed(result) + self.assertEqual(self.decision("node1")["switch"], "") + self.assertIn("node2 does not yet name its own LAN address", result.stdout) + # node2's own move needs only the others' certificates, which name theirs. + self.assertEqual(self.decision("node2")["switch"], "to_lan") + + def test_a_server_that_does_not_reach_the_address_keeps_the_member_on_the_mesh(self) -> None: + self.unreachable("node3", LAN["node1"]) + result = self.run_tasks(github_runner_cluster_etcd_peers_over_lan=True) + self.passed(result) + self.assertEqual(self.decision("node1"), {"peer_on_lan": False, "switch": "", "candidate": "", "target": LAN["node1"]}) + self.assertIn(f"node3 could not reach {LAN['node1']}", result.stdout) + self.assertEqual(self.decision("node2")["switch"], "to_lan") + + def test_a_member_on_the_lan_stays_there_and_the_next_one_moves(self) -> None: + self.members(on_lan=("node1",)) + self.passed(self.run_tasks(github_runner_cluster_etcd_peers_over_lan=True)) + self.assertEqual(self.decision("node1"), {"peer_on_lan": True, "switch": "", "candidate": "", "target": ""}) + self.assertEqual(self.decision("node2")["switch"], "to_lan") + self.assertFalse(self.decision("node3")["peer_on_lan"]) + + def test_turned_off_a_member_on_the_lan_moves_back_to_its_mesh_address(self) -> None: + self.members(on_lan=("node1", "node2")) + self.passed(self.run_tasks()) + self.assertEqual(self.decision("node1"), {"peer_on_lan": False, "switch": "to_mesh", "candidate": "to_mesh", "target": MESH["node1"]}) + self.assertEqual(self.decision("node2"), {"peer_on_lan": True, "switch": "", "candidate": "to_mesh", "target": MESH["node2"]}) + self.assertEqual(self.decision("node3")["candidate"], "") + + def test_the_address_cannot_be_removed_while_env_starts_etcd_on_it(self) -> None: + (self.work / "node1").mkdir() + (self.work / "node1" / ".env").write_text(f"K3S_TOKEN=secret\nK3S_ETCD_LAN_ADDRESS={LAN['node1']}\nK3S_ETCD_PEER_ADDRESS={LAN['node1']}\n") + result = self.run_tasks(addresses=False, limit="node1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("is no longer set", result.stdout) + self.assertNotIn("secret", result.stdout) + + def test_an_address_on_the_bridge_network_is_refused_before_anything_is_read(self) -> None: + (self.work / "inventory.json").unlink(missing_ok=True) + result = self.run_tasks(github_runner_cluster_network_mode="bridge") + self.assertNotEqual(result.returncode, 0) + self.assertIn("with github_runner_cluster_network_mode bridge", result.stdout) + self.assertEqual(self.calls(), []) + + def test_an_address_the_docker_host_does_not_have_is_refused_naming_the_ones_it_has(self) -> None: + result = self.run_tasks(limit="node1", stand_in_addresses="3: col0 inet 192.0.2.50/24 brd 192.0.2.255 scope global col0") + self.assertNotEqual(result.returncode, 0) + self.assertIn("its addresses in 192.0.2.0/24: 192.0.2.50", result.stdout) + self.assertEqual(self.calls(), []) + + def test_a_cluster_no_server_can_be_read_through_refuses(self) -> None: + (self.work / "etcd-down").touch() + result = self.run_tasks(github_runner_cluster_etcd_peers_over_lan=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("could not be read through any server", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_fleet_role_probe.py b/tests/unit/test_fleet_role_probe.py index 266d3018..9478847e 100644 --- a/tests/unit/test_fleet_role_probe.py +++ b/tests/unit/test_fleet_role_probe.py @@ -378,6 +378,13 @@ def test_a_peer_that_never_answers_costs_one_election_timeout_not_more(self) -> self.assertEqual(self.newest()["tcp"]["peer"], 400) self.assertLess(time.monotonic() - started, 5) + def test_the_peer_is_dialled_at_its_etcd_member_s_address_not_its_node_ip(self) -> None: + # A voter whose member moved onto its LAN address (github_runner_cluster_etcd_peers_over_lan) listens for peers there and not on its InternalIP, which stays the mesh address; an address reserved for documentation stands in for that InternalIP, where nothing answers. + with listener() as port: + completed = self.probe(voters=[node("peer", voter=True, address="192.0.2.1", etcd_address="127.0.0.1"), node("me", eligible=True)], ETCD_PEER_PORT=str(port), ELECTION_TIMEOUT_MS="400") + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertLess(self.newest()["tcp"]["peer"], 400) + def test_an_ipv6_peer_address_is_bracketed(self) -> None: try: with listener("::1") as port: diff --git a/tests/unit/test_server_recreate.py b/tests/unit/test_server_recreate.py index ab68ebf9..2346ace3 100644 --- a/tests/unit/test_server_recreate.py +++ b/tests/unit/test_server_recreate.py @@ -498,8 +498,9 @@ def test_servers_start_through_their_turns_and_agents_do_not(self) -> None: def test_a_turn_checks_before_it_starts_and_waits_only_afterwards(self) -> None: tasks = role_tasks("start_server.yml") order = [task.get("ansible.builtin.include_tasks") or next(key for key in task if key.startswith(("community.docker", "ansible.builtin."))) for task in tasks] - self.assertEqual(order, ["server_recreate_check.yml", "ansible.builtin.stat", "ansible.builtin.debug", "community.docker.docker_compose_v2", "server_ready.yml"]) - self.assertEqual(tasks[3]["community.docker.docker_compose_v2"]["profiles"], ["server"]) + self.assertEqual(order, ["server_recreate_check.yml", "ansible.builtin.stat", "ansible.builtin.debug", "server_compose_up.yml", "server_ready.yml", "etcd_lan_move.yml"]) + (start,) = role_tasks("server_compose_up.yml") + self.assertEqual(start["community.docker.docker_compose_v2"]["profiles"], ["server"]) self.assertIn("not ansible_check_mode", tasks[4]["when"]) def test_the_turns_loop_over_the_servers_in_the_play_and_run_for_one_host_each(self) -> None: