diff --git a/.env.bootstrap.example b/.env.bootstrap.example index a1ec05a2..bfe1df9e 100644 --- a/.env.bootstrap.example +++ b/.env.bootstrap.example @@ -5,13 +5,13 @@ # Arbitrary shared secret for node-join auth inside the k3s cluster, unrelated to GitHub (e.g. `openssl rand -hex 32`). The same value on every server/agent host. K3S_TOKEN= -# A reusable, pre-authorized Tailscale auth key: k3s's own --vpn-auth integration runs Tailscale inside each node's k3s container with it. See the README's Tailscale section for how it's provisioned. +# A reusable, pre-authorized Tailscale auth key: k3s's own --vpn-auth integration runs Tailscale inside each node's k3s container with it. See the Tailscale section of docs/architecture.md for how it's provisioned. K3S_VPN_AUTH_JOIN_KEY= # Optional: a Tailscale API token. When set, the role also ensures the tailnet's ACL grants cross-node pod traffic (see roles/github_runner_cluster/tasks/mesh/tailscale_api.yml). Without it the ACL is left as it is. TAILSCALE_API_TOKEN= -# Every server's own Tailscale hostname, space-separated, including this one's own. Leave empty on a single-server setup, which then uses this host's node name. A multi-server cluster needs every server's cert to cover every server's name; see the README's Architecture section. +# Every server's own Tailscale hostname, space-separated, including this one's own. Leave empty on a single-server setup, which then uses this host's node name. A multi-server cluster needs every server's cert to cover every server's name; see the Architecture section of docs/architecture.md. K3S_TLS_SAN_LIST= # This host's k3s node name, which is also its Tailscale device hostname. Leave empty to use this machine's own short hostname. @@ -60,7 +60,7 @@ GHCR_PULL_TOKEN= HEARTBEAT_GH_TOKEN= HEARTBEAT_GIST_ID= -# The usage-driven autoscaler (see scripts/autoscaler.sh and the README's Autoscaler section) manages the scale set only when AUTOSCALER_USABLE_BUDGET_GI is set, and then also needs AUTOSCALER_FLOOR: the memory runner pods may use across every node in GiB, and the maxRunners in the scale set's values, which every Helm upgrade reverts to. They depend on the machines, so they have no defaults. It works out the highest maxRunners it may set from the nodes on every poll; AUTOSCALER_MAX_CEILING, optional, bounds that. The rest default to the role's own values (roles/github_runner_arc/defaults/main.yml). +# The usage-driven autoscaler (see scripts/autoscaler.sh and docs/autoscaler.md) manages the scale set only when AUTOSCALER_USABLE_BUDGET_GI is set, and then also needs AUTOSCALER_FLOOR: the memory runner pods may use across every node in GiB, and the maxRunners in the scale set's values, which every Helm upgrade reverts to. They depend on the machines, so they have no defaults. It works out the highest maxRunners it may set from the nodes on every poll; AUTOSCALER_MAX_CEILING, optional, bounds that. The rest default to the role's own values (roles/github_runner_arc/defaults/main.yml). # AUTOSCALER_USABLE_BUDGET_GI= # AUTOSCALER_MAX_CEILING= # AUTOSCALER_FLOOR= diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 262c303d..0e82027b 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,6 +1,6 @@ self-hosted-runner: labels: - # The ARC runner scale set label the fleet smoke tests (test-arc-runner.yml, test-autoscaler.yml) target - see the project README's Architecture section. + # The ARC runner scale set label the fleet smoke tests (test-arc-runner.yml, test-autoscaler.yml) target - see docs/architecture.md. - exadev-runners # Configuration variables in array of strings defined in your repository or diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de565b18..93946753 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,6 @@ name: CI -# Lints every shell script, workflow file, YAML file, and Dockerfile in the repo. There is no application test suite here (this repo is fleet infrastructure config, not application code) - these checks are its actual verification surface, matching the README's Build/test/lint section. +# Lints every shell script, workflow file, YAML file, and Dockerfile in the repo. There is no application test suite here (this repo is fleet infrastructure config, not application code) - these checks are its actual verification surface, matching docs/build-and-test.md. # # Runs on GitHub-hosted runners, which are free for a public repository, so the checks of the fleet's own configuration don't depend on the fleet being up. The fleet's own behaviour is tested by test-arc-runner.yml and test-autoscaler.yml, which target it directly. # diff --git a/.github/workflows/test-autoscaler.yml b/.github/workflows/test-autoscaler.yml index e864d29a..4e78e18a 100644 --- a/.github/workflows/test-autoscaler.yml +++ b/.github/workflows/test-autoscaler.yml @@ -1,6 +1,6 @@ name: test-autoscaler -# Manual traffic generator for the usage-driven autoscaler: dispatches several jobs onto exadev-runners at once so there is real, concurrent memory usage to watch the autoscaler's dry-run output/status react to (see the project README's Autoscaler section). This workflow does not assert anything about the autoscaler itself - it only creates the load; watch `kubectl logs -n github-runner-platform deploy/autoscaler -f` and `kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.status\.json}'` while this runs. +# Manual traffic generator for the usage-driven autoscaler: dispatches several jobs onto exadev-runners at once so there is real, concurrent memory usage to watch the autoscaler's dry-run output/status react to (see docs/autoscaler.md). This workflow does not assert anything about the autoscaler itself - it only creates the load; watch `kubectl logs -n github-runner-platform deploy/autoscaler -f` and `kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.status\.json}'` while this runs. on: workflow_dispatch: inputs: diff --git a/README.md b/README.md index acc5eeb5..5e9c5aaf 100644 --- a/README.md +++ b/README.md @@ -6,20 +6,20 @@ ## Getting started -There are two ways to deploy. A fleet's hosts are managed from a control machine with Ansible, from the fleet's own inventory (see [Ansible](#ansible) below). `bootstrap.sh`, described here, brings up a single machine from a checkout of this repository without a control node or inventory. It runs the same Ansible role against the machine itself, so both paths deploy identically. +There are two ways to deploy. A fleet's hosts are managed from a control machine with Ansible, from the fleet's own inventory (see [Ansible](docs/ansible.md) below). `bootstrap.sh`, described here, brings up a single machine from a checkout of this repository without a control node or inventory. It runs the same Ansible role against the machine itself, so both paths deploy identically. Prerequisites on the host: Docker (or Colima) with the Compose plugin, any release but 2.37.1 to 2.38.x (the cluster role refuses those; see Docker Compose versions in `roles/github_runner_cluster/README.md`), `python3`, `jq`, `openssl`, and `gh`. On macOS, the playbook installs `helm`, `kubectl` and Docker through Homebrew itself, and the cluster role starts a Colima host's profile again after a reboot (the node's profile at boot, with nobody logged in, any other at login) and fails unless the runtime is up (Docker Desktop needs its own start-at-login setting; see Container runtime after a reboot in `roles/github_runner_cluster/README.md`). Elsewhere, install `helm` and `kubectl` from the host's package manager first. 1. **Create a GitHub App** on the organisation with **Self-hosted runners: read & write** permission, and install it on the organisation. `playbooks/github_app_setup.yml` does this through GitHub's manifest flow (see `roles/github_runner_arc/README.md`). For one repository's runners instead, which is how a repository of a personal account uses the fleet, the App needs repository **Administration: read & write** and **Metadata: read**, installed where it reaches that repository, and `RUNNER_REPOSITORY` names it (see Repository scope in `roles/github_runner_arc/README.md`). 2. **Configure the environment.** Copy `.env.bootstrap.example` to `.env.bootstrap` and fill in the following (the file's own comments cover the rest, such as joining an existing cluster or running as an agent): - `K3S_TOKEN`: any random string (`openssl rand -hex 32`). It authenticates node-join inside the cluster only; it has no relation to GitHub. - - `K3S_VPN_AUTH_JOIN_KEY`, `K3S_TLS_SAN_LIST`: see [Tailscale](#tailscale) below. + - `K3S_VPN_AUTH_JOIN_KEY`, `K3S_TLS_SAN_LIST`: see [Tailscale](docs/architecture.md#tailscale) below. - `RUNNER_ORG`, `RUNNER_APP_ID`, `RUNNER_APP_PRIVATE_KEY_PATH`: the organisation, the App ID and a path to its private key PEM file (for example under `./secrets/`, already gitignored). With `RUNNER_REPOSITORY` set, the runners register in that repository of `RUNNER_ORG`, which may then be a user. - `RUNNER_APP_INSTALLATION_ID`: optional. Leave it blank and the role resolves it automatically from the App ID and key. - `RUNNER_IMAGE`, and either `RUNNER_VALUES_FILE` (a Helm values template) or `RUNNER_MAX_RUNNERS` (the ARC role's own template with that many runners at most). - `GHCR_PULL_USERNAME`, `GHCR_PULL_TOKEN`: a registry credential, such as a GHCR personal access token with `read:packages` scope, to pull the runner image and the fleet-health platform's own images. - - `HEARTBEAT_GH_TOKEN`, `HEARTBEAT_GIST_ID`: see [Heartbeat](#heartbeat) below. -3. **Build and push the runner images.** A fleet spanning both `arm64` and `amd64` hosts needs a real multi-arch push, not a plain `docker build`: `docker buildx build --platform linux/arm64,linux/amd64 --push -t ghcr.io/exadev/github-runner:latest .` for the main runner image (see [Build, test, and smoke-test](#build-test-and-smoke-test) for why this is currently the only working path, not just a manual fallback), plus the same for `heartbeat/Dockerfile` → `ghcr.io/exadev/github-runner-heartbeat:latest` and `autoscaler/Dockerfile` → `ghcr.io/exadev/github-runner-autoscaler:latest` - both now run as in-cluster pods that can land on either architecture, so both need pulling from either too. + - `HEARTBEAT_GH_TOKEN`, `HEARTBEAT_GIST_ID`: see [Heartbeat](docs/heartbeat.md) below. +3. **Build and push the runner images.** A fleet spanning both `arm64` and `amd64` hosts needs a real multi-arch push, not a plain `docker build`: `docker buildx build --platform linux/arm64,linux/amd64 --push -t ghcr.io/exadev/github-runner:latest .` for the main runner image (see [Build, test, and smoke-test](docs/build-and-test.md) for why this is currently the only working path, not just a manual fallback), plus the same for `heartbeat/Dockerfile` → `ghcr.io/exadev/github-runner-heartbeat:latest` and `autoscaler/Dockerfile` → `ghcr.io/exadev/github-runner-autoscaler:latest` - both now run as in-cluster pods that can land on either architecture, so both need pulling from either too. 4. **Bootstrap everything:** ```bash ./bootstrap.sh @@ -28,7 +28,7 @@ Prerequisites on the host: Docker (or Colima) with the Compose plugin, any relea ## Ansible -The repository is laid out as an Ansible collection, `exadev.github_runner` (see `galaxy.yml`): its roles live in `roles/` and its playbooks in `playbooks/`: `site.yml` brings up the whole fleet, `arc.yml` installs only ARC into a cluster that already exists (from the control machine, with a kubeconfig), and `github_app_setup.yml` creates the runners' GitHub App through GitHub's manifest flow, and `recover_in_cluster_mesh.yml` recovers the in-cluster Headscale provider's bootstrap server. A fleet keeps its inventory, `host_vars` and Helm values in a repository of its own, installs the collection there, and runs the playbooks by name (`exadev.github_runner.site`); `examples/` has inventories to start from. `ansible/` holds a generic `ansible.cfg` and the collections the roles require, for running the playbooks from a checkout of this repository as `bootstrap.sh` and CI do; `playbooks/collections/` links back to the checkout, so a playbook run from it finds `exadev.github_runner` in place without installing it. Verified end to end against real infrastructure (three hosts, a real single-node-to-3-node etcd HA cutover, real jobs scheduling across all three). `bootstrap.sh` (see [Getting started](#getting-started)) is a thin wrapper that runs this same playbook locally. +The repository is an Ansible collection, `exadev.github_runner` (see `galaxy.yml`): roles in `roles/`, playbooks in `playbooks/` (`site.yml` for the whole fleet, `arc.yml`, `github_app_setup.yml`, `recover_in_cluster_mesh.yml`). A fleet keeps its inventory, `host_vars` and Helm values in its own repository, installs the collection there and runs the playbooks by name. `bootstrap.sh` is a thin wrapper that runs the same playbook locally. The roles, server election, scale-set profiles, host prerequisites, Unraid hosts, capabilities and the registry cache are in [docs/ansible.md](docs/ansible.md). ```bash # in the fleet's own repository, next to its inventory @@ -37,215 +37,23 @@ ansible-playbook -i inventory.yml exadev.github_runner.site --limit node1 --chec ansible-playbook -i inventory.yml exadev.github_runner.site --limit node1 # real run ``` -The playbook runs these roles, each with its own README: `github_runner_cluster` brings the host up as a k3s node, in a container under Docker Compose or, with `github_runner_cluster_node_mode: native`, directly under systemd on a Linux host, `github_runner_arc` installs ARC and the fleet-health platform, `github_runner_k8s_client` (included by the other two) gives a host the kubernetes Python client, and `github_runner_secrets_onepassword` reads the secrets from 1Password when `github_runner_secrets_source` is `onepassword`. The cluster and ARC roles read every secret (`K3S_TOKEN`, the mesh keys, GHCR credentials, the heartbeat PAT, each org's App private key) as a plain variable, so any source Ansible reads works: ansible-vault, SOPS, an env or file lookup, HashiCorp Vault or AWS SSM (see `examples/ansible-vault/`, `examples/env/` and `examples/onepassword/`). With `github_runner_secrets_source: onepassword`, every secret is read live from one 1Password item at task time; `roles/github_runner_secrets_onepassword/README.md` lists the item's fields and the `op://` references. `bootstrap.sh` sets the same variables directly from `.env.bootstrap`. The cluster role ships the node's `docker-compose.yml` and k3s image build context in `roles/github_runner_cluster/files/`, copies them on every run into each host's `github_runner_cluster_dir` (`~/.github-runner` by default), and templates the `.env` for their `k3s`/`k3s-agent` services there, so a host needs no checkout of this repository. The Compose project name is pinned to `github-runner` (`github_runner_cluster_compose_project`) rather than taken from the directory's name, because the named volumes holding the k3s datastore and each node's mesh identity are prefixed with it; the directory can move without losing them. - -Which hosts are k3s servers is worked out from the inventory, not set per host. The `github_runner_cluster` inventory group lists the cluster's hosts in order: the first N become servers (control-plane/etcd members running `k3s server`, genuine voting members of the raft quorum), where N is the largest odd number no greater than both the number of hosts and `github_runner_cluster_max_servers` (5 by default), so 1 or 2 hosts give 1 server, 3 or 4 give 3, and 5 or more give 5. The first server bootstraps the cluster (`--cluster-init`, embedded etcd) and every other host joins it, the rest as agents (`k3s agent`, schedulable capacity with no control-plane role). Join URLs and every server's `--tls-san` list are derived from each server's node name and `github_runner_cluster_tailnet_domain`. A host can still set `github_runner_cluster_node_role`, `github_runner_cluster_bootstrap`, `github_runner_cluster_server_url` or `github_runner_cluster_tls_sans` to override its part, and the run fails with an explanation if the result has an even number of servers, none at all, or more than one bootstrap host; `roles/github_runner_cluster/defaults/main.yml` describes each. Installing the ARC controller, each org's runner scale set, and the fleet-health platform is not tied to a node role at all: each is driven purely by whether the installing host's own `github_runner_arc_orgs`/`github_runner_arc_heartbeat_gist_id` is set, so any server host can be the one Ansible installs against, and moving which host does the installing is a config change (which host's `host_vars` sets it), not a role change. See the [Architecture](#architecture) section for why 3 servers is the minimum sane count for genuine HA (2 is worse than 1 for quorum math, not better). To add a server or agent, add the host to the `github_runner_cluster` group in the fleet's inventory and give it its own `host_vars/.yml`; keep the bootstrap host first. - -A `github_runner_arc_orgs` entry registers its scale sets in an organisation, or with `repository` in one repository of a user or an organisation (`githubConfigUrl` `https://github.com//`); a repository's scale sets take only that repository's jobs, so each repository needs its own entry (see Repository scope in `roles/github_runner_arc/README.md`). `github_runner_arc_orgs[].scale_set_profiles` is a list of `{suffix, values_file?, node_selector?, runs_on_label?, capabilities?, autoscale?, sizing?, burst?}` entries (see `roles/github_runner_arc/README.md` for every key) - one Helm release and one namespace per entry, both removed by the next run once the entry leaves the inventory (see Profiles that leave the inventory in `roles/github_runner_arc/README.md`), pooled under the org's own shared `runs-on` label via `scaleSetLabels` (not `runnerScaleSetName`, which is a scale set's own unique GitHub-side registration identity, not just a label - two releases sharing it collide outright) unless a profile sets its own `runs_on_label` to deliberately opt out of that pool - e.g. a dedicated image-building profile ordinary CI jobs have no business landing on. `node_selector` is optional per profile, not mandatory: omitting it leaves that profile's pods genuinely unpinned, letting the Kubernetes scheduler place them on whichever node in the fleet actually has room - real failover if one machine goes down, not just capacity pooling. Values files are rendered on the control node from `github_runner_arc_values_dir`, a directory in the fleet's own repository, so no fleet host needs them. ExaDev's own fleet uses two profiles: an ordinary CI profile unpinned across the whole fleet (sized with Guaranteed QoS for the smallest machine's real capacity), and a second, separately-labelled, Docker-in-Docker profile pinned to one host for building the runner image - a deliberate isolation choice, not a capacity constraint. - -### Prerequisites - -The collection brings up the k3s nodes and what runs on them; it does not set up the hosts underneath. Each host must already have: - -- Python 3, or `github_runner_standalone_python_install: true` for a host that has none (see [Unraid hosts](#unraid-hosts)). -- A connecting user that is not root and can become root without a password, or a play run with `--ask-become-pass`: the tasks of the cluster role that change the host, and those of the macOS Actions runner role, use `become`. -- Docker with the Compose plugin for a node under Docker Compose (see the Prerequisites under [Getting started](#getting-started)). On macOS that includes Homebrew: the playbook installs `helm`, `kubectl`, Docker and Compose through it, and the macOS Actions runner role's `PATH` names both of its prefixes. -- A host that stays awake while it carries a node. A Mac needs its AC power settings set so that it never sleeps (and, with the lid closed, `pmset disablesleep`), and automatic restart after a power failure if it should come back by itself. A Linux laptop needs its suspend targets masked and its lid switch ignored, and on Wi-Fi its radio's power save turned off. The cluster role sets none of these. -- For a native node, nothing more where `github_runner_cluster_native_tailscale_install` is `true` (the default), since the role installs the pinned Tailscale package. With `false`, a Tailscale client already installed and logged in to the cluster's tailnet. A node under Docker Compose has its own Tailscale device inside the container and needs no client on the host. - -`github_runner_cluster_k3s_prerequisites` and `github_runner_cluster_containerd_apparmor_stacking_fix` are the two host steps the cluster role does offer, because they exist only for k3s (see Host preparation in `roles/github_runner_cluster/README.md`). - -### Unraid hosts - -An Unraid host runs the same cluster role as any other Docker host (it ships Docker and the Compose plugin), with three differences. It has no Python, so `github_runner_standalone_python_install: true` in its `host_vars` makes `site.yml` install a pinned one first (`roles/github_runner_standalone_python/README.md`), at `github_runner_standalone_python_install_dir`, with `ansible_python_interpreter` pointing at it. Its root filesystem and home directory live in RAM, so `github_runner_cluster_dir` and that install directory must be on persistent storage (under `/mnt/user/appdata`), never the default `~/.github-runner`. And Docker only runs once the array is started, so the node comes back after a reboot only when the array does. A host that is a k3s server also needs a k3s data volume on storage with fast synchronous writes, because etcd waits for the disk on every write: Docker's data directory on a spinning array disk is not enough, and a server there makes the whole control plane slow. Such a host belongs in the inventory after the servers, as an agent, until Docker's data directory sits on an SSD pool. - -### Capabilities - -A profile can also add tools, libraries and setup to its runner pods when they start, rather than baking them into a runner image someone has to build, publish and patch: `capabilities` names entries of `github_runner_arc_capabilities`. A tool-cache capability is a public, pinned image holding a tool in the Actions tool-cache layout; an init container copies it into the runner's `RUNNER_TOOL_CACHE`, where `actions/setup-node`, `setup-tflint` and other setup actions that consult the cache find it and skip their download, and it can also be put on `PATH` for a tool no setup action looks up. A system-library capability is a public, pinned image holding a relocatable install prefix (shared libraries, headers, pkg-config files, programs) for what cannot live in the tool cache, such as an MPI implementation, a solver library or a database client library; an init container copies it into a sysroot (`/opt/sysroot` by default), and the job-started hook puts the sysroot on `PATH`, `LD_LIBRARY_PATH`, `LIBRARY_PATH`, `CPATH`, `PKG_CONFIG_PATH` and `CMAKE_PREFIX_PATH` and sets any relocation variable the library needs. A hook capability is a script run before each job's first step, through a job-started hook mounted from a ConfigMap. A node-cache capability is a directory of the node shared by the profile's runner pods there, for a package manager's store, which a pruner keeps under a size limit. A profile with its own `runs_on_label` then works as a named runner, for example one with Terraform, TFLint and the AWS CLI, or one with Open MPI and a solver library, that workflows select with `runs-on`, all from one values file and the stock `actions-runner` image. `tool-images/` has Dockerfiles for reference tool images (Node.js, Terraform, TFLint, the AWS CLI) reference system-library images (Open MPI, HiGHS, GLPK, libpq, MariaDB Connector/C) and a reference build runner image (the stock runner image with a C toolchain, for jobs that compile against those libraries), which `.github/workflows/publish-tool-images.yml` publishes when started by hand or by a `tool-images-*` tag; `roles/github_runner_arc/README.md`'s Capabilities section has the tool-image and payload contracts, how each reference library is declared and proven to work from a relocated prefix, how to build an image of your own, and where capabilities apply. - -### Registry cache - -Each Docker-in-Docker runner pod starts with an empty Docker daemon, so every job pulls its base images, service containers and container actions from the registries again. `github_runner_arc_registry_cache_enabled: true` installs a pull-through cache (CNCF Distribution in proxy mode, one container per registry behind one Service, on persistent or ephemeral storage) and points every dind profile's daemon at it through a `hosts.toml` mirror per registry, and the `docker-container` builders `docker/setup-buildx-action` creates at it through buildx's default `buildkitd.default.toml`, so nothing in a workflow changes. Registries are a list (Docker Hub, ghcr.io and quay.io by default), each optionally pulling with a read-only credential from a Secret, and pulls fall back to the registry whenever the cache is down or failing. It is off by default, and off renders exactly what the role rendered without it. `roles/github_runner_arc/README.md`'s Registry cache section says which pull paths it covers, which bypass it, and what a credential on it exposes. - ## Build, test, and smoke-test -- **Build the runner image locally (the only working path right now):** `docker buildx build --platform linux/arm64,linux/amd64 --push -t ghcr.io/exadev/github-runner:latest .` from a machine with buildx/QEMU cross-platform support (Docker Desktop has this built in). A plain `docker build` with no `--platform` only produces an image for the building machine's own architecture, which silently breaks scheduling on the fleet's other architecture. The fleet-health platform's own two images need the identical treatment, for the identical reason (they're genuinely unpinned pods now too - see [Heartbeat](#heartbeat)/[Autoscaler](#autoscaler)): `docker buildx build --platform linux/arm64,linux/amd64 --push -f heartbeat/Dockerfile -t ghcr.io/exadev/github-runner-heartbeat:latest .` and the same with `autoscaler/Dockerfile` → `ghcr.io/exadev/github-runner-autoscaler:latest`, `pull-secret-renewer/Dockerfile` → `ghcr.io/exadev/github-runner-pull-secret-renewer:latest` (the image of the ARC role's App-sourced pull Secret renewal), `node-recovery/Dockerfile` → `ghcr.io/exadev/github-runner-node-recovery:latest` (see [Node recovery](#node-recovery)), and `fleet-role/Dockerfile` → `ghcr.io/exadev/github-runner-fleet-role:latest` (see [Role controller](#role-controller)). -- **Build and push via CI:** `.github/workflows/release.yml`'s `build-images` job builds and pushes every image (this one, `heartbeat`, `autoscaler`, `pull-secret-renewer`, `node-recovery` and `fleet-role`) multi-arch on GitHub-hosted `ubuntu-latest` whenever a release is published - see [Releases](#releases) below. -- **Smoke-test the ARC scale set** (confirms a job gets a real ephemeral pod): `gh workflow run test-arc-runner.yml`, then watch `kubectl get pods -n arc-runners- -w` (`arc-runners---` for a repository entry). A pod must appear only once the job is queued, run to completion, and be deleted within seconds. -- **Smoke-test `ubuntu-latest` routing** (confirms `runner-fallback-action` still reaches GitHub-hosted runners when needed): `gh workflow run test-ubuntu-latest.yml`. -- **Generate load for the autoscaler** (see [Autoscaler](#autoscaler) below): `gh workflow run test-autoscaler.yml`. -- **Verify cluster health:** - ```bash - kubectl get nodes # must show every server/agent host as Ready - kubectl get pods -n actions-runner-controller # controller pod and the scale set's listener pod both run here - kubectl get pods -n arc-runners- # zero runner pods when idle is expected, not a bug - ``` -- **Confirm no disk accumulates across repeated runner jobs:** the Docker host's disk usage (inside the VM, on a host that runs Docker in one) must stay flat across repeated `test-arc-runner.yml` runs. -- **Test capabilities:** `tests/capabilities/run.sh` builds the reference Node.js tool image and libpq system-library image, renders a profile with capabilities through the ARC role and the scale-set chart, and runs a pod from the chart's runner template in a throwaway kind cluster to check that the tool reaches the tool cache and `PATH` and the library reaches the sysroot and its search paths; `.github/workflows/capabilities.yml` runs it on pull requests. `tests/system_libraries/run.sh [openmpi|highs|libpq|glpk|libmariadb]...` proves each reference system-library image works relocated: it copies the payload to `/opt/sysroot` as the role's init container does, applies the role's job-started hook, and runs the payload's programs in the stock `actions-runner` image and a program compiled against it in the reference build runner image (`tool-images/build`, the stock image with `build-essential` and `pkg-config`), including a two-process `mpirun` for Open MPI; `.github/workflows/system-libraries.yml` runs it on amd64 and arm64 on pull requests. The images under `tool-images/` build with `docker buildx build --platform linux/arm64,linux/amd64 tool-images/`; `.github/workflows/publish-tool-images.yml` publishes them (see Releases), and the release workflow does not. -- **Test the build runner image's ids:** `tests/runner_build/run.sh` builds `tool-images/build` with its default runner ids and with `RUNNER_UID` and `RUNNER_GID` set to another, and checks the runner user in each; `.github/workflows/runner-build.yml` runs it on amd64 and arm64 for changes to the image. -- **Canary a published fleet-role image:** `gh workflow run published-image-canary.yml --ref main -f image='ghcr.io/exadev/github-runner-fleet-role@sha256:' -f scenario=role_controller` runs the chosen role-controller scenario in the disposable mesh on a GitHub-hosted Linux amd64 runner. The image must be publicly pullable and digest-pinned. The scenario choices are `role_controller`, `role_controller_swap`, `role_swap`, `role_swap_paused`, `role_swap_blocked_peer` and `role_swap_refused`; each tests only its own behaviour and the image selected for that runner's architecture, not other platform images. -- **Compare two published fleet-role images:** `gh workflow run published-image-canary.yml --ref main -f image='ghcr.io/exadev/github-runner-fleet-role@sha256:' -f comparison_image='ghcr.io/exadev/github-runner-fleet-role@sha256:' -f scenario=role_controller` runs the selected scenario in ABBA order on one GitHub-hosted runner, rebuilding the disposable mesh each time. The `published-image-comparison` artifact contains each run's log and a TSV of image, scenario, outcome and elapsed seconds. The run fails if any scenario fails; timings include mesh setup and teardown, may reflect caching and runner contention, and are not a fleet benchmark. The job's GitHub-hosted limit can end a comparison before all runs finish. -- **Test the registry cache:** `tests/registry_cache/run.sh` installs the cache as the ARC role renders it in a throwaway kind cluster, with Docker Hub, ghcr.io and a private registry behind basic auth, runs a pod from the chart's runner template as the role renders a Docker-in-Docker profile with the cache on, and checks in the cache's access logs that `docker pull` and a `docker-container` buildx builder pulled through it from all three without logging in, then scales the cache to nothing and checks that pulls and builds still work directly; `.github/workflows/registry-cache.yml` runs it on pull requests. -- **Lint:** `.github/workflows/ci.yml` runs Shellcheck, actionlint, yamllint, hadolint, an Ansible syntax check, ansible-lint (at its `production` profile), and a collection build check on every push and pull request, plus commitlint (on every commit in a pull request, and on the pull request's own title) on pull requests specifically - all gated by one `required-checks` job so a repository ruleset only ever has to require that one check regardless of how many lint jobs exist. hadolint's warning-level pinning advice is visible but non-blocking (`--failure-threshold error`): this repo deliberately floats several images/packages on `latest` rather than pinning. yamllint's config lives in `.yamllint`, relaxing the three default rules that conflict with this project's own conventions (long single-line comments, no leading `---`, GitHub Actions' `on:` key). -- **Commit messages:** `.husky/commit-msg` runs commitlint locally (after `npm ci`), the same check CI's commitlint job applies, so a message semantic-release would mishandle is rejected before it is pushed. The machine-wide hooks dispatcher hands off to it; without one, `git config core.hooksPath .husky` enables it. - -There is no application test suite in this repo; the checks above are the project's actual verification surface. +There is no application test suite; the checks are the project's verification surface. Run the repository's own lint and checks before pushing: `.github/workflows/ci.yml` runs Shellcheck, actionlint, yamllint, hadolint, an Ansible syntax check, ansible-lint (`production` profile) and a collection build check, gated by one `required-checks` job. `.husky/commit-msg` runs commitlint locally after `npm ci` (without a machine-wide hooks dispatcher, `git config core.hooksPath .husky` enables it). The image builds (local `docker buildx` is the only working path for the runner image), the smoke tests (`gh workflow run test-arc-runner.yml`, `test-ubuntu-latest.yml`, `test-autoscaler.yml`), the cluster health commands, `tests/*/run.sh` for capabilities, the build image's ids and the registry cache, the published-image canaries, and the full lint notes are in [docs/build-and-test.md](docs/build-and-test.md). ## Releases -Every push to `main` runs `.github/workflows/release.yml`, driven by [semantic-release](https://semantic-release.gitbook.io/) (`.releaserc.json`) against [Conventional Commits](https://www.conventionalcommits.org/) (`commitlint.config.js`, enforced by the `commitlint` CI job above, which checks every commit and the pull request title). Almost every conventional-commit type triggers at least a patch release (see `.releaserc.json`'s `commit-analyzer` `releaseRules`: `feat` is minor, a breaking change (`!` or a `BREAKING CHANGE:` footer) is major, everything else conventional is patch), so in practice nearly every push to `main` produces a release. - -A release: - -1. Stamps the new version into `galaxy.yml`'s `version:` field (which stays `0.0.0` in source between releases) and into the published-image tag defaults in `roles/github_runner_arc/defaults/main.yml` (`github_runner_arc_heartbeat_image`, `github_runner_arc_autoscaler_image`, `github_runner_arc_image_pull_secret_renewer_image`, `github_runner_arc_node_recovery_image`), and into each role's `*_collection_version` default, which the deployed-state stamps record - `scripts/release/stamp_version.py` does this by matching each variable's own name, not a line number, so it stays correct even while those files are edited concurrently by unrelated work. -2. Builds the collection tarball (`ansible-galaxy collection build --force`) and attaches it to the GitHub Release. -3. Updates `CHANGELOG.md` and commits it, `galaxy.yml`, and the stamped defaults file back to `main` as `chore(release): [skip ci]`. -4. Builds and pushes every container image (`Dockerfile`, `heartbeat/Dockerfile`, `autoscaler/Dockerfile`, `pull-secret-renewer/Dockerfile`, `node-recovery/Dockerfile`, `fleet-role/Dockerfile`) multi-arch (`linux/arm64,linux/amd64`) to their `ghcr.io/exadev/github-runner*` repositories, each tagged with the exact version, the floating major version, and `latest` - all three tags on the same manifest, via `docker/metadata-action`'s `type=semver` patterns. To pin a host to a specific release rather than floating on `latest`, set `roles/github_runner_arc/defaults/main.yml`'s image variables (or a host's own override) to an exact version tag instead. -5. Optionally publishes the collection to Ansible Galaxy, only if a `GALAXY_API_KEY` secret is present - the `[skip ci]` release commit and the image build both happen unconditionally, in this same run, regardless of whether Galaxy publishing is configured. - -The `[skip ci]` in the release commit message is deliberate: the image build already happens in this same run using the exact version semantic-release just decided, so there is nothing useful for a second, separately-triggered run of this workflow (or of `ci.yml`) to do against a commit that only changed a version stamp and a changelog - `ci.yml`'s own lint jobs already ran against every commit before it reached `main`. - -The reference capability images in `tool-images/` are not part of a release. `.github/workflows/publish-tool-images.yml` builds each one natively on amd64 and arm64 runners and pushes it to `ghcr.io//github-runner-toolcache-`, `ghcr.io//github-runner-sysroot-` or `ghcr.io//github-runner-runner-build`, tagged with the version its Dockerfile pins, when started by hand (`gh workflow run publish-tool-images.yml`) or by pushing a `tool-images-` tag. It leaves a version tag that already exists alone, and its run summary gives each pushed image's `:@sha256:` reference to pin in a capability or as a runner image. A new GitHub Container Registry package starts private; make it public in its package settings so pods can pull it without a credential. A build runner entry with a `runner_id` publishes the same image with its runner user at that uid and gid, tagged `-uid` (the ARC role README says when a fleet needs one). - -**What the release push needs:** a deploy key with write access on `ExaDev/github-runner`, whose private key is the `RELEASE_DEPLOY_KEY` repository secret `release.yml` checks out with. If `main` gains a ruleset, add deploy keys as its bypass actor so the release commit can land. Publishing to Ansible Galaxy additionally needs a `GALAXY_API_KEY` repository secret (an API key from `https://galaxy.ansible.com/me/preferences`); without it, the publish step logs that it's skipping and the release still completes. +Every push to `main` runs `.github/workflows/release.yml` (semantic-release): it stamps the version, builds and attaches the collection tarball, commits the changelog back with `[skip ci]`, and builds and pushes every container image. `galaxy.yml`'s `version:` stays `0.0.0` in source. Commits must follow Conventional Commits. The release steps, the `[skip ci]` reasoning, the tool images and the deploy key the release push needs are in [docs/releases.md](docs/releases.md). ## Architecture -An earlier version of this repo ran a fixed number of long-lived containers via Docker Compose, cycling the GitHub runner *registration* between jobs (`EPHEMERAL=true`) but never resetting the container's own filesystem. Two problems followed: build artifacts, `node_modules`, and Playwright browser caches accumulated indefinitely inside the same container and filled the host's Docker disk allocation, and a fixed replica count meant jobs queued whenever more were in flight than there were replicas, with no way to burst beyond it. - -ARC's `gha-runner-scale-set` solves both by design: it runs one fresh pod per job and deletes it afterwards, and it autoscales on queue depth. ARC needs a real Kubernetes cluster to run in. [k3s ships an official Docker image](https://hub.docker.com/r/rancher/k3s) that runs the whole control plane and kubelet inside one privileged container, a "cluster in a box". This makes the cluster itself just two Compose services (`k3s` on every control-plane host - the same service definition whether it bootstraps or joins, see the [Ansible](#ansible) section's node-role split - and `k3s-agent` on a pure-worker host), portable to any machine with Docker (plus `helm` and `kubectl` on the host), with no Colima-specific flags, no separately installed `kind` or `k3d` CLI. ARC itself (the controller, the runner scale sets, and the ephemeral runner pods), plus the fleet-health platform (heartbeat/autoscaler - see [Heartbeat](#heartbeat)/[Autoscaler](#autoscaler) below), are not more Compose services; they're all Kubernetes-native resources installed by the Ansible role (Helm and `kubernetes.core.k8s`) into this cluster (see `roles/github_runner_arc/`). - -The control plane itself runs genuine embedded-etcd HA across three `server`-role hosts, not a single node with worker capacity bolted on: one host bootstraps with `--cluster-init` (the one supported migration path from k3s's default single-node SQLite datastore to embedded etcd), and two further hosts join as full `k3s server` members - a real vote each in etcd's own raft quorum, not just schedulable capacity. 3 is the practical minimum: etcd tolerates the loss of any *minority* of its members without losing quorum, and with 2 servers that minority is zero - a second server can only ever make quorum strictly worse than staying at one, never better, so "2-node HA" isn't a real intermediate step, it's a trap. Getting there over Tailscale rather than a shared LAN needed real, non-obvious work - see the [Non-obvious behaviour](#non-obvious-behaviour) section's etcd-specific entries for what actually broke and why. - -Hosts on different networks can't reach each other's LAN addresses, and even hosts that share a LAN can't from inside their Docker VMs by default: each node's k3s container sits behind its VM's NAT, so its own Tailscale client has no address to offer its peers and every pair of nodes is relayed. So cross-node cluster traffic (a node joining, cross-node pod-to-pod traffic, the API server's proxy to a remote kubelet, and etcd's own peer traffic between `server` hosts) runs over a real overlay with mesh addresses - see [Tailscale](#tailscale) below for how that overlay is built and the DNS pitfall it introduces. Where hosts do share a LAN, `github_runner_cluster_network_mode: host`, on a Docker VM that has an address of its own on that LAN, lets their Tailscale clients reach each other directly over it, with the relay as the fallback when a host leaves; nothing k3s uses changes (see `roles/github_runner_cluster/README.md`, Networking). - -The custom runner toolchain (GCC 11/G++ for C++20, Bun, the `gh` CLI — see `Dockerfile`) carries over from the previous design, rebased onto ARC's own [`actions/actions-runner`](https://github.com/actions/runner/pkgs/container/actions-runner) base image instead of `catthehacker/ubuntu:act-latest`. ARC's runner pods use their own registration flow, so the previous design's vendored [myoung34/docker-github-actions-runner](https://github.com/myoung34/docker-github-actions-runner) scripts (registration, GitHub App JWT signing, ephemeral re-registration) are not needed. - -The images are built on GitHub-hosted runners rather than on the fleet, so a broken image can never take down the pool it would need to be rebuilt on. See [Releases](#releases) below for how a release builds them. - -`.github/workflows/ci.yml`, `mesh-integration.yml` and `litestream.yml` run on GitHub-hosted `ubuntu-latest`, which is free for a public repository, so checking the fleet's configuration never depends on the fleet itself being up. `test-arc-runner.yml` and `test-autoscaler.yml` are the ones that target the fleet, since the fleet's own behaviour is what they test. - -### Tailscale - -Each node's k3s process joins a Tailscale tailnet as its own device, using k3s's own built-in (experimental) [`--vpn-auth` integration](https://docs.k3s.io/networking/distributed-multicloud): `--vpn-auth="name=tailscale,joinKey="` on both `k3s server` and `k3s agent`. This runs Tailscale *inside* the k3s container/process itself, separate from the host machine's own Tailscale identity if it has one, and flannel uses Tailscale's own subnet-route advertisement for pod-to-pod traffic instead of its usual VXLAN encapsulation - no manual `--node-ip`, no Docker port-publishing, no host-level Tailscale dependency at all. The Mac's own Tailscale client, if it has one, is a separate device that this collection does not manage. A native node (a Linux host with `github_runner_cluster_node_mode: native`, which has no container to put a client in) is the exception: the host's own Tailscale client is the node's device, and the host is dedicated to the cluster; `roles/github_runner_cluster/README.md`'s Native nodes section says why. `k3s/Dockerfile` layers the `tailscale`/`tailscaled` binaries from the official Tailscale image onto `rancher/k3s`'s own minimal base (which has no package manager to install them at runtime), and `k3s/node-entrypoint.sh` (the Compose services' entrypoint, bind-mounted from the node's Compose project directory; both live in `roles/github_runner_cluster/files/`) starts `tailscaled` itself before execing k3s, since k3s's vpn-auth integration only runs `tailscale up` and assumes the daemon is already running. - -The mesh's control server does not have to be Tailscale's. The cluster role's `github_runner_cluster_mesh` switches it to Headscale: a server someone else runs, one the role runs under Compose outside k3s, or one it runs inside the cluster itself; see `roles/github_runner_cluster/README.md`. The rest of this section describes the default, Tailscale. - -Provisioning the join key: create a reusable, pre-authorized Tailscale auth key (Tailscale admin console → Settings → Keys, or the `POST /api/v2/tailnet/-/keys` API), tag it so an ACL grant can target it, and set `K3S_VPN_AUTH_JOIN_KEY` in every host's `.env` (or, via Ansible, set `github_runner_cluster_tailscale_join_key`, which the 1Password adapter reads from a `tailscale-join-key` field, as `roles/github_runner_secrets_onepassword/README.md` describes). The tailnet's ACL needs a `grants` entry permitting traffic between the nodes' own tag *and* the pod CIDR itself (e.g. `10.42.0.0/16`), not just the tag - Tailscale evaluates a subnet-routed packet's actual embedded source IP, not just the sending device's identity, so a grant scoped to the tag alone silently blocks cross-node pod traffic ([k3s-io/k3s#8372](https://github.com/k3s-io/k3s/issues/8372)). - -**Tailscale's own DNS management is a real trap here.** By default (`--accept-dns=true`, Tailscale's own default), `tailscale up` overwrites the container's `/etc/resolv.conf` with MagicDNS's resolver (a fixed, well-known `100.100.100.100`) plus whatever search domains the tailnet's own DNS settings configure - which can include search domains meant for entirely unrelated, personal use of the same tailnet. Kubernetes' `dnsPolicy: ClusterFirst` then copies the *node's* search domains into every pod's own `/etc/resolv.conf` alongside the cluster's own three, and the default `ndots:5` means a pod's resolver tries every search-suffixed name before the bare hostname for anything with fewer than 5 dots - which is essentially every real-world hostname. If any of those inherited search domains carries a wildcard DNS record, a pod's lookup of a real external hostname can resolve to that wildcard's target instead, while the application still sends the original hostname as its TLS SNI - silent certificate-verification failures for arbitrary outbound HTTPS traffic, indistinguishable at a glance from a hung network. Both `k3s server`/`k3s agent` invocations set `--resolv-conf` to a synthetic file containing only `nameserver 100.100.100.100` and no search domains, so pods still resolve everything correctly through Tailscale's own DNS proxy without inheriting a search list they have no use for. - -### Heartbeat - -ARC's autoscaling means there is normally no pre-existing "online runner" to check the way the previous design's fallback logic did (querying `/orgs/{org}/actions/runners` for an online match); with `minRunners: 0`, pods exist only while a job is running. To still support falling back to `ubuntu-latest` if the whole fleet goes down, `heartbeat` (an in-cluster Deployment - see `roles/github_runner_arc/tasks/install_platform.yml`) checks every `HEARTBEAT_INTERVAL_SECONDS` (default 180s) that a k3s node and the ARC controller are healthy and that no runner pod has been unschedulable for `HEARTBEAT_UNSCHEDULABLE_AFTER_SECONDS` (default: one interval), since a fleet with nowhere to place a runner only queues the work sent to it, and if so, refreshes a single secret gist with a unix timestamp set `HEARTBEAT_WINDOW_SECONDS` (default 600s) into the future. [`exadev/runner-fallback-action`](https://github.com/ExaDev/runner-fallback-action) reads that gist over the unauthenticated GitHub API and routes to `exadev-runners` while the timestamp is still fresh, or to `ubuntu-latest` otherwise. Every tick, healthy or not, it also writes `fleet-health.json` to the same gist (`HEALTH_GIST_FILE`): whether the fleet is vouched for, the reasons it is not, and any node reporting a pressure condition such as `DiskPressure`. A node under pressure does not make the fleet unhealthy, since the other nodes still serve jobs, but it is named there so lost capacity is visible. See that repo's `docs/spec.md` for the algorithm. It also writes `job-peak-summary.json`, the distribution of the peak memory jobs have measured (job count, how many distinct repositories they came from, the period they cover, median, 95th percentile and maximum in MiB, next to the runner pod's memory limit; the count and period are there because a large job count from one busy repository on one day says little), so whether the pods' request can come down is a lookup there and not a query of the cluster. It holds only figures, never a repository, workflow or job name, because the gist is readable by anyone holding its link. - -`fleet-health.json` also carries `deployed`: the collection version each playbook scope last recorded, so a scope that a run never reached shows as behind the version the fleet repository pins instead of looking like one that is current. The ARC and platform scope (`arc.yml`) records it in the platform namespace's `deployed-state` ConfigMap at the end of a run, as `platform`; each server host (`site.yml`) annotates its own node with `github-runner.exadev.com/deployed-version` once the node has registered, as an entry in `nodes`. A host annotates only its own node, so a `--limit` run never claims a version for hosts it did not touch, and an agent host, which has no API of its own, records nothing and shows `null`. The stamp records what a run applied, not what is running: a pod that failed to start after a run still leaves the stamp, so it complements the health reasons and does not replace them. `arc.yml` does not touch hosts, so a release that changes the Compose directory (the `docker-prune` service was one) shows the nodes behind until `site.yml` runs. - -Runs as a genuinely unpinned Kubernetes Deployment (`replicas: 1`, no `nodeSelector`), the same pattern ARC's own controller pods already use, not a host-pinned Docker Compose service — Kubernetes' own scheduler places and reschedules it on any server host with no config to move if one goes down. `kubectl` needs no `KUBECONFIG`: running as a pod with a mounted ServiceAccount token, in-cluster config is auto-detected. RBAC is scoped to exactly what it reads: cluster-wide `nodes` get/list, `deployments` get/list in the `actions-runner-controller` namespace, and `get` on the autoscaler's own status ConfigMap and the `deployed-state` ConfigMap by name (see Autoscaler below) — no wildcards. - -Create the secret gist once: -```bash -echo "$(($(date +%s) + 600))" | gh gist create --filename arc-healthy-until --desc "ARC fleet health heartbeat" - -# put the hex id from the returned gist URL into the installing host's github_runner_arc_heartbeat_gist_id -# (or .env.bootstrap's HEARTBEAT_GIST_ID for bootstrap.sh), -# and into whatever reads it, such as runner-fallback-action's gist-id input -``` -Confirm it refreshes: `gh gist view "$HEARTBEAT_GIST_ID"` must show a timestamp near `now + 600`, advancing roughly every 3 minutes. - -#### Fleet-down notification - -`runner-fallback-action` reroutes silently, so nothing otherwise tells a person that the fleet has no runners. `fleet-health.json` carries what a notifier needs to judge how long that has lasted: `at` (when the heartbeat last wrote), `unhealthy_since` (when it stopped vouching for the fleet, `null` while it does) and `not_ready_nodes` (each node whose Ready condition is not True, whether it is a control-plane server, the `regrade_level` memory-pressure regrading has stepped it to if it has, and since when). The heartbeat carries each onset over from the document its previous tick published, read back from the gist, so a tick that cannot read it restarts the onsets at now, which delays an alert and never raises a false one. - -`scripts/fleet-alert.sh` reads that document and announces an outage when any of these has held for at least `FLEET_ALERT_AFTER_SECONDS` (default 900, longer than the heartbeat's own 600 second window): the heartbeat has not written for that long, a control-plane server has not been Ready for that long (except one regrading has stepped down, which is not Ready on purpose), or the heartbeat has not vouched for the fleet for that long (a scale set missing or without a running listener, no ready controller, no Ready node, runner pods that cannot be scheduled; the reasons are in the notification). A worker node that is not Ready is not an outage by itself. An outage is announced once, and a recovery note follows when every condition has cleared. A heartbeat image older than the onset fields makes the script fail with a message saying so, rather than never alerting. - -Delivery is independent per channel, and with none configured the script only prints what it found: - -| Channel | Set | Notes | -|---|---|---| -| Issue | `FLEET_ALERT_ISSUE_REPO` (`owner/repo`) and `FLEET_ALERT_ISSUE_TOKEN` | Opens an issue and finds it again by a hidden marker in its body, so the open issue is the record that the outage was announced; closes it with a comment on recovery. The heartbeat's `HEARTBEAT_GH_TOKEN` is a gist-scoped token and cannot write issues, so this needs its own token with issue write access to that repository; a workflow's own `GITHUB_TOKEN` with `issues: write` suffices for the repository the workflow lives in. | -| Webhook | `FLEET_ALERT_WEBHOOK_URL` and `FLEET_ALERT_STATE_FILE` | POSTs JSON: `event` (`firing` or `resolved`), `text` (a one-line summary), `conditions` (each with `kind`, `detail` and `since`) and `heartbeat_at`. It names no chat product, so a receiver maps `text` or `conditions` to its own format. The state file holds only whether the outage was announced; a delivery that fails is not recorded and the next pass retries. | - -Both the URL and the tokens are passed to `curl` on its standard input, never its command line. Issue bodies name nodes and the heartbeat's reasons, so use a private repository. - -**Where it runs matters.** A check that runs only inside the cluster cannot report that the cluster is down, and a check that runs on the fleet's own runners does not start when the fleet has no runners. So the check reads the gist, which the heartbeat writes, from outside: `examples/fleet-alert/fleet-alert.yml` is a scheduled workflow for a repository not served by the fleet, on a GitHub-hosted runner, and the script runs just as well from cron on any host outside the fleet. The tradeoff is that it depends on GitHub (Actions scheduling, which can be delayed and is limited to a run every five minutes, and the gist API) and on the heartbeat's write path: if GitHub is unreachable from the fleet the heartbeat goes stale and the alert fires, which is also what the fleet's users experience. It is not a cluster Deployment, deliberately. A repository with no GitHub-hosted runners available cannot use the workflow and should use cron. The workflow template keeps the webhook's state in `actions/cache`, which can be evicted; the worst case is one repeated announcement. - -### Autoscaler - -A scale set's static `maxRunners` has to be safe for the pool's real worst case, every pod at its limit at once, so it leaves real idle capacity unused whenever the jobs in flight need less than that. `autoscaler` (an in-cluster Deployment, same shape/placement as heartbeat above) raises and lowers `maxRunners` across every profile with `autoscale: true` (one or more, possibly from different orgs, named by `AUTOSCALER_TARGETS`), based on real, current memory usage and pressure, never on job identity or GitHub Actions queue depth: every consuming repo's CI keeps using its scale set's own `runs-on` label unchanged, and no job is ever classified as light or heavy. When more than one profile is pooled, they share one combined budget rather than each getting a fixed slice: the algorithm below runs against the pool's combined `currentRunners`/`maxRunners`, and a change is handed to whichever pooled profile's own headroom (its `maxRunners` minus its `currentRunners`) makes it the right target: raises go to the profile with the *least* headroom (closest to genuinely needing more), lowers take from the profile with the *most* (its capacity is going unused), one runner at a time, and a lower never takes a profile below its own `currentRunners`. RBAC is scoped to exactly what it reads/writes: cluster-wide `nodes.metrics.k8s.io` and nodes get/list, and pods get/list for the ceiling; per pooled profile, `get`/`patch` on that `AutoscalingRunnerSet` by name, `get`/`list` on `ephemeralrunners` (to count the running runners and to name the job a runner is running), `get`/`list` on events and pods, and `pods.metrics.k8s.io` get/list in its own namespace; and `get`/`patch` on its own status ConfigMap by name, with no wildcards. - -Each poll (`AUTOSCALER_POLL_SECONDS`, default 45s): -- For every pooled profile, counts its running runners from its `EphemeralRunner` resources (every one that has not finished: a phase other than `Succeeded` or `Failed`, so a pending runner counts, because its pod already holds a slot) and reads its per-pod hard memory limit (`spec.template.spec.containers[0].resources.limits.memory`) straight from its live `AutoscalingRunnerSet`, never a static config value, so both always match what is actually deployed. The running count is not read from the `AutoscalingRunnerSet`'s status: in the ARC release this role pins (0.15) that status carries only `phase` and `observedGeneration`, so a count read from it would always be empty. A failed read of the runners stops the poll instead of counting zero. Sums the running counts into one combined total, and takes the largest of the pod memory limits as the threshold a raise or lower must leave room for. -- Sums real, current memory usage of every pooled profile's running runner pods, from the metrics API's pod list for each profile's own namespace (`scripts/autoscaler-usage.jq`); this, not a static per-pod request or limit, is the entire point of "usage-driven". It does not use `kubectl top pod`, which fails a whole namespace while any pod in it has no metrics, as every new runner pod has none until metrics-server first samples it. A runner pod placed on a node, not finished and missing from the list is unmeasured, and assumed to use its own memory limit (its scale set's limit when it sets none), the most it can reach. Lowering is decided on the measured usage, so a new pod never lowers the pool by itself; a raise also needs a pod's room with every unmeasured pod at its limit, and the pool holds until they are measured otherwise. -- Reads cluster-wide memory-availability pressure via `kubectl top nodes`, taking the **worst-case (minimum)** available-memory percentage across the nodes runner pods may use, not an average — a pool average can look healthy while the specific node a new runner pod would actually land on is not, consistent with the algorithm's own bias toward lowering eagerly (lowering never disrupts in-flight jobs). With the node eligibility label set (`github_runner_arc_node_label_key`, passed as `AUTOSCALER_NODE_SELECTOR`) only nodes carrying it are read, otherwise every node: a node outside the label runs no runner pod, so its pressure says nothing about where the next one lands, and when its kubelet drops out its missing metrics would otherwise fail safe on every poll. The same holds for a node inside the label that is not `Ready` (its kubelet stopped reporting or reported a fault): it is left out of this check by the definition the ceiling uses (`scripts/autoscaler-node.jq`), since it places no runner pod and `kubectl top nodes` prints `` for it. This is a genuine improvement over the pod's own earlier host-pinned design, which could only ever see one fixed machine's pressure via `/proc/meminfo`, not the whole pool's. **No swap-pressure signal**: metrics-server's API has no swap field at all, and the only alternative (the kubelet's own Summary API) needs a materially broader RBAC grant for a signal most kubelet versions don't even surface unless an off-by-default feature gate is on — dropping swap detection is a deliberate, documented trade-off once the pod is genuinely unpinned to any node, not an oversight; memory-availability pressure remains the dominant signal (the incident the values file's own comments were tuned from). -- Works out the ceiling, how many pooled runner pods the nodes can place now, from the live cluster (`scripts/autoscaler-ceiling.jq`). It reads every pooled profile's runner pod spec from its `AutoscalingRunnerSet` and takes the largest request among them, per resource, as the runner pod (a container that sets only a limit counts at its limit, as it will once its pod is created, and sidecars, init containers and pod overhead count as the scheduler counts them). A node counts when every pooled profile's runner pods could be placed on it now: it carries their `nodeSelector`, is `Ready`, is not cordoned, and their tolerations match each of its `NoSchedule` and `NoExecute` taints; a node that is not ready or unreachable is left out until it is back. On each, allocatable CPU and memory less the requests of the pods on it that are not pooled runner pods, divided by the runner pod's request, and its pod capacity (`allocatable.pods`) less those pods, give how many fit; the smallest of the three, rounded down, is the node's figure, and the ceiling is the sum. The pooled runner pods are left out so that runners already running never lower the ceiling they count towards; every other pod still holding requests counts, a non-pooled scale set's runners included. Node affinity is not read: the role gives an autoscaled profile none, and a pod one would keep off a counted node is caught as unschedulable below. `AUTOSCALER_MAX_CEILING` (`github_runner_arc_autoscaler_max_ceiling`, or, when exactly one pooled profile sets `sizing`, that profile's own derived figure) is an optional upper bound on it; empty leaves it unbounded. The ceiling worked out is kept in the status ConfigMap's `ceiling` key, and `status.json` reports `ceiling`, `ceiling_bound`, `ceiling_measured` and, per node, the runners it places or why it is not counted. When the nodes, the pods or a template cannot be read, or a figure in them is missing or is not a Kubernetes quantity, the poll prints why on stderr, keeps the last ceiling it worked out (or the bound, before it has one) for lowering and rebalancing, and does not raise the pool. Reading every node's pods needs `get`/`list` on pods cluster-wide, which the autoscaler's service account holds for this alone, as the heartbeat's does. -- Raises the combined `maxRunners` by exactly +1 per cycle (never straight to a computed target) once headroom has covered a full pod's hard limit for `AUTOSCALER_RAISE_CONFIRM_POLLS` (default 2) consecutive polls, capped at the ceiling, and only in a poll that worked the ceiling out. -- Lowers the combined `maxRunners` to the ceiling when the pooled total is above it, even when nothing else is wrong: each profile's own static `maxRunners` can sum to more than the ceiling, every Helm upgrade restores those, and the ceiling falls when a node goes, so the cap is enforced rather than only bounding raises. -- Rebalances a pool that is healthy and already holds as many runners as it may: the pooled total is at the ceiling, memory headroom covers a pod, there is no pressure and no unschedulable runner pod. Slots are fixed per profile, so without this a profile that has used all of its own while another sits idle would queue for capacity the pool has. When one profile's running count has reached its own `maxRunners` and another has at least one spare slot, it moves exactly one slot from the profile with the most spare slots to the saturated one, never changing the combined total, never taking the donor below its own running count and never below one runner (so a quiet profile can always start a job), and only once the same imbalance has been seen for `AUTOSCALER_RAISE_CONFIRM_POLLS` consecutive polls, so a burst that ends within a poll or two moves nothing. The count of consecutive polls is kept in the status ConfigMap's `rebalance-confirm-count` key beside `raise-confirm-count`, and every poll that is not a confirmed imbalance resets it to zero. Under pressure, with an unschedulable pod, above the ceiling or below it (where a raise hands the new slot to the profile with the least headroom) the slot-moving never runs. -- Lowers the combined `maxRunners` immediately, with no delay or averaging, the moment headroom drops below a pod's hard limit or real pressure is detected — lowering never disrupts in-flight jobs, since ARC only gates new claims. Never patches any pooled profile below its own currently-running count, nor below one runner: a scale set at `maxRunners` 0 asks for no runner, so a job queued for it would wait with nothing to start it, and the cluster keeps no record of a queued job the autoscaler could see. A pool therefore never goes below one runner per pooled profile, whatever the floor, pressure or ceiling say, and a profile left at 0 by something else counts as saturated, so rebalancing gives it a slot back. -- Treats any unschedulable runner pod (`PodScheduled=False`, reason `Unschedulable`) like pressure: the memory budget sums every node's capacity, so it cannot see a node that is tainted for disk pressure or a request that fits no single node. It lowers the combined `maxRunners` to the running count and never raises while one exists. -- Also collects the peak memory each job's runner measured for itself: the runner image's job-completed hook (`runner-hooks/job-completed.sh`, set through `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`) reads the container cgroup's `memory.peak`, which a spike shorter than the poll cannot escape, and posts it as a `JobMemoryPeak` Event in the pod's namespace. The autoscaler reads those Events each poll and keeps the maximum per repository, workflow ref and job id under the status ConfigMap's `job-peak` key (`kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.job-peak}'`), in MiB. Keeping the maximum means a forged low figure lowers nothing and re-reading an Event changes nothing. The hook never fails the job it ends. To post an Event the runner pods' service account, which otherwise holds no permission, gets `create` on `events` in its own namespace and nothing else (`github_runner_arc_record_job_peaks`, default on; `false` removes the grant). A job's code runs in the same container and can read the same token, so the worst it can do with that permission is post events in its own runner namespace. -- Records, per job, the highest memory a runner pod running it has been seen at, in the status ConfigMap's `job-memory` key (`kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.job-memory}'`), keyed by repository, workflow ref and job name with the peak in MiB, the number of observations and when it was last seen. It is sampled once per poll from the metrics server, so it is a lower bound that a spike shorter than the poll escapes; it shows how far below a pod's request jobs sit, which is what decides whether an accurate in-pod measurement (see the per-job sizing issue) is worth building. Entries unseen for 30 days are dropped, and a failure to keep the record never affects a capacity decision. -- Optionally reads per-node pressure from the nodes' own `/proc/pressure` (`github_runner_arc_autoscaler_node_pressure_enabled`). A `psi-reporter` DaemonSet (`scripts/psi-reporter.sh`, in the autoscaler's image) runs on each node runner pods may use, tolerating every taint, and patches that node's own key of one ConfigMap with the `avg10` of its io, cpu and memory pressure. The reading is not taken from the kubelet's summary API, because on a cgroup v1 node that reports every pressure field as 0 whatever the node is doing. The autoscaler lowers the combined `maxRunners` by exactly one each poll, never below the running count, while any node's I/O "full" stall is at or over `AUTOSCALER_IO_PRESSURE_PCT` (default 10) or, when set above 0, its cpu "some" stall is at or over `AUTOSCALER_CPU_PRESSURE_PCT`, and it does not raise the pool while any node has no report newer than three polls, since whether that node can take more is then unknown. This is the signal for a pool whose nodes differ: memory can have room on a node whose disk is already the limit. -- Fails safe on any memory measurement error (`kubectl top nodes` or the metrics API's pod list failing, or the node list failing, or a `Ready` node's figure that cannot be parsed), but not on a pod that merely has no metrics yet, nor on a node that is not `Ready`: lowers the combined total toward `AUTOSCALER_FLOOR` (`github_runner_arc_autoscaler_floor`) immediately rather than skipping the cycle silently. A failure reading the scale sets' own state or their pods exits non-zero instead, since fail-safe needs that state to lower anything and the API server is then unreachable anyway. This combined floor need not equal any one profile's own values-file `maxRunners`, since each profile's Helm upgrade reverts to its own static floor independently of how the pool's combined floor is set. - -Because the ceiling is the tighter of CPU and memory on each node, shrinking the runner pod's CPU request raises it only on nodes where CPU is what decides. Halving a 2 CPU, 7Gi pod to 1 CPU was measured on a two-node pool and not taken: memory then decided on both nodes, so it bought one runner (a ceiling of 5 became 6), while every job that keeps more than one core busy would be throttled to one, and the autoscaler records memory only, so there was no CPU history to weigh that against. Revisit it when CPU history exists (the node pressure reporter's cpu figures, or a per-job CPU record), when one more runner is worth that throttling, or when a node with too few cores for the larger pod needs a 1 CPU pod to take work. - -Ships with `AUTOSCALER_DRY_RUN=true` by default: it computes and logs the target and writes its own status, but never patches. Generate real concurrent traffic to watch it react: `gh workflow run test-autoscaler.yml`, then watch `kubectl logs -n github-runner-platform deploy/autoscaler -f` and `kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.status\.json}'`. Only set `AUTOSCALER_DRY_RUN=false` after watching real dry-run output across genuine CI traffic. Status and its raise-confirm counter live in a shared Kubernetes ConfigMap (`autoscaler-status`, pre-created empty so neither pod's own ServiceAccount ever needs `create` RBAC on it, only `get`/`patch`) rather than a bind-mounted directory, since the two pods can land on different nodes now — `scripts/heartbeat.sh` reads that same ConfigMap back out and republishes it as a second file in the heartbeat gist, reusing its existing gist-write credential rather than giving the autoscaler its own. - -A Helm upgrade of a scale set (the role runs one when the release's rendered values or chart differ from what is installed) reverts `maxRunners` to the values file's safe floor. On Helm 4, which applies releases server-side, the autoscaler's patches make it the manager of `maxRunners`, so the role forces conflicts on the controller and scale-set releases (`force_conflicts`, only over the fields the chart sets); without that the upgrade would fail instead of reverting. The role's `install_org.yml` triggers one immediate autoscaler poll after every such upgrade (`kubectl exec deploy/autoscaler -- ...`, reaching the pod through the API server regardless of which node it's on), so the safe-floor window after a redeploy is seconds, not a full poll interval. - -### Node recovery - -When a node stops (its host is switched off or asleep, Docker quits, or the k3s container is stopped), the node controller marks it `Unknown` once its kubelet has missed heartbeats for the node-monitor grace period, and after the pods' five-minute unreachable toleration marks them for deletion. Only the node's kubelet can confirm a deletion, so they then stay `Terminating` indefinitely. A Deployment's pods are replaced regardless, which is why the ARC controller comes back on its own, but ARC creates each scale set's listener as a single pod and waits for the old one to go, so a listener on the stopped node is never replaced and every job for that scale set queues until someone force-deletes it. - -`node-recovery` (an in-cluster Deployment in the platform namespace, see `roles/github_runner_arc/templates/node-recovery.yaml.j2` and `scripts/node-recovery.sh`) applies Kubernetes' [non-graceful node shutdown](https://kubernetes.io/docs/concepts/cluster-administration/node-shutdown/#non-graceful-node-shutdown): every `NODE_RECOVERY_POLL_SECONDS` (default 15s) it gives each node whose `Ready` condition has been `Unknown` for `github_runner_arc_node_recovery_after_seconds` (default 120s) the `node.kubernetes.io/out-of-service=nodeshutdown:NoExecute` taint. The control plane then evicts every pod on that node at once, unless it tolerates that taint, and the pod garbage collector force-deletes the ones already terminating there, so ARC starts the listener again on a healthy node. When the node reports `Ready` again the watcher removes the taint, and the node's kubelet stops the containers of the pods that were deleted while it was away. With the defaults a stopped node stops holding up job pickup within roughly four minutes: the grace period (50s on current Kubernetes), the threshold, one poll, the garbage collector's 20-second sweep, and the listener's own restart. `github_runner_arc_node_recovery_enabled: false` removes the watcher and any taint it applied. - -The watcher only acts on a node whose kubelet has gone silent (`Unknown`), never on one reporting `NotReady` (`False`), whose kubelet is alive and finishes its pods' deletion itself. It never taints the node it runs on, removes only taints it applied (recognised by its `github-runner.exadev/out-of-service-applied-at` annotation), and sends each change as a JSON patch that first tests the node's `resourceVersion`, so a node that came back since it was read is left for the next poll. RBAC is `get`, `list` and `patch` on nodes, with no `delete` unless pruning is switched on (below): the taint does not touch k3s's etcd membership, which only deleting a server's Node object changes. Its own pod tolerates an unreachable or not-ready node for only 10 seconds, so if it was running on the node that stopped, its ReplicaSet starts a replacement elsewhere well before the threshold passes. +ARC's `gha-runner-scale-set` on a k3s cluster (embedded-etcd HA across three `server`-role hosts) that spans arm64 and amd64 hosts on different networks over Tailscale or Headscale; one fresh pod per job. Each part has its own page: -The threshold is also where a stopped node's runner pods are given up, earlier than the five-minute default eviction would. Their containers stopped with the node, so their jobs are lost either way unless the node returns within that time; lower the threshold for faster recovery, or raise it to wait longer for a node that is only asleep. - -It also leaves alone the nodes a voter swap or a regrading step is moving through, since those stay silent on purpose: while the Lease `kube-system/fleet-regrade` is unexpired it neither taints nor prunes a node its `github-runner.exadev.com/swap-nodes` annotation lists or its holder names, and it never taints a node carrying the regrade level label. See the role README's Node recovery section. - -Pruning is opt-in (`github_runner_arc_node_recovery_prune_after_seconds`, empty by default). When it is set, the watcher also deletes the Node object of a node that has been `Unknown` for that long and carries the `github-runner.exadev.io/removable=true` label (`github_runner_arc_node_recovery_prune_label`), so a device that is never coming back stops being listed. It never deletes an unlabelled node, a node that is a control-plane or etcd member (even if labelled), its own node or a node without a `Ready` condition, and it deletes nothing unless at least three `Ready` nodes carrying the etcd role label would remain, because k3s removes a server's etcd member when its Node is deleted. It sends the delete with the node's `uid` and `resourceVersion` as preconditions so a node that has come back since it was listed is left alone. Turning it on gives the watcher a cluster-wide `delete` on nodes. Its pod is then confined to control-plane nodes, since a token that can delete nodes should not be readable from a worker. A pruned node comes back only when its k3s agent process restarts: a running kubelet does not register itself again once its Node object is gone, so pruning is safe only for nodes that restart their agent when their Node vanishes or that reboot, which the node has to arrange. See the role README's Node recovery section for the details. - -Its image, `github_runner_arc_node_recovery_image`, is pulled without a pull Secret, so the published package must be public. - -### Role controller - -`fleet-role-probe` and `fleet-role` (an optional DaemonSet and Deployment in the platform namespace, see `roles/github_runner_arc/templates/fleet-role.yaml.j2`, `scripts/fleet-role-probe.sh` and `scripts/fleet-role.sh`; `github_runner_cluster_role_controller`, off by default, which `github_runner_arc_role_controller_enabled` takes as its default) measure which hosts are fit to hold an etcd vote, report the swap of an unfit voter for a fit standby that the controller would make and, when asked or when `github_runner_arc_role_controller_dry_run` is false, carry the swap out ([Voter swap](#voter-swap)). The probe only reports: its ClusterRole can `get` and `list` nodes and nothing else, and the only object it writes is its own ConfigMap. The controller runs unchanged when no host is marked eligible, when it reports the voters' fitness alone. - -The probe runs on every Linux node that is an etcd voter (k3s's `node-role.kubernetes.io/etcd` label) or is marked eligible and has a kubelet (a server regraded to etcd-only has none, and its side is read from its peers), on the node's own network, through a node affinity on those two labels, so a node that is neither (a Unraid host, an image-flashed agent) has no pod and nothing is asked of its kernel. A pod stopped on a node that has lost both labels, or is gone, removes the node's entry as it ends, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the control plane and not with the fleet. Every `github_runner_arc_role_controller_sample_seconds` (15) it appends one sample to its node's key of the `fleet-role-probe` ConfigMap, which keeps the last window of them so that a replaced pod loses nothing. A sample holds the TCP connect time to every voter's etcd peer port (a connect that does not finish within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what its etcd metrics (the `127.0.0.1:2381` listener) gained since the last sample: the WAL fsync and backend commit histograms, the round trip to each peer, leader changes, and whether it has a leader, is a learner and knows as many members as there are voters (a learner or a member that has not started shows as an extra known member); on a node that is not a voter, an fdatasync reading from fio with the write size etcd's own disk check uses, once per window; the memory pressure (`/proc/pressure/memory` of the pod's own procfs, which serves the whole node's, so nothing is mounted) and `MemAvailable` the regrade core reads, with the core's own functions, where a node on a kernel without pressure stall information (built or booted without it, which has no `/proc/pressure` or, on older kernels, has the files and refuses to read them) gives `MemAvailable` and `MemTotal` and no pressure figure; and the medium of the node's link, `wired`, `wifi` or `unknown`: the value of the node's `github-runner.exadev.com/link` label (`github_runner_arc_role_controller_link_label`), which the cluster role reads on the host itself (see the cluster role's README) since a node in a container or a VM sees only its own virtual interface, and otherwise the medium of the interface the pod's default route leaves by (an interface that is virtual, such as the k3s container's veth, hides the link beneath it), each sample saying which (`link_source`, `label` or `interface`). The pod's own reading is taken from the interface's `/sys/class/net` entry and never by walking `/sys`, whose symlinks loop. `scripts/fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of these readings alone (the link as the pod's own interface gives it), with no cluster access, to compare with a reading taken by hand on the host. - -The controller reads the Nodes and the probes' samples every sample period and publishes `fleet-role-status`, which the heartbeat republishes in its gist beside the autoscaler's status. For every voter and every eligible host it gives a verdict, `fit`, `unfit` or `unknown`, with each reason and the numbers behind it. Every limit derives from etcd's guidance and the cluster's own timing, not chosen numbers: - -- A voter is unfit when its tail (99th percentile) round trip to any other voter exceeds a tenth of the election timeout, etcd's rule that the election timeout is at least ten round trips. A standby is fit only at or below a twentieth of it, the 0.5 end of etcd's heartbeat band; the gap between the two is the hysteresis. The election timeout is `github_runner_cluster_etcd_election_timeout`, or k3s's 1000 ms when that is empty. A pair's figure is the worse of the TCP connect tail and etcd's own round trip tail. -- A voter's WAL fsync tail must be under 10 ms and its backend commit tail under 25 ms (etcd's FAQ); a standby's fdatasync tail under 10 ms. -- Memory is never over the regrade limit (`github_runner_cluster_regrade_pressure_threshold` and `github_runner_cluster_regrade_available_floor_percent`) in any sample of the window, and `MemAvailable` is at least etcd's storage quota (what the voters report, 2 GiB by default). A host none of whose samples has a pressure reading cannot be over the regrade limit, which needs both, so it is judged by `MemAvailable` against the quota alone, and its status says so: `unknown_signals` lists `memory pressure` (and the log adds `memory pressure unknown` to its verdict), beside `figures.psi_avg300_max`, the worst pressure of the window, which is then `null`. -- Stability: no `NotReady` spell in the window and no k3s restart, seen on a voter as etcd's counters going backwards. On a node that is not a voter the only sign of a restart is its probe starting again, so a new probe pod counts as one. -- A window is 100 samples, the fewest that give the 99th percentile a sample of its own, so one outlier never decides; a host with fewer is `unknown`, as is one whose probe has not reported for three sample periods. Readings from before the last change of the voter set are not counted. -- Hosts that pass are ranked wired before Wi-Fi before unknown, then by round trip, then by disk. The link ranks hosts and is never a gate; the status gives each host's link and where it was read (`link` and `link_source`), and the log names it beside the verdict. - -A slow link makes both of its ends unfit, since a voter is unfit when its round trip to any other voter is, so every unfit verdict is attributed to the node at fault for it (`attributed_to` on each host, and `attribution`, the hosts each node accounts for). A host's own disk, memory, readiness or restarts are its own fault. Its round trip is the link to its worst peer, and the end at fault for a slow link is the one whose links to the other voters are slower too, compared by the best of them, since a node on a poor path is slow to every peer and a node on a good one is fast to its others; when the two ends' other links are as fast as each other's, or one end has none, the link is attributed to both. The voter to replace is the unfit voter the most hosts' unfit verdicts are attributed to, the worst-ranked among those with as many, and the log names the attribution beside each unfit verdict. A swap is reported only when that voter is eligible and not pinned (when it is not, replacing a voter that shares a slow link with it would fix nothing, so nothing is swapped) and a fit eligible standby ranks above it. A candidate is judged against the voters that would remain, since a slow link to the very voter it would replace says nothing against it. A voter is a node that carries k3s's etcd label and whose `EtcdIsVoter` condition is True: k3s labels a server the moment it joins etcd as a learner and never removes the label, so a labelled node without the condition is a member that does not vote: it is not counted as a voter and holds a swap back (`member but not a voter` in the status). It is held back, with the reason in the status and the log, while the voter count is not `github_runner_arc_role_controller_voters`, any voter is below `full`, is not Ready, has no leader, is a learner or knows a member that is not a voter, is not connected to every other voter, fewer hosts are eligible than the target, or the cooldown after a change of the voter set has not passed (k3s's five minutes to remove a stalled learner plus one window). The target must be odd, at least 3, and at most `github_runner_cluster_max_servers` minus one, which leaves a swap one place of headroom; the role refuses anything else. A host is eligible, and a voter pinned, by a Node label with the value `true` (`github_runner_arc_role_controller_eligible_label` and `_pinned_label`), which the cluster role publishes from `github_runner_cluster_voter_eligible` and `github_runner_cluster_voter_pinned` per host (and the target from `github_runner_cluster_voters`, refused unless odd, at least 3 and below `github_runner_cluster_max_servers`; see the cluster role's README); a voter not marked eligible is never swapped out, since nothing has prepared it to come back. - -#### Voter swap - -A swap brings an eligible standby in as a voter and takes an unfit voter out, as one operation under the `fleet-regrade` Lease, which the controller holds as `fleet-role` and renews on every pass for as long as the swap runs. A regrade step, a run of the cluster role and node recovery all leave the nodes of a swap alone while it does. The controller takes at most one action in a pass and reads the step a swap is at from the Nodes, the hosts' role actuators' reports and the Lease, so a controller pod that is rescheduled resumes where the swap is. The state machine is `scripts/fleet-role-swap.jq`, a pure function over that state, and `scripts/fleet-role.sh` carries out the one action it names. - -A swap starts in one of two ways. `kubectl -n github-runner-platform exec deploy/fleet-role -- /app/scripts/fleet-role.sh swap OUT IN` asks for it by command: the command exits 0 when every gate holds and the request is recorded for the next pass, 1 when it is refused (each reason is printed) and 2 for a node that is unknown, not marked eligible or not the voter and the standby it is taken for. A request is carried out whatever the verdicts say, and inside the cooldown, but never takes out a pinned voter and honours every gate. The controller starts the swap it decides on only when `FLEET_ROLE_DRY_RUN` is `false` (`github_runner_arc_role_controller_dry_run`, true by default, which reports such a swap and carries out requests alone). A swap already under way is finished or rolled back whatever the setting, so an even count of voters is never left behind. `fleet-role.sh status` prints the swap and its step. - -The gates are those of the epic: every voter at `full`, Ready, with a fresh probe report, etcd with a leader, no learner and no member that has not started, the voter count the target, and the regrade Lease free. The steps: - -1. A snapshot is asked of a remaining voter through its Node (`snapshot-request`) and waited for until k3s publishes an `ETCDSnapshotFile` of that name that reports `readyToUse`. Nothing has changed yet, so a snapshot that is not ready within the operation timeout abandons the swap. -2. The candidate is cordoned and the controller waits for its runner pods to finish (a planned change never cuts a job); past the runner wait the swap is abandoned, the candidate uncordoned and the Lease released, with no cooldown. Otherwise it is drained and asked to promote: `desired-role=server`, with `role-swap`, `role-step=promote` and `role-join-node` naming a remaining voter. Its actuator writes the role file, a join URL and the unit changes, and starts the server, which k3s joins as a learner that the etcd leader promotes once it has caught up. The step ends when the candidate is a voter (`EtcdIsVoter`) and Ready on a kubelet heartbeat after the server started, and the candidate is uncordoned. -3. The gates are held again, now with one more voter than the target. If the outgoing voter is down they are held to the other voters alone. -4. The outgoing voter is tainted and cordoned, its runner pods waited for (past the runner wait it is drained anyway, as regrading drains a server) and drained, labelled `regrade-level=etcd-only` and asked to step down (`desired-role=agent`, `role-step=step-down`). Its actuator brings its server to etcd-only and writes the role file for the agent. When it reports done, or when the node is down and has not reported within the operation timeout, the controller annotates `etcd.k3s.cattle.io/remove=true` while the remaining voters are healthy, and again if k3s clears the annotation, and waits for `removed-node-name`. A step-down the actuator reports failed on a node that stays Ready is held for the owner. The member counts as removed once k3s records it, or once every remaining voter's probe reports etcd listing as many members as there are voters, which holds when a server still running on the host has put its etcd label back; the controller then marks the Node with `role-member-gone` naming the swap, the proof its actuator finishes on, since k3s's own record can name another member than the host's datastore. The actuator then finishes the demotion (the server stopped, its datastore moved aside, the agent started) and the controller removes the `control-plane`, `etcd` and `master` labels, the `etcd.k3s.cattle.io/*` annotations, the level label and the taint, waits for the node to be Ready as an agent and uncordons it. A host that has not come back within the operation timeout is left to node recovery and the swap records `outgoing_pending`. -5. The swap annotations are cleared and the Lease is released. The cooldown starts from the membership change, as for a decision. - -A candidate that is not a voter by the deadline (k3s's learner stall time plus the operation timeout), or whose actuator reports the promotion failed, is rolled back: the controller annotates the removal of its learner if it joined, asks its actuator to put the host back to an agent (`role-step=rollback`), waits until every voter's probe lists exactly the target number of members, then cleans the Node up and uncordons it. A refused promotion, which changed nothing on the host, abandons the swap. Every wait is derived from the operation timeout, the runner wait and the learner stall time, which the role takes from the cluster role's own settings (`github_runner_arc_role_controller_operation_timeout_seconds`, `github_runner_arc_role_controller_runner_wait_seconds` and `github_runner_arc_role_controller_lease_seconds`). - -`fleet-role-status` carries `swap` (`id`, `out`, `in`, `step`, `started_at`, `step_since`, `deadline`, `manual`, `snapshot`), `waiting` (why the swap is not acting this pass), `last_swap` (`outcome`, `completed`, `rolled-back` or `abandoned`, with its reasons), the `request` pending and the real `dry_run`. A manual request is kept in the status ConfigMap under `request.json`, a key of its own, so that the command and the controller's pass never overwrite each other. The controller's ClusterRole adds to the probe's `patch` on nodes, `get` and `list` on pods, `create` on `pods/eviction`, `get` on daemon sets and `get` and `list` on k3s's `etcdsnapshotfiles`; its Roles let it `create` Leases and `get` and `update` the regrade Lease in `kube-system`, `get` the kubelets' Leases and `get` the actuators' ConfigMap. What stays manual is removing a voter that is down and not coming back, restoring after quorum loss and changing the target count. - -The controller and the probe run one image, `github_runner_arc_role_controller_image`, pulled without a pull Secret, so the published package must be public. +- [Architecture and Tailscale](docs/architecture.md): why the design replaced fixed Compose containers, HA, the mesh, the Tailscale DNS trap, the join key. +- [Heartbeat and fleet-down notification](docs/heartbeat.md): the health gist `runner-fallback-action` reads, `fleet-health.json`, `scripts/fleet-alert.sh`, where the check must run. +- [Autoscaler](docs/autoscaler.md): the combined `maxRunners` ceiling, pressure handling, dry-run default, Helm upgrades of scale sets. +- [Node recovery](docs/node-recovery.md): what happens when a node's host stops, pruning, the recovery image. +- [Role controller and voter swap](docs/role-controller.md): the probe, fitness rules, regrading, the swap procedure and its status. ## Conventions @@ -266,25 +74,29 @@ Commits follow Conventional Commits (`fix:`, `feat:`, `tune:`, and so on). ## Non-obvious behaviour -- **A scale-set's listener pod can crash-loop forever with `ephemeralrunnersets... not found` after a Helm upgrade, even though the release itself succeeds.** ARC names the `EphemeralRunnerSet` it creates for a release with a spec hash suffix, and gives it a new name whenever the pod template changes (a `helm upgrade` that touches resource limits, `nodeSelector`, or similar). The `AutoscalingListener` is supposed to track this rename, but can be left pointing at the old, now-deleted name - it then restarts every second or two, each time failing at startup with `could not patch ephemeral runner set ... not found`, and the scale set's own job queue never gets serviced even though every other release's listener stays healthy. Diagnose with `kubectl get autoscalinglisteners -n actions-runner-controller -o custom-columns='NAME:.metadata.name,ERS:.spec.ephemeralRunnerSetName'` against `kubectl get ephemeralrunnersets -A` - a mismatch confirms it. Fix: `kubectl delete autoscalinglistener -n actions-runner-controller`; the controller recreates it immediately with the current `EphemeralRunnerSet` name. -- **k3s cannot overwrite an existing kubeconfig through Colima's macOS share.** The container-root-to-Mac-user file mapping leaves `/output/kubeconfig.yaml` owner-read-only, so k3s crash-loops with "permission denied" on every restart (for example after a Mac reboot) unless the file is removed first. `k3s/node-entrypoint.sh` deletes it before starting k3s each time — do not remove that step. -- **Pods can lose every cluster service IP, cluster DNS included, whenever the Colima Linux VM has not loaded `br_netfilter` (a restarted VM, after a host reboot for example).** Without the module (`/proc/sys/net/bridge` is missing in the VM) packets bridged between pods never traverse iptables, so the service DNAT never applies: a pod reaches another pod's IP and the node's resolver directly but times out on `10.43.0.10` when the Service's only backend pod (CoreDNS, say) is on the same node as the caller, since a bridged hairpin reply skips the reverse NAT; backends on other nodes still answer, which is why it can stay latent until such a pod is rescheduled onto the node. Nothing looks wrong at first, because ARC keeps working from cached state; it fails only when it next needs a fresh GitHub token, when the controller logs `lookup api.github.com: i/o timeout`, a scale set's listener is not recreated, its `AutoscalingRunnerSet` stays `Pending` and the heartbeat reports `has no running listener`. The cluster role does this on every run for a Colima host (`tasks/colima_kernel.yml`, see its README's Colima kernel paragraph): it loads the module, writes `/etc/modules-load.d/br_netfilter.conf` and a sysctl file in the VM and sets the bridge sysctls. To repair a host before the playbook has run, load it with `colima ssh -- sudo modprobe br_netfilter`; check with `colima ssh -- ls /proc/sys/net/bridge`. -- **The k3s node name must stay stable across container recreations.** Without a pinned `hostname`, k3s derives the node name from the container's hostname (its ID), so every recreation (for example on an image pull) registers a new node and leaves the old one stale in etcd. The cluster role's `docker-compose.yml` pins every server host's own hostname to `${K3S_NODE_NAME:-k3s-server}`, Ansible-templated per host (defaulting to that host's own inventory name; `github_runner_cluster_node_name` overrides it, for example to keep a pre-existing device identity rather than forcing a disruptive rename). -- **`kubectl get nodes` can succeed with zero nodes.** It returns an empty, still-successful list before the node object has registered. The role polls for at least one node before calling `kubectl wait`, which fails outright ("no matching resources found") against zero nodes. It then waits for the inventory's nodes by name, never for every node of the cluster, so a node outside the inventory that is `NotReady` is reported and does not fail the run. -- **Zero runner pods in an `arc-runners-` namespace (or its per-profile `-` variant) when idle is expected, not a bug** — `minRunners: 0` means pods exist only while a job is running. -- **An org's Settings > Actions > Runners page shows the runner scale set itself** (a repository entry's shows in that repository's own Settings > Actions > Runners), not individual long-lived runner names the way the previous design did. ARC's ephemeral runners do not persist between jobs, so there is nothing to list while idle. -- **An autoscaled profile's static `maxRunners` and per-pod resource limits have to come from the pool's measured capacity**, not arbitrary defaults: a higher `maxRunners` without matching headroom leads to out-of-memory kills and pod evictions (ExaDev's fleet repository records the incident its own values were tuned from). `scripts/autoscaler.sh` depends on this value staying the safe floor it reverts to. -- **The metrics API's pod list and `kubectl top nodes` require metrics-server.** k3s bundles it by default (the cluster role's `docker-compose.yml` only disables `traefik`), but confirm it is actually running during the first real autoscaler dry run — `scripts/autoscaler.sh` fails safe to the static floor if either is unreachable. A very short-lived job (seconds, not minutes) can complete and its pod be garbage-collected before metrics-server's own scrape interval ever captures its usage - confirmed live against `test-arc-runner.yml`'s ~15-20s jobs, where `kubectl top pod` genuinely never returns data. This correctly triggers the fail-safe rather than a wrong reading, and doesn't matter in practice: a job that short never accumulates meaningful memory pressure anyway. `test-autoscaler.yml`'s own probe jobs hold for 120s specifically so metrics-server has time to observe them. -- **A replaced node can stay permanently `NotReady` with "Node password rejected" in its logs.** k3s keeps a node's password in `/etc/rancher/node/password` and the cluster holds a Secret for it (`kube-system/.node-password.k3s`). A recreated container keeps its password, because `/etc/rancher/node` is a named volume that the cluster role seeds from the running container before the first recreate that mounts it (see the role README's "Node passwords and recreating servers"), so this only happens to a node that is genuinely new under an old name: a replaced host, or a node whose volume was removed. Its container then generates a password the bootstrap host's Secret does not match, and the bootstrap host rejects the rejoin. Fix: on the bootstrap host, `kubectl delete secret .node-password.k3s -n kube-system` (and, for an agent, `kubectl delete node `), then restart the container so it registers fresh. Never delete a server's Node object: k3s then removes that server's etcd member, and a removed member's etcd refuses to start until its local datastore (`/var/lib/rancher/k3s/server/db`) is moved aside, after which it rejoins as a new member. `roles/github_runner_cluster/tasks/main.yml` self-heals the password mismatch automatically, only for hosts whose node is missing or not Ready, and deletes Node objects only for agents. -- **etcd's own peer traffic does not follow vpn-auth's `--node-ip`/`--advertise-address` the way every other kind of cluster traffic does - it stays on the container's own internal, cross-machine-unroutable Docker-bridge IP by default.** k3s logs this outright ("Etcd IP (PrivateIP) remains the local IP. Running etcd traffic over VPN is not recommended due to performance issues") and deliberately avoids the VPN interface for etcd specifically, assuming a real local network is the better alternative - an assumption that doesn't hold here, since a container's own bridge IP is never reachable from another machine at all. Confirmed live: a second server's join failed outright with `MemberAdd request timed out`, because it was trying to reach the first server's own unroutable bridge IP. An `--etcd-arg="initial-advertise-peer-urls=..."` passthrough looks like the fix but isn't: it only overrides etcd's own raw flag, while k3s independently builds its own `--initial-cluster` string from `config.PrivateIP`, computed separately and earlier - the two then disagree and etcd's bootstrap validation rejects the mismatch outright (`"--initial-cluster has ...= but missing from --initial-advertise-peer-urls="`). The actual fix, traced through k3s's own source (`pkg/cli/server/server.go`, `pkg/executor/embed/embed.go`): `config.PrivateIP` is populated from `--node-ip`'s own value if set, and this happens *before* `--vpn-auth`'s own executor-layer code later rebuilds `NodeIP` from its own Tailscale detection - so passing `--node-ip` explicitly ourselves, to the same real Tailscale IP vpn-auth would itself have detected, is what actually makes `PrivateIP` (and therefore etcd's peer-advertise address) correct, with no second, disagreeing source of truth for the same value. `docker-compose.yml`'s `k3s` service entrypoint brings Tailscale up itself and reads `tailscale ip -4` before exec'ing k3s specifically to learn this IP first, since `exec` replaces the script's own process and nothing can react afterward to what `--vpn-auth`'s own internal call would have produced. -- **A global `--etcd-arg` override is not scoped to the "real" server's own etcd instance - it also applies to k3s's separate, internal "temporary etcd", spun up on every *restart* (never on first boot) to reconcile with an already-initialized on-disk datastore before the real instance starts.** An earlier `--etcd-arg="listen-peer-urls=https://0.0.0.0:2380"` (kept as a "harmless" belt-and-braces bind override after fixing the peer-URL issue above) forced that temporary instance onto the same TLS `0.0.0.0:2380` bind it has no certificates for by design (it normally listens on a local, ephemeral, non-TLS port). Confirmed live: this only broke on a restart against an already-initialized datastore, never on a fresh `--cluster-init`, which is exactly why it wasn't caught until well after the first working deploy - fatal: `"cannot listen on TLS for [::]:2380: KeyFile and CertFile are not presented"`. `--node-ip` alone (above) is sufficient; etcd's own default `listen-peer-urls` already derives from the same `PrivateIP` correctly, so there's nothing left for an explicit override to add. -- **Calling `tailscale up --authkey=...` unconditionally on every container restart re-registers a brand new tailnet device each time, instead of reconnecting the persisted one - even with the join key's own state volume intact.** A reusable Tailscale auth key is designed to authorize provisioning *new* devices, and a naive `tailscale up` call on every restart (needed here to learn this node's own Tailscale IP before `exec`ing k3s - see above) doesn't check whether the daemon is already authenticated first, so it keeps minting new devices under names like `node1`, `node1-1`, `node1-2`... k3s's own `--vpn-auth` integration avoids exactly this from v1.35.0 on, by checking the Tailscale backend state first and skipping its own `tailscale up` call entirely once already `"Running"` (`StartVPN()` in `pkg/vpn/vpn.go`; the pinned v1.34 releases run `tailscale up --authkey ... --reset` on every start whatever the state) - the manual pre-empt call needs the identical guard: check `tailscale ip -4` succeeds *before* calling `tailscale up` at all, only calling it when not already authenticated. -- **Tailscale assigns an IP and wires up its own MagicDNS resolver as two separate steps, not atomically - a node having a real IP is not proof DNS resolution against another tailnet member is working yet.** A joining server needs to resolve the bootstrap host's own MagicDNS hostname (`K3S_JOIN_SERVER_URL`) before it can bootstrap against it at all. Confirmed live: k3s failed outright with `"dial tcp: lookup k3s-server.example.ts.net: no such host"` immediately after `tailscale ip -4` had already returned a real address for the *local* node. Fix: wait for a real `nslookup` success against the join server's own hostname specifically, not just local Tailscale readiness, before invoking k3s (the `k3s` service's entrypoint only does this when it has a server to join, `K3S_JOIN_SERVER_URL` or the role directory's `join-url` - the bootstrap host has nothing else to resolve). -- **A server that memory-pressure regrading has stepped to `etcd-only` is `NotReady` by design, and must stay registered.** With `github_runner_cluster_regrade` on, a server under sustained memory pressure steps down (`full`, `control-plane`, `etcd-only`) while it keeps its etcd member and vote; at `etcd-only` k3s runs with `--disable-apiserver --disable-controller-manager --disable-scheduler --disable-agent`, so there is no kubelet to keep its Node Ready, and the Node goes `NotReady` with `Unknown` conditions and stays so. That is not a failure to clear. Its Node object must not be deleted (k3s removes that server's etcd member with it), so anything that culls long-`NotReady` nodes must leave a Node carrying the `github-runner.exadev.com/regrade-level` label alone, and the role's wait for every node to be Ready already does. `kubectl` on such a server's own container is unreliable (its API server is off); the regrade service always reaches the cluster through another server. See the cluster role's README, Memory-pressure regrading. Confirmed against a three-server k3s cluster: the same etcd member ids before, during and after, the cluster taking writes with the stepped-down server as one of the two live voters, and the same Node and node password after stepping back up. -- **A single remaining healthy etcd server can get stuck indefinitely if another member is registered but permanently unreachable (e.g. a partial join that never completed), even though that member is a non-voting learner and shouldn't block quorum.** Symptom: `"failed to publish local member to cluster through raft"` with `context deadline exceeded`, repeating forever, alongside `kubectl` returning `503 the server is currently unable to handle the request`. The fix is k3s's own documented recovery mechanism, run once against the affected node: stop the container, then `k3s server --cluster-reset` (with the SAME flags the real service uses, including `--node-ip` set to the node's own persisted Tailscale IP - getting this wrong re-registers the single remaining member under the *wrong* address, e.g. the invoking container's own ad-hoc bridge IP, and the real service then fails to start with `"this server is a not a member of the etcd cluster"`) - this resets *etcd cluster membership* back to a single member using the *existing local data*, it does not wipe anything. It's a one-shot operation: k3s writes a guard file after a successful reset and refuses to run `--cluster-reset` again until it's manually deleted (`rm /var/lib/rancher/k3s/server/db/reset-flag`) if a second reset is genuinely needed. After a reset, restart the service normally (without the flag) and clean up the stale member's own now-orphaned Kubernetes `Node` object and `node-password` Secret before it rejoins fresh. +Each entry is a trap with a diagnosis and fix in [docs/non-obvious-behaviour.md](docs/non-obvious-behaviour.md); read the entry before acting on the symptom. + +- **A scale-set's listener pod can crash-loop forever with `ephemeralrunnersets... not found` after a Helm upgrade, even though the release itself succeeds.** +- **k3s cannot overwrite an existing kubeconfig through Colima's macOS share.** +- **Pods can lose every cluster service IP, cluster DNS included, whenever the Colima Linux VM has not loaded `br_netfilter` (a restarted VM, after a host reboot for example).** +- **The k3s node name must stay stable across container recreations.** +- **`kubectl get nodes` can succeed with zero nodes.** +- **Zero runner pods in an `arc-runners-` namespace (or its per-profile `-` variant) when idle is expected, not a bug** +- **An org's Settings > Actions > Runners page shows the runner scale set itself** +- **An autoscaled profile's static `maxRunners` and per-pod resource limits have to come from the pool's measured capacity** +- **The metrics API's pod list and `kubectl top nodes` require metrics-server.** +- **A replaced node can stay permanently `NotReady` with "Node password rejected" in its logs.** +- **etcd's own peer traffic does not follow vpn-auth's `--node-ip`/`--advertise-address` the way every other kind of cluster traffic does - it stays on the container's own internal, cross-machine-unroutable Docker-bridge IP by default.** +- **A global `--etcd-arg` override is not scoped to the "real" server's own etcd instance - it also applies to k3s's separate, internal "temporary etcd", spun up on every *restart* (never on first boot) to reconcile with an already-initialized on-disk datastore before the real instance starts.** +- **Calling `tailscale up --authkey=...` unconditionally on every container restart re-registers a brand new tailnet device each time, instead of reconnecting the persisted one - even with the join key's own state volume intact.** +- **Tailscale assigns an IP and wires up its own MagicDNS resolver as two separate steps, not atomically - a node having a real IP is not proof DNS resolution against another tailnet member is working yet.** +- **A server that memory-pressure regrading has stepped to `etcd-only` is `NotReady` by design, and must stay registered.** +- **A single remaining healthy etcd server can get stuck indefinitely if another member is registered but permanently unreachable (e.g. a partial join that never completed), even though that member is a non-voting learner and shouldn't block quorum.** ## References +Further detail lives in `docs/`: [ansible.md](docs/ansible.md) (roles, server election, profiles, host prerequisites, Unraid, capabilities, registry cache), [build-and-test.md](docs/build-and-test.md) (builds, smoke tests, canaries, lint), [releases.md](docs/releases.md) (release steps and the deploy key), [architecture.md](docs/architecture.md) (design and Tailscale), [heartbeat.md](docs/heartbeat.md) (health gist and alerts), [autoscaler.md](docs/autoscaler.md), [node-recovery.md](docs/node-recovery.md), [role-controller.md](docs/role-controller.md) (voter regrading and swap), [non-obvious-behaviour.md](docs/non-obvious-behaviour.md) (read when a cluster, mesh or etcd symptom is odd). Each role also has its own README under `roles/`. + - [`ExaDev/runner-fallback-action`](https://github.com/ExaDev/runner-fallback-action) — reads the heartbeat gist; its `docs/spec.md` documents the routing algorithm. - [`actions/actions-runner-controller`](https://github.com/actions/actions-runner-controller) — upstream ARC project. - [`actions/actions-runner-controller` gha-runner-scale-set-controller docs](https://github.com/actions/actions-runner-controller/blob/master/docs/gha-runner-scale-set-controller/README.md) — the `AutoscalingRunnerSet` CRD (`actions.github.com/v1alpha1`) `scripts/autoscaler.sh` reads and patches. diff --git a/docs/ansible.md b/docs/ansible.md new file mode 100644 index 00000000..fbb5b36d --- /dev/null +++ b/docs/ansible.md @@ -0,0 +1,42 @@ +# Ansible collection layout, roles and host prerequisites + +## Ansible + +The repository is laid out as an Ansible collection, `exadev.github_runner` (see `galaxy.yml`): its roles live in `roles/` and its playbooks in `playbooks/`: `site.yml` brings up the whole fleet, `arc.yml` installs only ARC into a cluster that already exists (from the control machine, with a kubeconfig), and `github_app_setup.yml` creates the runners' GitHub App through GitHub's manifest flow, and `recover_in_cluster_mesh.yml` recovers the in-cluster Headscale provider's bootstrap server. A fleet keeps its inventory, `host_vars` and Helm values in a repository of its own, installs the collection there, and runs the playbooks by name (`exadev.github_runner.site`); `examples/` has inventories to start from. `ansible/` holds a generic `ansible.cfg` and the collections the roles require, for running the playbooks from a checkout of this repository as `bootstrap.sh` and CI do; `playbooks/collections/` links back to the checkout, so a playbook run from it finds `exadev.github_runner` in place without installing it. Verified end to end against real infrastructure (three hosts, a real single-node-to-3-node etcd HA cutover, real jobs scheduling across all three). `bootstrap.sh` (see [Getting started](../README.md#getting-started)) is a thin wrapper that runs this same playbook locally. + +```bash +# in the fleet's own repository, next to its inventory +ansible-galaxy collection install git+https://github.com/ExaDev/github-runner.git +ansible-playbook -i inventory.yml exadev.github_runner.site --limit node1 --check # dry run +ansible-playbook -i inventory.yml exadev.github_runner.site --limit node1 # real run +``` + +The playbook runs these roles, each with its own README: `github_runner_cluster` brings the host up as a k3s node, in a container under Docker Compose or, with `github_runner_cluster_node_mode: native`, directly under systemd on a Linux host, `github_runner_arc` installs ARC and the fleet-health platform, `github_runner_k8s_client` (included by the other two) gives a host the kubernetes Python client, and `github_runner_secrets_onepassword` reads the secrets from 1Password when `github_runner_secrets_source` is `onepassword`. The cluster and ARC roles read every secret (`K3S_TOKEN`, the mesh keys, GHCR credentials, the heartbeat PAT, each org's App private key) as a plain variable, so any source Ansible reads works: ansible-vault, SOPS, an env or file lookup, HashiCorp Vault or AWS SSM (see `examples/ansible-vault/`, `examples/env/` and `examples/onepassword/`). With `github_runner_secrets_source: onepassword`, every secret is read live from one 1Password item at task time; `roles/github_runner_secrets_onepassword/README.md` lists the item's fields and the `op://` references. `bootstrap.sh` sets the same variables directly from `.env.bootstrap`. The cluster role ships the node's `docker-compose.yml` and k3s image build context in `roles/github_runner_cluster/files/`, copies them on every run into each host's `github_runner_cluster_dir` (`~/.github-runner` by default), and templates the `.env` for their `k3s`/`k3s-agent` services there, so a host needs no checkout of this repository. The Compose project name is pinned to `github-runner` (`github_runner_cluster_compose_project`) rather than taken from the directory's name, because the named volumes holding the k3s datastore and each node's mesh identity are prefixed with it; the directory can move without losing them. + +Which hosts are k3s servers is worked out from the inventory, not set per host. The `github_runner_cluster` inventory group lists the cluster's hosts in order: the first N become servers (control-plane/etcd members running `k3s server`, genuine voting members of the raft quorum), where N is the largest odd number no greater than both the number of hosts and `github_runner_cluster_max_servers` (5 by default), so 1 or 2 hosts give 1 server, 3 or 4 give 3, and 5 or more give 5. The first server bootstraps the cluster (`--cluster-init`, embedded etcd) and every other host joins it, the rest as agents (`k3s agent`, schedulable capacity with no control-plane role). Join URLs and every server's `--tls-san` list are derived from each server's node name and `github_runner_cluster_tailnet_domain`. A host can still set `github_runner_cluster_node_role`, `github_runner_cluster_bootstrap`, `github_runner_cluster_server_url` or `github_runner_cluster_tls_sans` to override its part, and the run fails with an explanation if the result has an even number of servers, none at all, or more than one bootstrap host; `roles/github_runner_cluster/defaults/main.yml` describes each. Installing the ARC controller, each org's runner scale set, and the fleet-health platform is not tied to a node role at all: each is driven purely by whether the installing host's own `github_runner_arc_orgs`/`github_runner_arc_heartbeat_gist_id` is set, so any server host can be the one Ansible installs against, and moving which host does the installing is a config change (which host's `host_vars` sets it), not a role change. See the [Architecture](architecture.md#architecture) section for why 3 servers is the minimum sane count for genuine HA (2 is worse than 1 for quorum math, not better). To add a server or agent, add the host to the `github_runner_cluster` group in the fleet's inventory and give it its own `host_vars/.yml`; keep the bootstrap host first. + +A `github_runner_arc_orgs` entry registers its scale sets in an organisation, or with `repository` in one repository of a user or an organisation (`githubConfigUrl` `https://github.com//`); a repository's scale sets take only that repository's jobs, so each repository needs its own entry (see Repository scope in `roles/github_runner_arc/README.md`). `github_runner_arc_orgs[].scale_set_profiles` is a list of `{suffix, values_file?, node_selector?, runs_on_label?, capabilities?, autoscale?, sizing?, burst?}` entries (see `roles/github_runner_arc/README.md` for every key) - one Helm release and one namespace per entry, both removed by the next run once the entry leaves the inventory (see Profiles that leave the inventory in `roles/github_runner_arc/README.md`), pooled under the org's own shared `runs-on` label via `scaleSetLabels` (not `runnerScaleSetName`, which is a scale set's own unique GitHub-side registration identity, not just a label - two releases sharing it collide outright) unless a profile sets its own `runs_on_label` to deliberately opt out of that pool - e.g. a dedicated image-building profile ordinary CI jobs have no business landing on. `node_selector` is optional per profile, not mandatory: omitting it leaves that profile's pods genuinely unpinned, letting the Kubernetes scheduler place them on whichever node in the fleet actually has room - real failover if one machine goes down, not just capacity pooling. Values files are rendered on the control node from `github_runner_arc_values_dir`, a directory in the fleet's own repository, so no fleet host needs them. ExaDev's own fleet uses two profiles: an ordinary CI profile unpinned across the whole fleet (sized with Guaranteed QoS for the smallest machine's real capacity), and a second, separately-labelled, Docker-in-Docker profile pinned to one host for building the runner image - a deliberate isolation choice, not a capacity constraint. + +### Prerequisites + +The collection brings up the k3s nodes and what runs on them; it does not set up the hosts underneath. Each host must already have: + +- Python 3, or `github_runner_standalone_python_install: true` for a host that has none (see [Unraid hosts](#unraid-hosts)). +- A connecting user that is not root and can become root without a password, or a play run with `--ask-become-pass`: the tasks of the cluster role that change the host, and those of the macOS Actions runner role, use `become`. +- Docker with the Compose plugin for a node under Docker Compose (see the Prerequisites under [Getting started](../README.md#getting-started)). On macOS that includes Homebrew: the playbook installs `helm`, `kubectl`, Docker and Compose through it, and the macOS Actions runner role's `PATH` names both of its prefixes. +- A host that stays awake while it carries a node. A Mac needs its AC power settings set so that it never sleeps (and, with the lid closed, `pmset disablesleep`), and automatic restart after a power failure if it should come back by itself. A Linux laptop needs its suspend targets masked and its lid switch ignored, and on Wi-Fi its radio's power save turned off. The cluster role sets none of these. +- For a native node, nothing more where `github_runner_cluster_native_tailscale_install` is `true` (the default), since the role installs the pinned Tailscale package. With `false`, a Tailscale client already installed and logged in to the cluster's tailnet. A node under Docker Compose has its own Tailscale device inside the container and needs no client on the host. + +`github_runner_cluster_k3s_prerequisites` and `github_runner_cluster_containerd_apparmor_stacking_fix` are the two host steps the cluster role does offer, because they exist only for k3s (see Host preparation in `roles/github_runner_cluster/README.md`). + +### Unraid hosts + +An Unraid host runs the same cluster role as any other Docker host (it ships Docker and the Compose plugin), with three differences. It has no Python, so `github_runner_standalone_python_install: true` in its `host_vars` makes `site.yml` install a pinned one first (`roles/github_runner_standalone_python/README.md`), at `github_runner_standalone_python_install_dir`, with `ansible_python_interpreter` pointing at it. Its root filesystem and home directory live in RAM, so `github_runner_cluster_dir` and that install directory must be on persistent storage (under `/mnt/user/appdata`), never the default `~/.github-runner`. And Docker only runs once the array is started, so the node comes back after a reboot only when the array does. A host that is a k3s server also needs a k3s data volume on storage with fast synchronous writes, because etcd waits for the disk on every write: Docker's data directory on a spinning array disk is not enough, and a server there makes the whole control plane slow. Such a host belongs in the inventory after the servers, as an agent, until Docker's data directory sits on an SSD pool. + +### Capabilities + +A profile can also add tools, libraries and setup to its runner pods when they start, rather than baking them into a runner image someone has to build, publish and patch: `capabilities` names entries of `github_runner_arc_capabilities`. A tool-cache capability is a public, pinned image holding a tool in the Actions tool-cache layout; an init container copies it into the runner's `RUNNER_TOOL_CACHE`, where `actions/setup-node`, `setup-tflint` and other setup actions that consult the cache find it and skip their download, and it can also be put on `PATH` for a tool no setup action looks up. A system-library capability is a public, pinned image holding a relocatable install prefix (shared libraries, headers, pkg-config files, programs) for what cannot live in the tool cache, such as an MPI implementation, a solver library or a database client library; an init container copies it into a sysroot (`/opt/sysroot` by default), and the job-started hook puts the sysroot on `PATH`, `LD_LIBRARY_PATH`, `LIBRARY_PATH`, `CPATH`, `PKG_CONFIG_PATH` and `CMAKE_PREFIX_PATH` and sets any relocation variable the library needs. A hook capability is a script run before each job's first step, through a job-started hook mounted from a ConfigMap. A node-cache capability is a directory of the node shared by the profile's runner pods there, for a package manager's store, which a pruner keeps under a size limit. A profile with its own `runs_on_label` then works as a named runner, for example one with Terraform, TFLint and the AWS CLI, or one with Open MPI and a solver library, that workflows select with `runs-on`, all from one values file and the stock `actions-runner` image. `tool-images/` has Dockerfiles for reference tool images (Node.js, Terraform, TFLint, the AWS CLI) reference system-library images (Open MPI, HiGHS, GLPK, libpq, MariaDB Connector/C) and a reference build runner image (the stock runner image with a C toolchain, for jobs that compile against those libraries), which `.github/workflows/publish-tool-images.yml` publishes when started by hand or by a `tool-images-*` tag; `roles/github_runner_arc/README.md`'s Capabilities section has the tool-image and payload contracts, how each reference library is declared and proven to work from a relocated prefix, how to build an image of your own, and where capabilities apply. + +### Registry cache + +Each Docker-in-Docker runner pod starts with an empty Docker daemon, so every job pulls its base images, service containers and container actions from the registries again. `github_runner_arc_registry_cache_enabled: true` installs a pull-through cache (CNCF Distribution in proxy mode, one container per registry behind one Service, on persistent or ephemeral storage) and points every dind profile's daemon at it through a `hosts.toml` mirror per registry, and the `docker-container` builders `docker/setup-buildx-action` creates at it through buildx's default `buildkitd.default.toml`, so nothing in a workflow changes. Registries are a list (Docker Hub, ghcr.io and quay.io by default), each optionally pulling with a read-only credential from a Secret, and pulls fall back to the registry whenever the cache is down or failing. It is off by default, and off renders exactly what the role rendered without it. `roles/github_runner_arc/README.md`'s Registry cache section says which pull paths it covers, which bypass it, and what a credential on it exposes. diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000..d4df6ab0 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,27 @@ +# Architecture and Tailscale + +## Architecture + +An earlier version of this repo ran a fixed number of long-lived containers via Docker Compose, cycling the GitHub runner *registration* between jobs (`EPHEMERAL=true`) but never resetting the container's own filesystem. Two problems followed: build artifacts, `node_modules`, and Playwright browser caches accumulated indefinitely inside the same container and filled the host's Docker disk allocation, and a fixed replica count meant jobs queued whenever more were in flight than there were replicas, with no way to burst beyond it. + +ARC's `gha-runner-scale-set` solves both by design: it runs one fresh pod per job and deletes it afterwards, and it autoscales on queue depth. ARC needs a real Kubernetes cluster to run in. [k3s ships an official Docker image](https://hub.docker.com/r/rancher/k3s) that runs the whole control plane and kubelet inside one privileged container, a "cluster in a box". This makes the cluster itself just two Compose services (`k3s` on every control-plane host - the same service definition whether it bootstraps or joins, see the [Ansible](ansible.md#ansible) section's node-role split - and `k3s-agent` on a pure-worker host), portable to any machine with Docker (plus `helm` and `kubectl` on the host), with no Colima-specific flags, no separately installed `kind` or `k3d` CLI. ARC itself (the controller, the runner scale sets, and the ephemeral runner pods), plus the fleet-health platform (heartbeat/autoscaler - see [Heartbeat](heartbeat.md#heartbeat)/[Autoscaler](autoscaler.md#autoscaler) below), are not more Compose services; they're all Kubernetes-native resources installed by the Ansible role (Helm and `kubernetes.core.k8s`) into this cluster (see `roles/github_runner_arc/`). + +The control plane itself runs genuine embedded-etcd HA across three `server`-role hosts, not a single node with worker capacity bolted on: one host bootstraps with `--cluster-init` (the one supported migration path from k3s's default single-node SQLite datastore to embedded etcd), and two further hosts join as full `k3s server` members - a real vote each in etcd's own raft quorum, not just schedulable capacity. 3 is the practical minimum: etcd tolerates the loss of any *minority* of its members without losing quorum, and with 2 servers that minority is zero - a second server can only ever make quorum strictly worse than staying at one, never better, so "2-node HA" isn't a real intermediate step, it's a trap. Getting there over Tailscale rather than a shared LAN needed real, non-obvious work - see the [Non-obvious behaviour](non-obvious-behaviour.md#non-obvious-behaviour) section's etcd-specific entries for what actually broke and why. + +Hosts on different networks can't reach each other's LAN addresses, and even hosts that share a LAN can't from inside their Docker VMs by default: each node's k3s container sits behind its VM's NAT, so its own Tailscale client has no address to offer its peers and every pair of nodes is relayed. So cross-node cluster traffic (a node joining, cross-node pod-to-pod traffic, the API server's proxy to a remote kubelet, and etcd's own peer traffic between `server` hosts) runs over a real overlay with mesh addresses - see [Tailscale](#tailscale) below for how that overlay is built and the DNS pitfall it introduces. Where hosts do share a LAN, `github_runner_cluster_network_mode: host`, on a Docker VM that has an address of its own on that LAN, lets their Tailscale clients reach each other directly over it, with the relay as the fallback when a host leaves; nothing k3s uses changes (see `roles/github_runner_cluster/README.md`, Networking). + +The custom runner toolchain (GCC 11/G++ for C++20, Bun, the `gh` CLI — see `Dockerfile`) carries over from the previous design, rebased onto ARC's own [`actions/actions-runner`](https://github.com/actions/runner/pkgs/container/actions-runner) base image instead of `catthehacker/ubuntu:act-latest`. ARC's runner pods use their own registration flow, so the previous design's vendored [myoung34/docker-github-actions-runner](https://github.com/myoung34/docker-github-actions-runner) scripts (registration, GitHub App JWT signing, ephemeral re-registration) are not needed. + +The images are built on GitHub-hosted runners rather than on the fleet, so a broken image can never take down the pool it would need to be rebuilt on. See [Releases](releases.md#releases) below for how a release builds them. + +`.github/workflows/ci.yml`, `mesh-integration.yml` and `litestream.yml` run on GitHub-hosted `ubuntu-latest`, which is free for a public repository, so checking the fleet's configuration never depends on the fleet itself being up. `test-arc-runner.yml` and `test-autoscaler.yml` are the ones that target the fleet, since the fleet's own behaviour is what they test. + +### Tailscale + +Each node's k3s process joins a Tailscale tailnet as its own device, using k3s's own built-in (experimental) [`--vpn-auth` integration](https://docs.k3s.io/networking/distributed-multicloud): `--vpn-auth="name=tailscale,joinKey="` on both `k3s server` and `k3s agent`. This runs Tailscale *inside* the k3s container/process itself, separate from the host machine's own Tailscale identity if it has one, and flannel uses Tailscale's own subnet-route advertisement for pod-to-pod traffic instead of its usual VXLAN encapsulation - no manual `--node-ip`, no Docker port-publishing, no host-level Tailscale dependency at all. The Mac's own Tailscale client, if it has one, is a separate device that this collection does not manage. A native node (a Linux host with `github_runner_cluster_node_mode: native`, which has no container to put a client in) is the exception: the host's own Tailscale client is the node's device, and the host is dedicated to the cluster; `roles/github_runner_cluster/README.md`'s Native nodes section says why. `k3s/Dockerfile` layers the `tailscale`/`tailscaled` binaries from the official Tailscale image onto `rancher/k3s`'s own minimal base (which has no package manager to install them at runtime), and `k3s/node-entrypoint.sh` (the Compose services' entrypoint, bind-mounted from the node's Compose project directory; both live in `roles/github_runner_cluster/files/`) starts `tailscaled` itself before execing k3s, since k3s's vpn-auth integration only runs `tailscale up` and assumes the daemon is already running. + +The mesh's control server does not have to be Tailscale's. The cluster role's `github_runner_cluster_mesh` switches it to Headscale: a server someone else runs, one the role runs under Compose outside k3s, or one it runs inside the cluster itself; see `roles/github_runner_cluster/README.md`. The rest of this section describes the default, Tailscale. + +Provisioning the join key: create a reusable, pre-authorized Tailscale auth key (Tailscale admin console → Settings → Keys, or the `POST /api/v2/tailnet/-/keys` API), tag it so an ACL grant can target it, and set `K3S_VPN_AUTH_JOIN_KEY` in every host's `.env` (or, via Ansible, set `github_runner_cluster_tailscale_join_key`, which the 1Password adapter reads from a `tailscale-join-key` field, as `roles/github_runner_secrets_onepassword/README.md` describes). The tailnet's ACL needs a `grants` entry permitting traffic between the nodes' own tag *and* the pod CIDR itself (e.g. `10.42.0.0/16`), not just the tag - Tailscale evaluates a subnet-routed packet's actual embedded source IP, not just the sending device's identity, so a grant scoped to the tag alone silently blocks cross-node pod traffic ([k3s-io/k3s#8372](https://github.com/k3s-io/k3s/issues/8372)). + +**Tailscale's own DNS management is a real trap here.** By default (`--accept-dns=true`, Tailscale's own default), `tailscale up` overwrites the container's `/etc/resolv.conf` with MagicDNS's resolver (a fixed, well-known `100.100.100.100`) plus whatever search domains the tailnet's own DNS settings configure - which can include search domains meant for entirely unrelated, personal use of the same tailnet. Kubernetes' `dnsPolicy: ClusterFirst` then copies the *node's* search domains into every pod's own `/etc/resolv.conf` alongside the cluster's own three, and the default `ndots:5` means a pod's resolver tries every search-suffixed name before the bare hostname for anything with fewer than 5 dots - which is essentially every real-world hostname. If any of those inherited search domains carries a wildcard DNS record, a pod's lookup of a real external hostname can resolve to that wildcard's target instead, while the application still sends the original hostname as its TLS SNI - silent certificate-verification failures for arbitrary outbound HTTPS traffic, indistinguishable at a glance from a hung network. Both `k3s server`/`k3s agent` invocations set `--resolv-conf` to a synthetic file containing only `nameserver 100.100.100.100` and no search domains, so pods still resolve everything correctly through Tailscale's own DNS proxy without inheriting a search list they have no use for. diff --git a/docs/autoscaler.md b/docs/autoscaler.md new file mode 100644 index 00000000..9a7876f7 --- /dev/null +++ b/docs/autoscaler.md @@ -0,0 +1,26 @@ +# Autoscaler + +### Autoscaler + +A scale set's static `maxRunners` has to be safe for the pool's real worst case, every pod at its limit at once, so it leaves real idle capacity unused whenever the jobs in flight need less than that. `autoscaler` (an in-cluster Deployment, same shape/placement as heartbeat above) raises and lowers `maxRunners` across every profile with `autoscale: true` (one or more, possibly from different orgs, named by `AUTOSCALER_TARGETS`), based on real, current memory usage and pressure, never on job identity or GitHub Actions queue depth: every consuming repo's CI keeps using its scale set's own `runs-on` label unchanged, and no job is ever classified as light or heavy. When more than one profile is pooled, they share one combined budget rather than each getting a fixed slice: the algorithm below runs against the pool's combined `currentRunners`/`maxRunners`, and a change is handed to whichever pooled profile's own headroom (its `maxRunners` minus its `currentRunners`) makes it the right target: raises go to the profile with the *least* headroom (closest to genuinely needing more), lowers take from the profile with the *most* (its capacity is going unused), one runner at a time, and a lower never takes a profile below its own `currentRunners`. RBAC is scoped to exactly what it reads/writes: cluster-wide `nodes.metrics.k8s.io` and nodes get/list, and pods get/list for the ceiling; per pooled profile, `get`/`patch` on that `AutoscalingRunnerSet` by name, `get`/`list` on `ephemeralrunners` (to count the running runners and to name the job a runner is running), `get`/`list` on events and pods, and `pods.metrics.k8s.io` get/list in its own namespace; and `get`/`patch` on its own status ConfigMap by name, with no wildcards. + +Each poll (`AUTOSCALER_POLL_SECONDS`, default 45s): +- For every pooled profile, counts its running runners from its `EphemeralRunner` resources (every one that has not finished: a phase other than `Succeeded` or `Failed`, so a pending runner counts, because its pod already holds a slot) and reads its per-pod hard memory limit (`spec.template.spec.containers[0].resources.limits.memory`) straight from its live `AutoscalingRunnerSet`, never a static config value, so both always match what is actually deployed. The running count is not read from the `AutoscalingRunnerSet`'s status: in the ARC release this role pins (0.15) that status carries only `phase` and `observedGeneration`, so a count read from it would always be empty. A failed read of the runners stops the poll instead of counting zero. Sums the running counts into one combined total, and takes the largest of the pod memory limits as the threshold a raise or lower must leave room for. +- Sums real, current memory usage of every pooled profile's running runner pods, from the metrics API's pod list for each profile's own namespace (`scripts/autoscaler-usage.jq`); this, not a static per-pod request or limit, is the entire point of "usage-driven". It does not use `kubectl top pod`, which fails a whole namespace while any pod in it has no metrics, as every new runner pod has none until metrics-server first samples it. A runner pod placed on a node, not finished and missing from the list is unmeasured, and assumed to use its own memory limit (its scale set's limit when it sets none), the most it can reach. Lowering is decided on the measured usage, so a new pod never lowers the pool by itself; a raise also needs a pod's room with every unmeasured pod at its limit, and the pool holds until they are measured otherwise. +- Reads cluster-wide memory-availability pressure via `kubectl top nodes`, taking the **worst-case (minimum)** available-memory percentage across the nodes runner pods may use, not an average — a pool average can look healthy while the specific node a new runner pod would actually land on is not, consistent with the algorithm's own bias toward lowering eagerly (lowering never disrupts in-flight jobs). With the node eligibility label set (`github_runner_arc_node_label_key`, passed as `AUTOSCALER_NODE_SELECTOR`) only nodes carrying it are read, otherwise every node: a node outside the label runs no runner pod, so its pressure says nothing about where the next one lands, and when its kubelet drops out its missing metrics would otherwise fail safe on every poll. The same holds for a node inside the label that is not `Ready` (its kubelet stopped reporting or reported a fault): it is left out of this check by the definition the ceiling uses (`scripts/autoscaler-node.jq`), since it places no runner pod and `kubectl top nodes` prints `` for it. This is a genuine improvement over the pod's own earlier host-pinned design, which could only ever see one fixed machine's pressure via `/proc/meminfo`, not the whole pool's. **No swap-pressure signal**: metrics-server's API has no swap field at all, and the only alternative (the kubelet's own Summary API) needs a materially broader RBAC grant for a signal most kubelet versions don't even surface unless an off-by-default feature gate is on — dropping swap detection is a deliberate, documented trade-off once the pod is genuinely unpinned to any node, not an oversight; memory-availability pressure remains the dominant signal (the incident the values file's own comments were tuned from). +- Works out the ceiling, how many pooled runner pods the nodes can place now, from the live cluster (`scripts/autoscaler-ceiling.jq`). It reads every pooled profile's runner pod spec from its `AutoscalingRunnerSet` and takes the largest request among them, per resource, as the runner pod (a container that sets only a limit counts at its limit, as it will once its pod is created, and sidecars, init containers and pod overhead count as the scheduler counts them). A node counts when every pooled profile's runner pods could be placed on it now: it carries their `nodeSelector`, is `Ready`, is not cordoned, and their tolerations match each of its `NoSchedule` and `NoExecute` taints; a node that is not ready or unreachable is left out until it is back. On each, allocatable CPU and memory less the requests of the pods on it that are not pooled runner pods, divided by the runner pod's request, and its pod capacity (`allocatable.pods`) less those pods, give how many fit; the smallest of the three, rounded down, is the node's figure, and the ceiling is the sum. The pooled runner pods are left out so that runners already running never lower the ceiling they count towards; every other pod still holding requests counts, a non-pooled scale set's runners included. Node affinity is not read: the role gives an autoscaled profile none, and a pod one would keep off a counted node is caught as unschedulable below. `AUTOSCALER_MAX_CEILING` (`github_runner_arc_autoscaler_max_ceiling`, or, when exactly one pooled profile sets `sizing`, that profile's own derived figure) is an optional upper bound on it; empty leaves it unbounded. The ceiling worked out is kept in the status ConfigMap's `ceiling` key, and `status.json` reports `ceiling`, `ceiling_bound`, `ceiling_measured` and, per node, the runners it places or why it is not counted. When the nodes, the pods or a template cannot be read, or a figure in them is missing or is not a Kubernetes quantity, the poll prints why on stderr, keeps the last ceiling it worked out (or the bound, before it has one) for lowering and rebalancing, and does not raise the pool. Reading every node's pods needs `get`/`list` on pods cluster-wide, which the autoscaler's service account holds for this alone, as the heartbeat's does. +- Raises the combined `maxRunners` by exactly +1 per cycle (never straight to a computed target) once headroom has covered a full pod's hard limit for `AUTOSCALER_RAISE_CONFIRM_POLLS` (default 2) consecutive polls, capped at the ceiling, and only in a poll that worked the ceiling out. +- Lowers the combined `maxRunners` to the ceiling when the pooled total is above it, even when nothing else is wrong: each profile's own static `maxRunners` can sum to more than the ceiling, every Helm upgrade restores those, and the ceiling falls when a node goes, so the cap is enforced rather than only bounding raises. +- Rebalances a pool that is healthy and already holds as many runners as it may: the pooled total is at the ceiling, memory headroom covers a pod, there is no pressure and no unschedulable runner pod. Slots are fixed per profile, so without this a profile that has used all of its own while another sits idle would queue for capacity the pool has. When one profile's running count has reached its own `maxRunners` and another has at least one spare slot, it moves exactly one slot from the profile with the most spare slots to the saturated one, never changing the combined total, never taking the donor below its own running count and never below one runner (so a quiet profile can always start a job), and only once the same imbalance has been seen for `AUTOSCALER_RAISE_CONFIRM_POLLS` consecutive polls, so a burst that ends within a poll or two moves nothing. The count of consecutive polls is kept in the status ConfigMap's `rebalance-confirm-count` key beside `raise-confirm-count`, and every poll that is not a confirmed imbalance resets it to zero. Under pressure, with an unschedulable pod, above the ceiling or below it (where a raise hands the new slot to the profile with the least headroom) the slot-moving never runs. +- Lowers the combined `maxRunners` immediately, with no delay or averaging, the moment headroom drops below a pod's hard limit or real pressure is detected — lowering never disrupts in-flight jobs, since ARC only gates new claims. Never patches any pooled profile below its own currently-running count, nor below one runner: a scale set at `maxRunners` 0 asks for no runner, so a job queued for it would wait with nothing to start it, and the cluster keeps no record of a queued job the autoscaler could see. A pool therefore never goes below one runner per pooled profile, whatever the floor, pressure or ceiling say, and a profile left at 0 by something else counts as saturated, so rebalancing gives it a slot back. +- Treats any unschedulable runner pod (`PodScheduled=False`, reason `Unschedulable`) like pressure: the memory budget sums every node's capacity, so it cannot see a node that is tainted for disk pressure or a request that fits no single node. It lowers the combined `maxRunners` to the running count and never raises while one exists. +- Also collects the peak memory each job's runner measured for itself: the runner image's job-completed hook (`runner-hooks/job-completed.sh`, set through `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`) reads the container cgroup's `memory.peak`, which a spike shorter than the poll cannot escape, and posts it as a `JobMemoryPeak` Event in the pod's namespace. The autoscaler reads those Events each poll and keeps the maximum per repository, workflow ref and job id under the status ConfigMap's `job-peak` key (`kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.job-peak}'`), in MiB. Keeping the maximum means a forged low figure lowers nothing and re-reading an Event changes nothing. The hook never fails the job it ends. To post an Event the runner pods' service account, which otherwise holds no permission, gets `create` on `events` in its own namespace and nothing else (`github_runner_arc_record_job_peaks`, default on; `false` removes the grant). A job's code runs in the same container and can read the same token, so the worst it can do with that permission is post events in its own runner namespace. +- Records, per job, the highest memory a runner pod running it has been seen at, in the status ConfigMap's `job-memory` key (`kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.job-memory}'`), keyed by repository, workflow ref and job name with the peak in MiB, the number of observations and when it was last seen. It is sampled once per poll from the metrics server, so it is a lower bound that a spike shorter than the poll escapes; it shows how far below a pod's request jobs sit, which is what decides whether an accurate in-pod measurement (see the per-job sizing issue) is worth building. Entries unseen for 30 days are dropped, and a failure to keep the record never affects a capacity decision. +- Optionally reads per-node pressure from the nodes' own `/proc/pressure` (`github_runner_arc_autoscaler_node_pressure_enabled`). A `psi-reporter` DaemonSet (`scripts/psi-reporter.sh`, in the autoscaler's image) runs on each node runner pods may use, tolerating every taint, and patches that node's own key of one ConfigMap with the `avg10` of its io, cpu and memory pressure. The reading is not taken from the kubelet's summary API, because on a cgroup v1 node that reports every pressure field as 0 whatever the node is doing. The autoscaler lowers the combined `maxRunners` by exactly one each poll, never below the running count, while any node's I/O "full" stall is at or over `AUTOSCALER_IO_PRESSURE_PCT` (default 10) or, when set above 0, its cpu "some" stall is at or over `AUTOSCALER_CPU_PRESSURE_PCT`, and it does not raise the pool while any node has no report newer than three polls, since whether that node can take more is then unknown. This is the signal for a pool whose nodes differ: memory can have room on a node whose disk is already the limit. +- Fails safe on any memory measurement error (`kubectl top nodes` or the metrics API's pod list failing, or the node list failing, or a `Ready` node's figure that cannot be parsed), but not on a pod that merely has no metrics yet, nor on a node that is not `Ready`: lowers the combined total toward `AUTOSCALER_FLOOR` (`github_runner_arc_autoscaler_floor`) immediately rather than skipping the cycle silently. A failure reading the scale sets' own state or their pods exits non-zero instead, since fail-safe needs that state to lower anything and the API server is then unreachable anyway. This combined floor need not equal any one profile's own values-file `maxRunners`, since each profile's Helm upgrade reverts to its own static floor independently of how the pool's combined floor is set. + +Because the ceiling is the tighter of CPU and memory on each node, shrinking the runner pod's CPU request raises it only on nodes where CPU is what decides. Halving a 2 CPU, 7Gi pod to 1 CPU was measured on a two-node pool and not taken: memory then decided on both nodes, so it bought one runner (a ceiling of 5 became 6), while every job that keeps more than one core busy would be throttled to one, and the autoscaler records memory only, so there was no CPU history to weigh that against. Revisit it when CPU history exists (the node pressure reporter's cpu figures, or a per-job CPU record), when one more runner is worth that throttling, or when a node with too few cores for the larger pod needs a 1 CPU pod to take work. + +Ships with `AUTOSCALER_DRY_RUN=true` by default: it computes and logs the target and writes its own status, but never patches. Generate real concurrent traffic to watch it react: `gh workflow run test-autoscaler.yml`, then watch `kubectl logs -n github-runner-platform deploy/autoscaler -f` and `kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.status\.json}'`. Only set `AUTOSCALER_DRY_RUN=false` after watching real dry-run output across genuine CI traffic. Status and its raise-confirm counter live in a shared Kubernetes ConfigMap (`autoscaler-status`, pre-created empty so neither pod's own ServiceAccount ever needs `create` RBAC on it, only `get`/`patch`) rather than a bind-mounted directory, since the two pods can land on different nodes now — `scripts/heartbeat.sh` reads that same ConfigMap back out and republishes it as a second file in the heartbeat gist, reusing its existing gist-write credential rather than giving the autoscaler its own. + +A Helm upgrade of a scale set (the role runs one when the release's rendered values or chart differ from what is installed) reverts `maxRunners` to the values file's safe floor. On Helm 4, which applies releases server-side, the autoscaler's patches make it the manager of `maxRunners`, so the role forces conflicts on the controller and scale-set releases (`force_conflicts`, only over the fields the chart sets); without that the upgrade would fail instead of reverting. The role's `install_org.yml` triggers one immediate autoscaler poll after every such upgrade (`kubectl exec deploy/autoscaler -- ...`, reaching the pod through the API server regardless of which node it's on), so the safe-floor window after a redeploy is seconds, not a full poll interval. diff --git a/docs/build-and-test.md b/docs/build-and-test.md new file mode 100644 index 00000000..62036a24 --- /dev/null +++ b/docs/build-and-test.md @@ -0,0 +1,25 @@ +# Build, test and smoke-test + +## Build, test, and smoke-test + +- **Build the runner image locally (the only working path right now):** `docker buildx build --platform linux/arm64,linux/amd64 --push -t ghcr.io/exadev/github-runner:latest .` from a machine with buildx/QEMU cross-platform support (Docker Desktop has this built in). A plain `docker build` with no `--platform` only produces an image for the building machine's own architecture, which silently breaks scheduling on the fleet's other architecture. The fleet-health platform's own two images need the identical treatment, for the identical reason (they're genuinely unpinned pods now too - see [Heartbeat](heartbeat.md#heartbeat)/[Autoscaler](autoscaler.md#autoscaler)): `docker buildx build --platform linux/arm64,linux/amd64 --push -f heartbeat/Dockerfile -t ghcr.io/exadev/github-runner-heartbeat:latest .` and the same with `autoscaler/Dockerfile` → `ghcr.io/exadev/github-runner-autoscaler:latest`, `pull-secret-renewer/Dockerfile` → `ghcr.io/exadev/github-runner-pull-secret-renewer:latest` (the image of the ARC role's App-sourced pull Secret renewal), `node-recovery/Dockerfile` → `ghcr.io/exadev/github-runner-node-recovery:latest` (see [Node recovery](node-recovery.md#node-recovery)), and `fleet-role/Dockerfile` → `ghcr.io/exadev/github-runner-fleet-role:latest` (see [Role controller](role-controller.md#role-controller)). +- **Build and push via CI:** `.github/workflows/release.yml`'s `build-images` job builds and pushes every image (this one, `heartbeat`, `autoscaler`, `pull-secret-renewer`, `node-recovery` and `fleet-role`) multi-arch on GitHub-hosted `ubuntu-latest` whenever a release is published - see [Releases](releases.md#releases) below. +- **Smoke-test the ARC scale set** (confirms a job gets a real ephemeral pod): `gh workflow run test-arc-runner.yml`, then watch `kubectl get pods -n arc-runners- -w` (`arc-runners---` for a repository entry). A pod must appear only once the job is queued, run to completion, and be deleted within seconds. +- **Smoke-test `ubuntu-latest` routing** (confirms `runner-fallback-action` still reaches GitHub-hosted runners when needed): `gh workflow run test-ubuntu-latest.yml`. +- **Generate load for the autoscaler** (see [Autoscaler](autoscaler.md#autoscaler) below): `gh workflow run test-autoscaler.yml`. +- **Verify cluster health:** + ```bash + kubectl get nodes # must show every server/agent host as Ready + kubectl get pods -n actions-runner-controller # controller pod and the scale set's listener pod both run here + kubectl get pods -n arc-runners- # zero runner pods when idle is expected, not a bug + ``` +- **Confirm no disk accumulates across repeated runner jobs:** the Docker host's disk usage (inside the VM, on a host that runs Docker in one) must stay flat across repeated `test-arc-runner.yml` runs. +- **Test capabilities:** `tests/capabilities/run.sh` builds the reference Node.js tool image and libpq system-library image, renders a profile with capabilities through the ARC role and the scale-set chart, and runs a pod from the chart's runner template in a throwaway kind cluster to check that the tool reaches the tool cache and `PATH` and the library reaches the sysroot and its search paths; `.github/workflows/capabilities.yml` runs it on pull requests. `tests/system_libraries/run.sh [openmpi|highs|libpq|glpk|libmariadb]...` proves each reference system-library image works relocated: it copies the payload to `/opt/sysroot` as the role's init container does, applies the role's job-started hook, and runs the payload's programs in the stock `actions-runner` image and a program compiled against it in the reference build runner image (`tool-images/build`, the stock image with `build-essential` and `pkg-config`), including a two-process `mpirun` for Open MPI; `.github/workflows/system-libraries.yml` runs it on amd64 and arm64 on pull requests. The images under `tool-images/` build with `docker buildx build --platform linux/arm64,linux/amd64 tool-images/`; `.github/workflows/publish-tool-images.yml` publishes them (see Releases), and the release workflow does not. +- **Test the build runner image's ids:** `tests/runner_build/run.sh` builds `tool-images/build` with its default runner ids and with `RUNNER_UID` and `RUNNER_GID` set to another, and checks the runner user in each; `.github/workflows/runner-build.yml` runs it on amd64 and arm64 for changes to the image. +- **Canary a published fleet-role image:** `gh workflow run published-image-canary.yml --ref main -f image='ghcr.io/exadev/github-runner-fleet-role@sha256:' -f scenario=role_controller` runs the chosen role-controller scenario in the disposable mesh on a GitHub-hosted Linux amd64 runner. The image must be publicly pullable and digest-pinned. The scenario choices are `role_controller`, `role_controller_swap`, `role_swap`, `role_swap_paused`, `role_swap_blocked_peer` and `role_swap_refused`; each tests only its own behaviour and the image selected for that runner's architecture, not other platform images. +- **Compare two published fleet-role images:** `gh workflow run published-image-canary.yml --ref main -f image='ghcr.io/exadev/github-runner-fleet-role@sha256:' -f comparison_image='ghcr.io/exadev/github-runner-fleet-role@sha256:' -f scenario=role_controller` runs the selected scenario in ABBA order on one GitHub-hosted runner, rebuilding the disposable mesh each time. The `published-image-comparison` artifact contains each run's log and a TSV of image, scenario, outcome and elapsed seconds. The run fails if any scenario fails; timings include mesh setup and teardown, may reflect caching and runner contention, and are not a fleet benchmark. The job's GitHub-hosted limit can end a comparison before all runs finish. +- **Test the registry cache:** `tests/registry_cache/run.sh` installs the cache as the ARC role renders it in a throwaway kind cluster, with Docker Hub, ghcr.io and a private registry behind basic auth, runs a pod from the chart's runner template as the role renders a Docker-in-Docker profile with the cache on, and checks in the cache's access logs that `docker pull` and a `docker-container` buildx builder pulled through it from all three without logging in, then scales the cache to nothing and checks that pulls and builds still work directly; `.github/workflows/registry-cache.yml` runs it on pull requests. +- **Lint:** `.github/workflows/ci.yml` runs Shellcheck, actionlint, yamllint, hadolint, an Ansible syntax check, ansible-lint (at its `production` profile), and a collection build check on every push and pull request, plus commitlint (on every commit in a pull request, and on the pull request's own title) on pull requests specifically - all gated by one `required-checks` job so a repository ruleset only ever has to require that one check regardless of how many lint jobs exist. hadolint's warning-level pinning advice is visible but non-blocking (`--failure-threshold error`): this repo deliberately floats several images/packages on `latest` rather than pinning. yamllint's config lives in `.yamllint`, relaxing the three default rules that conflict with this project's own conventions (long single-line comments, no leading `---`, GitHub Actions' `on:` key). +- **Commit messages:** `.husky/commit-msg` runs commitlint locally (after `npm ci`), the same check CI's commitlint job applies, so a message semantic-release would mishandle is rejected before it is pushed. The machine-wide hooks dispatcher hands off to it; without one, `git config core.hooksPath .husky` enables it. + +There is no application test suite in this repo; the checks above are the project's actual verification surface. diff --git a/docs/heartbeat.md b/docs/heartbeat.md new file mode 100644 index 00000000..c3ee87cd --- /dev/null +++ b/docs/heartbeat.md @@ -0,0 +1,35 @@ +# Heartbeat and fleet-down notification + +### Heartbeat + +ARC's autoscaling means there is normally no pre-existing "online runner" to check the way the previous design's fallback logic did (querying `/orgs/{org}/actions/runners` for an online match); with `minRunners: 0`, pods exist only while a job is running. To still support falling back to `ubuntu-latest` if the whole fleet goes down, `heartbeat` (an in-cluster Deployment - see `roles/github_runner_arc/tasks/install_platform.yml`) checks every `HEARTBEAT_INTERVAL_SECONDS` (default 180s) that a k3s node and the ARC controller are healthy and that no runner pod has been unschedulable for `HEARTBEAT_UNSCHEDULABLE_AFTER_SECONDS` (default: one interval), since a fleet with nowhere to place a runner only queues the work sent to it, and if so, refreshes a single secret gist with a unix timestamp set `HEARTBEAT_WINDOW_SECONDS` (default 600s) into the future. [`exadev/runner-fallback-action`](https://github.com/ExaDev/runner-fallback-action) reads that gist over the unauthenticated GitHub API and routes to `exadev-runners` while the timestamp is still fresh, or to `ubuntu-latest` otherwise. Every tick, healthy or not, it also writes `fleet-health.json` to the same gist (`HEALTH_GIST_FILE`): whether the fleet is vouched for, the reasons it is not, and any node reporting a pressure condition such as `DiskPressure`. A node under pressure does not make the fleet unhealthy, since the other nodes still serve jobs, but it is named there so lost capacity is visible. See that repo's `docs/spec.md` for the algorithm. It also writes `job-peak-summary.json`, the distribution of the peak memory jobs have measured (job count, how many distinct repositories they came from, the period they cover, median, 95th percentile and maximum in MiB, next to the runner pod's memory limit; the count and period are there because a large job count from one busy repository on one day says little), so whether the pods' request can come down is a lookup there and not a query of the cluster. It holds only figures, never a repository, workflow or job name, because the gist is readable by anyone holding its link. + +`fleet-health.json` also carries `deployed`: the collection version each playbook scope last recorded, so a scope that a run never reached shows as behind the version the fleet repository pins instead of looking like one that is current. The ARC and platform scope (`arc.yml`) records it in the platform namespace's `deployed-state` ConfigMap at the end of a run, as `platform`; each server host (`site.yml`) annotates its own node with `github-runner.exadev.com/deployed-version` once the node has registered, as an entry in `nodes`. A host annotates only its own node, so a `--limit` run never claims a version for hosts it did not touch, and an agent host, which has no API of its own, records nothing and shows `null`. The stamp records what a run applied, not what is running: a pod that failed to start after a run still leaves the stamp, so it complements the health reasons and does not replace them. `arc.yml` does not touch hosts, so a release that changes the Compose directory (the `docker-prune` service was one) shows the nodes behind until `site.yml` runs. + +Runs as a genuinely unpinned Kubernetes Deployment (`replicas: 1`, no `nodeSelector`), the same pattern ARC's own controller pods already use, not a host-pinned Docker Compose service — Kubernetes' own scheduler places and reschedules it on any server host with no config to move if one goes down. `kubectl` needs no `KUBECONFIG`: running as a pod with a mounted ServiceAccount token, in-cluster config is auto-detected. RBAC is scoped to exactly what it reads: cluster-wide `nodes` get/list, `deployments` get/list in the `actions-runner-controller` namespace, and `get` on the autoscaler's own status ConfigMap and the `deployed-state` ConfigMap by name (see Autoscaler below) — no wildcards. + +Create the secret gist once: +```bash +echo "$(($(date +%s) + 600))" | gh gist create --filename arc-healthy-until --desc "ARC fleet health heartbeat" - +# put the hex id from the returned gist URL into the installing host's github_runner_arc_heartbeat_gist_id +# (or .env.bootstrap's HEARTBEAT_GIST_ID for bootstrap.sh), +# and into whatever reads it, such as runner-fallback-action's gist-id input +``` +Confirm it refreshes: `gh gist view "$HEARTBEAT_GIST_ID"` must show a timestamp near `now + 600`, advancing roughly every 3 minutes. + +#### Fleet-down notification + +`runner-fallback-action` reroutes silently, so nothing otherwise tells a person that the fleet has no runners. `fleet-health.json` carries what a notifier needs to judge how long that has lasted: `at` (when the heartbeat last wrote), `unhealthy_since` (when it stopped vouching for the fleet, `null` while it does) and `not_ready_nodes` (each node whose Ready condition is not True, whether it is a control-plane server, the `regrade_level` memory-pressure regrading has stepped it to if it has, and since when). The heartbeat carries each onset over from the document its previous tick published, read back from the gist, so a tick that cannot read it restarts the onsets at now, which delays an alert and never raises a false one. + +`scripts/fleet-alert.sh` reads that document and announces an outage when any of these has held for at least `FLEET_ALERT_AFTER_SECONDS` (default 900, longer than the heartbeat's own 600 second window): the heartbeat has not written for that long, a control-plane server has not been Ready for that long (except one regrading has stepped down, which is not Ready on purpose), or the heartbeat has not vouched for the fleet for that long (a scale set missing or without a running listener, no ready controller, no Ready node, runner pods that cannot be scheduled; the reasons are in the notification). A worker node that is not Ready is not an outage by itself. An outage is announced once, and a recovery note follows when every condition has cleared. A heartbeat image older than the onset fields makes the script fail with a message saying so, rather than never alerting. + +Delivery is independent per channel, and with none configured the script only prints what it found: + +| Channel | Set | Notes | +|---|---|---| +| Issue | `FLEET_ALERT_ISSUE_REPO` (`owner/repo`) and `FLEET_ALERT_ISSUE_TOKEN` | Opens an issue and finds it again by a hidden marker in its body, so the open issue is the record that the outage was announced; closes it with a comment on recovery. The heartbeat's `HEARTBEAT_GH_TOKEN` is a gist-scoped token and cannot write issues, so this needs its own token with issue write access to that repository; a workflow's own `GITHUB_TOKEN` with `issues: write` suffices for the repository the workflow lives in. | +| Webhook | `FLEET_ALERT_WEBHOOK_URL` and `FLEET_ALERT_STATE_FILE` | POSTs JSON: `event` (`firing` or `resolved`), `text` (a one-line summary), `conditions` (each with `kind`, `detail` and `since`) and `heartbeat_at`. It names no chat product, so a receiver maps `text` or `conditions` to its own format. The state file holds only whether the outage was announced; a delivery that fails is not recorded and the next pass retries. | + +Both the URL and the tokens are passed to `curl` on its standard input, never its command line. Issue bodies name nodes and the heartbeat's reasons, so use a private repository. + +**Where it runs matters.** A check that runs only inside the cluster cannot report that the cluster is down, and a check that runs on the fleet's own runners does not start when the fleet has no runners. So the check reads the gist, which the heartbeat writes, from outside: `examples/fleet-alert/fleet-alert.yml` is a scheduled workflow for a repository not served by the fleet, on a GitHub-hosted runner, and the script runs just as well from cron on any host outside the fleet. The tradeoff is that it depends on GitHub (Actions scheduling, which can be delayed and is limited to a run every five minutes, and the gist API) and on the heartbeat's write path: if GitHub is unreachable from the fleet the heartbeat goes stale and the alert fires, which is also what the fleet's users experience. It is not a cluster Deployment, deliberately. A repository with no GitHub-hosted runners available cannot use the workflow and should use cron. The workflow template keeps the webhook's state in `actions/cache`, which can be evicted; the worst case is one repeated announcement. diff --git a/docs/node-recovery.md b/docs/node-recovery.md new file mode 100644 index 00000000..66364a40 --- /dev/null +++ b/docs/node-recovery.md @@ -0,0 +1,17 @@ +# Node recovery + +### Node recovery + +When a node stops (its host is switched off or asleep, Docker quits, or the k3s container is stopped), the node controller marks it `Unknown` once its kubelet has missed heartbeats for the node-monitor grace period, and after the pods' five-minute unreachable toleration marks them for deletion. Only the node's kubelet can confirm a deletion, so they then stay `Terminating` indefinitely. A Deployment's pods are replaced regardless, which is why the ARC controller comes back on its own, but ARC creates each scale set's listener as a single pod and waits for the old one to go, so a listener on the stopped node is never replaced and every job for that scale set queues until someone force-deletes it. + +`node-recovery` (an in-cluster Deployment in the platform namespace, see `roles/github_runner_arc/templates/node-recovery.yaml.j2` and `scripts/node-recovery.sh`) applies Kubernetes' [non-graceful node shutdown](https://kubernetes.io/docs/concepts/cluster-administration/node-shutdown/#non-graceful-node-shutdown): every `NODE_RECOVERY_POLL_SECONDS` (default 15s) it gives each node whose `Ready` condition has been `Unknown` for `github_runner_arc_node_recovery_after_seconds` (default 120s) the `node.kubernetes.io/out-of-service=nodeshutdown:NoExecute` taint. The control plane then evicts every pod on that node at once, unless it tolerates that taint, and the pod garbage collector force-deletes the ones already terminating there, so ARC starts the listener again on a healthy node. When the node reports `Ready` again the watcher removes the taint, and the node's kubelet stops the containers of the pods that were deleted while it was away. With the defaults a stopped node stops holding up job pickup within roughly four minutes: the grace period (50s on current Kubernetes), the threshold, one poll, the garbage collector's 20-second sweep, and the listener's own restart. `github_runner_arc_node_recovery_enabled: false` removes the watcher and any taint it applied. + +The watcher only acts on a node whose kubelet has gone silent (`Unknown`), never on one reporting `NotReady` (`False`), whose kubelet is alive and finishes its pods' deletion itself. It never taints the node it runs on, removes only taints it applied (recognised by its `github-runner.exadev/out-of-service-applied-at` annotation), and sends each change as a JSON patch that first tests the node's `resourceVersion`, so a node that came back since it was read is left for the next poll. RBAC is `get`, `list` and `patch` on nodes, with no `delete` unless pruning is switched on (below): the taint does not touch k3s's etcd membership, which only deleting a server's Node object changes. Its own pod tolerates an unreachable or not-ready node for only 10 seconds, so if it was running on the node that stopped, its ReplicaSet starts a replacement elsewhere well before the threshold passes. + +The threshold is also where a stopped node's runner pods are given up, earlier than the five-minute default eviction would. Their containers stopped with the node, so their jobs are lost either way unless the node returns within that time; lower the threshold for faster recovery, or raise it to wait longer for a node that is only asleep. + +It also leaves alone the nodes a voter swap or a regrading step is moving through, since those stay silent on purpose: while the Lease `kube-system/fleet-regrade` is unexpired it neither taints nor prunes a node its `github-runner.exadev.com/swap-nodes` annotation lists or its holder names, and it never taints a node carrying the regrade level label. See the role README's Node recovery section. + +Pruning is opt-in (`github_runner_arc_node_recovery_prune_after_seconds`, empty by default). When it is set, the watcher also deletes the Node object of a node that has been `Unknown` for that long and carries the `github-runner.exadev.io/removable=true` label (`github_runner_arc_node_recovery_prune_label`), so a device that is never coming back stops being listed. It never deletes an unlabelled node, a node that is a control-plane or etcd member (even if labelled), its own node or a node without a `Ready` condition, and it deletes nothing unless at least three `Ready` nodes carrying the etcd role label would remain, because k3s removes a server's etcd member when its Node is deleted. It sends the delete with the node's `uid` and `resourceVersion` as preconditions so a node that has come back since it was listed is left alone. Turning it on gives the watcher a cluster-wide `delete` on nodes. Its pod is then confined to control-plane nodes, since a token that can delete nodes should not be readable from a worker. A pruned node comes back only when its k3s agent process restarts: a running kubelet does not register itself again once its Node object is gone, so pruning is safe only for nodes that restart their agent when their Node vanishes or that reboot, which the node has to arrange. See the role README's Node recovery section for the details. + +Its image, `github_runner_arc_node_recovery_image`, is pulled without a pull Secret, so the published package must be public. diff --git a/docs/non-obvious-behaviour.md b/docs/non-obvious-behaviour.md new file mode 100644 index 00000000..92f7785b --- /dev/null +++ b/docs/non-obvious-behaviour.md @@ -0,0 +1,20 @@ +# Non-obvious behaviour + +## Non-obvious behaviour + +- **A scale-set's listener pod can crash-loop forever with `ephemeralrunnersets... not found` after a Helm upgrade, even though the release itself succeeds.** ARC names the `EphemeralRunnerSet` it creates for a release with a spec hash suffix, and gives it a new name whenever the pod template changes (a `helm upgrade` that touches resource limits, `nodeSelector`, or similar). The `AutoscalingListener` is supposed to track this rename, but can be left pointing at the old, now-deleted name - it then restarts every second or two, each time failing at startup with `could not patch ephemeral runner set ... not found`, and the scale set's own job queue never gets serviced even though every other release's listener stays healthy. Diagnose with `kubectl get autoscalinglisteners -n actions-runner-controller -o custom-columns='NAME:.metadata.name,ERS:.spec.ephemeralRunnerSetName'` against `kubectl get ephemeralrunnersets -A` - a mismatch confirms it. Fix: `kubectl delete autoscalinglistener -n actions-runner-controller`; the controller recreates it immediately with the current `EphemeralRunnerSet` name. +- **k3s cannot overwrite an existing kubeconfig through Colima's macOS share.** The container-root-to-Mac-user file mapping leaves `/output/kubeconfig.yaml` owner-read-only, so k3s crash-loops with "permission denied" on every restart (for example after a Mac reboot) unless the file is removed first. `k3s/node-entrypoint.sh` deletes it before starting k3s each time — do not remove that step. +- **Pods can lose every cluster service IP, cluster DNS included, whenever the Colima Linux VM has not loaded `br_netfilter` (a restarted VM, after a host reboot for example).** Without the module (`/proc/sys/net/bridge` is missing in the VM) packets bridged between pods never traverse iptables, so the service DNAT never applies: a pod reaches another pod's IP and the node's resolver directly but times out on `10.43.0.10` when the Service's only backend pod (CoreDNS, say) is on the same node as the caller, since a bridged hairpin reply skips the reverse NAT; backends on other nodes still answer, which is why it can stay latent until such a pod is rescheduled onto the node. Nothing looks wrong at first, because ARC keeps working from cached state; it fails only when it next needs a fresh GitHub token, when the controller logs `lookup api.github.com: i/o timeout`, a scale set's listener is not recreated, its `AutoscalingRunnerSet` stays `Pending` and the heartbeat reports `has no running listener`. The cluster role does this on every run for a Colima host (`tasks/colima_kernel.yml`, see its README's Colima kernel paragraph): it loads the module, writes `/etc/modules-load.d/br_netfilter.conf` and a sysctl file in the VM and sets the bridge sysctls. To repair a host before the playbook has run, load it with `colima ssh -- sudo modprobe br_netfilter`; check with `colima ssh -- ls /proc/sys/net/bridge`. +- **The k3s node name must stay stable across container recreations.** Without a pinned `hostname`, k3s derives the node name from the container's hostname (its ID), so every recreation (for example on an image pull) registers a new node and leaves the old one stale in etcd. The cluster role's `docker-compose.yml` pins every server host's own hostname to `${K3S_NODE_NAME:-k3s-server}`, Ansible-templated per host (defaulting to that host's own inventory name; `github_runner_cluster_node_name` overrides it, for example to keep a pre-existing device identity rather than forcing a disruptive rename). +- **`kubectl get nodes` can succeed with zero nodes.** It returns an empty, still-successful list before the node object has registered. The role polls for at least one node before calling `kubectl wait`, which fails outright ("no matching resources found") against zero nodes. It then waits for the inventory's nodes by name, never for every node of the cluster, so a node outside the inventory that is `NotReady` is reported and does not fail the run. +- **Zero runner pods in an `arc-runners-` namespace (or its per-profile `-` variant) when idle is expected, not a bug** — `minRunners: 0` means pods exist only while a job is running. +- **An org's Settings > Actions > Runners page shows the runner scale set itself** (a repository entry's shows in that repository's own Settings > Actions > Runners), not individual long-lived runner names the way the previous design did. ARC's ephemeral runners do not persist between jobs, so there is nothing to list while idle. +- **An autoscaled profile's static `maxRunners` and per-pod resource limits have to come from the pool's measured capacity**, not arbitrary defaults: a higher `maxRunners` without matching headroom leads to out-of-memory kills and pod evictions (ExaDev's fleet repository records the incident its own values were tuned from). `scripts/autoscaler.sh` depends on this value staying the safe floor it reverts to. +- **The metrics API's pod list and `kubectl top nodes` require metrics-server.** k3s bundles it by default (the cluster role's `docker-compose.yml` only disables `traefik`), but confirm it is actually running during the first real autoscaler dry run — `scripts/autoscaler.sh` fails safe to the static floor if either is unreachable. A very short-lived job (seconds, not minutes) can complete and its pod be garbage-collected before metrics-server's own scrape interval ever captures its usage - confirmed live against `test-arc-runner.yml`'s ~15-20s jobs, where `kubectl top pod` genuinely never returns data. This correctly triggers the fail-safe rather than a wrong reading, and doesn't matter in practice: a job that short never accumulates meaningful memory pressure anyway. `test-autoscaler.yml`'s own probe jobs hold for 120s specifically so metrics-server has time to observe them. +- **A replaced node can stay permanently `NotReady` with "Node password rejected" in its logs.** k3s keeps a node's password in `/etc/rancher/node/password` and the cluster holds a Secret for it (`kube-system/.node-password.k3s`). A recreated container keeps its password, because `/etc/rancher/node` is a named volume that the cluster role seeds from the running container before the first recreate that mounts it (see the role README's "Node passwords and recreating servers"), so this only happens to a node that is genuinely new under an old name: a replaced host, or a node whose volume was removed. Its container then generates a password the bootstrap host's Secret does not match, and the bootstrap host rejects the rejoin. Fix: on the bootstrap host, `kubectl delete secret .node-password.k3s -n kube-system` (and, for an agent, `kubectl delete node `), then restart the container so it registers fresh. Never delete a server's Node object: k3s then removes that server's etcd member, and a removed member's etcd refuses to start until its local datastore (`/var/lib/rancher/k3s/server/db`) is moved aside, after which it rejoins as a new member. `roles/github_runner_cluster/tasks/main.yml` self-heals the password mismatch automatically, only for hosts whose node is missing or not Ready, and deletes Node objects only for agents. +- **etcd's own peer traffic does not follow vpn-auth's `--node-ip`/`--advertise-address` the way every other kind of cluster traffic does - it stays on the container's own internal, cross-machine-unroutable Docker-bridge IP by default.** k3s logs this outright ("Etcd IP (PrivateIP) remains the local IP. Running etcd traffic over VPN is not recommended due to performance issues") and deliberately avoids the VPN interface for etcd specifically, assuming a real local network is the better alternative - an assumption that doesn't hold here, since a container's own bridge IP is never reachable from another machine at all. Confirmed live: a second server's join failed outright with `MemberAdd request timed out`, because it was trying to reach the first server's own unroutable bridge IP. An `--etcd-arg="initial-advertise-peer-urls=..."` passthrough looks like the fix but isn't: it only overrides etcd's own raw flag, while k3s independently builds its own `--initial-cluster` string from `config.PrivateIP`, computed separately and earlier - the two then disagree and etcd's bootstrap validation rejects the mismatch outright (`"--initial-cluster has ...= but missing from --initial-advertise-peer-urls="`). The actual fix, traced through k3s's own source (`pkg/cli/server/server.go`, `pkg/executor/embed/embed.go`): `config.PrivateIP` is populated from `--node-ip`'s own value if set, and this happens *before* `--vpn-auth`'s own executor-layer code later rebuilds `NodeIP` from its own Tailscale detection - so passing `--node-ip` explicitly ourselves, to the same real Tailscale IP vpn-auth would itself have detected, is what actually makes `PrivateIP` (and therefore etcd's peer-advertise address) correct, with no second, disagreeing source of truth for the same value. `docker-compose.yml`'s `k3s` service entrypoint brings Tailscale up itself and reads `tailscale ip -4` before exec'ing k3s specifically to learn this IP first, since `exec` replaces the script's own process and nothing can react afterward to what `--vpn-auth`'s own internal call would have produced. +- **A global `--etcd-arg` override is not scoped to the "real" server's own etcd instance - it also applies to k3s's separate, internal "temporary etcd", spun up on every *restart* (never on first boot) to reconcile with an already-initialized on-disk datastore before the real instance starts.** An earlier `--etcd-arg="listen-peer-urls=https://0.0.0.0:2380"` (kept as a "harmless" belt-and-braces bind override after fixing the peer-URL issue above) forced that temporary instance onto the same TLS `0.0.0.0:2380` bind it has no certificates for by design (it normally listens on a local, ephemeral, non-TLS port). Confirmed live: this only broke on a restart against an already-initialized datastore, never on a fresh `--cluster-init`, which is exactly why it wasn't caught until well after the first working deploy - fatal: `"cannot listen on TLS for [::]:2380: KeyFile and CertFile are not presented"`. `--node-ip` alone (above) is sufficient; etcd's own default `listen-peer-urls` already derives from the same `PrivateIP` correctly, so there's nothing left for an explicit override to add. +- **Calling `tailscale up --authkey=...` unconditionally on every container restart re-registers a brand new tailnet device each time, instead of reconnecting the persisted one - even with the join key's own state volume intact.** A reusable Tailscale auth key is designed to authorize provisioning *new* devices, and a naive `tailscale up` call on every restart (needed here to learn this node's own Tailscale IP before `exec`ing k3s - see above) doesn't check whether the daemon is already authenticated first, so it keeps minting new devices under names like `node1`, `node1-1`, `node1-2`... k3s's own `--vpn-auth` integration avoids exactly this from v1.35.0 on, by checking the Tailscale backend state first and skipping its own `tailscale up` call entirely once already `"Running"` (`StartVPN()` in `pkg/vpn/vpn.go`; the pinned v1.34 releases run `tailscale up --authkey ... --reset` on every start whatever the state) - the manual pre-empt call needs the identical guard: check `tailscale ip -4` succeeds *before* calling `tailscale up` at all, only calling it when not already authenticated. +- **Tailscale assigns an IP and wires up its own MagicDNS resolver as two separate steps, not atomically - a node having a real IP is not proof DNS resolution against another tailnet member is working yet.** A joining server needs to resolve the bootstrap host's own MagicDNS hostname (`K3S_JOIN_SERVER_URL`) before it can bootstrap against it at all. Confirmed live: k3s failed outright with `"dial tcp: lookup k3s-server.example.ts.net: no such host"` immediately after `tailscale ip -4` had already returned a real address for the *local* node. Fix: wait for a real `nslookup` success against the join server's own hostname specifically, not just local Tailscale readiness, before invoking k3s (the `k3s` service's entrypoint only does this when it has a server to join, `K3S_JOIN_SERVER_URL` or the role directory's `join-url` - the bootstrap host has nothing else to resolve). +- **A server that memory-pressure regrading has stepped to `etcd-only` is `NotReady` by design, and must stay registered.** With `github_runner_cluster_regrade` on, a server under sustained memory pressure steps down (`full`, `control-plane`, `etcd-only`) while it keeps its etcd member and vote; at `etcd-only` k3s runs with `--disable-apiserver --disable-controller-manager --disable-scheduler --disable-agent`, so there is no kubelet to keep its Node Ready, and the Node goes `NotReady` with `Unknown` conditions and stays so. That is not a failure to clear. Its Node object must not be deleted (k3s removes that server's etcd member with it), so anything that culls long-`NotReady` nodes must leave a Node carrying the `github-runner.exadev.com/regrade-level` label alone, and the role's wait for every node to be Ready already does. `kubectl` on such a server's own container is unreliable (its API server is off); the regrade service always reaches the cluster through another server. See the cluster role's README, Memory-pressure regrading. Confirmed against a three-server k3s cluster: the same etcd member ids before, during and after, the cluster taking writes with the stepped-down server as one of the two live voters, and the same Node and node password after stepping back up. +- **A single remaining healthy etcd server can get stuck indefinitely if another member is registered but permanently unreachable (e.g. a partial join that never completed), even though that member is a non-voting learner and shouldn't block quorum.** Symptom: `"failed to publish local member to cluster through raft"` with `context deadline exceeded`, repeating forever, alongside `kubectl` returning `503 the server is currently unable to handle the request`. The fix is k3s's own documented recovery mechanism, run once against the affected node: stop the container, then `k3s server --cluster-reset` (with the SAME flags the real service uses, including `--node-ip` set to the node's own persisted Tailscale IP - getting this wrong re-registers the single remaining member under the *wrong* address, e.g. the invoking container's own ad-hoc bridge IP, and the real service then fails to start with `"this server is a not a member of the etcd cluster"`) - this resets *etcd cluster membership* back to a single member using the *existing local data*, it does not wipe anything. It's a one-shot operation: k3s writes a guard file after a successful reset and refuses to run `--cluster-reset` again until it's manually deleted (`rm /var/lib/rancher/k3s/server/db/reset-flag`) if a second reset is genuinely needed. After a reset, restart the service normally (without the flag) and clean up the stale member's own now-orphaned Kubernetes `Node` object and `node-password` Secret before it rejoins fresh. diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 00000000..93e26c3f --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,19 @@ +# Releases + +## Releases + +Every push to `main` runs `.github/workflows/release.yml`, driven by [semantic-release](https://semantic-release.gitbook.io/) (`.releaserc.json`) against [Conventional Commits](https://www.conventionalcommits.org/) (`commitlint.config.js`, enforced by the `commitlint` CI job above, which checks every commit and the pull request title). Almost every conventional-commit type triggers at least a patch release (see `.releaserc.json`'s `commit-analyzer` `releaseRules`: `feat` is minor, a breaking change (`!` or a `BREAKING CHANGE:` footer) is major, everything else conventional is patch), so in practice nearly every push to `main` produces a release. + +A release: + +1. Stamps the new version into `galaxy.yml`'s `version:` field (which stays `0.0.0` in source between releases) and into the published-image tag defaults in `roles/github_runner_arc/defaults/main.yml` (`github_runner_arc_heartbeat_image`, `github_runner_arc_autoscaler_image`, `github_runner_arc_image_pull_secret_renewer_image`, `github_runner_arc_node_recovery_image`), and into each role's `*_collection_version` default, which the deployed-state stamps record - `scripts/release/stamp_version.py` does this by matching each variable's own name, not a line number, so it stays correct even while those files are edited concurrently by unrelated work. +2. Builds the collection tarball (`ansible-galaxy collection build --force`) and attaches it to the GitHub Release. +3. Updates `CHANGELOG.md` and commits it, `galaxy.yml`, and the stamped defaults file back to `main` as `chore(release): [skip ci]`. +4. Builds and pushes every container image (`Dockerfile`, `heartbeat/Dockerfile`, `autoscaler/Dockerfile`, `pull-secret-renewer/Dockerfile`, `node-recovery/Dockerfile`, `fleet-role/Dockerfile`) multi-arch (`linux/arm64,linux/amd64`) to their `ghcr.io/exadev/github-runner*` repositories, each tagged with the exact version, the floating major version, and `latest` - all three tags on the same manifest, via `docker/metadata-action`'s `type=semver` patterns. To pin a host to a specific release rather than floating on `latest`, set `roles/github_runner_arc/defaults/main.yml`'s image variables (or a host's own override) to an exact version tag instead. +5. Optionally publishes the collection to Ansible Galaxy, only if a `GALAXY_API_KEY` secret is present - the `[skip ci]` release commit and the image build both happen unconditionally, in this same run, regardless of whether Galaxy publishing is configured. + +The `[skip ci]` in the release commit message is deliberate: the image build already happens in this same run using the exact version semantic-release just decided, so there is nothing useful for a second, separately-triggered run of this workflow (or of `ci.yml`) to do against a commit that only changed a version stamp and a changelog - `ci.yml`'s own lint jobs already ran against every commit before it reached `main`. + +The reference capability images in `tool-images/` are not part of a release. `.github/workflows/publish-tool-images.yml` builds each one natively on amd64 and arm64 runners and pushes it to `ghcr.io//github-runner-toolcache-`, `ghcr.io//github-runner-sysroot-` or `ghcr.io//github-runner-runner-build`, tagged with the version its Dockerfile pins, when started by hand (`gh workflow run publish-tool-images.yml`) or by pushing a `tool-images-` tag. It leaves a version tag that already exists alone, and its run summary gives each pushed image's `:@sha256:` reference to pin in a capability or as a runner image. A new GitHub Container Registry package starts private; make it public in its package settings so pods can pull it without a credential. A build runner entry with a `runner_id` publishes the same image with its runner user at that uid and gid, tagged `-uid` (the ARC role README says when a fleet needs one). + +**What the release push needs:** a deploy key with write access on `ExaDev/github-runner`, whose private key is the `RELEASE_DEPLOY_KEY` repository secret `release.yml` checks out with. If `main` gains a ruleset, add deploy keys as its bypass actor so the release commit can land. Publishing to Ansible Galaxy additionally needs a `GALAXY_API_KEY` repository secret (an API key from `https://galaxy.ansible.com/me/preferences`); without it, the publish step logs that it's skipping and the release still completes. diff --git a/docs/role-controller.md b/docs/role-controller.md new file mode 100644 index 00000000..557a32b8 --- /dev/null +++ b/docs/role-controller.md @@ -0,0 +1,38 @@ +# Role controller and voter swap + +### Role controller + +`fleet-role-probe` and `fleet-role` (an optional DaemonSet and Deployment in the platform namespace, see `roles/github_runner_arc/templates/fleet-role.yaml.j2`, `scripts/fleet-role-probe.sh` and `scripts/fleet-role.sh`; `github_runner_cluster_role_controller`, off by default, which `github_runner_arc_role_controller_enabled` takes as its default) measure which hosts are fit to hold an etcd vote, report the swap of an unfit voter for a fit standby that the controller would make and, when asked or when `github_runner_arc_role_controller_dry_run` is false, carry the swap out ([Voter swap](#voter-swap)). The probe only reports: its ClusterRole can `get` and `list` nodes and nothing else, and the only object it writes is its own ConfigMap. The controller runs unchanged when no host is marked eligible, when it reports the voters' fitness alone. + +The probe runs on every Linux node that is an etcd voter (k3s's `node-role.kubernetes.io/etcd` label) or is marked eligible and has a kubelet (a server regraded to etcd-only has none, and its side is read from its peers), on the node's own network, through a node affinity on those two labels, so a node that is neither (a Unraid host, an image-flashed agent) has no pod and nothing is asked of its kernel. A pod stopped on a node that has lost both labels, or is gone, removes the node's entry as it ends, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the control plane and not with the fleet. Every `github_runner_arc_role_controller_sample_seconds` (15) it appends one sample to its node's key of the `fleet-role-probe` ConfigMap, which keeps the last window of them so that a replaced pod loses nothing. A sample holds the TCP connect time to every voter's etcd peer port (a connect that does not finish within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what its etcd metrics (the `127.0.0.1:2381` listener) gained since the last sample: the WAL fsync and backend commit histograms, the round trip to each peer, leader changes, and whether it has a leader, is a learner and knows as many members as there are voters (a learner or a member that has not started shows as an extra known member); on a node that is not a voter, an fdatasync reading from fio with the write size etcd's own disk check uses, once per window; the memory pressure (`/proc/pressure/memory` of the pod's own procfs, which serves the whole node's, so nothing is mounted) and `MemAvailable` the regrade core reads, with the core's own functions, where a node on a kernel without pressure stall information (built or booted without it, which has no `/proc/pressure` or, on older kernels, has the files and refuses to read them) gives `MemAvailable` and `MemTotal` and no pressure figure; and the medium of the node's link, `wired`, `wifi` or `unknown`: the value of the node's `github-runner.exadev.com/link` label (`github_runner_arc_role_controller_link_label`), which the cluster role reads on the host itself (see the cluster role's README) since a node in a container or a VM sees only its own virtual interface, and otherwise the medium of the interface the pod's default route leaves by (an interface that is virtual, such as the k3s container's veth, hides the link beneath it), each sample saying which (`link_source`, `label` or `interface`). The pod's own reading is taken from the interface's `/sys/class/net` entry and never by walking `/sys`, whose symlinks loop. `scripts/fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of these readings alone (the link as the pod's own interface gives it), with no cluster access, to compare with a reading taken by hand on the host. + +The controller reads the Nodes and the probes' samples every sample period and publishes `fleet-role-status`, which the heartbeat republishes in its gist beside the autoscaler's status. For every voter and every eligible host it gives a verdict, `fit`, `unfit` or `unknown`, with each reason and the numbers behind it. Every limit derives from etcd's guidance and the cluster's own timing, not chosen numbers: + +- A voter is unfit when its tail (99th percentile) round trip to any other voter exceeds a tenth of the election timeout, etcd's rule that the election timeout is at least ten round trips. A standby is fit only at or below a twentieth of it, the 0.5 end of etcd's heartbeat band; the gap between the two is the hysteresis. The election timeout is `github_runner_cluster_etcd_election_timeout`, or k3s's 1000 ms when that is empty. A pair's figure is the worse of the TCP connect tail and etcd's own round trip tail. +- A voter's WAL fsync tail must be under 10 ms and its backend commit tail under 25 ms (etcd's FAQ); a standby's fdatasync tail under 10 ms. +- Memory is never over the regrade limit (`github_runner_cluster_regrade_pressure_threshold` and `github_runner_cluster_regrade_available_floor_percent`) in any sample of the window, and `MemAvailable` is at least etcd's storage quota (what the voters report, 2 GiB by default). A host none of whose samples has a pressure reading cannot be over the regrade limit, which needs both, so it is judged by `MemAvailable` against the quota alone, and its status says so: `unknown_signals` lists `memory pressure` (and the log adds `memory pressure unknown` to its verdict), beside `figures.psi_avg300_max`, the worst pressure of the window, which is then `null`. +- Stability: no `NotReady` spell in the window and no k3s restart, seen on a voter as etcd's counters going backwards. On a node that is not a voter the only sign of a restart is its probe starting again, so a new probe pod counts as one. +- A window is 100 samples, the fewest that give the 99th percentile a sample of its own, so one outlier never decides; a host with fewer is `unknown`, as is one whose probe has not reported for three sample periods. Readings from before the last change of the voter set are not counted. +- Hosts that pass are ranked wired before Wi-Fi before unknown, then by round trip, then by disk. The link ranks hosts and is never a gate; the status gives each host's link and where it was read (`link` and `link_source`), and the log names it beside the verdict. + +A slow link makes both of its ends unfit, since a voter is unfit when its round trip to any other voter is, so every unfit verdict is attributed to the node at fault for it (`attributed_to` on each host, and `attribution`, the hosts each node accounts for). A host's own disk, memory, readiness or restarts are its own fault. Its round trip is the link to its worst peer, and the end at fault for a slow link is the one whose links to the other voters are slower too, compared by the best of them, since a node on a poor path is slow to every peer and a node on a good one is fast to its others; when the two ends' other links are as fast as each other's, or one end has none, the link is attributed to both. The voter to replace is the unfit voter the most hosts' unfit verdicts are attributed to, the worst-ranked among those with as many, and the log names the attribution beside each unfit verdict. A swap is reported only when that voter is eligible and not pinned (when it is not, replacing a voter that shares a slow link with it would fix nothing, so nothing is swapped) and a fit eligible standby ranks above it. A candidate is judged against the voters that would remain, since a slow link to the very voter it would replace says nothing against it. A voter is a node that carries k3s's etcd label and whose `EtcdIsVoter` condition is True: k3s labels a server the moment it joins etcd as a learner and never removes the label, so a labelled node without the condition is a member that does not vote: it is not counted as a voter and holds a swap back (`member but not a voter` in the status). It is held back, with the reason in the status and the log, while the voter count is not `github_runner_arc_role_controller_voters`, any voter is below `full`, is not Ready, has no leader, is a learner or knows a member that is not a voter, is not connected to every other voter, fewer hosts are eligible than the target, or the cooldown after a change of the voter set has not passed (k3s's five minutes to remove a stalled learner plus one window). The target must be odd, at least 3, and at most `github_runner_cluster_max_servers` minus one, which leaves a swap one place of headroom; the role refuses anything else. A host is eligible, and a voter pinned, by a Node label with the value `true` (`github_runner_arc_role_controller_eligible_label` and `_pinned_label`), which the cluster role publishes from `github_runner_cluster_voter_eligible` and `github_runner_cluster_voter_pinned` per host (and the target from `github_runner_cluster_voters`, refused unless odd, at least 3 and below `github_runner_cluster_max_servers`; see the cluster role's README); a voter not marked eligible is never swapped out, since nothing has prepared it to come back. + +#### Voter swap + +A swap brings an eligible standby in as a voter and takes an unfit voter out, as one operation under the `fleet-regrade` Lease, which the controller holds as `fleet-role` and renews on every pass for as long as the swap runs. A regrade step, a run of the cluster role and node recovery all leave the nodes of a swap alone while it does. The controller takes at most one action in a pass and reads the step a swap is at from the Nodes, the hosts' role actuators' reports and the Lease, so a controller pod that is rescheduled resumes where the swap is. The state machine is `scripts/fleet-role-swap.jq`, a pure function over that state, and `scripts/fleet-role.sh` carries out the one action it names. + +A swap starts in one of two ways. `kubectl -n github-runner-platform exec deploy/fleet-role -- /app/scripts/fleet-role.sh swap OUT IN` asks for it by command: the command exits 0 when every gate holds and the request is recorded for the next pass, 1 when it is refused (each reason is printed) and 2 for a node that is unknown, not marked eligible or not the voter and the standby it is taken for. A request is carried out whatever the verdicts say, and inside the cooldown, but never takes out a pinned voter and honours every gate. The controller starts the swap it decides on only when `FLEET_ROLE_DRY_RUN` is `false` (`github_runner_arc_role_controller_dry_run`, true by default, which reports such a swap and carries out requests alone). A swap already under way is finished or rolled back whatever the setting, so an even count of voters is never left behind. `fleet-role.sh status` prints the swap and its step. + +The gates are those of the epic: every voter at `full`, Ready, with a fresh probe report, etcd with a leader, no learner and no member that has not started, the voter count the target, and the regrade Lease free. The steps: + +1. A snapshot is asked of a remaining voter through its Node (`snapshot-request`) and waited for until k3s publishes an `ETCDSnapshotFile` of that name that reports `readyToUse`. Nothing has changed yet, so a snapshot that is not ready within the operation timeout abandons the swap. +2. The candidate is cordoned and the controller waits for its runner pods to finish (a planned change never cuts a job); past the runner wait the swap is abandoned, the candidate uncordoned and the Lease released, with no cooldown. Otherwise it is drained and asked to promote: `desired-role=server`, with `role-swap`, `role-step=promote` and `role-join-node` naming a remaining voter. Its actuator writes the role file, a join URL and the unit changes, and starts the server, which k3s joins as a learner that the etcd leader promotes once it has caught up. The step ends when the candidate is a voter (`EtcdIsVoter`) and Ready on a kubelet heartbeat after the server started, and the candidate is uncordoned. +3. The gates are held again, now with one more voter than the target. If the outgoing voter is down they are held to the other voters alone. +4. The outgoing voter is tainted and cordoned, its runner pods waited for (past the runner wait it is drained anyway, as regrading drains a server) and drained, labelled `regrade-level=etcd-only` and asked to step down (`desired-role=agent`, `role-step=step-down`). Its actuator brings its server to etcd-only and writes the role file for the agent. When it reports done, or when the node is down and has not reported within the operation timeout, the controller annotates `etcd.k3s.cattle.io/remove=true` while the remaining voters are healthy, and again if k3s clears the annotation, and waits for `removed-node-name`. A step-down the actuator reports failed on a node that stays Ready is held for the owner. The member counts as removed once k3s records it, or once every remaining voter's probe reports etcd listing as many members as there are voters, which holds when a server still running on the host has put its etcd label back; the controller then marks the Node with `role-member-gone` naming the swap, the proof its actuator finishes on, since k3s's own record can name another member than the host's datastore. The actuator then finishes the demotion (the server stopped, its datastore moved aside, the agent started) and the controller removes the `control-plane`, `etcd` and `master` labels, the `etcd.k3s.cattle.io/*` annotations, the level label and the taint, waits for the node to be Ready as an agent and uncordons it. A host that has not come back within the operation timeout is left to node recovery and the swap records `outgoing_pending`. +5. The swap annotations are cleared and the Lease is released. The cooldown starts from the membership change, as for a decision. + +A candidate that is not a voter by the deadline (k3s's learner stall time plus the operation timeout), or whose actuator reports the promotion failed, is rolled back: the controller annotates the removal of its learner if it joined, asks its actuator to put the host back to an agent (`role-step=rollback`), waits until every voter's probe lists exactly the target number of members, then cleans the Node up and uncordons it. A refused promotion, which changed nothing on the host, abandons the swap. Every wait is derived from the operation timeout, the runner wait and the learner stall time, which the role takes from the cluster role's own settings (`github_runner_arc_role_controller_operation_timeout_seconds`, `github_runner_arc_role_controller_runner_wait_seconds` and `github_runner_arc_role_controller_lease_seconds`). + +`fleet-role-status` carries `swap` (`id`, `out`, `in`, `step`, `started_at`, `step_since`, `deadline`, `manual`, `snapshot`), `waiting` (why the swap is not acting this pass), `last_swap` (`outcome`, `completed`, `rolled-back` or `abandoned`, with its reasons), the `request` pending and the real `dry_run`. A manual request is kept in the status ConfigMap under `request.json`, a key of its own, so that the command and the controller's pass never overwrite each other. The controller's ClusterRole adds to the probe's `patch` on nodes, `get` and `list` on pods, `create` on `pods/eviction`, `get` on daemon sets and `get` and `list` on k3s's `etcdsnapshotfiles`; its Roles let it `create` Leases and `get` and `update` the regrade Lease in `kube-system`, `get` the kubelets' Leases and `get` the actuators' ConfigMap. What stays manual is removing a voter that is down and not coming back, restoring after quorum loss and changing the target count. + +The controller and the probe run one image, `github_runner_arc_role_controller_image`, pulled without a pull Secret, so the published package must be public. diff --git a/examples/fleet-alert/fleet-alert.yml b/examples/fleet-alert/fleet-alert.yml index 07e11548..354caee7 100644 --- a/examples/fleet-alert/fleet-alert.yml +++ b/examples/fleet-alert/fleet-alert.yml @@ -1,6 +1,6 @@ name: Fleet alert -# Copy this file to .github/workflows/ in a repository that is not served by the fleet, and set the repository variables and secrets listed below. It reads the heartbeat's fleet-health.json from its gist and announces an outage that has lasted longer than a window (see "Fleet-down notification" in the collection's README). +# Copy this file to .github/workflows/ in a repository that is not served by the fleet, and set the repository variables and secrets listed below. It reads the heartbeat's fleet-health.json from its gist and announces an outage that has lasted longer than a window (see "Fleet-down notification" in docs/heartbeat.md). # # It runs on GitHub-hosted ubuntu-latest on purpose: a job routed to the fleet would not start when the fleet is down, which is exactly when this has to run. Do not change runs-on to the fleet's label or to runner-fallback-action's output. # diff --git a/examples/native-agent/inventory.yml b/examples/native-agent/inventory.yml index fc64b5c0..2df39ac9 100644 --- a/examples/native-agent/inventory.yml +++ b/examples/native-agent/inventory.yml @@ -21,5 +21,5 @@ all: github_runner_cluster_node_mode: native # The host is dedicated to the cluster, so the role may log its client out of the tailnet it was in. github_runner_cluster_native_tailscale_leave_other_tailnet: true - # The kernel modules and sysctls k3s needs, set by the cluster role. What else a laptop dedicated to the cluster needs of its host (no suspend when the lid closes, no Wi-Fi power save, a charge limit on its always-on battery, no desktop, security updates that never reboot it) is the operator's to provide before the play runs; see the README's Prerequisites. + # The kernel modules and sysctls k3s needs, set by the cluster role. What else a laptop dedicated to the cluster needs of its host (no suspend when the lid closes, no Wi-Fi power save, a charge limit on its always-on battery, no desktop, security updates that never reboot it) is the operator's to provide before the play runs; see the Prerequisites section of docs/ansible.md. github_runner_cluster_k3s_prerequisites: true diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index 16dc508e..e411e689 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -5,8 +5,8 @@ # API), and bash. All available from Alpine's apk, so the image is # arch-agnostic (no gh binary to fetch per-arch) and small. Build context is # the repo root so it can COPY scripts/heartbeat.sh; built and pushed -# multi-arch the same manual way as the main runner image (see the README's -# Build/test section) so the image is pullable from whichever node the +# multi-arch the same manual way as the main runner image (see +# docs/build-and-test.md) so the image is pullable from whichever node the # scheduler places this pod on. FROM alpine:3.20 diff --git a/playbooks/site.yml b/playbooks/site.yml index abf77127..fa3165db 100644 --- a/playbooks/site.yml +++ b/playbooks/site.yml @@ -26,7 +26,7 @@ name: exadev.github_runner.github_runner_arc tasks_from: validate.yml -# The host baseline is the operator's, applied before this play and never by it: Homebrew on a Mac (the pre_tasks below and the macOS runner role use it), a connecting user who can become root without a password or a play run with --ask-become-pass, a host that does not sleep while the node runs (a Mac on AC power with sleep off, a laptop that ignores its lid), and, where a native node asks for one, a Tailscale client on the host. The README's Prerequisites section lists what each kind of host must already have. +# The host baseline is the operator's, applied before this play and never by it: Homebrew on a Mac (the pre_tasks below and the macOS runner role use it), a connecting user who can become root without a password or a play run with --ask-become-pass, a host that does not sleep while the node runs (a Mac on AC power with sleep off, a laptop that ignores its lid), and, where a native node asks for one, a Tailscale client on the host. The Prerequisites section of docs/ansible.md lists what each kind of host must already have. - name: GitHub Actions runner fleet # The cluster group, whose inventory order decides which hosts are k3s servers (see roles/github_runner_cluster/defaults/main.yml). bootstrap.sh sets github_runner_cluster_group to all, since its one-host inventory has no groups. hosts: "{{ github_runner_cluster_group | default('github_runner_cluster') }}" diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 4809d0c3..1f01ee97 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -55,7 +55,7 @@ Helm 4 and server-side apply. The role reads `helm version --short` and, on Helm - `github_runner_arc_image_pull_secret_source`: where a pull Secret the role writes gets its credential: `static` (the default) from `github_runner_arc_ghcr_username` and `github_runner_arc_ghcr_token`, or `app` from each org's GitHub App, renewed in the cluster (see [Image pull Secret from the GitHub App](#image-pull-secret-from-the-github-app)), or `none` when every image is public (see [Public images without a pull Secret](#public-images-without-a-pull-secret)). `github_runner_arc_image_pull_secret_renewal_schedule` (default every 15 minutes) and `github_runner_arc_image_pull_secret_renewer_image` configure the renewal. - `github_runner_arc_image_pull_registry`, `github_runner_arc_image_pull_secret_name`, `github_runner_arc_verify_image_pull`: the pull Secret's registry and name, and whether to prove the credential before writing it. - `github_runner_arc_heartbeat_gist_id`: install the fleet-health platform from this host, refreshing this gist. `github_runner_arc_heartbeat_bootstrap_gist`, `github_runner_arc_heartbeat_gist_description` and `github_runner_arc_heartbeat_gist_consumer` control the bootstrap described above. -- `github_runner_arc_autoscaler_usable_budget_gi`, `github_runner_arc_autoscaler_floor`: required when a profile sets `autoscale: true` and the platform is installed. The floor must equal the autoscaled profile's `maxRunners`, which every Helm upgrade reverts to. The autoscaler works its ceiling out from the nodes on every poll (see the project README's Autoscaler section); `github_runner_arc_autoscaler_max_ceiling`, optional, is an upper bound on it and must be no lower than the floor. The other `github_runner_arc_autoscaler_*` and `github_runner_arc_heartbeat_*` settings have defaults; see `defaults/main.yml`. +- `github_runner_arc_autoscaler_usable_budget_gi`, `github_runner_arc_autoscaler_floor`: required when a profile sets `autoscale: true` and the platform is installed. The floor must equal the autoscaled profile's `maxRunners`, which every Helm upgrade reverts to. The autoscaler works its ceiling out from the nodes on every poll (see `docs/autoscaler.md`); `github_runner_arc_autoscaler_max_ceiling`, optional, is an upper bound on it and must be no lower than the floor. The other `github_runner_arc_autoscaler_*` and `github_runner_arc_heartbeat_*` settings have defaults; see `defaults/main.yml`. - `github_runner_arc_node_recovery_enabled` (default `true`), `github_runner_arc_node_recovery_after_seconds` (default 120), `github_runner_arc_node_recovery_poll_seconds` (default 15), `github_runner_arc_node_recovery_prune_after_seconds` (default empty, which leaves pruning off), `github_runner_arc_node_recovery_prune_label` (default `github-runner.exadev.io/removable`), `github_runner_arc_node_recovery_level_label` (default the cluster role's `github_runner_cluster_regrade_level_label`, else `github-runner.exadev.com/regrade-level`), `github_runner_arc_node_recovery_image`: the node recovery watcher (see [Node recovery](#node-recovery)). - `github_runner_arc_role_controller_enabled` (default the cluster role's `github_runner_cluster_role_controller`, itself `false`), `github_runner_arc_role_controller_image`, `github_runner_arc_role_controller_sample_seconds` (default 15), `github_runner_arc_role_controller_voters` (default 3), `github_runner_arc_role_controller_max_servers`, `github_runner_arc_role_controller_election_timeout_ms`, `github_runner_arc_role_controller_pressure_threshold`, `github_runner_arc_role_controller_available_floor_percent`, `github_runner_arc_role_controller_eligible_label`, `github_runner_arc_role_controller_pinned_label`, `github_runner_arc_role_controller_level_label`, `github_runner_arc_role_controller_link_label`, `github_runner_arc_role_controller_probe_configmap`, `github_runner_arc_role_controller_status_configmap`, `github_runner_arc_role_controller_data_dir`, `github_runner_arc_role_controller_dry_run` (default true), `github_runner_arc_role_controller_operation_timeout_seconds`, `github_runner_arc_role_controller_runner_wait_seconds`, `github_runner_arc_role_controller_lease_seconds`, `github_runner_arc_role_controller_actuator_configmap`, `github_runner_arc_role_controller_lease_namespace` and `github_runner_arc_role_controller_lease_name`: the role controller, which reports which hosts are fit to hold an etcd vote and carries out a voter swap (see [Role controller](#role-controller)). Those that mirror the cluster role (`voters`, `max_servers`, `election_timeout_ms`, `pressure_threshold`, `available_floor_percent`, `eligible_label`, `pinned_label`, `level_label`, `link_label`, `operation_timeout_seconds`, `runner_wait_seconds`, `lease_seconds`, `actuator_configmap`, `lease_namespace`, `lease_name`) take the cluster role's own variable when it is defined. - `github_runner_arc_runner_reaper_enabled` (default `false`), `github_runner_arc_runner_reaper_max_job_seconds` (default 432000, five days), `github_runner_arc_runner_reaper_completed_grace_seconds` (default 300), `github_runner_arc_runner_reaper_schedule` (default every 5 minutes), `github_runner_arc_runner_reaper_dry_run` (default `false`), `github_runner_arc_runner_reaper_image` (default the renewal's image): the runner reaper (see [Runner reaper](#runner-reaper)). @@ -434,7 +434,7 @@ The names a repository entry derives join the login and the repository with a do The scope (`name`, or `name/repository`) identifies an entry. Validation refuses the same scope twice, in any letter case, whether on one host or on two hosts of one cluster, and checks release names and cross-cluster registrations per scope, since GitHub keeps an organisation's scale set names apart from a repository's. The role records each entry's installation id and App-minted pull token under its scope, so an organisation's entry and any number of its repositories' entries can sit side by side, each with its own App or sharing one. With the `github_runner_secrets_onepassword` adapter each `private_key_op_reference` is read once, whichever entries name it. -A repository's scale set takes jobs from that repository only, so every repository that should use the fleet needs an entry of its own, with its own releases, its own listener in the controller's namespace and its own App Secret in each of its namespaces. Its runners' `runs-on` label serves that repository alone, unlike an organisation entry's pooled label. An autoscaled repository profile joins the one pooled memory budget every autoscaled profile shares (see the repository README's Autoscaler section); nothing about the pool is per scope. Likewise the heartbeat counts a repository's scale sets among those it expects (each must have a running listener before it vouches for the fleet), and anything reading its gist sees one fleet, not one health per scope. +A repository's scale set takes jobs from that repository only, so every repository that should use the fleet needs an entry of its own, with its own releases, its own listener in the controller's namespace and its own App Secret in each of its namespaces. Its runners' `runs-on` label serves that repository alone, unlike an organisation entry's pooled label. An autoscaled repository profile joins the one pooled memory budget every autoscaled profile shares (see `docs/autoscaler.md`); nothing about the pool is per scope. Likewise the heartbeat counts a repository's scale sets among those it expects (each must have a running listener before it vouches for the fleet), and anything reading its gist sees one fleet, not one health per scope. The entry's GitHub App needs the permissions ARC documents for the repository scope ([Authenticating ARC to the GitHub API](https://docs.github.com/en/actions/how-tos/manage-runners/use-actions-runner-controller/authenticate-to-the-api)): @@ -493,7 +493,7 @@ github_runner_arc_orgs: ## Node recovery -Every host that installs anything also installs a small watcher, the `node-recovery` Deployment in `github_runner_arc_platform_namespace`, so that a node that stops does not leave its pods `Terminating` for ever. ARC waits for a listener's old pod to go before starting a new one, so a listener on a stopped node otherwise stalls its scale set until the pod is force-deleted by hand. Once a node's `Ready` condition has been `Unknown` for `github_runner_arc_node_recovery_after_seconds`, the watcher gives it the `node.kubernetes.io/out-of-service=nodeshutdown:NoExecute` taint of Kubernetes' non-graceful node shutdown, which makes the control plane evict the node's pods and force-delete the terminating ones; it removes the taint when the node reports `Ready` again. It never acts on a node reporting `NotReady`, never on its own node, and never removes a taint it did not apply. Its ClusterRole allows `get`, `list` and `patch` on nodes and nothing else, unless pruning is switched on (below). `github_runner_arc_node_recovery_enabled: false` removes the watcher, its RBAC and any taint it left behind. Its image is pulled without a pull Secret, so it must be public. The repository README's Node recovery section explains the default threshold. +Every host that installs anything also installs a small watcher, the `node-recovery` Deployment in `github_runner_arc_platform_namespace`, so that a node that stops does not leave its pods `Terminating` for ever. ARC waits for a listener's old pod to go before starting a new one, so a listener on a stopped node otherwise stalls its scale set until the pod is force-deleted by hand. Once a node's `Ready` condition has been `Unknown` for `github_runner_arc_node_recovery_after_seconds`, the watcher gives it the `node.kubernetes.io/out-of-service=nodeshutdown:NoExecute` taint of Kubernetes' non-graceful node shutdown, which makes the control plane evict the node's pods and force-delete the terminating ones; it removes the taint when the node reports `Ready` again. It never acts on a node reporting `NotReady`, never on its own node, and never removes a taint it did not apply. Its ClusterRole allows `get`, `list` and `patch` on nodes and nothing else, unless pruning is switched on (below). `github_runner_arc_node_recovery_enabled: false` removes the watcher, its RBAC and any taint it left behind. Its image is pulled without a pull Secret, so it must be public. `docs/node-recovery.md` explains the default threshold. The watcher also leaves alone the nodes a voter swap or a regrading step is moving through, because those are meant to be silent for a while: the host stops one k3s unit and starts the other, or a server runs at etcd-only with no kubelet, and an out-of-service taint would evict pods that the controller is draining on purpose. It reads the Lease `kube-system/fleet-regrade`, which serialises both, through a Role that allows `get` on that one name and nothing else in the namespace. While the Lease is unexpired (its renew time plus its duration has not passed, the rule `lease_state` applies in the cluster role's regrade core) it neither taints nor prunes a node that the Lease's `github-runner.exadev.com/swap-nodes` annotation lists (comma separated) or that its holder names, which is a node name when regrading holds the Lease. A node carrying the regrade level label (`github_runner_arc_node_recovery_level_label`, `github-runner.exadev.com/regrade-level` unless the cluster role is configured otherwise) is never tainted, whether or not any Lease is held. A taint the watcher applied earlier is still removed once such a node reports `Ready`. A Lease that cannot be read fails the pass before it changes anything, since tainting on a guess could cut a swap short. @@ -505,7 +505,7 @@ A pruned node returns only when its k3s agent process restarts. A running k3s ag ## Role controller -With `github_runner_cluster_role_controller: true` (or `github_runner_arc_role_controller_enabled: true`, for a host that does not run the cluster role) the role installs `fleet-role-probe`, a DaemonSet on every Linux node that is an etcd voter or is marked eligible (`github_runner_arc_role_controller_eligible_label`) and has a kubelet, and `fleet-role`, a Deployment, both in `github_runner_arc_platform_namespace` and both from `github_runner_arc_role_controller_image`. Together they report, for every etcd voter and every host marked eligible, whether it is fit to hold an etcd vote and why, and the swap of an unfit voter for a fit standby that the controller would make. The probe changes nothing in the cluster: its ClusterRole can `get` and `list` nodes and nothing else, and its Role reaches only its own ConfigMap by name (`get` and `patch`, never `create`; the role creates both ConfigMaps empty and keeps them). The controller carries out a voter swap when it is asked to (`fleet-role.sh swap OUT IN`, run with `kubectl exec deploy/fleet-role`) or, with `github_runner_arc_role_controller_dry_run` false, when it decides on one, and always finishes a swap already under way. It therefore also has `patch` on nodes, `get` and `list` on pods in every namespace, `create` on `pods/eviction`, `get` on daemon sets and `get` and `list` on k3s's `etcdsnapshotfiles` in its ClusterRole, `get` on the actuators' ConfigMap, and two Roles of its own outside the platform namespace: in the regrade Lease's namespace (`create` on Leases, and `get` and `update` on the regrade Lease by name) and in `kube-node-lease` (`get`, to tell a restarted node's kubelet is running). The dry run is on by default and the role never turns it off by itself. `false` removes the workloads, their RBAC and both ConfigMaps. The repository README's Role controller section explains each signal, how it is read, the thresholds and when a swap is held back. +With `github_runner_cluster_role_controller: true` (or `github_runner_arc_role_controller_enabled: true`, for a host that does not run the cluster role) the role installs `fleet-role-probe`, a DaemonSet on every Linux node that is an etcd voter or is marked eligible (`github_runner_arc_role_controller_eligible_label`) and has a kubelet, and `fleet-role`, a Deployment, both in `github_runner_arc_platform_namespace` and both from `github_runner_arc_role_controller_image`. Together they report, for every etcd voter and every host marked eligible, whether it is fit to hold an etcd vote and why, and the swap of an unfit voter for a fit standby that the controller would make. The probe changes nothing in the cluster: its ClusterRole can `get` and `list` nodes and nothing else, and its Role reaches only its own ConfigMap by name (`get` and `patch`, never `create`; the role creates both ConfigMaps empty and keeps them). The controller carries out a voter swap when it is asked to (`fleet-role.sh swap OUT IN`, run with `kubectl exec deploy/fleet-role`) or, with `github_runner_arc_role_controller_dry_run` false, when it decides on one, and always finishes a swap already under way. It therefore also has `patch` on nodes, `get` and `list` on pods in every namespace, `create` on `pods/eviction`, `get` on daemon sets and `get` and `list` on k3s's `etcdsnapshotfiles` in its ClusterRole, `get` on the actuators' ConfigMap, and two Roles of its own outside the platform namespace: in the regrade Lease's namespace (`create` on Leases, and `get` and `update` on the regrade Lease by name) and in `kube-node-lease` (`get`, to tell a restarted node's kubelet is running). The dry run is on by default and the role never turns it off by itself. `false` removes the workloads, their RBAC and both ConfigMaps. `docs/role-controller.md` explains each signal, how it is read, the thresholds and when a swap is held back. The probe runs with the node's network and as root with every capability dropped, because the directory it measures the disk in (`github_runner_arc_role_controller_data_dir`, on the filesystem k3s keeps its data on) is created by the kubelet and owned by root. It mounts nothing for memory pressure: `/proc/pressure/memory` of its own procfs is the node's, as `/proc/meminfo` is, and a `hostPath` mount of `/proc/pressure` would stop the pod starting on a kernel without pressure stall information, since the container runtime then tries to create the directory. The controller runs as an unprivileged user with a read-only root filesystem. The heartbeat, when it is installed, reads the controller's status ConfigMap too and republishes it in its gist as `fleet-role-status.json`. diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index dd90f738..9dce5af0 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -167,7 +167,7 @@ github_runner_arc_platform_priority_value: 1000000 github_runner_arc_platform_ephemeral_storage_request: "64Mi" github_runner_arc_platform_ephemeral_storage_limit: "128Mi" -# Usage-driven autoscaler for the one scale-set profile that sets autoscale: true (see scripts/autoscaler.sh and the README's Autoscaler section). It is installed only when such a profile exists and this host installs the platform. The memory budget and floor have no defaults because they depend on the pool: the budget is the memory runner pods may use across every node, and the floor must equal the maxRunners in that profile's values, which every Helm upgrade reverts to. The ceiling on the pooled maxRunners is worked out from the nodes every poll; github_runner_arc_autoscaler_max_ceiling, which has no default, only bounds it (the autoscaled profile's sizing, when set, supplies the bound instead). +# Usage-driven autoscaler for the one scale-set profile that sets autoscale: true (see scripts/autoscaler.sh and docs/autoscaler.md). It is installed only when such a profile exists and this host installs the platform. The memory budget and floor have no defaults because they depend on the pool: the budget is the memory runner pods may use across every node, and the floor must equal the maxRunners in that profile's values, which every Helm upgrade reverts to. The ceiling on the pooled maxRunners is worked out from the nodes every poll; github_runner_arc_autoscaler_max_ceiling, which has no default, only bounds it (the autoscaled profile's sizing, when set, supplies the bound instead). github_runner_arc_autoscaler_dry_run: true github_runner_arc_autoscaler_poll_seconds: 45 github_runner_arc_autoscaler_raise_confirm_polls: 2 diff --git a/roles/github_runner_arc/tasks/install_scale_set_profile.yml b/roles/github_runner_arc/tasks/install_scale_set_profile.yml index 3ccb5df6..9d348628 100644 --- a/roles/github_runner_arc/tasks/install_scale_set_profile.yml +++ b/roles/github_runner_arc/tasks/install_scale_set_profile.yml @@ -129,7 +129,7 @@ environment: HELM_DIFF_USE_UPGRADE_DRY_RUN: "true" -# A Helm upgrade that changes the pod template renames the EphemeralRunnerSet the release owns (it carries a spec-hash suffix). The AutoscalingListener should follow the rename but can be left pointing at the deleted name, crash-looping on every patch while the scale set's queue goes unserviced; see the README's Non-obvious behaviour section. Deleting the stale listener makes the controller recreate it against the EphemeralRunnerSet that exists now. +# A Helm upgrade that changes the pod template renames the EphemeralRunnerSet the release owns (it carries a spec-hash suffix). The AutoscalingListener should follow the rename but can be left pointing at the deleted name, crash-looping on every patch while the scale set's queue goes unserviced; see docs/non-obvious-behaviour.md. Deleting the stale listener makes the controller recreate it against the EphemeralRunnerSet that exists now. - name: Grant the runner pods the permission to record job peaks for {{ profile.label }} ansible.builtin.include_tasks: install_job_peak_rbac.yml diff --git a/roles/github_runner_arc/tasks/validate.yml b/roles/github_runner_arc/tasks/validate.yml index 303e2a94..a523b196 100644 --- a/roles/github_runner_arc/tasks/validate.yml +++ b/roles/github_runner_arc/tasks/validate.yml @@ -244,7 +244,7 @@ and (github_runner_arc_ceiling | string is not match('^[0-9]+$') or (github_runner_arc_autoscaler_floor | int) > (github_runner_arc_ceiling | int))) -# The platform installs from whichever host sets github_runner_arc_heartbeat_gist_id, so the question is whether ANY host in the inventory sets one. scripts/heartbeat.sh checks HEARTBEAT_GIST_ID at container start, but that only shows up as a crash-looping pod, and with no gist id anywhere the platform is not installed at all. This creates the gist the same way the README's manual instructions do, then fails on purpose: its id has to be recorded by hand in a host's host_vars (and in any consumer named in github_runner_arc_heartbeat_gist_consumer), and carrying on silently would leave that undone. +# The platform installs from whichever host sets github_runner_arc_heartbeat_gist_id, so the question is whether ANY host in the inventory sets one. scripts/heartbeat.sh checks HEARTBEAT_GIST_ID at container start, but that only shows up as a crash-looping pod, and with no gist id anywhere the platform is not installed at all. This creates the gist the same way the manual instructions in docs/heartbeat.md do, then fails on purpose: its id has to be recorded by hand in a host's host_vars (and in any consumer named in github_runner_arc_heartbeat_gist_consumer), and carrying on silently would leave that undone. - name: Bootstrap the heartbeat gist if no host in the fleet has one yet run_once: true when: diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 72a7e350..f4baba59 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -185,7 +185,7 @@ A host that got the agent from release 2.19.0 moves over on its next run: the ro **End checks.** On every macOS host the role checks, before anything reads the daemon and again after it has started the node's Compose project, that the runtime answers (`docker info` through the host's Docker context), and after starting the project that the node's k3s container (`github-runner-k3s`, or `github-runner-k3s-agent` on an agent) is running. A converge of a host that rebooted therefore repairs a stopped Colima or fails with a message that says what to do, and never reports success over a node that is not up. A `--check` run does not fail on a stopped runtime or container it has not started. -**Colima kernel.** On every Colima host, whether or not `github_runner_cluster_runtime_autostart` is on, once the VM answers (`tasks/colima_kernel.yml`), the role makes the VM's kernel pass bridged pod traffic through iptables, now and at every VM boot: it writes `/etc/modules-load.d/br_netfilter.conf` and `/etc/sysctl.d/90-github-runner-br-netfilter.conf` in the VM, runs `modprobe br_netfilter` when the VM has no `/proc/sys/net/bridge`, and sets `net.bridge.bridge-nf-call-iptables` and `net.bridge.bridge-nf-call-ip6tables` to 1 when they differ. Without the module a pod that talks to a Service whose only backend pod is on the same node gets no reply, because the Service's reverse NAT never applies to the bridged hairpin traffic; pods on other nodes keep working, so it stays latent until a single-backend Service such as CoreDNS lands on the node (the `br_netfilter` note under Non-obvious behaviour in the repository README has the symptoms). Everything runs in the VM through `colima ssh --profile `, as the VM's user with `sudo`, and no `become` on the host. A file is written only when its content differs, so a rerun changes nothing, and the module file takes the name the README's note told people to create by hand, so an existing one is replaced by the managed one. Under `--check` each change is reported and none is made, and nothing is read from a VM that a check run would only have started. Docker Desktop's VM is not reachable by the role and other runtimes are not touched. +**Colima kernel.** On every Colima host, whether or not `github_runner_cluster_runtime_autostart` is on, once the VM answers (`tasks/colima_kernel.yml`), the role makes the VM's kernel pass bridged pod traffic through iptables, now and at every VM boot: it writes `/etc/modules-load.d/br_netfilter.conf` and `/etc/sysctl.d/90-github-runner-br-netfilter.conf` in the VM, runs `modprobe br_netfilter` when the VM has no `/proc/sys/net/bridge`, and sets `net.bridge.bridge-nf-call-iptables` and `net.bridge.bridge-nf-call-ip6tables` to 1 when they differ. Without the module a pod that talks to a Service whose only backend pod is on the same node gets no reply, because the Service's reverse NAT never applies to the bridged hairpin traffic; pods on other nodes keep working, so it stays latent until a single-backend Service such as CoreDNS lands on the node (the `br_netfilter` note in `docs/non-obvious-behaviour.md` has the symptoms). Everything runs in the VM through `colima ssh --profile `, as the VM's user with `sudo`, and no `become` on the host. A file is written only when its content differs, so a rerun changes nothing, and the module file takes the name the br_netfilter note in `docs/non-obvious-behaviour.md` told people to create by hand, so an existing one is replaced by the managed one. Under `--check` each change is reported and none is made, and nothing is read from a VM that a check run would only have started. Docker Desktop's VM is not reachable by the role and other runtimes are not touched. **Colima route guard.** On every Colima host, once the VM answers (`tasks/colima_route_guard.yml`), the role installs `files/route-guard.sh` in the VM as `github_runner_cluster_colima_route_guard_script` with a systemd service and timer of the same name (`github_runner_cluster_colima_route_guard_unit`) that run it at every VM boot and every `github_runner_cluster_colima_route_guard_interval_seconds` seconds (30 by default). A bridged Colima interface that loses its link keeps its default routes, and when they carry a lower metric than the working interface's, the VM's outbound traffic leaves by the dead one, so the node loses its peers and, with enough nodes cut off, the cluster loses its datastore quorum. The guard removes a default route whose gateway answers neither an ICMP echo through its own interface nor ARP (a resolved neighbour entry), and only while another default route's gateway answers, so it never leaves the VM with none and never judges a route that has no gateway. A DHCP renewal may add a removed route back, and the next tick removes it again. The files are written only when their content differs, and under `--check` each change is reported and none is made. @@ -322,7 +322,7 @@ The decision logic lives in `files/regrade/fleet-regrade-core.sh` and calls only ## Role controller -`github_runner_cluster_role_controller: true` turns on the ARC role's role controller, which measures which hosts are fit to hold an etcd vote, reports the swap of an unfit voter for a fit standby that it would make and carries a swap out when asked (see the ARC role's README, Role controller, and the repository README's Voter swap). Its dry run is on unless the ARC role's `github_runner_arc_role_controller_dry_run` says otherwise. While it holds the `fleet-regrade` Lease for a swap, every run of this role refuses to change a node. On a native server or eligible native host the role also runs the role actuator that a swap switches the host through ([Role actuator](#role-actuator)). It is off by default. This role holds the controller's inventory: +`github_runner_cluster_role_controller: true` turns on the ARC role's role controller, which measures which hosts are fit to hold an etcd vote, reports the swap of an unfit voter for a fit standby that it would make and carries a swap out when asked (see the ARC role's README, Role controller, and `docs/role-controller.md`, Voter swap). Its dry run is on unless the ARC role's `github_runner_arc_role_controller_dry_run` says otherwise. While it holds the `fleet-regrade` Lease for a swap, every run of this role refuses to change a node. On a native server or eligible native host the role also runs the role actuator that a swap switches the host through ([Role actuator](#role-actuator)). It is off by default. This role holds the controller's inventory: - `github_runner_cluster_voter_eligible` (per host, default false) marks a host the controller may promote to a voter or demote from one. The role prepares no host for promotion by marking it, so mark only a host that is, or has been staged as, a server. - `github_runner_cluster_voter_pinned` (per host, default false) marks a voter the controller must never demote. Set each pin in the inventory with a comment saying why. diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index d81aeecc..7c6e71ce 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -264,7 +264,7 @@ github_runner_cluster_colima_kernel_module: br_netfilter github_runner_cluster_colima_sysctl_settings: net.bridge.bridge-nf-call-iptables: 1 net.bridge.bridge-nf-call-ip6tables: 1 -# Not settings: the directory that exists in the VM exactly while the module is loaded (the bridge sysctls live under it), and the files in the VM that load the module and set the sysctls at boot. The module file keeps the name the repository README's note has long told people to create by hand, so a host that has one is adopted, not duplicated. +# Not settings: the directory that exists in the VM exactly while the module is loaded (the bridge sysctls live under it), and the files in the VM that load the module and set the sysctls at boot. The module file keeps the name the note in docs/non-obvious-behaviour.md has long told people to create by hand, so a host that has one is adopted, not duplicated. github_runner_cluster_colima_bridge_proc_dir: /proc/sys/net/bridge github_runner_cluster_colima_modules_load_file: /etc/modules-load.d/br_netfilter.conf github_runner_cluster_colima_sysctl_file: /etc/sysctl.d/90-github-runner-br-netfilter.conf diff --git a/roles/github_runner_cluster/files/docker-compose.yml b/roles/github_runner_cluster/files/docker-compose.yml index becc6d07..02e515c1 100644 --- a/roles/github_runner_cluster/files/docker-compose.yml +++ b/roles/github_runner_cluster/files/docker-compose.yml @@ -4,7 +4,7 @@ # # Two roles, selected via Compose profiles rather than separate compose files: # - `--profile server`: a control-plane/etcd member (`k3s` service, running -# `k3s server`). Every server host runs the SAME service - whether it bootstraps the cluster (`--cluster-init`) or joins an existing one (`--server `) is decided at runtime by whether it has a server to join (K3S_JOIN_SERVER_URL, or the role directory's join-url that docker-compose.fleet-role.yml mounts), not by a separate service/profile. See the project README's Architecture section for why 3 servers is the right minimum (etcd quorum math: 2 is worse than 1, not better) and why every server needs every server's own hostname on every server's own --tls-san list, not just its own - see K3S_TLS_SAN_LIST below. +# `k3s server`). Every server host runs the SAME service - whether it bootstraps the cluster (`--cluster-init`) or joins an existing one (`--server `) is decided at runtime by whether it has a server to join (K3S_JOIN_SERVER_URL, or the role directory's join-url that docker-compose.fleet-role.yml mounts), not by a separate service/profile. See docs/architecture.md for why 3 servers is the right minimum (etcd quorum math: 2 is worse than 1, not better) and why every server needs every server's own hostname on every server's own --tls-san list, not just its own - see K3S_TLS_SAN_LIST below. # - `--profile agent`: a pure worker (`k3s-agent` service, running # `k3s agent`) - schedulable capacity with no control-plane role at all. # diff --git a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh index 3836c3cf..3893ad9a 100755 --- a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh +++ b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh @@ -128,7 +128,7 @@ ip -6 route replace unreachable fd7a:115c:a1e0::/48 # Only a self-hosted bootstrap server may start without a join key: its first start is what brings up the Headscale server that issues it. if [ -z "$join_key" ] && [ -z "$self_hosted_health_url" ]; then - echo "K3S_VPN_AUTH_JOIN_KEY must be set in .env: a reusable, pre-authorised join key for the mesh (see the README's Tailscale section for how it's provisioned)." >&2 + echo "K3S_VPN_AUTH_JOIN_KEY must be set in .env: a reusable, pre-authorised join key for the mesh (see the Tailscale section of docs/architecture.md for how it's provisioned)." >&2 exit 1 fi diff --git a/roles/github_runner_cluster/tasks/colima_kernel.yml b/roles/github_runner_cluster/tasks/colima_kernel.yml index 5279a80a..d2c60238 100644 --- a/roles/github_runner_cluster/tasks/colima_kernel.yml +++ b/roles/github_runner_cluster/tasks/colima_kernel.yml @@ -1,5 +1,5 @@ --- -# Colima hosts only (runtime.yml includes it once the runtime answers). Makes the Linux VM the node runs in load br_netfilter and set the bridge sysctls, now and at every VM boot: a modules-load.d file and a sysctl.d file in the VM for boot, and modprobe and sysctl -w for now. Without the module, bridged pod traffic skips iptables, so a pod that talks to a Service whose only backend pod is on the same node gets no reply (the Service's reverse NAT never applies); pods on other nodes keep working, which hides it until a single-backend Service such as CoreDNS is rescheduled onto this node. See the repository README's note on br_netfilter. Idempotent: a file is written only when its content differs, the module is loaded only when the VM has no bridge sysctl directory, and a sysctl is set only when its value differs. Under --check each change is reported and none is made. Every command runs in the VM through colima ssh, as the VM's user with sudo, and needs no become on the host. +# Colima hosts only (runtime.yml includes it once the runtime answers). Makes the Linux VM the node runs in load br_netfilter and set the bridge sysctls, now and at every VM boot: a modules-load.d file and a sysctl.d file in the VM for boot, and modprobe and sysctl -w for now. Without the module, bridged pod traffic skips iptables, so a pod that talks to a Service whose only backend pod is on the same node gets no reply (the Service's reverse NAT never applies); pods on other nodes keep working, which hides it until a single-backend Service such as CoreDNS is rescheduled onto this node. See the br_netfilter note in docs/non-obvious-behaviour.md. Idempotent: a file is written only when its content differs, the module is loaded only when the VM has no bridge sysctl directory, and a sysctl is set only when its value differs. Under --check each change is reported and none is made. Every command runs in the VM through colima ssh, as the VM's user with sudo, and needs no become on the host. - name: Work out the files the VM keeps for the module and the sysctls ansible.builtin.set_fact: diff --git a/scripts/autoscaler.sh b/scripts/autoscaler.sh index a04cc3f8..1f2b49ae 100755 --- a/scripts/autoscaler.sh +++ b/scripts/autoscaler.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Usage-driven maxRunners autoscaler. Runs inside the `autoscaler` in-cluster Deployment (see autoscaler/loop.sh for the poll loop). Never inspects job identity or the GitHub Actions queue: it only watches real, current memory usage and pressure, and adjusts spec.maxRunners across every AutoscalingRunnerSet named in AUTOSCALER_TARGETS, pooling them against one combined memory budget rather than budgeting each separately. See the project README's Autoscaler section for the full algorithm and rationale. +# Usage-driven maxRunners autoscaler. Runs inside the `autoscaler` in-cluster Deployment (see autoscaler/loop.sh for the poll loop). Never inspects job identity or the GitHub Actions queue: it only watches real, current memory usage and pressure, and adjusts spec.maxRunners across every AutoscalingRunnerSet named in AUTOSCALER_TARGETS, pooling them against one combined memory budget rather than budgeting each separately. See docs/autoscaler.md for the full algorithm and rationale. # # Environment (provided by the Deployment - see roles/github_runner_arc/tasks/install_platform.yml): # - kubectl needs no KUBECONFIG here: running as a pod with a mounted ServiceAccount token, client-go auto-detects in-cluster config. diff --git a/tests/unit/test_runtime_autostart.py b/tests/unit/test_runtime_autostart.py index be6e5825..0b21059c 100644 --- a/tests/unit/test_runtime_autostart.py +++ b/tests/unit/test_runtime_autostart.py @@ -108,7 +108,7 @@ def test_the_readmes_say_what_the_role_does_to_the_vm(self) -> None: self.assertIn("**Colima kernel.**", readme) self.assertIn(MODULES_FILE, readme) self.assertIn(SYSCTL_FILE, readme) - self.assertIn("colima_kernel.yml", (ROOT / "README.md").read_text()) + self.assertIn("colima_kernel.yml", (ROOT / "docs" / "non-obvious-behaviour.md").read_text()) def test_both_includes_run_on_macos_only(self) -> None: tasks = yaml.safe_load((ROLE / "tasks" / "main.yml").read_text())