From d3459a304198c7a8371a30f336ff08c55db2e794 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 14 Nov 2026 15:07:54 +0000 Subject: [PATCH 1/7] fix(github_runner_cluster): stop a wedged Colima profile before starting it A Colima VM that Lima's VZ driver has put into an error state is still listed as running, with its host agent alive and its Docker socket dead. The role's start found the profile running and did nothing, so the runtime tasks failed and nothing in the role could recover the host. When the runtime does not answer and `colima list` shows the profile as running, the role now ends it with `colima stop --force` and starts it again from the same disk. Nothing in the VM's disk, containers or volumes is removed. A profile that is listed as stopped, one whose runtime answers and a run under --check are left as before. --- roles/github_runner_cluster/README.md | 2 +- roles/github_runner_cluster/tasks/runtime.yml | 20 ++++++ tests/unit/test_runtime_autostart.py | 66 ++++++++++++++++++- 3 files changed, 85 insertions(+), 3 deletions(-) diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index f4baba59..3cafd066 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -166,7 +166,7 @@ On a Mac the container runtime is a Linux VM (Colima, or Docker Desktop) that st Which runtime a Mac runs is taken from the Docker context the role drives (see [Docker contexts](#docker-contexts)): `colima`, or `colima-` for another profile, is Colima, and `desktop-linux` is Docker Desktop. `github_runner_cluster_runtime` says so explicitly for a host whose context does not tell, for example one that uses `DOCKER_HOST`, and `github_runner_cluster_colima_profile` names the profile to start when the context does not. -**Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role starts the profile if it is stopped, with `colima start --profile --activate=false`, so the profile does not make its context the host's current one, and installs the launchd job that starts it again after a reboot. There are two kinds, chosen by `github_runner_cluster_runtime_autostart_at` (`login`, `boot` or `auto`, the default). `auto` is `boot` for the profile named by `github_runner_cluster_runtime_boot_profile` (default `node`, the profile the bridged k3s node runs in) and `login` for any other, such as a `default` profile a host keeps for other workloads. A profile has one kind at a time: installing either retires the other's property list, unloading it first, so the profile is not started twice. +**Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role starts the profile if it is stopped, with `colima start --profile --activate=false`, so the profile does not make its context the host's current one. A profile that `colima list` shows as running while its Docker runtime does not answer is a wedged VM (Lima's VZ driver has put it in an error state, and a start would find it running and do nothing), so the role first ends it with `colima stop --profile --force`, which removes no disk, container or volume, and then starts it from the same disk. It installs the launchd job that starts it again after a reboot. There are two kinds, chosen by `github_runner_cluster_runtime_autostart_at` (`login`, `boot` or `auto`, the default). `auto` is `boot` for the profile named by `github_runner_cluster_runtime_boot_profile` (default `node`, the profile the bridged k3s node runs in) and `login` for any other, such as a `default` profile a host keeps for other workloads. A profile has one kind at a time: installing either retires the other's property list, unloading it first, so the profile is not started twice. **At login (an agent).** The role: diff --git a/roles/github_runner_cluster/tasks/runtime.yml b/roles/github_runner_cluster/tasks/runtime.yml index 9ef9c027..a7715564 100644 --- a/roles/github_runner_cluster/tasks/runtime.yml +++ b/roles/github_runner_cluster/tasks/runtime.yml @@ -72,6 +72,26 @@ - github_runner_cluster_effective_runtime.kind == 'colima' - github_runner_cluster_runtime_autostart | bool block: + - name: Read whether Colima lists the profile as running + # colima list prints one JSON object per profile, whatever profile is asked for. Only read when the runtime did not answer. + ansible.builtin.command: + argv: [colima, list, --json] + environment: "{{ github_runner_cluster_runtime_environment }}" + register: github_runner_cluster_runtime_colima_list + changed_when: false + check_mode: false + when: not github_runner_cluster_runtime_up | bool + + - name: Stop a profile Colima lists as running while its runtime does not answer + # A VM that has gone into an error state (Lima's VZ driver reports it as "error", and the lima host agent logs `Invalid virtual machine state`) is still listed as running with its host agent alive, so the start below would find the profile running and do nothing, and its Docker socket never answers again. A forced stop (colima stop --force) ends the VM and its host agent, and the start then boots it from the same disk: nothing in the VM's disk, its containers or its volumes is removed. + ansible.builtin.command: + argv: [colima, stop, --profile, "{{ github_runner_cluster_effective_runtime.colima_profile }}", --force] + environment: "{{ github_runner_cluster_runtime_environment }}" + changed_when: true + when: + - not github_runner_cluster_runtime_up | bool + - github_runner_cluster_runtime_colima_list.stdout_lines | map('from_json') | selectattr('name', 'equalto', github_runner_cluster_effective_runtime.colima_profile) | selectattr('status', 'equalto', 'Running') | list | length > 0 + - name: Start the Colima profile if it is stopped # --activate=false keeps the profile from making its Docker context the host's current one, which the role promises never to change (see Docker contexts in the README). Starting a profile that already runs would do nothing, but this only runs when the runtime did not answer. Skipped under --check, which reports the start as skipped, not as a failure. ansible.builtin.command: diff --git a/tests/unit/test_runtime_autostart.py b/tests/unit/test_runtime_autostart.py index 0b21059c..99475c7e 100644 --- a/tests/unit/test_runtime_autostart.py +++ b/tests/unit/test_runtime_autostart.py @@ -380,12 +380,26 @@ def sysctl_file(key): print(content, end="") else: sys.exit("unexpected colima ssh call: " + " ".join(command)) +elif name == "colima" and args[0] == "list": + # One JSON object per profile, whatever profile is asked for: this host has the profile the test names and another one beside it, which is always running. + profile = os.environ.get("RUNTIME_PROFILE", "default") + print('{{"name":"other","status":"Running","runtime":"docker"}}') + print('{{"name":"' + profile + '","status":"' + ("Running" if exists("colima-listed-running") else "Stopped") + '","runtime":"docker"}}') +elif name == "colima" and args[0] == "stop": + if args != ["stop", "--profile", os.environ.get("RUNTIME_PROFILE", "default"), "--force"]: + sys.exit("unexpected colima stop call: " + " ".join(args)) + for file in ("colima-listed-running", "runtime-up"): + if exists(file): + os.remove(os.path.join(stub, file)) elif name == "colima": if args[0] != "start" or args[-1] != "--activate=false": sys.exit("unexpected colima call: " + " ".join(args)) if exists("colima-start-fails"): sys.exit("colima: the VM did not start") - touch("runtime-up") + # Starting a profile Colima already lists as running does nothing, so a wedged VM stays wedged. + if not exists("colima-listed-running"): + touch("colima-listed-running") + touch("runtime-up") elif name == "launchctl": system = args[1] == "system" or args[1].startswith("system/") if args[0] == "print": @@ -471,7 +485,7 @@ def run_role(self, *extra: str, tasks: str = "runtime,end_checks", context: str inventory = self.work / "inventory.yml" inventory.write_text(yaml.safe_dump({"all": {"hosts": {"node1": host}}})) # The stand-ins come first on the playbook's own PATH as well as the tasks', so a tool a task runs without the role's PATH still cannot reach this machine's real launchd, Colima or Docker. - env = ansible_env() | {"RUNTIME_STUB": str(self.stub), "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}"} + env = ansible_env() | {"RUNTIME_STUB": str(self.stub), "RUNTIME_PROFILE": profile, "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}"} return subprocess.run([ANSIBLE_PLAYBOOK, "-i", str(inventory), str(RUN), *extra], capture_output=True, text=True, check=False, env=env, cwd=self.work) def calls(self) -> list[str]: @@ -524,6 +538,54 @@ def test_a_stopped_colima_is_started_and_the_agent_installed_and_loaded(self) -> self.assertEqual(self.plist.stat().st_mode & 0o777, 0o644) self.assertTrue((self.stub / "loaded").exists()) + def stops(self) -> list[str]: + return [call for call in self.calls() if re.match(r"\S+ colima stop", call)] + + def test_a_profile_listed_as_running_whose_runtime_does_not_answer_is_stopped_and_started_again(self) -> None: + # A wedged VM: the profile is listed as running, the Docker socket does not answer and a start would do nothing. + self.state("session", "colima-listed-running") + result = self.run_role() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), ["colima colima stop --profile default --force"]) + starts = [call for call in self.calls() if re.match(r"\S+ colima start", call)] + self.assertEqual(starts, ["colima colima start --profile default --activate=false"]) + self.assertLess(self.calls().index(self.stops()[0]), self.calls().index(starts[0])) + self.assertTrue((self.stub / "runtime-up").exists()) + + def test_a_second_run_after_the_recovery_stops_and_starts_nothing(self) -> None: + self.state("session", "colima-listed-running") + self.run_role() + before = len(self.calls()) + result = self.run_role() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual([call for call in self.calls()[before:] if re.match(r"\S+ colima (stop|start)", call)], []) + + def test_a_stopped_profile_is_started_without_being_stopped_first(self) -> None: + self.state("session") + result = self.run_role() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), []) + + def test_a_running_runtime_is_neither_listed_nor_stopped(self) -> None: + self.state("session", "runtime-up", "colima-listed-running") + result = self.run_role() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), []) + self.assertEqual([call for call in self.calls() if re.match(r"\S+ colima list", call)], []) + + def test_only_the_hosts_own_profile_is_stopped_while_another_listed_profile_runs(self) -> None: + self.state("session") + result = self.run_role(context="colima-example", context_host=f"unix://{self.home}/.colima-example/example/docker.sock") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), []) + + def test_check_mode_stops_nothing_in_a_wedged_profile(self) -> None: + self.state("session", "colima-listed-running") + result = self.run_role("--check") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), []) + self.assertTrue((self.stub / "colima-listed-running").exists()) + def test_every_docker_call_uses_the_hosts_docker_context(self) -> None: self.state("session") self.run_role(context="colima-example") From 4558a29fce66bd418089c40ec731a6a983a4f52e Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 14 Nov 2026 15:40:14 +0000 Subject: [PATCH 2/7] fix(github_runner_cluster): confirm a wedged Colima profile before stopping it A single failed docker info is no longer enough to stop a VM. A profile that Colima lists as running while its runtime is silent is stopped only when docker info keeps failing on repeated bounded tries, a command in the guest fails, the VM's host agent is older than a boot grace period and the Docker context points at that profile's own socket. The stop is colima stop first and colima stop --force only if that fails. The stop has its own switch, github_runner_cluster_runtime_force_stop_wedged, separate from autostart. Switched off, or when a condition does not hold, the run fails with the signals it saw and the commands to recover by hand. Under --check every read is made and nothing is stopped. --- roles/github_runner_cluster/README.md | 3 +- roles/github_runner_cluster/defaults/main.yml | 11 ++ roles/github_runner_cluster/tasks/runtime.yml | 24 +-- .../tasks/runtime_wedged.yml | 168 ++++++++++++++++++ tests/unit/test_runtime_autostart.py | 140 +++++++++++++-- 5 files changed, 306 insertions(+), 40 deletions(-) create mode 100644 roles/github_runner_cluster/tasks/runtime_wedged.yml diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 3cafd066..423dd740 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -166,7 +166,7 @@ On a Mac the container runtime is a Linux VM (Colima, or Docker Desktop) that st Which runtime a Mac runs is taken from the Docker context the role drives (see [Docker contexts](#docker-contexts)): `colima`, or `colima-` for another profile, is Colima, and `desktop-linux` is Docker Desktop. `github_runner_cluster_runtime` says so explicitly for a host whose context does not tell, for example one that uses `DOCKER_HOST`, and `github_runner_cluster_colima_profile` names the profile to start when the context does not. -**Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role starts the profile if it is stopped, with `colima start --profile --activate=false`, so the profile does not make its context the host's current one. A profile that `colima list` shows as running while its Docker runtime does not answer is a wedged VM (Lima's VZ driver has put it in an error state, and a start would find it running and do nothing), so the role first ends it with `colima stop --profile --force`, which removes no disk, container or volume, and then starts it from the same disk. It installs the launchd job that starts it again after a reboot. There are two kinds, chosen by `github_runner_cluster_runtime_autostart_at` (`login`, `boot` or `auto`, the default). `auto` is `boot` for the profile named by `github_runner_cluster_runtime_boot_profile` (default `node`, the profile the bridged k3s node runs in) and `login` for any other, such as a `default` profile a host keeps for other workloads. A profile has one kind at a time: installing either retires the other's property list, unloading it first, so the profile is not started twice. +**Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role starts the profile if it is stopped, with `colima start --profile --activate=false`, so the profile does not make its context the host's current one. A profile that `colima list` shows as running while its Docker runtime does not answer is not started again, because a start finds it running and does nothing. The role stops it, with `colima stop` and then `colima stop --force` only if that fails, and starts it again from the same disk, but only when `github_runner_cluster_runtime_force_stop_wedged` is on (the default) and all of these hold: `docker info` fails on `github_runner_cluster_runtime_probe_attempts` further tries (default 5), each given `github_runner_cluster_runtime_probe_timeout` seconds (default 20) and `github_runner_cluster_runtime_probe_delay` seconds apart (default 10); a command in the guest (`colima ssh -- true`) fails within the same timeout; the VM's host agent was started more than `github_runner_cluster_runtime_boot_grace` seconds ago (default 600), read from the modification time of its pid file in the profile's Lima instance directory under the Colima home; and the Docker context the probes used points at that profile's own socket. A host that is only loaded, a Docker daemon that is restarting, a VM that is still booting and a context that points at another runtime each fail one of these, so nothing is stopped. A stop does not shut the guest down gracefully when the clean stop fails or times out (`github_runner_cluster_runtime_clean_stop_timeout`, default 120 seconds): every process in the guest, k3s and etcd included, ends, and the VM's disk, containers and volumes are kept, so on an etcd voter the member is out until the start completes. Switched off, or when a condition does not hold, the run fails with the signals it saw and the commands to recover by hand. Under `--check` the reads are made, nothing is stopped and the report says what a real run would do. It installs the launchd job that starts it again after a reboot. There are two kinds, chosen by `github_runner_cluster_runtime_autostart_at` (`login`, `boot` or `auto`, the default). `auto` is `boot` for the profile named by `github_runner_cluster_runtime_boot_profile` (default `node`, the profile the bridged k3s node runs in) and `login` for any other, such as a `default` profile a host keeps for other workloads. A profile has one kind at a time: installing either retires the other's property list, unloading it first, so the profile is not started twice. **At login (an agent).** The role: @@ -491,6 +491,7 @@ The role controller holds the regrade Lease (`kube-system/fleet-regrade`, the Le - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)). - `github_runner_cluster_network_mode` (default `bridge`, or `host`): the network the node's container runs on, and with `host`, `github_runner_cluster_lan_cidr` (required: the LAN the nodes share) and `github_runner_cluster_network_probe_image` (default `busybox:1.37`, for reading the host's addresses) (see [Networking](#networking)). - `github_runner_cluster_runtime_autostart_at` (default `auto`; `login` or `boot`) and `github_runner_cluster_runtime_boot_profile` (default `node`): whether the Colima profile is started by a launchd agent at login or by a launchd daemon at boot, and which profile `auto` starts at boot (see [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). +- `github_runner_cluster_runtime_force_stop_wedged` (default true), `github_runner_cluster_runtime_probe_attempts` (default 5), `github_runner_cluster_runtime_probe_delay` (default 10), `github_runner_cluster_runtime_probe_timeout` (default 20), `github_runner_cluster_runtime_boot_grace` (default 600) and `github_runner_cluster_runtime_clean_stop_timeout` (default 120): when a Colima profile listed as running with a silent runtime is stopped, see the Colima paragraph above. - `github_runner_cluster_runtime_autostart` (default true), `github_runner_cluster_runtime` (default `auto`; `colima`, `docker_desktop` or `other`), `github_runner_cluster_colima_profile` (default empty, taken from the Docker context), `github_runner_cluster_colima_home` (default empty: the `COLIMA_HOME` is read from the endpoint of the profile's Docker context, and is unset for Colima's own `~/.colima`; set it only for a context that does not say, and the role passes it to colima and to the launchd agent), `github_runner_cluster_runtime_path` (default both Homebrew prefixes and the system directories): on a macOS host, start Colima at login through a launchd agent and check the runtime (see [Container runtime at login](#container-runtime-after-a-reboot-macos)). - `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus` (default empty, keeping the profile's own): a Colima host's VM memory in GiB and its CPUs, each a positive whole number, applied one node at a time through the quorum guard when the VM differs (see Colima VM size under [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). `github_runner_cluster_colima_memory_reserved_gib` (default empty) derives the memory instead, or bounds one set explicitly, from the host's memory less it and `github_runner_cluster_colima_memory_host_reserve_gib` (default 10, macOS's own), never below `github_runner_cluster_colima_guest_baseline_gib` (default 3.4) plus `github_runner_cluster_colima_runner_memory_limit_gib` (default empty, required by a derivation) rounded up. `github_runner_cluster_colima_resize_lease_seconds`, `github_runner_cluster_colima_resize_lease_holder`, `github_runner_cluster_regrade_lease_name` and `github_runner_cluster_regrade_lease_namespace` are derived, never set directly. `github_runner_cluster_native_runner_wait_timeout` and `github_runner_cluster_native_ready_timeout` bound the wait for the node's runner pods and for the node to come back, as they do for a native node. - `github_runner_cluster_container_name` (default `github-runner-k3s`) and `github_runner_cluster_agent_container_name` (default `github-runner-k3s-agent`): the names of the node's server and agent containers, which a second cluster on the same Docker hosts sets to names of its own (see [Several clusters on one host](#several-clusters-on-one-host)). diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index 4ac8fd81..57e1172b 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -237,6 +237,17 @@ github_runner_cluster_server_ready_timeout: 600 github_runner_cluster_runtime_autostart: true # When the role starts a Colima profile: login installs a per-user launchd agent (a property list in the connecting user's ~/Library/LaunchAgents, loaded into the user's login session), which runs only once that user has logged in; boot installs a launchd daemon (/Library/LaunchDaemons, owned by root, loaded into the system domain, run as the connecting user through UserName), which runs when the Mac starts, with nobody logged in, so a node recovers from a power loss or a reboot at the login window; auto is boot for the profile named by github_runner_cluster_runtime_boot_profile and login for any other. Only one of the two exists for a profile at a time: installing either removes the other's property list, and unloads it, so the profile is not started twice. A daemon takes root (become) to write and to load, which the agent never does. github_runner_cluster_runtime_autostart_at: auto +# Whether the role may stop a Colima profile that Colima lists as running while its container runtime does not answer, so that the start after it can boot it again. It does so only when tasks/runtime_wedged.yml has confirmed the failure (repeated probes, a failing guest command, a host agent older than the boot grace and a Docker context that points at that profile's socket). Off, such a profile fails the run with what was seen and the commands to recover by hand. Separate from github_runner_cluster_runtime_autostart, which starts a stopped profile and installs the launchd job. +github_runner_cluster_runtime_force_stop_wedged: true +# How many times docker info is tried, and the seconds between tries, before a profile listed as running with a silent runtime is looked into further. Together with the timeout they set the window in which a loaded host or a restarting Docker daemon can still answer. +github_runner_cluster_runtime_probe_attempts: 5 +github_runner_cluster_runtime_probe_delay: 10 +# The seconds each docker info, and the command run in the guest, is given before it counts as failed. +github_runner_cluster_runtime_probe_timeout: 20 +# The seconds a VM's host agent must have been running before a silent runtime counts as wedged rather than booting. +github_runner_cluster_runtime_boot_grace: 600 +# The seconds colima stop is given before colima stop --force follows it. +github_runner_cluster_runtime_clean_stop_timeout: 120 # The Colima profile that runs this collection's k3s node, which must come back with no one logged in: the one github_runner_cluster_runtime_autostart_at auto starts at boot. Any other profile on the host (the mini's default profile for its own workloads, say) keeps the login agent. github_runner_cluster_runtime_boot_profile: node # Which container runtime a macOS host runs: auto takes it from the Docker context the role drives (colima or colima- is Colima, desktop-linux is Docker Desktop, anything else is other), colima, docker_desktop and other say so for a host whose context does not tell (DOCKER_HOST set, a context with another name). diff --git a/roles/github_runner_cluster/tasks/runtime.yml b/roles/github_runner_cluster/tasks/runtime.yml index a7715564..0825d973 100644 --- a/roles/github_runner_cluster/tasks/runtime.yml +++ b/roles/github_runner_cluster/tasks/runtime.yml @@ -31,7 +31,7 @@ | exadev.github_runner.cluster_container_runtime(github_runner_cluster_runtime, github_runner_cluster_colima_profile) }} - name: Read the Docker host of the Colima profile's context - # Where the profile lives: Colima points the context at //docker.sock. Needed only when github_runner_cluster_colima_home does not say. Read-only and answered by the context file alone, so it works with the VM stopped and under --check. It runs with the environment of every other docker call, so it reads the same context store. + # Where the profile lives: Colima points the context at //docker.sock. Needed for the Colima home when github_runner_cluster_colima_home does not say, and by runtime_wedged.yml to prove that a profile it would stop is the one the Docker probe measured. Read-only and answered by the context file alone, so it works with the VM stopped and under --check. It runs with the environment of every other docker call, so it reads the same context store. ansible.builtin.command: argv: [docker, context, inspect, --format, "{{ '{{' }}.Endpoints.docker.Host{{ '}}' }}", "{{ github_runner_cluster_docker_cli_context }}"] environment: "{{ github_runner_cluster_docker_cli_environment | combine({'PATH': github_runner_cluster_runtime_path}) }}" @@ -40,7 +40,6 @@ check_mode: false when: - github_runner_cluster_effective_runtime.kind == 'colima' - - github_runner_cluster_colima_home | length == 0 - name: Work out the Colima home the profile lives in # The variable when a host sets it, else the one its Docker context names, empty for Colima's own default. Nothing outside Colima has one. @@ -72,26 +71,11 @@ - github_runner_cluster_effective_runtime.kind == 'colima' - github_runner_cluster_runtime_autostart | bool block: - - name: Read whether Colima lists the profile as running - # colima list prints one JSON object per profile, whatever profile is asked for. Only read when the runtime did not answer. - ansible.builtin.command: - argv: [colima, list, --json] - environment: "{{ github_runner_cluster_runtime_environment }}" - register: github_runner_cluster_runtime_colima_list - changed_when: false - check_mode: false + - name: Deal with a profile Colima lists as running while its runtime does not answer + # A start would find such a profile running and do nothing. See runtime_wedged.yml for what is required before the VM is stopped. + ansible.builtin.include_tasks: runtime_wedged.yml when: not github_runner_cluster_runtime_up | bool - - name: Stop a profile Colima lists as running while its runtime does not answer - # A VM that has gone into an error state (Lima's VZ driver reports it as "error", and the lima host agent logs `Invalid virtual machine state`) is still listed as running with its host agent alive, so the start below would find the profile running and do nothing, and its Docker socket never answers again. A forced stop (colima stop --force) ends the VM and its host agent, and the start then boots it from the same disk: nothing in the VM's disk, its containers or its volumes is removed. - ansible.builtin.command: - argv: [colima, stop, --profile, "{{ github_runner_cluster_effective_runtime.colima_profile }}", --force] - environment: "{{ github_runner_cluster_runtime_environment }}" - changed_when: true - when: - - not github_runner_cluster_runtime_up | bool - - github_runner_cluster_runtime_colima_list.stdout_lines | map('from_json') | selectattr('name', 'equalto', github_runner_cluster_effective_runtime.colima_profile) | selectattr('status', 'equalto', 'Running') | list | length > 0 - - name: Start the Colima profile if it is stopped # --activate=false keeps the profile from making its Docker context the host's current one, which the role promises never to change (see Docker contexts in the README). Starting a profile that already runs would do nothing, but this only runs when the runtime did not answer. Skipped under --check, which reports the start as skipped, not as a failure. ansible.builtin.command: diff --git a/roles/github_runner_cluster/tasks/runtime_wedged.yml b/roles/github_runner_cluster/tasks/runtime_wedged.yml new file mode 100644 index 00000000..a637dc8f --- /dev/null +++ b/roles/github_runner_cluster/tasks/runtime_wedged.yml @@ -0,0 +1,168 @@ +--- +# Reached on a Colima host whose container runtime did not answer. A profile that Colima lists as stopped is only started (runtime.yml). A profile it lists as running is either booting, slow, or stuck, and a start does nothing for it, so the VM could stay down for ever. This file ends such a VM with colima stop, and only when every one of these holds: +# +# 1. github_runner_cluster_runtime_force_stop_wedged is on; +# 2. docker info still fails on every one of github_runner_cluster_runtime_probe_attempts further tries, each given github_runner_cluster_runtime_probe_timeout seconds and github_runner_cluster_runtime_probe_delay seconds apart, so a host that is only loaded or a dockerd that is restarting answers somewhere in that window and nothing is stopped; +# 3. a command run in the guest (colima ssh -- true) fails within github_runner_cluster_runtime_probe_timeout seconds, so a guest that works while its Docker daemon does not is left alone; +# 4. the VM's host agent was started more than github_runner_cluster_runtime_boot_grace seconds ago, read from the modification time of its pid file in the profile's Lima instance directory, so a VM that is still booting is left alone. An agent whose age cannot be read does not count as old; +# 5. the Docker context the probes used points at the socket of the profile that would be stopped (//docker.sock), so a failing probe of one runtime never stops another's VM. +# +# The stop is first a clean one (colima stop), given github_runner_cluster_runtime_clean_stop_timeout seconds, and a forced one (colima stop --force, which ends the VM without a graceful shutdown) only when that fails. Either way the VM's disk is kept and nothing is removed, so the start that follows boots it from the same disk. When any condition fails, or the switch is off, the run fails here and says which signals it saw and how to recover. Under --check every read is made and nothing is stopped; the report says what a real run would do. +# +# Needs github_runner_cluster_runtime_environment, github_runner_cluster_runtime_colima_home and github_runner_cluster_runtime_context_host from runtime.yml. + +- name: Read whether Colima lists the profile as running + # colima list prints one JSON object per profile, whatever profile is asked for. + ansible.builtin.command: + argv: [colima, list, --json] + environment: "{{ github_runner_cluster_runtime_environment }}" + register: github_runner_cluster_runtime_colima_list + changed_when: false + check_mode: false + +- name: Note whether the profile is listed as running + ansible.builtin.set_fact: + github_runner_cluster_runtime_listed_running: >- + {{ github_runner_cluster_runtime_colima_list.stdout_lines | map('from_json') + | selectattr('name', 'equalto', github_runner_cluster_effective_runtime.colima_profile) + | selectattr('status', 'equalto', 'Running') | list | length > 0 }} + +- name: Look into a profile that is listed as running + when: github_runner_cluster_runtime_listed_running | bool + block: + - name: Ask the Docker daemon again, a bounded number of times + # perl's alarm bounds each try: macOS has no timeout command, and docker info has no timeout of its own. + ansible.builtin.command: + argv: + - perl + - -e + - alarm shift; exec @ARGV + - "{{ github_runner_cluster_runtime_probe_timeout }}" + - docker + - info + - --format + - "{{ '{{' }}.ServerVersion{{ '}}' }}" + environment: "{{ github_runner_cluster_runtime_environment }}" + register: github_runner_cluster_runtime_reprobe + changed_when: false + failed_when: false + check_mode: false + until: github_runner_cluster_runtime_reprobe.rc == 0 + retries: "{{ github_runner_cluster_runtime_probe_attempts | int - 1 }}" + delay: "{{ github_runner_cluster_runtime_probe_delay }}" + + - name: Note that the runtime answered after all + ansible.builtin.set_fact: + github_runner_cluster_runtime_up: true + when: github_runner_cluster_runtime_reprobe.rc == 0 + + - name: Gather the other signals while the runtime stays silent + when: github_runner_cluster_runtime_reprobe.rc != 0 + block: + - name: Run a command in the guest + ansible.builtin.command: + argv: + - perl + - -e + - alarm shift; exec @ARGV + - "{{ github_runner_cluster_runtime_probe_timeout }}" + - colima + - ssh + - --profile + - "{{ github_runner_cluster_effective_runtime.colima_profile }}" + - -- + - "true" + environment: "{{ github_runner_cluster_runtime_environment }}" + register: github_runner_cluster_runtime_guest + changed_when: false + failed_when: false + check_mode: false + + - name: Find the VM's host agent pid file + # Colima keeps one Lima instance per profile under /_lima: colima for the default profile, colima- for another. + ansible.builtin.stat: + path: >- + {{ (github_runner_cluster_runtime_colima_home if github_runner_cluster_runtime_colima_home | length > 0 else ansible_facts['env']['HOME'] ~ '/.colima') + ~ '/_lima/' ~ ('colima' if github_runner_cluster_effective_runtime.colima_profile == 'default' else 'colima-' ~ github_runner_cluster_effective_runtime.colima_profile) + ~ '/ha.pid' }} + get_checksum: false + register: github_runner_cluster_runtime_agent_pid + + - name: Read the time on the host + ansible.builtin.command: + argv: [date, +%s] + register: github_runner_cluster_runtime_now + changed_when: false + check_mode: false + + - name: Work out what the signals say + ansible.builtin.set_fact: + github_runner_cluster_runtime_agent_age: >- + {{ (github_runner_cluster_runtime_now.stdout | trim | int - github_runner_cluster_runtime_agent_pid.stat.mtime | int) + if github_runner_cluster_runtime_agent_pid.stat.exists else -1 }} + github_runner_cluster_runtime_context_socket_ok: >- + {{ github_runner_cluster_runtime_context_host.stdout | trim + == 'unix://' ~ (github_runner_cluster_runtime_colima_home if github_runner_cluster_runtime_colima_home | length > 0 else ansible_facts['env']['HOME'] ~ '/.colima') + ~ '/' ~ github_runner_cluster_effective_runtime.colima_profile ~ '/docker.sock' }} + + - name: Decide whether the VM is wedged + ansible.builtin.set_fact: + github_runner_cluster_runtime_wedged: >- + {{ github_runner_cluster_runtime_guest.rc != 0 + and github_runner_cluster_runtime_agent_age | int > github_runner_cluster_runtime_boot_grace | int + and github_runner_cluster_runtime_context_socket_ok | bool }} + github_runner_cluster_runtime_wedged_report: >- + Colima lists profile {{ github_runner_cluster_effective_runtime.colima_profile }} as running, but docker info failed on {{ github_runner_cluster_runtime_probe_attempts }} tries + {{ github_runner_cluster_runtime_probe_delay }} seconds apart; a command in the guest {{ 'failed' if github_runner_cluster_runtime_guest.rc != 0 else 'succeeded' }}; + the VM's host agent was started {{ 'more than ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds ago' if github_runner_cluster_runtime_agent_age | int > github_runner_cluster_runtime_boot_grace | int + else ('within the last ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds' if github_runner_cluster_runtime_agent_age | int >= 0 else 'at a time that could not be read') }}; + the Docker context {{ 'points at' if github_runner_cluster_runtime_context_socket_ok | bool else 'does not point at' }} the profile's own socket. + + - name: Say what a real run would do + ansible.builtin.debug: + msg: >- + {{ github_runner_cluster_runtime_wedged_report }} + {{ 'A real run would stop the profile (colima stop, then colima stop --force if that fails) and start it again.' + if github_runner_cluster_runtime_wedged | bool and github_runner_cluster_runtime_force_stop_wedged | bool + else 'A real run would stop nothing and fail here.' }} + changed_when: github_runner_cluster_runtime_wedged | bool and github_runner_cluster_runtime_force_stop_wedged | bool + when: ansible_check_mode + + - name: Fail where the VM is not shown to be wedged or the stop is off + ansible.builtin.fail: + msg: >- + {{ github_runner_cluster_runtime_wedged_report }} + {{ 'Nothing was stopped, because ' ~ ('github_runner_cluster_runtime_force_stop_wedged is off.' + if github_runner_cluster_runtime_wedged | bool + else 'the signals do not show a wedged VM (a VM that is still booting, or whose guest works, is left alone).') }} + To recover by hand: colima stop --profile {{ github_runner_cluster_effective_runtime.colima_profile }} --force, then colima start --profile {{ github_runner_cluster_effective_runtime.colima_profile }}. + when: + - not ansible_check_mode + - not (github_runner_cluster_runtime_wedged | bool and github_runner_cluster_runtime_force_stop_wedged | bool) + + - name: Stop the wedged profile cleanly + ansible.builtin.command: + argv: + - perl + - -e + - alarm shift; exec @ARGV + - "{{ github_runner_cluster_runtime_clean_stop_timeout }}" + - colima + - stop + - --profile + - "{{ github_runner_cluster_effective_runtime.colima_profile }}" + environment: "{{ github_runner_cluster_runtime_environment }}" + register: github_runner_cluster_runtime_clean_stop + changed_when: true + failed_when: false + when: not ansible_check_mode + + - name: Stop the wedged profile without a graceful shutdown, since the clean stop failed + # colima stop --force ends the VM without a graceful shutdown (colima stop --help: "stop without graceful shutdown"): every process in the guest, k3s and etcd included, is killed. The VM's disk and everything on it are kept. + ansible.builtin.command: + argv: [colima, stop, --profile, "{{ github_runner_cluster_effective_runtime.colima_profile }}", --force] + environment: "{{ github_runner_cluster_runtime_environment }}" + changed_when: true + when: + - not ansible_check_mode + - github_runner_cluster_runtime_clean_stop.rc != 0 diff --git a/tests/unit/test_runtime_autostart.py b/tests/unit/test_runtime_autostart.py index 99475c7e..547dc655 100644 --- a/tests/unit/test_runtime_autostart.py +++ b/tests/unit/test_runtime_autostart.py @@ -13,6 +13,7 @@ import subprocess import sys import tempfile +import time import unittest from pathlib import Path from typing import Any @@ -80,9 +81,15 @@ def test_the_node_container_follows_the_node_s_role(self) -> None: self.assertIn("github_runner_cluster_container_name", template) self.assertIn("github_runner_cluster_agent_container_name", template) + def test_the_stop_of_a_silent_runtime_has_its_own_switch_and_bounded_probes(self) -> None: + self.assertIs(DEFAULTS["github_runner_cluster_runtime_force_stop_wedged"], True) + self.assertGreater(DEFAULTS["github_runner_cluster_runtime_probe_attempts"], 1) + self.assertGreater(DEFAULTS["github_runner_cluster_runtime_probe_timeout"], 0) + self.assertGreater(DEFAULTS["github_runner_cluster_runtime_boot_grace"], DEFAULTS["github_runner_cluster_runtime_probe_attempts"] * (DEFAULTS["github_runner_cluster_runtime_probe_timeout"] + DEFAULTS["github_runner_cluster_runtime_probe_delay"])) + def test_every_new_variable_is_in_the_readme(self) -> None: readme = (ROLE / "README.md").read_text() - for name in ("github_runner_cluster_runtime_autostart", "github_runner_cluster_runtime_autostart_at", "github_runner_cluster_runtime_boot_profile", "github_runner_cluster_runtime", "github_runner_cluster_colima_profile", "github_runner_cluster_runtime_path"): + for name in ("github_runner_cluster_runtime_force_stop_wedged", "github_runner_cluster_runtime_probe_attempts", "github_runner_cluster_runtime_probe_delay", "github_runner_cluster_runtime_probe_timeout", "github_runner_cluster_runtime_boot_grace", "github_runner_cluster_runtime_clean_stop_timeout", "github_runner_cluster_runtime_autostart", "github_runner_cluster_runtime_autostart_at", "github_runner_cluster_runtime_boot_profile", "github_runner_cluster_runtime", "github_runner_cluster_colima_profile", "github_runner_cluster_runtime_path"): with self.subTest(name=name): self.assertIn(f"`{name}`", readme) @@ -306,6 +313,14 @@ def need_root(): sys.exit("context inspect: no context-host file") print(open(os.path.join(stub, "context-host")).read(), end="") elif name == "docker": + if not exists("runtime-up") and args[:1] == ["info"] and exists("docker-answers-after"): + # A daemon that fails its first tries and then answers, as one that is restarting or a loaded host does. + counter = os.path.join(stub, "docker-info-failures") + seen = int(open(counter).read()) if os.path.exists(counter) else 0 + if seen >= int(open(os.path.join(stub, "docker-answers-after")).read()): + touch("runtime-up") + else: + open(counter, "w").write(str(seen + 1)) if not exists("runtime-up"): sys.exit("Cannot connect to the Docker daemon") if args[0] == "info": @@ -317,6 +332,8 @@ def need_root(): else: sys.exit("unexpected docker call: " + " ".join(args)) elif name == "colima" and args[0] == "ssh": + if exists("guest-answers") and args[args.index("--") + 1:] == ["true"]: + sys.exit(0) if not exists("runtime-up"): sys.exit("colima: the VM is not running") command = args[args.index("--") + 1:] @@ -386,8 +403,10 @@ def sysctl_file(key): print('{{"name":"other","status":"Running","runtime":"docker"}}') print('{{"name":"' + profile + '","status":"' + ("Running" if exists("colima-listed-running") else "Stopped") + '","runtime":"docker"}}') elif name == "colima" and args[0] == "stop": - if args != ["stop", "--profile", os.environ.get("RUNTIME_PROFILE", "default"), "--force"]: + if args[:3] != ["stop", "--profile", os.environ.get("RUNTIME_PROFILE", "default")] or args[3:] not in ([], ["--force"]): sys.exit("unexpected colima stop call: " + " ".join(args)) + if args[3:] == [] and exists("clean-stop-fails"): + sys.exit("colima: the VM does not stop") for file in ("colima-listed-running", "runtime-up"): if exists(file): os.remove(os.path.join(stub, file)) @@ -538,25 +557,105 @@ def test_a_stopped_colima_is_started_and_the_agent_installed_and_loaded(self) -> self.assertEqual(self.plist.stat().st_mode & 0o777, 0o644) self.assertTrue((self.stub / "loaded").exists()) + # A profile Colima lists as running while its runtime does not answer. + + FAST = {"github_runner_cluster_runtime_probe_attempts": 3, "github_runner_cluster_runtime_probe_delay": 0, "github_runner_cluster_runtime_probe_timeout": 5} + def stops(self) -> list[str]: return [call for call in self.calls() if re.match(r"\S+ colima stop", call)] - def test_a_profile_listed_as_running_whose_runtime_does_not_answer_is_stopped_and_started_again(self) -> None: - # A wedged VM: the profile is listed as running, the Docker socket does not answer and a start would do nothing. + def starts(self) -> list[str]: + return [call for call in self.calls() if re.match(r"\S+ colima start", call)] + + def listed_running(self, agent_age: int | None = 7200) -> None: + """A VM Colima lists as running whose Docker daemon is silent, with its host agent started `agent_age` seconds ago (None for a pid file that is not there).""" self.state("session", "colima-listed-running") - result = self.run_role() + if agent_age is not None: + pid = self.home / ".colima" / "_lima" / "colima" / "ha.pid" + pid.parent.mkdir(parents=True, exist_ok=True) + pid.write_text("1") + started = time.time() - agent_age + os.utime(pid, (started, started)) + + def info_calls(self) -> list[str]: + return [call for call in self.calls() if re.match(r"\S+ docker info", call)] + + def test_a_vm_whose_runtime_stays_silent_and_whose_guest_and_agent_confirm_it_is_stopped_cleanly_and_started_again(self) -> None: + self.listed_running() + result = self.run_role(**self.FAST) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertEqual(self.stops(), ["colima colima stop --profile default --force"]) - starts = [call for call in self.calls() if re.match(r"\S+ colima start", call)] - self.assertEqual(starts, ["colima colima start --profile default --activate=false"]) - self.assertLess(self.calls().index(self.stops()[0]), self.calls().index(starts[0])) + self.assertEqual(self.stops(), ["colima colima stop --profile default"]) + self.assertEqual(self.starts(), ["colima colima start --profile default --activate=false"]) + self.assertLess(self.calls().index(self.stops()[0]), self.calls().index(self.starts()[0])) + # The first probe and every further try. + self.assertGreaterEqual(len(self.info_calls()), 4) self.assertTrue((self.stub / "runtime-up").exists()) + def test_a_clean_stop_that_fails_is_followed_by_a_forced_one(self) -> None: + self.listed_running() + self.state("clean-stop-fails") + result = self.run_role(**self.FAST) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), ["colima colima stop --profile default", "colima colima stop --profile default --force"]) + self.assertEqual(len(self.starts()), 1) + + def test_only_the_hosts_own_profile_is_stopped_while_another_listed_profile_runs(self) -> None: + self.listed_running() + self.run_role(**self.FAST) + self.assertEqual([call for call in self.stops() if "--profile other" in call], []) + self.assertTrue(self.stops()) + + def test_a_daemon_that_answers_on_a_later_try_is_never_stopped(self) -> None: + # The first probe and one further try fail, as on a loaded host or while dockerd restarts, and the next answers. + self.listed_running() + (self.stub / "docker-answers-after").write_text("2") + result = self.run_role(**self.FAST) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), []) + self.assertEqual(self.starts(), []) + + def assert_left_alone(self, result: subprocess.CompletedProcess[str], *words: str) -> None: + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertEqual(failed_task(result.stdout), "Fail where the VM is not shown to be wedged or the stop is off") + self.assertEqual(self.stops(), []) + self.assertEqual(self.starts(), []) + self.assertIn("colima stop --profile default --force", result.stdout, "the recovery by hand is named") + for word in words: + self.assertIn(word, result.stdout) + + def test_a_guest_that_answers_a_command_is_left_alone_although_docker_is_silent(self) -> None: + self.listed_running() + self.state("guest-answers") + self.assert_left_alone(self.run_role(**self.FAST), "a command in the guest succeeded") + + def test_a_vm_that_is_still_booting_is_left_alone(self) -> None: + self.listed_running(agent_age=30) + self.assert_left_alone(self.run_role(**self.FAST), "within the last 600 seconds") + + def test_an_agent_whose_age_cannot_be_read_does_not_count_as_old(self) -> None: + self.listed_running(agent_age=None) + self.assert_left_alone(self.run_role(**self.FAST), "could not be read") + + def test_a_docker_context_that_points_at_another_socket_never_leads_to_a_stop(self) -> None: + self.listed_running() + result = self.run_role(context_host=f"unix://{self.home}/.elsewhere/default/docker.sock", **self.FAST) + self.assert_left_alone(result, "does not point at") + + def test_with_the_stop_switched_off_a_confirmed_wedge_fails_naming_the_switch(self) -> None: + self.listed_running() + self.assert_left_alone(self.run_role(github_runner_cluster_runtime_force_stop_wedged=False, **self.FAST), "github_runner_cluster_runtime_force_stop_wedged is off") + + def test_the_stop_switch_is_independent_of_autostart(self) -> None: + self.listed_running() + result = self.run_role(github_runner_cluster_runtime_force_stop_wedged=False, github_runner_cluster_runtime_autostart=True, **self.FAST) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.stops(), []) + def test_a_second_run_after_the_recovery_stops_and_starts_nothing(self) -> None: - self.state("session", "colima-listed-running") - self.run_role() + self.listed_running() + self.run_role(**self.FAST) before = len(self.calls()) - result = self.run_role() + result = self.run_role(**self.FAST) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual([call for call in self.calls()[before:] if re.match(r"\S+ colima (stop|start)", call)], []) @@ -573,18 +672,21 @@ def test_a_running_runtime_is_neither_listed_nor_stopped(self) -> None: self.assertEqual(self.stops(), []) self.assertEqual([call for call in self.calls() if re.match(r"\S+ colima list", call)], []) - def test_only_the_hosts_own_profile_is_stopped_while_another_listed_profile_runs(self) -> None: - self.state("session") - result = self.run_role(context="colima-example", context_host=f"unix://{self.home}/.colima-example/example/docker.sock") + def test_check_mode_reads_everything_stops_nothing_and_says_what_a_real_run_would_do(self) -> None: + self.listed_running() + result = self.run_role("--check", **self.FAST) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(self.stops(), []) + self.assertTrue((self.stub / "colima-listed-running").exists()) + self.assertIn("A real run would stop the profile", result.stdout) - def test_check_mode_stops_nothing_in_a_wedged_profile(self) -> None: - self.state("session", "colima-listed-running") - result = self.run_role("--check") + def test_check_mode_on_a_vm_that_is_not_shown_to_be_wedged_says_nothing_would_be_stopped(self) -> None: + self.listed_running() + self.state("guest-answers") + result = self.run_role("--check", **self.FAST) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(self.stops(), []) - self.assertTrue((self.stub / "colima-listed-running").exists()) + self.assertIn("A real run would stop nothing and fail here", result.stdout) def test_every_docker_call_uses_the_hosts_docker_context(self) -> None: self.state("session") From 9a8566eaa6f4b390ae265755506758b9c1b954b4 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 14 Nov 2026 16:05:34 +0000 Subject: [PATCH 3/7] fix(github_runner_cluster): bound the wedge probes and read the agent's age live A timer set with alarm before exec does not end a Go program, which ignores SIGALRM, so a runtime that accepts a connection and never replies held the run, and a clean stop could block for ever. The probes, the guest command and the clean stop now run under files/run_bounded.pl, which gives the command its own process group, sends TERM at the deadline and KILL after a grace period, and reports a timeout distinctly. The first probe of the runtime is bounded the same way. The age of the VM's host agent now comes from the live process the pid file names, and only when that process is Lima's host agent. A pid file that outlived its process, as one from before a reboot does, or one that names another program, leaves the age unknown, which never counts as old. --- roles/github_runner_cluster/README.md | 4 +- roles/github_runner_cluster/defaults/main.yml | 8 +- .../files/run_bounded.pl | 52 ++++++ .../tasks/runtime_probe.yml | 12 +- .../tasks/runtime_wedged.yml | 46 +++-- tests/unit/test_runtime_autostart.py | 176 ++++++++++++++++-- 6 files changed, 263 insertions(+), 35 deletions(-) create mode 100644 roles/github_runner_cluster/files/run_bounded.pl diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 423dd740..94435557 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -166,7 +166,7 @@ On a Mac the container runtime is a Linux VM (Colima, or Docker Desktop) that st Which runtime a Mac runs is taken from the Docker context the role drives (see [Docker contexts](#docker-contexts)): `colima`, or `colima-` for another profile, is Colima, and `desktop-linux` is Docker Desktop. `github_runner_cluster_runtime` says so explicitly for a host whose context does not tell, for example one that uses `DOCKER_HOST`, and `github_runner_cluster_colima_profile` names the profile to start when the context does not. -**Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role starts the profile if it is stopped, with `colima start --profile --activate=false`, so the profile does not make its context the host's current one. A profile that `colima list` shows as running while its Docker runtime does not answer is not started again, because a start finds it running and does nothing. The role stops it, with `colima stop` and then `colima stop --force` only if that fails, and starts it again from the same disk, but only when `github_runner_cluster_runtime_force_stop_wedged` is on (the default) and all of these hold: `docker info` fails on `github_runner_cluster_runtime_probe_attempts` further tries (default 5), each given `github_runner_cluster_runtime_probe_timeout` seconds (default 20) and `github_runner_cluster_runtime_probe_delay` seconds apart (default 10); a command in the guest (`colima ssh -- true`) fails within the same timeout; the VM's host agent was started more than `github_runner_cluster_runtime_boot_grace` seconds ago (default 600), read from the modification time of its pid file in the profile's Lima instance directory under the Colima home; and the Docker context the probes used points at that profile's own socket. A host that is only loaded, a Docker daemon that is restarting, a VM that is still booting and a context that points at another runtime each fail one of these, so nothing is stopped. A stop does not shut the guest down gracefully when the clean stop fails or times out (`github_runner_cluster_runtime_clean_stop_timeout`, default 120 seconds): every process in the guest, k3s and etcd included, ends, and the VM's disk, containers and volumes are kept, so on an etcd voter the member is out until the start completes. Switched off, or when a condition does not hold, the run fails with the signals it saw and the commands to recover by hand. Under `--check` the reads are made, nothing is stopped and the report says what a real run would do. It installs the launchd job that starts it again after a reboot. There are two kinds, chosen by `github_runner_cluster_runtime_autostart_at` (`login`, `boot` or `auto`, the default). `auto` is `boot` for the profile named by `github_runner_cluster_runtime_boot_profile` (default `node`, the profile the bridged k3s node runs in) and `login` for any other, such as a `default` profile a host keeps for other workloads. A profile has one kind at a time: installing either retires the other's property list, unloading it first, so the profile is not started twice. +**Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role starts the profile if it is stopped, with `colima start --profile --activate=false`, so the profile does not make its context the host's current one. A profile that `colima list` shows as running while its Docker runtime does not answer is not started again, because a start finds it running and does nothing. The role stops it, with `colima stop` and then `colima stop --force` only if that fails, and starts it again from the same disk, but only when `github_runner_cluster_runtime_force_stop_wedged` is on (the default) and all of these hold: `docker info` fails on `github_runner_cluster_runtime_probe_attempts` further tries (default 5), each given `github_runner_cluster_runtime_probe_timeout` seconds (default 20) and `github_runner_cluster_runtime_probe_delay` seconds apart (default 10); a command in the guest (`colima ssh -- true`) fails within the same timeout; the VM's host agent has been running for more than `github_runner_cluster_runtime_boot_grace` seconds (default 600), read from the elapsed time of the live process that the pid file in the profile's Lima instance directory names, and only when that process's command is Lima's host agent (a missing file, a dead pid, another program or an unreadable time leaves the age unknown, which never counts as old, whatever the file's date); and the Docker context the probes used points at that profile's own socket. Every probe, the guest command and the clean stop run under `files/run_bounded.pl`, which owns the deadline: it sends TERM to the command's process group when the time is up and KILL `github_runner_cluster_runtime_kill_grace` seconds later (default 5), so a program that ignores signals or never replies (docker and colima ignore `SIGALRM`) ends the try as failed and cannot hold the run; the first probe of the runtime is bounded the same way. A host that is only loaded, a Docker daemon that is restarting, a VM that is still booting and a context that points at another runtime each fail one of these, so nothing is stopped. A stop does not shut the guest down gracefully when the clean stop fails or times out (`github_runner_cluster_runtime_clean_stop_timeout`, default 120 seconds): every process in the guest, k3s and etcd included, ends, and the VM's disk, containers and volumes are kept, so on an etcd voter the member is out until the start completes. Switched off, or when a condition does not hold, the run fails with the signals it saw and the commands to recover by hand. With all of this in force the stop never follows a single failed or slow probe, never touches a VM whose guest answers or whose agent process is young or unknown, and never ends in a hang; that is why the switch is on by default. Under `--check` the reads are made, nothing is stopped and the report says what a real run would do. It installs the launchd job that starts it again after a reboot. There are two kinds, chosen by `github_runner_cluster_runtime_autostart_at` (`login`, `boot` or `auto`, the default). `auto` is `boot` for the profile named by `github_runner_cluster_runtime_boot_profile` (default `node`, the profile the bridged k3s node runs in) and `login` for any other, such as a `default` profile a host keeps for other workloads. A profile has one kind at a time: installing either retires the other's property list, unloading it first, so the profile is not started twice. **At login (an agent).** The role: @@ -491,7 +491,7 @@ The role controller holds the regrade Lease (`kube-system/fleet-regrade`, the Le - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)). - `github_runner_cluster_network_mode` (default `bridge`, or `host`): the network the node's container runs on, and with `host`, `github_runner_cluster_lan_cidr` (required: the LAN the nodes share) and `github_runner_cluster_network_probe_image` (default `busybox:1.37`, for reading the host's addresses) (see [Networking](#networking)). - `github_runner_cluster_runtime_autostart_at` (default `auto`; `login` or `boot`) and `github_runner_cluster_runtime_boot_profile` (default `node`): whether the Colima profile is started by a launchd agent at login or by a launchd daemon at boot, and which profile `auto` starts at boot (see [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). -- `github_runner_cluster_runtime_force_stop_wedged` (default true), `github_runner_cluster_runtime_probe_attempts` (default 5), `github_runner_cluster_runtime_probe_delay` (default 10), `github_runner_cluster_runtime_probe_timeout` (default 20), `github_runner_cluster_runtime_boot_grace` (default 600) and `github_runner_cluster_runtime_clean_stop_timeout` (default 120): when a Colima profile listed as running with a silent runtime is stopped, see the Colima paragraph above. +- `github_runner_cluster_runtime_force_stop_wedged` (default true), `github_runner_cluster_runtime_probe_attempts` (default 5), `github_runner_cluster_runtime_probe_delay` (default 10), `github_runner_cluster_runtime_probe_timeout` (default 20), `github_runner_cluster_runtime_boot_grace` (default 600), `github_runner_cluster_runtime_kill_grace` (default 5) and `github_runner_cluster_runtime_clean_stop_timeout` (default 120): when a Colima profile listed as running with a silent runtime is stopped, see the Colima paragraph above. - `github_runner_cluster_runtime_autostart` (default true), `github_runner_cluster_runtime` (default `auto`; `colima`, `docker_desktop` or `other`), `github_runner_cluster_colima_profile` (default empty, taken from the Docker context), `github_runner_cluster_colima_home` (default empty: the `COLIMA_HOME` is read from the endpoint of the profile's Docker context, and is unset for Colima's own `~/.colima`; set it only for a context that does not say, and the role passes it to colima and to the launchd agent), `github_runner_cluster_runtime_path` (default both Homebrew prefixes and the system directories): on a macOS host, start Colima at login through a launchd agent and check the runtime (see [Container runtime at login](#container-runtime-after-a-reboot-macos)). - `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus` (default empty, keeping the profile's own): a Colima host's VM memory in GiB and its CPUs, each a positive whole number, applied one node at a time through the quorum guard when the VM differs (see Colima VM size under [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). `github_runner_cluster_colima_memory_reserved_gib` (default empty) derives the memory instead, or bounds one set explicitly, from the host's memory less it and `github_runner_cluster_colima_memory_host_reserve_gib` (default 10, macOS's own), never below `github_runner_cluster_colima_guest_baseline_gib` (default 3.4) plus `github_runner_cluster_colima_runner_memory_limit_gib` (default empty, required by a derivation) rounded up. `github_runner_cluster_colima_resize_lease_seconds`, `github_runner_cluster_colima_resize_lease_holder`, `github_runner_cluster_regrade_lease_name` and `github_runner_cluster_regrade_lease_namespace` are derived, never set directly. `github_runner_cluster_native_runner_wait_timeout` and `github_runner_cluster_native_ready_timeout` bound the wait for the node's runner pods and for the node to come back, as they do for a native node. - `github_runner_cluster_container_name` (default `github-runner-k3s`) and `github_runner_cluster_agent_container_name` (default `github-runner-k3s-agent`): the names of the node's server and agent containers, which a second cluster on the same Docker hosts sets to names of its own (see [Several clusters on one host](#several-clusters-on-one-host)). diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index 57e1172b..b881d742 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -242,13 +242,15 @@ github_runner_cluster_runtime_force_stop_wedged: true # How many times docker info is tried, and the seconds between tries, before a profile listed as running with a silent runtime is looked into further. Together with the timeout they set the window in which a loaded host or a restarting Docker daemon can still answer. github_runner_cluster_runtime_probe_attempts: 5 github_runner_cluster_runtime_probe_delay: 10 -# The seconds each docker info, and the command run in the guest, is given before it counts as failed. +# The seconds each docker info, and the command run in the guest, is given before it counts as failed (a run that does not end by then is ended by files/run_bounded.pl). github_runner_cluster_runtime_probe_timeout: 20 -# The seconds a VM's host agent must have been running before a silent runtime counts as wedged rather than booting. +# The seconds a VM's host agent process must have been running before a silent runtime counts as wedged rather than booting. github_runner_cluster_runtime_boot_grace: 600 +# The seconds a probe, a guest command or a clean stop that has passed its deadline is given to end after TERM before it is killed. +github_runner_cluster_runtime_kill_grace: 5 # The seconds colima stop is given before colima stop --force follows it. github_runner_cluster_runtime_clean_stop_timeout: 120 -# The Colima profile that runs this collection's k3s node, which must come back with no one logged in: the one github_runner_cluster_runtime_autostart_at auto starts at boot. Any other profile on the host (the mini's default profile for its own workloads, say) keeps the login agent. +# The Colima profile that runs this collection's k3s node, which must come back with no one logged in: the one github_runner_cluster_runtime_autostart_at auto starts at boot. Any other profile on the host (a `default` profile a host keeps for other workloads, say) keeps the login agent. github_runner_cluster_runtime_boot_profile: node # Which container runtime a macOS host runs: auto takes it from the Docker context the role drives (colima or colima- is Colima, desktop-linux is Docker Desktop, anything else is other), colima, docker_desktop and other say so for a host whose context does not tell (DOCKER_HOST set, a context with another name). github_runner_cluster_runtime: auto diff --git a/roles/github_runner_cluster/files/run_bounded.pl b/roles/github_runner_cluster/files/run_bounded.pl new file mode 100644 index 00000000..e869167e --- /dev/null +++ b/roles/github_runner_cluster/files/run_bounded.pl @@ -0,0 +1,52 @@ +#!/usr/bin/perl +# Runs a command with a deadline that binds whatever the command does with signals. Usage: run_bounded.pl DEADLINE GRACE COMMAND [ARGUMENT...] (both in seconds). +# +# The command runs in a process group of its own. When DEADLINE passes, the group gets TERM; if any member is still there GRACE seconds later it gets KILL. A timer set with alarm before exec does not do this: a Go program such as docker or colima does not exit on SIGALRM, and a program may ignore TERM too. +# +# The exit status is the command's own, 128 plus the signal number when a signal ended it, 124 when the deadline passed (as timeout(1) reports it) and 127 when the command could not be started. Its standard input, output and error are the caller's. +use strict; +use warnings; +use POSIX qw(WNOHANG setsid); +use Time::HiRes qw(sleep time); + +my ($deadline, $grace, @command) = @ARGV; +die "usage: run_bounded.pl DEADLINE GRACE COMMAND [ARGUMENT...]\n" unless @command && defined $grace; + +my $child = fork(); +die "fork: $!\n" unless defined $child; +if ($child == 0) { + setsid(); + exec { $command[0] } @command or exit 127; +} + +# The status of the child once it has ended, in the shell's convention. +sub status_of { + my ($status) = @_; + return $status & 127 ? 128 + ($status & 127) : $status >> 8; +} + +# True once the child has ended, with its status in $main::status. +our $status; +sub ended { + my $done = waitpid($child, WNOHANG); + return 0 if $done == 0; + $status = status_of($?); + return 1; +} + +my $end = time + $deadline; +while (time < $end) { + exit $status if ended(); + sleep 0.05; +} + +kill 'TERM', -$child; +my $kill_at = time + $grace; +while (time < $kill_at) { + last if ended(); + sleep 0.05; +} +# The group is killed whether or not its leader ended, so that nothing it started outlives the deadline. +kill 'KILL', -$child; +waitpid($child, 0) unless defined $status; +exit 124; diff --git a/roles/github_runner_cluster/tasks/runtime_probe.yml b/roles/github_runner_cluster/tasks/runtime_probe.yml index 92ec41b5..76dbe6f3 100644 --- a/roles/github_runner_cluster/tasks/runtime_probe.yml +++ b/roles/github_runner_cluster/tasks/runtime_probe.yml @@ -2,8 +2,18 @@ # Reads whether the container runtime this host's Docker context points at answers, and publishes github_runner_cluster_runtime_up. Read-only, so it runs under --check too. Needs github_runner_cluster_runtime_environment from runtime.yml. A daemon that is down is an answer here, not a failure: the callers decide what to do about it. - name: Read whether the container runtime answers + # Bounded by files/run_bounded.pl: a daemon that accepts the connection and never replies counts as not answering after github_runner_cluster_runtime_probe_timeout seconds, instead of holding the run. ansible.builtin.command: - argv: [docker, info, --format, "{{ '{{' }}.ServerVersion{{ '}}' }}"] + argv: + - perl + - -e + - "{{ lookup('ansible.builtin.file', 'run_bounded.pl') }}" + - "{{ github_runner_cluster_runtime_probe_timeout }}" + - "{{ github_runner_cluster_runtime_kill_grace }}" + - docker + - info + - --format + - "{{ '{{' }}.ServerVersion{{ '}}' }}" environment: "{{ github_runner_cluster_runtime_environment }}" register: github_runner_cluster_runtime_probe changed_when: false diff --git a/roles/github_runner_cluster/tasks/runtime_wedged.yml b/roles/github_runner_cluster/tasks/runtime_wedged.yml index a637dc8f..8b58052b 100644 --- a/roles/github_runner_cluster/tasks/runtime_wedged.yml +++ b/roles/github_runner_cluster/tasks/runtime_wedged.yml @@ -2,9 +2,9 @@ # Reached on a Colima host whose container runtime did not answer. A profile that Colima lists as stopped is only started (runtime.yml). A profile it lists as running is either booting, slow, or stuck, and a start does nothing for it, so the VM could stay down for ever. This file ends such a VM with colima stop, and only when every one of these holds: # # 1. github_runner_cluster_runtime_force_stop_wedged is on; -# 2. docker info still fails on every one of github_runner_cluster_runtime_probe_attempts further tries, each given github_runner_cluster_runtime_probe_timeout seconds and github_runner_cluster_runtime_probe_delay seconds apart, so a host that is only loaded or a dockerd that is restarting answers somewhere in that window and nothing is stopped; +# 2. docker info still fails on every one of github_runner_cluster_runtime_probe_attempts further tries, each given github_runner_cluster_runtime_probe_timeout seconds (files/run_bounded.pl ends one that has not finished, with TERM and then KILL after github_runner_cluster_runtime_kill_grace seconds, so a program that ignores signals cannot hold the run) and github_runner_cluster_runtime_probe_delay seconds apart, so a host that is only loaded or a dockerd that is restarting answers somewhere in that window and nothing is stopped; # 3. a command run in the guest (colima ssh -- true) fails within github_runner_cluster_runtime_probe_timeout seconds, so a guest that works while its Docker daemon does not is left alone; -# 4. the VM's host agent was started more than github_runner_cluster_runtime_boot_grace seconds ago, read from the modification time of its pid file in the profile's Lima instance directory, so a VM that is still booting is left alone. An agent whose age cannot be read does not count as old; +# 4. the VM's host agent has been running for more than github_runner_cluster_runtime_boot_grace seconds, read from the elapsed time of the live process the pid file in the profile's Lima instance directory names (and only if its command is Lima's host agent), so a VM that is still booting is left alone. A missing file, a dead pid, another program or an unreadable time means the agent's age is unknown and does not count as old, whatever the file's date; # 5. the Docker context the probes used points at the socket of the profile that would be stopped (//docker.sock), so a failing probe of one runtime never stops another's VM. # # The stop is first a clean one (colima stop), given github_runner_cluster_runtime_clean_stop_timeout seconds, and a forced one (colima stop --force, which ends the VM without a graceful shutdown) only when that fails. Either way the VM's disk is kept and nothing is removed, so the start that follows boots it from the same disk. When any condition fails, or the switch is off, the run fails here and says which signals it saw and how to recover. Under --check every read is made and nothing is stopped; the report says what a real run would do. @@ -31,13 +31,14 @@ when: github_runner_cluster_runtime_listed_running | bool block: - name: Ask the Docker daemon again, a bounded number of times - # perl's alarm bounds each try: macOS has no timeout command, and docker info has no timeout of its own. + # files/run_bounded.pl bounds each try: macOS has no timeout command, docker info has no timeout of its own, and a timer set before exec does not stop a Go program (docker, colima), which ignores SIGALRM. ansible.builtin.command: argv: - perl - -e - - alarm shift; exec @ARGV + - "{{ lookup('ansible.builtin.file', 'run_bounded.pl') }}" - "{{ github_runner_cluster_runtime_probe_timeout }}" + - "{{ github_runner_cluster_runtime_kill_grace }}" - docker - info - --format @@ -64,8 +65,9 @@ argv: - perl - -e - - alarm shift; exec @ARGV + - "{{ lookup('ansible.builtin.file', 'run_bounded.pl') }}" - "{{ github_runner_cluster_runtime_probe_timeout }}" + - "{{ github_runner_cluster_runtime_kill_grace }}" - colima - ssh - --profile @@ -80,26 +82,37 @@ - name: Find the VM's host agent pid file # Colima keeps one Lima instance per profile under /_lima: colima for the default profile, colima- for another. - ansible.builtin.stat: - path: >- + ansible.builtin.slurp: + src: >- {{ (github_runner_cluster_runtime_colima_home if github_runner_cluster_runtime_colima_home | length > 0 else ansible_facts['env']['HOME'] ~ '/.colima') ~ '/_lima/' ~ ('colima' if github_runner_cluster_effective_runtime.colima_profile == 'default' else 'colima-' ~ github_runner_cluster_effective_runtime.colima_profile) ~ '/ha.pid' }} - get_checksum: false register: github_runner_cluster_runtime_agent_pid + failed_when: false - - name: Read the time on the host + - name: Read the live process the pid file names + # A pid file can outlive the process (a reboot, a crash), so the age is the running process's own: its elapsed time, and only when its command is Lima's host agent. ansible.builtin.command: - argv: [date, +%s] - register: github_runner_cluster_runtime_now + argv: [ps, -o, etime=, -o, command=, -p, "{{ (github_runner_cluster_runtime_agent_pid.content | b64decode | trim) if github_runner_cluster_runtime_agent_pid.content is defined else '0' }}"] + register: github_runner_cluster_runtime_agent_process changed_when: false + failed_when: false check_mode: false - name: Work out what the signals say + vars: + # ps prints [[days-]hours:]minutes:seconds, then the command. Anything else, a dead pid or another program leaves the age unknown (-1). + github_runner_cluster_runtime_agent_words: "{{ github_runner_cluster_runtime_agent_process.stdout | trim | split(None, 1) }}" ansible.builtin.set_fact: github_runner_cluster_runtime_agent_age: >- - {{ (github_runner_cluster_runtime_now.stdout | trim | int - github_runner_cluster_runtime_agent_pid.stat.mtime | int) - if github_runner_cluster_runtime_agent_pid.stat.exists else -1 }} + {%- set words = github_runner_cluster_runtime_agent_words -%} + {%- if github_runner_cluster_runtime_agent_process.rc == 0 and words | length == 2 and 'hostagent' in words[1] and words[0] is match('^([0-9]+-)?([0-9]+:)?[0-9]+:[0-9]+$') -%} + {%- set days_and_rest = words[0].split('-') -%} + {%- set days = days_and_rest[0] | int if days_and_rest | length > 1 else 0 -%} + {%- set parts = (days_and_rest[-1].split(':') | map('int') | list) -%} + {%- set clock = parts[-1] + 60 * parts[-2] + (3600 * parts[-3] if parts | length > 2 else 0) -%} + {{ days * 86400 + clock }} + {%- else -%}-1{%- endif -%} github_runner_cluster_runtime_context_socket_ok: >- {{ github_runner_cluster_runtime_context_host.stdout | trim == 'unix://' ~ (github_runner_cluster_runtime_colima_home if github_runner_cluster_runtime_colima_home | length > 0 else ansible_facts['env']['HOME'] ~ '/.colima') @@ -114,8 +127,8 @@ github_runner_cluster_runtime_wedged_report: >- Colima lists profile {{ github_runner_cluster_effective_runtime.colima_profile }} as running, but docker info failed on {{ github_runner_cluster_runtime_probe_attempts }} tries {{ github_runner_cluster_runtime_probe_delay }} seconds apart; a command in the guest {{ 'failed' if github_runner_cluster_runtime_guest.rc != 0 else 'succeeded' }}; - the VM's host agent was started {{ 'more than ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds ago' if github_runner_cluster_runtime_agent_age | int > github_runner_cluster_runtime_boot_grace | int - else ('within the last ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds' if github_runner_cluster_runtime_agent_age | int >= 0 else 'at a time that could not be read') }}; + the VM's host agent has been running {{ 'for more than ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds ago' if github_runner_cluster_runtime_agent_age | int > github_runner_cluster_runtime_boot_grace | int + else ('within the last ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds' if github_runner_cluster_runtime_agent_age | int >= 0 else 'for a time that could not be read') }}; the Docker context {{ 'points at' if github_runner_cluster_runtime_context_socket_ok | bool else 'does not point at' }} the profile's own socket. - name: Say what a real run would do @@ -145,8 +158,9 @@ argv: - perl - -e - - alarm shift; exec @ARGV + - "{{ lookup('ansible.builtin.file', 'run_bounded.pl') }}" - "{{ github_runner_cluster_runtime_clean_stop_timeout }}" + - "{{ github_runner_cluster_runtime_kill_grace }}" - colima - stop - --profile diff --git a/tests/unit/test_runtime_autostart.py b/tests/unit/test_runtime_autostart.py index 547dc655..7b0d1dc8 100644 --- a/tests/unit/test_runtime_autostart.py +++ b/tests/unit/test_runtime_autostart.py @@ -85,11 +85,12 @@ def test_the_stop_of_a_silent_runtime_has_its_own_switch_and_bounded_probes(self self.assertIs(DEFAULTS["github_runner_cluster_runtime_force_stop_wedged"], True) self.assertGreater(DEFAULTS["github_runner_cluster_runtime_probe_attempts"], 1) self.assertGreater(DEFAULTS["github_runner_cluster_runtime_probe_timeout"], 0) + self.assertGreater(DEFAULTS["github_runner_cluster_runtime_kill_grace"], 0) self.assertGreater(DEFAULTS["github_runner_cluster_runtime_boot_grace"], DEFAULTS["github_runner_cluster_runtime_probe_attempts"] * (DEFAULTS["github_runner_cluster_runtime_probe_timeout"] + DEFAULTS["github_runner_cluster_runtime_probe_delay"])) def test_every_new_variable_is_in_the_readme(self) -> None: readme = (ROLE / "README.md").read_text() - for name in ("github_runner_cluster_runtime_force_stop_wedged", "github_runner_cluster_runtime_probe_attempts", "github_runner_cluster_runtime_probe_delay", "github_runner_cluster_runtime_probe_timeout", "github_runner_cluster_runtime_boot_grace", "github_runner_cluster_runtime_clean_stop_timeout", "github_runner_cluster_runtime_autostart", "github_runner_cluster_runtime_autostart_at", "github_runner_cluster_runtime_boot_profile", "github_runner_cluster_runtime", "github_runner_cluster_colima_profile", "github_runner_cluster_runtime_path"): + for name in ("github_runner_cluster_runtime_force_stop_wedged", "github_runner_cluster_runtime_probe_attempts", "github_runner_cluster_runtime_probe_delay", "github_runner_cluster_runtime_probe_timeout", "github_runner_cluster_runtime_boot_grace", "github_runner_cluster_runtime_kill_grace", "github_runner_cluster_runtime_clean_stop_timeout", "github_runner_cluster_runtime_autostart", "github_runner_cluster_runtime_autostart_at", "github_runner_cluster_runtime_boot_profile", "github_runner_cluster_runtime", "github_runner_cluster_colima_profile", "github_runner_cluster_runtime_path"): with self.subTest(name=name): self.assertIn(f"`{name}`", readme) @@ -302,6 +303,15 @@ def test_the_comment_cannot_end_the_document_early(self) -> None: homes.write(name + " " + " ".join(args) + " " + os.environ.get("COLIMA_HOME", "-") + "\\n") def exists(file): return os.path.exists(os.path.join(stub, file)) +def hang(): + # A program that never answers and does not die on SIGALRM, as a Go binary does not; with ignore-term it ignores TERM as well, so only KILL ends it. + import signal, time + signal.signal(signal.SIGALRM, signal.SIG_IGN) + if exists("ignore-term"): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + with open(os.path.join(stub, "hung.pids"), "a") as pids: + pids.write(str(os.getpid()) + "\\n") + time.sleep(300) def touch(file): open(os.path.join(stub, file), "w").close() def need_root(): @@ -313,6 +323,8 @@ def need_root(): sys.exit("context inspect: no context-host file") print(open(os.path.join(stub, "context-host")).read(), end="") elif name == "docker": + if args[:1] == ["info"] and exists("docker-hangs"): + hang() if not exists("runtime-up") and args[:1] == ["info"] and exists("docker-answers-after"): # A daemon that fails its first tries and then answers, as one that is restarting or a loaded host does. counter = os.path.join(stub, "docker-info-failures") @@ -332,6 +344,8 @@ def need_root(): else: sys.exit("unexpected docker call: " + " ".join(args)) elif name == "colima" and args[0] == "ssh": + if exists("guest-hangs") and args[args.index("--") + 1:] == ["true"]: + hang() if exists("guest-answers") and args[args.index("--") + 1:] == ["true"]: sys.exit(0) if not exists("runtime-up"): @@ -405,9 +419,11 @@ def sysctl_file(key): elif name == "colima" and args[0] == "stop": if args[:3] != ["stop", "--profile", os.environ.get("RUNTIME_PROFILE", "default")] or args[3:] not in ([], ["--force"]): sys.exit("unexpected colima stop call: " + " ".join(args)) + if args[3:] == [] and exists("clean-stop-hangs"): + hang() if args[3:] == [] and exists("clean-stop-fails"): sys.exit("colima: the VM does not stop") - for file in ("colima-listed-running", "runtime-up"): + for file in ("colima-listed-running", "runtime-up", "docker-hangs", "guest-hangs"): if exists(file): os.remove(os.path.join(stub, file)) elif name == "colima": @@ -419,6 +435,11 @@ def sysctl_file(key): if not exists("colima-listed-running"): touch("colima-listed-running") touch("runtime-up") +elif name == "ps": + # The host agent as ps shows it, from the file agent-ps (" "); a pid that is not running prints nothing and fails. + if not exists("agent-ps"): + sys.exit(1) + print(open(os.path.join(stub, "agent-ps")).read(), end="") elif name == "launchctl": system = args[1] == "system" or args[1].startswith("system/") if args[0] == "print": @@ -455,6 +476,55 @@ def failed_task(stdout: str) -> str | None: return None +class BoundedRunTest(unittest.TestCase): + """files/run_bounded.pl, which gives probes, the guest command and the clean stop a deadline that holds whatever the command does with signals.""" + + SCRIPT = ROLE / "files" / "run_bounded.pl" + + def run_bounded(self, deadline: float, grace: float, *command: str) -> tuple[subprocess.CompletedProcess[str], float]: + began = time.monotonic() + result = subprocess.run(["perl", str(self.SCRIPT), str(deadline), str(grace), *command], capture_output=True, text=True, check=False, timeout=60) + return result, time.monotonic() - began + + def test_the_commands_status_and_output_pass_through(self) -> None: + result, _ = self.run_bounded(10, 1, "sh", "-c", "echo out; echo err >&2; exit 3") + self.assertEqual((result.returncode, result.stdout, result.stderr), (3, "out\n", "err\n")) + + def test_a_signal_that_ends_the_command_is_reported_as_the_shell_does(self) -> None: + result, _ = self.run_bounded(10, 1, "sh", "-c", "kill -9 $$") + self.assertEqual(result.returncode, 137) + + def test_a_command_that_ignores_sigalrm_is_ended_at_the_deadline(self) -> None: + # What a Go program does with the timer a bare alarm would set. + result, elapsed = self.run_bounded(1, 1, sys.executable, "-c", "import signal, time; signal.signal(signal.SIGALRM, signal.SIG_IGN); time.sleep(60)") + self.assertEqual(result.returncode, 124) + self.assertLess(elapsed, 10) + + def test_a_command_that_ignores_term_is_killed_after_the_grace(self) -> None: + result, elapsed = self.run_bounded(1, 1, sys.executable, "-c", "import signal, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(60)") + self.assertEqual(result.returncode, 124) + self.assertGreaterEqual(elapsed, 2) + self.assertLess(elapsed, 15) + + def test_what_the_command_started_does_not_outlive_the_deadline(self) -> None: + marker = Path(tempfile.mkdtemp(prefix="bounded-")) / "pid" + self.addCleanup(shutil.rmtree, marker.parent, True) + result, _ = self.run_bounded(1, 1, "sh", "-c", f"trap '' TERM; sleep 60 & echo $! > {marker}; wait") + self.assertEqual(result.returncode, 124) + pid = int(marker.read_text()) + for _ in range(50): + try: + os.kill(pid, 0) + except ProcessLookupError: + return + time.sleep(0.1) + self.fail("a process the command started is still running") + + def test_a_command_that_cannot_be_started_exits_127(self) -> None: + result, _ = self.run_bounded(5, 1, "/nonexistent/command") + self.assertEqual(result.returncode, 127) + + @unittest.skipIf(shutil.which(ANSIBLE_PLAYBOOK) is None, "ansible-playbook is not installed") class RoleTest(unittest.TestCase): def setUp(self) -> None: @@ -466,7 +536,7 @@ def setUp(self) -> None: self.home.mkdir() self.bin = self.work / "bin" self.bin.mkdir() - for name in ("docker", "colima", "launchctl", "plutil", "sudo"): + for name in ("docker", "colima", "launchctl", "plutil", "sudo", "ps"): tool = self.bin / name tool.write_text(STUB.format(python=sys.executable)) tool.chmod(0o755) @@ -567,15 +637,43 @@ def stops(self) -> list[str]: def starts(self) -> list[str]: return [call for call in self.calls() if re.match(r"\S+ colima start", call)] - def listed_running(self, agent_age: int | None = 7200) -> None: - """A VM Colima lists as running whose Docker daemon is silent, with its host agent started `agent_age` seconds ago (None for a pid file that is not there).""" + AGENT = "/usr/local/bin/limactl hostagent --pidfile /example/ha.pid --socket /example/ha.sock" + + @staticmethod + def etime(seconds: int) -> str: + """ps's elapsed time, [[days-]hours:]minutes:seconds.""" + days, rest = divmod(seconds, 86400) + hours, rest = divmod(rest, 3600) + minutes, secs = divmod(rest, 60) + return (f"{days}-" if days else "") + (f"{hours:02d}:" if days or hours else "") + f"{minutes:02d}:{secs:02d}" + + def listed_running(self, agent_age: int | None = 7200, command: str | None = None, stale_file: bool = False) -> None: + """A VM Colima lists as running whose Docker daemon is silent, with a live host agent that has run for `agent_age` seconds (None for no pid file at all). `stale_file` leaves a pid file that is old but names no running process, as one from before a reboot does; `command` is what the process runs.""" self.state("session", "colima-listed-running") - if agent_age is not None: - pid = self.home / ".colima" / "_lima" / "colima" / "ha.pid" - pid.parent.mkdir(parents=True, exist_ok=True) - pid.write_text("1") - started = time.time() - agent_age - os.utime(pid, (started, started)) + if agent_age is None: + return + pid = self.home / ".colima" / "_lima" / "colima" / "ha.pid" + pid.parent.mkdir(parents=True, exist_ok=True) + pid.write_text("4242\n") + old = time.time() - 7 * 86400 + os.utime(pid, (old, old)) + if not stale_file: + (self.stub / "agent-ps").write_text(f"{self.etime(agent_age)} {command or self.AGENT}\n") + + def hung_pids(self) -> list[int]: + log = self.stub / "hung.pids" + return [int(line) for line in log.read_text().split()] if log.exists() else [] + + def assert_all_ended(self) -> None: + for pid in self.hung_pids(): + for _ in range(50): + try: + os.kill(pid, 0) + except ProcessLookupError: + break + time.sleep(0.1) + else: + self.fail(f"the hung stand-in {pid} is still running") def info_calls(self) -> list[str]: return [call for call in self.calls() if re.match(r"\S+ docker info", call)] @@ -628,9 +726,25 @@ def test_a_guest_that_answers_a_command_is_left_alone_although_docker_is_silent( self.state("guest-answers") self.assert_left_alone(self.run_role(**self.FAST), "a command in the guest succeeded") - def test_a_vm_that_is_still_booting_is_left_alone(self) -> None: + def test_a_vm_whose_agent_started_moments_ago_is_left_alone_however_old_its_pid_file_is(self) -> None: + # The pid file is a week old, from a boot long gone, and the process it names started thirty seconds ago. self.listed_running(agent_age=30) - self.assert_left_alone(self.run_role(**self.FAST), "within the last 600 seconds") + self.assert_left_alone(self.run_role(**self.FAST), "for less than 600 seconds") + + def test_a_pid_file_naming_no_running_process_leaves_the_age_unknown_and_nothing_is_stopped(self) -> None: + # An old pid file whose process is gone, as after a reboot: its date says the agent is old, and that must not count. + self.listed_running(stale_file=True) + self.assert_left_alone(self.run_role(**self.FAST), "could not be read") + + def test_a_pid_file_naming_another_program_leaves_the_age_unknown(self) -> None: + self.listed_running(command="/usr/sbin/cron -f") + self.assert_left_alone(self.run_role(**self.FAST), "could not be read") + + def test_an_agent_that_has_run_for_days_counts_as_old(self) -> None: + self.listed_running(agent_age=3 * 86400 + 3600 + 61) + result = self.run_role(**self.FAST) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(self.stops()), 1) def test_an_agent_whose_age_cannot_be_read_does_not_count_as_old(self) -> None: self.listed_running(agent_age=None) @@ -672,6 +786,42 @@ def test_a_running_runtime_is_neither_listed_nor_stopped(self) -> None: self.assertEqual(self.stops(), []) self.assertEqual([call for call in self.calls() if re.match(r"\S+ colima list", call)], []) + HANG = {"github_runner_cluster_runtime_probe_attempts": 2, "github_runner_cluster_runtime_probe_delay": 0, "github_runner_cluster_runtime_probe_timeout": 2, "github_runner_cluster_runtime_kill_grace": 1, "github_runner_cluster_runtime_clean_stop_timeout": 2} + + def test_probes_that_never_reply_and_ignore_sigalrm_are_ended_and_the_vm_is_stopped_and_started(self) -> None: + # docker info and the guest command never reply and ignore SIGALRM, as Go programs do; the clean stop never ends and ignores TERM too. + self.listed_running() + self.state("docker-hangs", "guest-hangs", "clean-stop-hangs", "ignore-term") + began = time.monotonic() + result = self.run_role(**self.HANG) + elapsed = time.monotonic() - began + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), ["colima colima stop --profile default", "colima colima stop --profile default --force"]) + self.assertEqual(len(self.starts()), 1) + # Unbounded, each stand-in sleeps for 300 seconds. + self.assertLess(elapsed, 120) + self.assertGreaterEqual(len(self.hung_pids()), 5) + self.assert_all_ended() + + def test_a_hung_docker_with_a_guest_that_answers_fails_the_run_within_the_bound_and_stops_nothing(self) -> None: + self.listed_running() + self.state("docker-hangs", "guest-answers") + began = time.monotonic() + result = self.run_role(**self.HANG) + self.assertLess(time.monotonic() - began, 120) + self.assert_left_alone(result, "a command in the guest succeeded") + self.assert_all_ended() + + def test_a_hung_first_probe_counts_as_the_runtime_not_answering(self) -> None: + # A stopped profile whose Docker socket accepts and never replies: the start follows, as for a socket that refuses. + self.state("session", "docker-hangs") + began = time.monotonic() + result = self.run_role(**self.HANG) + self.assertLess(time.monotonic() - began, 120) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(self.starts()), 1) + self.assert_all_ended() + def test_check_mode_reads_everything_stops_nothing_and_says_what_a_real_run_would_do(self) -> None: self.listed_running() result = self.run_role("--check", **self.FAST) From b435bc17dababf14173f4e49b6fefe1c07b7938a Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 14 Nov 2026 16:27:21 +0000 Subject: [PATCH 4/7] test(github_runner_cluster): test the colima home against the context correctly A context whose endpoint is not a Colima socket is refused earlier, in the filter that reads the home from it, so the stop's own comparison is reached only when the home variable names another home than the context's socket. The test now sets exactly that, and the test of the variable winning over the context no longer asserts that the context is not read, since the comparison reads it. --- tests/unit/test_runtime_autostart.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_runtime_autostart.py b/tests/unit/test_runtime_autostart.py index 7b0d1dc8..1d49fb35 100644 --- a/tests/unit/test_runtime_autostart.py +++ b/tests/unit/test_runtime_autostart.py @@ -750,9 +750,10 @@ def test_an_agent_whose_age_cannot_be_read_does_not_count_as_old(self) -> None: self.listed_running(agent_age=None) self.assert_left_alone(self.run_role(**self.FAST), "could not be read") - def test_a_docker_context_that_points_at_another_socket_never_leads_to_a_stop(self) -> None: + def test_a_colima_home_that_is_not_where_the_context_points_never_leads_to_a_stop(self) -> None: + # The variable names another Colima home than the one the Docker context's socket is in: the probes measured one runtime and a stop would act on another's profile. (A context whose endpoint is no Colima socket at all fails earlier, in the filter that reads the home from it.) self.listed_running() - result = self.run_role(context_host=f"unix://{self.home}/.elsewhere/default/docker.sock", **self.FAST) + result = self.run_role(github_runner_cluster_colima_home="/elsewhere", **self.FAST) self.assert_left_alone(result, "does not point at") def test_with_the_stop_switched_off_a_confirmed_wedge_fails_naming_the_switch(self) -> None: @@ -1036,12 +1037,12 @@ def test_autostart_off_leaves_an_installed_daemon_and_agent_as_they_are(self) -> self.assertEqual((self.node_daemon.read_text(), self.node_agent.read_text()), ("", "")) self.assertEqual(self.changes(), []) - def test_the_variable_wins_over_the_context_and_the_context_is_not_read(self) -> None: + def test_the_variable_wins_over_the_context_for_the_home_colima_runs_in(self) -> None: + # The context is still read, since a stop of a silent profile compares its socket with the home; here the runtime is down and nothing is listed as running, so the comparison is never used. self.state("session") result = self.run_role(context="colima-node", github_runner_cluster_colima_home="/elsewhere") self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assert_colima_ran_in("/elsewhere") - self.assertFalse([call for call in self.calls() if " docker context " in call]) def test_a_profile_in_the_default_home_runs_with_no_variable(self) -> None: self.state("session") From 3d663c2074c63f7485837453d3c7d24e1f5dccda Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 14 Nov 2026 16:37:58 +0000 Subject: [PATCH 5/7] fix(github_runner_cluster): say how young a booting agent is, and let a started VM answer The report of a VM whose host agent is young now says "for less than" the grace period, matching the other wording of the age. The test stand-in for a restarted VM clears the hang modes of the stopped one, as a real start does. --- roles/github_runner_cluster/tasks/runtime_wedged.yml | 2 +- tests/unit/test_runtime_autostart.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/github_runner_cluster/tasks/runtime_wedged.yml b/roles/github_runner_cluster/tasks/runtime_wedged.yml index 8b58052b..71a2c3aa 100644 --- a/roles/github_runner_cluster/tasks/runtime_wedged.yml +++ b/roles/github_runner_cluster/tasks/runtime_wedged.yml @@ -128,7 +128,7 @@ Colima lists profile {{ github_runner_cluster_effective_runtime.colima_profile }} as running, but docker info failed on {{ github_runner_cluster_runtime_probe_attempts }} tries {{ github_runner_cluster_runtime_probe_delay }} seconds apart; a command in the guest {{ 'failed' if github_runner_cluster_runtime_guest.rc != 0 else 'succeeded' }}; the VM's host agent has been running {{ 'for more than ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds ago' if github_runner_cluster_runtime_agent_age | int > github_runner_cluster_runtime_boot_grace | int - else ('within the last ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds' if github_runner_cluster_runtime_agent_age | int >= 0 else 'for a time that could not be read') }}; + else ('for less than ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds' if github_runner_cluster_runtime_agent_age | int >= 0 else 'for a time that could not be read') }}; the Docker context {{ 'points at' if github_runner_cluster_runtime_context_socket_ok | bool else 'does not point at' }} the profile's own socket. - name: Say what a real run would do diff --git a/tests/unit/test_runtime_autostart.py b/tests/unit/test_runtime_autostart.py index 1d49fb35..98245026 100644 --- a/tests/unit/test_runtime_autostart.py +++ b/tests/unit/test_runtime_autostart.py @@ -435,6 +435,10 @@ def sysctl_file(key): if not exists("colima-listed-running"): touch("colima-listed-running") touch("runtime-up") + # A VM that is started again answers. + for file in ("docker-hangs", "guest-hangs"): + if exists(file): + os.remove(os.path.join(stub, file)) elif name == "ps": # The host agent as ps shows it, from the file agent-ps (" "); a pid that is not running prints nothing and fails. if not exists("agent-ps"): From e784165436eefb29091c7be5c5b08513fe906554 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sun, 15 Nov 2026 04:42:30 +0000 Subject: [PATCH 6/7] fix(github_runner_cluster): keep the stop's context check from failing hosts The Docker context is read for the Colima home only when no home is named, as before, so a host that names its home because it has no context that says (DOCKER_HOST, an empty or unknown context) runs again. The stop reads the context itself and may fail to: an unreadable context means the socket is not proven to be the profile's own, so nothing is stopped and the report says so. The supervisor ends the command's process group when it is itself sent TERM, INT or HUP and exits 143. Only exit 124 or the tool's own non-zero status counts as a failed probe; a supervisor that cannot start the command (127), is misused (255) or has no perl fails the run and names it. The deadline uses the monotonic clock, a command that ends in the last poll interval reports its own status, and the host agent is matched on the program's basename and its hostagent argument. --- roles/github_runner_cluster/README.md | 2 +- .../files/run_bounded.pl | 43 ++++-- roles/github_runner_cluster/tasks/runtime.yml | 3 +- .../tasks/runtime_probe.yml | 9 ++ .../tasks/runtime_wedged.yml | 56 ++++++-- tests/unit/test_runtime_autostart.py | 132 +++++++++++++++++- 6 files changed, 214 insertions(+), 31 deletions(-) diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 94435557..7172f3bc 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -166,7 +166,7 @@ On a Mac the container runtime is a Linux VM (Colima, or Docker Desktop) that st Which runtime a Mac runs is taken from the Docker context the role drives (see [Docker contexts](#docker-contexts)): `colima`, or `colima-` for another profile, is Colima, and `desktop-linux` is Docker Desktop. `github_runner_cluster_runtime` says so explicitly for a host whose context does not tell, for example one that uses `DOCKER_HOST`, and `github_runner_cluster_colima_profile` names the profile to start when the context does not. -**Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role starts the profile if it is stopped, with `colima start --profile --activate=false`, so the profile does not make its context the host's current one. A profile that `colima list` shows as running while its Docker runtime does not answer is not started again, because a start finds it running and does nothing. The role stops it, with `colima stop` and then `colima stop --force` only if that fails, and starts it again from the same disk, but only when `github_runner_cluster_runtime_force_stop_wedged` is on (the default) and all of these hold: `docker info` fails on `github_runner_cluster_runtime_probe_attempts` further tries (default 5), each given `github_runner_cluster_runtime_probe_timeout` seconds (default 20) and `github_runner_cluster_runtime_probe_delay` seconds apart (default 10); a command in the guest (`colima ssh -- true`) fails within the same timeout; the VM's host agent has been running for more than `github_runner_cluster_runtime_boot_grace` seconds (default 600), read from the elapsed time of the live process that the pid file in the profile's Lima instance directory names, and only when that process's command is Lima's host agent (a missing file, a dead pid, another program or an unreadable time leaves the age unknown, which never counts as old, whatever the file's date); and the Docker context the probes used points at that profile's own socket. Every probe, the guest command and the clean stop run under `files/run_bounded.pl`, which owns the deadline: it sends TERM to the command's process group when the time is up and KILL `github_runner_cluster_runtime_kill_grace` seconds later (default 5), so a program that ignores signals or never replies (docker and colima ignore `SIGALRM`) ends the try as failed and cannot hold the run; the first probe of the runtime is bounded the same way. A host that is only loaded, a Docker daemon that is restarting, a VM that is still booting and a context that points at another runtime each fail one of these, so nothing is stopped. A stop does not shut the guest down gracefully when the clean stop fails or times out (`github_runner_cluster_runtime_clean_stop_timeout`, default 120 seconds): every process in the guest, k3s and etcd included, ends, and the VM's disk, containers and volumes are kept, so on an etcd voter the member is out until the start completes. Switched off, or when a condition does not hold, the run fails with the signals it saw and the commands to recover by hand. With all of this in force the stop never follows a single failed or slow probe, never touches a VM whose guest answers or whose agent process is young or unknown, and never ends in a hang; that is why the switch is on by default. Under `--check` the reads are made, nothing is stopped and the report says what a real run would do. It installs the launchd job that starts it again after a reboot. There are two kinds, chosen by `github_runner_cluster_runtime_autostart_at` (`login`, `boot` or `auto`, the default). `auto` is `boot` for the profile named by `github_runner_cluster_runtime_boot_profile` (default `node`, the profile the bridged k3s node runs in) and `login` for any other, such as a `default` profile a host keeps for other workloads. A profile has one kind at a time: installing either retires the other's property list, unloading it first, so the profile is not started twice. +**Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role starts the profile if it is stopped, with `colima start --profile --activate=false`, so the profile does not make its context the host's current one. A profile that `colima list` shows as running while its Docker runtime does not answer is not started again, because a start finds it running and does nothing. The role stops it, with `colima stop` and then `colima stop --force` only if that fails, and starts it again from the same disk, but only when `github_runner_cluster_runtime_force_stop_wedged` is on (the default) and all of these hold: `docker info` fails on `github_runner_cluster_runtime_probe_attempts` further tries (default 5), each given `github_runner_cluster_runtime_probe_timeout` seconds (default 20) and `github_runner_cluster_runtime_probe_delay` seconds apart (default 10); a command in the guest (`colima ssh -- true`) fails within the same timeout; the VM's host agent has been running for more than `github_runner_cluster_runtime_boot_grace` seconds (default 600), read from the elapsed time of the live process that the pid file in the profile's Lima instance directory names, and only when the program that process runs is Lima's host agent (named limactl, with the argument hostagent; a missing file, a dead pid, another program or an unreadable time leaves the age unknown, which never counts as old, whatever the file's date); and the Docker context the probes used can be read and points at that profile's own socket. A host that names its Colima home with `github_runner_cluster_colima_home` because it has no context that says (it uses `DOCKER_HOST`, say) runs as before, but its stop is never proven and the run fails with the signals it saw instead of stopping anything. Every probe, the guest command and the clean stop run under `files/run_bounded.pl`, which owns the deadline: it sends TERM to the command's process group when the time is up and KILL `github_runner_cluster_runtime_kill_grace` seconds later (default 5), so a program that ignores signals or never replies (docker and colima ignore `SIGALRM`) ends the try as failed and cannot hold the run; the first probe of the runtime is bounded the same way. If the supervisor itself is signalled (TERM, INT or HUP) it ends the command's process group the same way and exits 143; KILL of the supervisor cannot be handled and leaves the command running. Only the exit status 124 (deadline) or the tool's own non-zero status counts as a failed try: a supervisor that cannot start the command (127), is misused (255) or is missing (no `perl` on the PATH the role uses) fails the run and names `files/run_bounded.pl`, and is never evidence about the VM. A host that is only loaded, a Docker daemon that is restarting, a VM that is still booting and a context that points at another runtime each fail one of these, so nothing is stopped. A stop does not shut the guest down gracefully when the clean stop fails or times out (`github_runner_cluster_runtime_clean_stop_timeout`, default 120 seconds): every process in the guest, k3s and etcd included, ends, and the VM's disk, containers and volumes are kept, so on an etcd voter the member is out until the start completes. Switched off, or when a condition does not hold, the run fails with the signals it saw and the commands to recover by hand. With all of this in force the stop never follows a single failed or slow probe, never touches a VM whose guest answers or whose agent process is young or unknown, and never ends in a hang; that is why the switch is on by default. Under `--check` the reads are made, nothing is stopped and the report says what a real run would do. It installs the launchd job that starts it again after a reboot. There are two kinds, chosen by `github_runner_cluster_runtime_autostart_at` (`login`, `boot` or `auto`, the default). `auto` is `boot` for the profile named by `github_runner_cluster_runtime_boot_profile` (default `node`, the profile the bridged k3s node runs in) and `login` for any other, such as a `default` profile a host keeps for other workloads. A profile has one kind at a time: installing either retires the other's property list, unloading it first, so the profile is not started twice. **At login (an agent).** The role: diff --git a/roles/github_runner_cluster/files/run_bounded.pl b/roles/github_runner_cluster/files/run_bounded.pl index e869167e..82c84c74 100644 --- a/roles/github_runner_cluster/files/run_bounded.pl +++ b/roles/github_runner_cluster/files/run_bounded.pl @@ -1,17 +1,19 @@ #!/usr/bin/perl # Runs a command with a deadline that binds whatever the command does with signals. Usage: run_bounded.pl DEADLINE GRACE COMMAND [ARGUMENT...] (both in seconds). # -# The command runs in a process group of its own. When DEADLINE passes, the group gets TERM; if any member is still there GRACE seconds later it gets KILL. A timer set with alarm before exec does not do this: a Go program such as docker or colima does not exit on SIGALRM, and a program may ignore TERM too. +# The command runs in a process group of its own. When DEADLINE passes, the group gets TERM; if any member is still there GRACE seconds later it gets KILL. A timer set with alarm before exec does not do this: a Go program such as docker or colima does not exit on SIGALRM, and a program may ignore TERM too. The deadline is measured on the monotonic clock, so a change of the system time does not move it. # -# The exit status is the command's own, 128 plus the signal number when a signal ended it, 124 when the deadline passed (as timeout(1) reports it) and 127 when the command could not be started. Its standard input, output and error are the caller's. +# The exit status is the command's own, 128 plus the signal number when a signal ended it, 124 when the deadline passed (as timeout(1) reports it), 127 when the command could not be started and 255 when this script was misused. 143 means this supervisor itself was sent TERM, INT or HUP: the command's group is then ended the same way (TERM, then KILL after GRACE) so that nothing is left running on its own. KILL of the supervisor cannot be handled and leaves the command running. The command's standard input, output and error are the caller's. use strict; use warnings; use POSIX qw(WNOHANG setsid); -use Time::HiRes qw(sleep time); +use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC sleep); my ($deadline, $grace, @command) = @ARGV; die "usage: run_bounded.pl DEADLINE GRACE COMMAND [ARGUMENT...]\n" unless @command && defined $grace; +sub now { return clock_gettime(CLOCK_MONOTONIC); } + my $child = fork(); die "fork: $!\n" unless defined $child; if ($child == 0) { @@ -25,28 +27,39 @@ sub status_of { return $status & 127 ? 128 + ($status & 127) : $status >> 8; } -# True once the child has ended, with its status in $main::status. +# True once the child has ended, with its status in $status. our $status; sub ended { + return 1 if defined $status; my $done = waitpid($child, WNOHANG); return 0 if $done == 0; $status = status_of($?); return 1; } -my $end = time + $deadline; -while (time < $end) { - exit $status if ended(); - sleep 0.05; +# Ends the child's group: TERM, then KILL when GRACE has passed or the leader is already gone. The group is killed whether or not its leader ended, so that nothing it started outlives it. +sub end_group { + kill 'TERM', -$child; + my $kill_at = now() + $grace; + while (now() < $kill_at) { + last if ended(); + sleep 0.05; + } + kill 'KILL', -$child; + waitpid($child, 0) unless defined $status; } -kill 'TERM', -$child; -my $kill_at = time + $grace; -while (time < $kill_at) { - last if ended(); +for my $name (qw(TERM INT HUP)) { + $SIG{$name} = sub { end_group(); exit 143; }; +} + +my $end = now() + $deadline; +while (now() < $end) { + exit $status if ended(); sleep 0.05; } -# The group is killed whether or not its leader ended, so that nothing it started outlives the deadline. -kill 'KILL', -$child; -waitpid($child, 0) unless defined $status; +# One last look, so that a command that ended in the final poll interval is not reported as timed out. +exit $status if ended(); + +end_group(); exit 124; diff --git a/roles/github_runner_cluster/tasks/runtime.yml b/roles/github_runner_cluster/tasks/runtime.yml index 0825d973..d0be7d30 100644 --- a/roles/github_runner_cluster/tasks/runtime.yml +++ b/roles/github_runner_cluster/tasks/runtime.yml @@ -31,7 +31,7 @@ | exadev.github_runner.cluster_container_runtime(github_runner_cluster_runtime, github_runner_cluster_colima_profile) }} - name: Read the Docker host of the Colima profile's context - # Where the profile lives: Colima points the context at //docker.sock. Needed for the Colima home when github_runner_cluster_colima_home does not say, and by runtime_wedged.yml to prove that a profile it would stop is the one the Docker probe measured. Read-only and answered by the context file alone, so it works with the VM stopped and under --check. It runs with the environment of every other docker call, so it reads the same context store. + # Where the profile lives: Colima points the context at //docker.sock. Needed only when github_runner_cluster_colima_home does not say, which is also why a host whose context has no such socket (one that uses DOCKER_HOST) sets the variable. Read-only and answered by the context file alone, so it works with the VM stopped and under --check. It runs with the environment of every other docker call, so it reads the same context store. ansible.builtin.command: argv: [docker, context, inspect, --format, "{{ '{{' }}.Endpoints.docker.Host{{ '}}' }}", "{{ github_runner_cluster_docker_cli_context }}"] environment: "{{ github_runner_cluster_docker_cli_environment | combine({'PATH': github_runner_cluster_runtime_path}) }}" @@ -40,6 +40,7 @@ check_mode: false when: - github_runner_cluster_effective_runtime.kind == 'colima' + - github_runner_cluster_colima_home | length == 0 - name: Work out the Colima home the profile lives in # The variable when a host sets it, else the one its Docker context names, empty for Colima's own default. Nothing outside Colima has one. diff --git a/roles/github_runner_cluster/tasks/runtime_probe.yml b/roles/github_runner_cluster/tasks/runtime_probe.yml index 76dbe6f3..72d27b36 100644 --- a/roles/github_runner_cluster/tasks/runtime_probe.yml +++ b/roles/github_runner_cluster/tasks/runtime_probe.yml @@ -20,6 +20,15 @@ failed_when: false check_mode: false +- name: Check that the supervisor could run docker info + # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a missing perl as exit 2 with the message "Error executing command". None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. + ansible.builtin.assert: + that: not (github_runner_cluster_runtime_probe.rc in [127, 255] or (github_runner_cluster_runtime_probe.rc == 2 and 'Error executing command' in (github_runner_cluster_runtime_probe.msg | default('')))) + fail_msg: >- + files/run_bounded.pl could not run docker info on {{ inventory_hostname }} (exit {{ github_runner_cluster_runtime_probe.rc }}): {{ github_runner_cluster_runtime_probe.msg | default(github_runner_cluster_runtime_probe.stderr | default('')) | trim }}. + This is not a verdict on the runtime. perl must be on the PATH the role uses (github_runner_cluster_runtime_path), and the command must exist. + quiet: true + - name: Publish whether the container runtime answers ansible.builtin.set_fact: github_runner_cluster_runtime_up: "{{ github_runner_cluster_runtime_probe.rc == 0 }}" diff --git a/roles/github_runner_cluster/tasks/runtime_wedged.yml b/roles/github_runner_cluster/tasks/runtime_wedged.yml index 71a2c3aa..76b386fe 100644 --- a/roles/github_runner_cluster/tasks/runtime_wedged.yml +++ b/roles/github_runner_cluster/tasks/runtime_wedged.yml @@ -4,12 +4,12 @@ # 1. github_runner_cluster_runtime_force_stop_wedged is on; # 2. docker info still fails on every one of github_runner_cluster_runtime_probe_attempts further tries, each given github_runner_cluster_runtime_probe_timeout seconds (files/run_bounded.pl ends one that has not finished, with TERM and then KILL after github_runner_cluster_runtime_kill_grace seconds, so a program that ignores signals cannot hold the run) and github_runner_cluster_runtime_probe_delay seconds apart, so a host that is only loaded or a dockerd that is restarting answers somewhere in that window and nothing is stopped; # 3. a command run in the guest (colima ssh -- true) fails within github_runner_cluster_runtime_probe_timeout seconds, so a guest that works while its Docker daemon does not is left alone; -# 4. the VM's host agent has been running for more than github_runner_cluster_runtime_boot_grace seconds, read from the elapsed time of the live process the pid file in the profile's Lima instance directory names (and only if its command is Lima's host agent), so a VM that is still booting is left alone. A missing file, a dead pid, another program or an unreadable time means the agent's age is unknown and does not count as old, whatever the file's date; -# 5. the Docker context the probes used points at the socket of the profile that would be stopped (//docker.sock), so a failing probe of one runtime never stops another's VM. +# 4. the VM's host agent has been running for more than github_runner_cluster_runtime_boot_grace seconds, read from the elapsed time of the live process the pid file in the profile's Lima instance directory names (and only if the program it runs is Lima's host agent, named by the basename limactl and the argument hostagent), so a VM that is still booting is left alone. A missing file, a dead pid, another program or an unreadable time means the agent's age is unknown and does not count as old, whatever the file's date; +# 5. the Docker context the probes used can be read and points at the socket of the profile that would be stopped (//docker.sock), so a failing probe of one runtime never stops another's VM and a context that cannot be read never proves anything. # # The stop is first a clean one (colima stop), given github_runner_cluster_runtime_clean_stop_timeout seconds, and a forced one (colima stop --force, which ends the VM without a graceful shutdown) only when that fails. Either way the VM's disk is kept and nothing is removed, so the start that follows boots it from the same disk. When any condition fails, or the switch is off, the run fails here and says which signals it saw and how to recover. Under --check every read is made and nothing is stopped; the report says what a real run would do. # -# Needs github_runner_cluster_runtime_environment, github_runner_cluster_runtime_colima_home and github_runner_cluster_runtime_context_host from runtime.yml. +# Needs github_runner_cluster_runtime_environment and github_runner_cluster_runtime_colima_home from runtime.yml; it reads the context's endpoint itself, because a host that names its Colima home may have no context to read. - name: Read whether Colima lists the profile as running # colima list prints one JSON object per profile, whatever profile is asked for. @@ -48,10 +48,20 @@ changed_when: false failed_when: false check_mode: false - until: github_runner_cluster_runtime_reprobe.rc == 0 + # A supervisor that cannot run ends the loop at once: the check below fails the run. + until: github_runner_cluster_runtime_reprobe.rc == 0 or github_runner_cluster_runtime_reprobe.rc in [127, 255] or (github_runner_cluster_runtime_reprobe.rc == 2 and 'Error executing command' in (github_runner_cluster_runtime_reprobe.msg | default(''))) retries: "{{ github_runner_cluster_runtime_probe_attempts | int - 1 }}" delay: "{{ github_runner_cluster_runtime_probe_delay }}" + - name: Check that the supervisor could run docker info + # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a missing perl as exit 2 with the message "Error executing command". None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. + ansible.builtin.assert: + that: not (github_runner_cluster_runtime_reprobe.rc in [127, 255] or (github_runner_cluster_runtime_reprobe.rc == 2 and 'Error executing command' in (github_runner_cluster_runtime_reprobe.msg | default('')))) + fail_msg: >- + files/run_bounded.pl could not run docker info on {{ inventory_hostname }} (exit {{ github_runner_cluster_runtime_reprobe.rc }}): {{ github_runner_cluster_runtime_reprobe.msg | default(github_runner_cluster_runtime_reprobe.stderr | default('')) | trim }}. + This is not a verdict on the runtime. perl must be on the PATH the role uses (github_runner_cluster_runtime_path), and the command must exist. + quiet: true + - name: Note that the runtime answered after all ansible.builtin.set_fact: github_runner_cluster_runtime_up: true @@ -80,6 +90,25 @@ failed_when: false check_mode: false + - name: Check that the supervisor could run a command in the guest + # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a missing perl as exit 2 with the message "Error executing command". None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. + ansible.builtin.assert: + that: not (github_runner_cluster_runtime_guest.rc in [127, 255] or (github_runner_cluster_runtime_guest.rc == 2 and 'Error executing command' in (github_runner_cluster_runtime_guest.msg | default('')))) + fail_msg: >- + files/run_bounded.pl could not run a command in the guest on {{ inventory_hostname }} (exit {{ github_runner_cluster_runtime_guest.rc }}): {{ github_runner_cluster_runtime_guest.msg | default(github_runner_cluster_runtime_guest.stderr | default('')) | trim }}. + This is not a verdict on the runtime. perl must be on the PATH the role uses (github_runner_cluster_runtime_path), and the command must exist. + quiet: true + + - name: Read the Docker host of the context the probes used + # On its own and allowed to fail: a host whose context says nothing about a Colima socket (DOCKER_HOST, an empty or unknown context) sets github_runner_cluster_colima_home and may have no context to read. An unreadable context means the socket is not proven to be this profile's, so nothing is stopped. + ansible.builtin.command: + argv: [docker, context, inspect, --format, "{{ '{{' }}.Endpoints.docker.Host{{ '}}' }}", "{{ github_runner_cluster_docker_cli_context }}"] + environment: "{{ github_runner_cluster_docker_cli_environment | combine({'PATH': github_runner_cluster_runtime_path}) }}" + register: github_runner_cluster_runtime_context_endpoint + changed_when: false + failed_when: false + check_mode: false + - name: Find the VM's host agent pid file # Colima keeps one Lima instance per profile under /_lima: colima for the default profile, colima- for another. ansible.builtin.slurp: @@ -106,7 +135,7 @@ ansible.builtin.set_fact: github_runner_cluster_runtime_agent_age: >- {%- set words = github_runner_cluster_runtime_agent_words -%} - {%- if github_runner_cluster_runtime_agent_process.rc == 0 and words | length == 2 and 'hostagent' in words[1] and words[0] is match('^([0-9]+-)?([0-9]+:)?[0-9]+:[0-9]+$') -%} + {%- if github_runner_cluster_runtime_agent_process.rc == 0 and words | length == 2 and words[1].split()[0].split('/')[-1] == 'limactl' and 'hostagent' in words[1].split()[1:] and words[0] is match('^([0-9]+-)?([0-9]+:)?[0-9]+:[0-9]+$') -%} {%- set days_and_rest = words[0].split('-') -%} {%- set days = days_and_rest[0] | int if days_and_rest | length > 1 else 0 -%} {%- set parts = (days_and_rest[-1].split(':') | map('int') | list) -%} @@ -114,7 +143,8 @@ {{ days * 86400 + clock }} {%- else -%}-1{%- endif -%} github_runner_cluster_runtime_context_socket_ok: >- - {{ github_runner_cluster_runtime_context_host.stdout | trim + {{ github_runner_cluster_runtime_context_endpoint.rc == 0 + and github_runner_cluster_runtime_context_endpoint.stdout | trim == 'unix://' ~ (github_runner_cluster_runtime_colima_home if github_runner_cluster_runtime_colima_home | length > 0 else ansible_facts['env']['HOME'] ~ '/.colima') ~ '/' ~ github_runner_cluster_effective_runtime.colima_profile ~ '/docker.sock' }} @@ -127,9 +157,9 @@ github_runner_cluster_runtime_wedged_report: >- Colima lists profile {{ github_runner_cluster_effective_runtime.colima_profile }} as running, but docker info failed on {{ github_runner_cluster_runtime_probe_attempts }} tries {{ github_runner_cluster_runtime_probe_delay }} seconds apart; a command in the guest {{ 'failed' if github_runner_cluster_runtime_guest.rc != 0 else 'succeeded' }}; - the VM's host agent has been running {{ 'for more than ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds ago' if github_runner_cluster_runtime_agent_age | int > github_runner_cluster_runtime_boot_grace | int + the VM's host agent has been running {{ 'for more than ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds' if github_runner_cluster_runtime_agent_age | int > github_runner_cluster_runtime_boot_grace | int else ('for less than ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds' if github_runner_cluster_runtime_agent_age | int >= 0 else 'for a time that could not be read') }}; - the Docker context {{ 'points at' if github_runner_cluster_runtime_context_socket_ok | bool else 'does not point at' }} the profile's own socket. + the Docker context {{ 'points at the socket of the profile' if github_runner_cluster_runtime_context_socket_ok | bool else ('could not be read, so its socket is not proven to belong to the profile' if github_runner_cluster_runtime_context_endpoint.rc != 0 else 'does not point at the socket of the profile') }}. - name: Say what a real run would do ansible.builtin.debug: @@ -171,6 +201,16 @@ failed_when: false when: not ansible_check_mode + - name: Check that the supervisor could run colima stop + # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a missing perl as exit 2 with the message "Error executing command". None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. + ansible.builtin.assert: + that: not (github_runner_cluster_runtime_clean_stop.rc in [127, 255] or (github_runner_cluster_runtime_clean_stop.rc == 2 and 'Error executing command' in (github_runner_cluster_runtime_clean_stop.msg | default('')))) + fail_msg: >- + files/run_bounded.pl could not run colima stop on {{ inventory_hostname }} (exit {{ github_runner_cluster_runtime_clean_stop.rc }}): {{ github_runner_cluster_runtime_clean_stop.msg | default(github_runner_cluster_runtime_clean_stop.stderr | default('')) | trim }}. + This is not a verdict on the runtime. perl must be on the PATH the role uses (github_runner_cluster_runtime_path), and the command must exist. + quiet: true + when: not ansible_check_mode + - name: Stop the wedged profile without a graceful shutdown, since the clean stop failed # colima stop --force ends the VM without a graceful shutdown (colima stop --help: "stop without graceful shutdown"): every process in the guest, k3s and etcd included, is killed. The VM's disk and everything on it are kept. ansible.builtin.command: diff --git a/tests/unit/test_runtime_autostart.py b/tests/unit/test_runtime_autostart.py index 98245026..f97bb407 100644 --- a/tests/unit/test_runtime_autostart.py +++ b/tests/unit/test_runtime_autostart.py @@ -10,6 +10,7 @@ import pwd import re import shutil +import signal import subprocess import sys import tempfile @@ -318,13 +319,17 @@ def need_root(): if os.environ.get("RUNTIME_ROOT") != "1": sys.exit("launchctl: Operation not permitted while changing the system domain as a user") if name == "docker" and args[:2] == ["context", "inspect"]: - # Read from the context file alone, so it answers with the daemon down. + # Read from the context file alone, so it answers with the daemon down. An empty context name or an unknown one is an error, as docker's is. + if args[-1] == "" or exists("context-unknown"): + sys.exit("context " + repr(args[-1]) + ": context not found") if not exists("context-host"): sys.exit("context inspect: no context-host file") print(open(os.path.join(stub, "context-host")).read(), end="") elif name == "docker": if args[:1] == ["info"] and exists("docker-hangs"): hang() + if args[:1] == ["info"] and exists("docker-exit"): + sys.exit(int(open(os.path.join(stub, "docker-exit")).read())) if not exists("runtime-up") and args[:1] == ["info"] and exists("docker-answers-after"): # A daemon that fails its first tries and then answers, as one that is restarting or a loaded host does. counter = os.path.join(stub, "docker-info-failures") @@ -423,7 +428,7 @@ def sysctl_file(key): hang() if args[3:] == [] and exists("clean-stop-fails"): sys.exit("colima: the VM does not stop") - for file in ("colima-listed-running", "runtime-up", "docker-hangs", "guest-hangs"): + for file in ("colima-listed-running", "runtime-up", "docker-hangs", "guest-hangs", "docker-exit"): if exists(file): os.remove(os.path.join(stub, file)) elif name == "colima": @@ -436,7 +441,7 @@ def sysctl_file(key): touch("colima-listed-running") touch("runtime-up") # A VM that is started again answers. - for file in ("docker-hangs", "guest-hangs"): + for file in ("docker-hangs", "guest-hangs", "docker-exit"): if exists(file): os.remove(os.path.join(stub, file)) elif name == "ps": @@ -524,6 +529,39 @@ def test_what_the_command_started_does_not_outlive_the_deadline(self) -> None: time.sleep(0.1) self.fail("a process the command started is still running") + def test_a_supervisor_that_is_signalled_ends_its_command_and_what_it_started(self) -> None: + # Ansible or a person killing the supervisor must not leave the hung command (a colima stop mid-shutdown, say) running on its own. + for signal_name in ("TERM", "INT", "HUP"): + with self.subTest(signal=signal_name): + marker = Path(tempfile.mkdtemp(prefix="bounded-")) / "pids" + self.addCleanup(shutil.rmtree, marker.parent, True) + script = f"trap '' TERM INT HUP; sleep 60 & echo $$ $! > {marker}; wait" + supervisor = subprocess.Popen(["perl", str(self.SCRIPT), "60", "1", "sh", "-c", script]) + for _ in range(100): + if marker.exists() and marker.read_text().strip(): + break + time.sleep(0.05) + pids = [int(word) for word in marker.read_text().split()] + os.kill(supervisor.pid, getattr(signal, "SIG" + signal_name)) + self.assertEqual(supervisor.wait(timeout=30), 143) + for pid in pids: + for _ in range(50): + try: + os.kill(pid, 0) + except ProcessLookupError: + break + time.sleep(0.1) + else: + self.fail(f"process {pid} survived the {signal_name} sent to the supervisor") + + def test_a_command_that_ends_just_as_the_deadline_passes_reports_its_own_status(self) -> None: + # The deadline is checked after one last look at the child, so a command that has already ended is not reported as timed out. + result, _ = self.run_bounded(0.2, 1, "sh", "-c", "sleep 0.1; exit 5") + self.assertEqual(result.returncode, 5) + + def test_the_deadline_is_measured_on_the_monotonic_clock(self) -> None: + self.assertIn("CLOCK_MONOTONIC", self.SCRIPT.read_text()) + def test_a_command_that_cannot_be_started_exits_127(self) -> None: result, _ = self.run_bounded(5, 1, "/nonexistent/command") self.assertEqual(result.returncode, 127) @@ -635,6 +673,15 @@ def test_a_stopped_colima_is_started_and_the_agent_installed_and_loaded(self) -> FAST = {"github_runner_cluster_runtime_probe_attempts": 3, "github_runner_cluster_runtime_probe_delay": 0, "github_runner_cluster_runtime_probe_timeout": 5} + def forget(self) -> None: + """Start a sub-case from a clean stub directory and a clean home.""" + for entry in self.stub.iterdir(): + if entry.is_dir(): + shutil.rmtree(entry) + elif entry.name not in ("context-host", "github-runner-k3s.state"): + entry.unlink() + shutil.rmtree(self.home / ".colima", ignore_errors=True) + def stops(self) -> list[str]: return [call for call in self.calls() if re.match(r"\S+ colima stop", call)] @@ -744,6 +791,78 @@ def test_a_pid_file_naming_another_program_leaves_the_age_unknown(self) -> None: self.listed_running(command="/usr/sbin/cron -f") self.assert_left_alone(self.run_role(**self.FAST), "could not be read") + def test_a_pid_file_naming_a_program_whose_path_merely_contains_hostagent_leaves_the_age_unknown(self) -> None: + self.listed_running(command="/opt/hostagent/bin/cron -f") + self.assert_left_alone(self.run_role(**self.FAST), "could not be read") + + def test_a_report_of_an_old_agent_does_not_say_ago_twice(self) -> None: + self.listed_running(command="/usr/local/bin/limactl hostagent x", agent_age=7200) + self.state("guest-answers") + result = self.run_role(**self.FAST) + self.assertIn("for more than 600 seconds;", result.stdout) + self.assertNotIn("seconds ago", result.stdout) + + # The Docker context, which the stop reads for its socket check on its own. + + def test_a_host_that_names_its_colima_home_and_has_no_readable_context_still_runs(self) -> None: + # The README's route for a host whose context does not say where the profile is (DOCKER_HOST, say): the variable gives the home and the context cannot be read. Nothing is listed as running, so the play starts the profile as before. + for label, variables in (("an empty context", {"context": ""}), ("an unknown context", {})): + with self.subTest(label=label): + self.forget() + self.state("session") + if not variables: + self.state("context-unknown") + result = self.run_role(github_runner_cluster_colima_home="/elsewhere", github_runner_cluster_runtime="colima", **variables) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assert_colima_ran_in("/elsewhere") + self.assertEqual(self.stops(), []) + + def test_a_context_that_cannot_be_read_never_leads_to_a_stop_and_the_report_says_so(self) -> None: + for label, variables in (("an empty context", {"context": ""}), ("an unknown context", {"unknown": True})): + with self.subTest(label=label): + self.forget() + self.listed_running() + if variables.get("unknown"): + self.state("context-unknown") + result = self.run_role(github_runner_cluster_colima_home=f"{self.home}/.colima", github_runner_cluster_runtime="colima", **{k: v for k, v in variables.items() if k != "unknown"}, **self.FAST) + self.assert_left_alone(result, "so its socket is not proven to belong to the profile") + + def test_a_context_whose_endpoint_is_another_runtime_never_leads_to_a_stop(self) -> None: + for endpoint in ("tcp://remote.example.invalid:2375", "unix:///var/run/docker.sock"): + with self.subTest(endpoint=endpoint): + self.forget() + self.listed_running() + result = self.run_role(context_host=endpoint, github_runner_cluster_colima_home=f"{self.home}/.colima", **self.FAST) + self.assert_left_alone(result, "does not point at the socket of the profile") + + # The supervisor failing is not a failed probe. + + def test_a_supervisor_that_cannot_run_the_command_fails_the_run_naming_it_and_is_not_evidence(self) -> None: + for code in (127, 255): + with self.subTest(code=code): + self.forget() + self.listed_running() + (self.stub / "docker-exit").write_text(str(code)) + result = self.run_role(**self.FAST) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("run_bounded.pl", result.stdout) + self.assertEqual(self.stops(), []) + + def test_a_host_without_perl_fails_the_run_naming_the_supervisor(self) -> None: + self.listed_running() + result = self.run_role(github_runner_cluster_runtime_path=str(self.bin), **self.FAST) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("run_bounded.pl", result.stdout) + self.assertEqual(self.stops(), []) + + def test_a_tool_that_fails_by_itself_still_counts_as_a_failed_probe(self) -> None: + # docker exiting 1 (cannot connect) is the daemon's answer, not the supervisor's. + self.listed_running() + (self.stub / "docker-exit").write_text("1") + result = self.run_role(**self.FAST) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(self.stops()), 1) + def test_an_agent_that_has_run_for_days_counts_as_old(self) -> None: self.listed_running(agent_age=3 * 86400 + 3600 + 61) result = self.run_role(**self.FAST) @@ -758,7 +877,7 @@ def test_a_colima_home_that_is_not_where_the_context_points_never_leads_to_a_sto # The variable names another Colima home than the one the Docker context's socket is in: the probes measured one runtime and a stop would act on another's profile. (A context whose endpoint is no Colima socket at all fails earlier, in the filter that reads the home from it.) self.listed_running() result = self.run_role(github_runner_cluster_colima_home="/elsewhere", **self.FAST) - self.assert_left_alone(result, "does not point at") + self.assert_left_alone(result, "does not point at the socket of the profile") def test_with_the_stop_switched_off_a_confirmed_wedge_fails_naming_the_switch(self) -> None: self.listed_running() @@ -1041,12 +1160,13 @@ def test_autostart_off_leaves_an_installed_daemon_and_agent_as_they_are(self) -> self.assertEqual((self.node_daemon.read_text(), self.node_agent.read_text()), ("", "")) self.assertEqual(self.changes(), []) - def test_the_variable_wins_over_the_context_for_the_home_colima_runs_in(self) -> None: - # The context is still read, since a stop of a silent profile compares its socket with the home; here the runtime is down and nothing is listed as running, so the comparison is never used. + def test_the_variable_wins_over_the_context_and_the_context_is_not_read(self) -> None: + # A profile that is stopped never reaches the stop's own read of the context, so the home comes from the variable alone. self.state("session") result = self.run_role(context="colima-node", github_runner_cluster_colima_home="/elsewhere") self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assert_colima_ran_in("/elsewhere") + self.assertFalse([call for call in self.calls() if " docker context " in call]) def test_a_profile_in_the_default_home_runs_with_no_variable(self) -> None: self.state("session") From bf1bc1ac42b133266450e0a9383a78a97e4d3e68 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sun, 15 Nov 2026 05:24:30 +0000 Subject: [PATCH 7/7] fix(github_runner_cluster): recognise a supervisor that cannot run by its message When the command module cannot start perl it reports "Error executing command" with the operating system's error number as rc (2 for a missing file, 13 for a permission error) or 257, so keying on one rc missed all but the missing file. A filter, cluster_supervisor_failed, now says whether a supervised command's result shows that the supervisor could not run: the supervisor's own 127 and 255, or that message with any rc. The probe, the guest command and the stop all use it, in place of five copies of an expression. The test of a command that ends before its deadline is labelled as the regression guard it is. --- plugins/filter/container_runtime.py | 23 +++++++++++++-- .../tasks/runtime_probe.yml | 4 +-- .../tasks/runtime_wedged.yml | 14 +++++----- tests/unit/test_container_runtime.py | 28 +++++++++++++++++++ tests/unit/test_runtime_autostart.py | 19 +++++++++++-- 5 files changed, 74 insertions(+), 14 deletions(-) diff --git a/plugins/filter/container_runtime.py b/plugins/filter/container_runtime.py index 11876ebb..3c3f4242 100644 --- a/plugins/filter/container_runtime.py +++ b/plugins/filter/container_runtime.py @@ -98,9 +98,28 @@ def cluster_colima_home(endpoint: Any, profile: Any, user_home: Any) -> str: return colima_home +SUPERVISOR_FAILURE_STATUSES = (127, 255) +SUPERVISOR_FAILURE_MESSAGE = "Error executing command" + + +def cluster_supervisor_failed(result: Any) -> bool: + """Say whether the registered result of a command run under ``files/run_bounded.pl`` shows that the supervisor itself could not run, as opposed to the supervised command failing or timing out. + + The supervisor exits 127 when it cannot start the command and 255 when it is misused (and the command module's own exit 127 is the same case for a missing interpreter). When the command module cannot start ``perl`` at all (not on the PATH, not executable) it reports the failure with the message ``Error executing command`` and an ``rc`` that is the OS error number (2 for a missing file, 13 for a permission error) or 257, so the message is what identifies it, whatever the ``rc``. + + :param result: The registered result of the command task: a mapping with ``rc`` and, when the module could not start the command, ``msg``. + :returns: True when the run says nothing about the command, so it must fail the play and never count as the command failing. + :raises AnsibleFilterError: the value is not a mapping. + """ + if not isinstance(result, dict): + raise AnsibleFilterError(f"The result of a supervised command must be a mapping, not {type(result).__name__}") + message = result.get("msg") + return result.get("rc") in SUPERVISOR_FAILURE_STATUSES or (isinstance(message, str) and SUPERVISOR_FAILURE_MESSAGE in message) + + class FilterModule: - """Registers the container runtime filter with Ansible.""" + """Registers the container runtime filters with Ansible.""" def filters(self) -> dict[str, Any]: """Return the filters this plugin provides.""" - return {"cluster_container_runtime": cluster_container_runtime, "cluster_colima_home": cluster_colima_home} + return {"cluster_container_runtime": cluster_container_runtime, "cluster_colima_home": cluster_colima_home, "cluster_supervisor_failed": cluster_supervisor_failed} diff --git a/roles/github_runner_cluster/tasks/runtime_probe.yml b/roles/github_runner_cluster/tasks/runtime_probe.yml index 72d27b36..2dd0226b 100644 --- a/roles/github_runner_cluster/tasks/runtime_probe.yml +++ b/roles/github_runner_cluster/tasks/runtime_probe.yml @@ -21,9 +21,9 @@ check_mode: false - name: Check that the supervisor could run docker info - # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a missing perl as exit 2 with the message "Error executing command". None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. + # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a perl it cannot start (missing, not executable) with the message "Error executing command" and an rc that is an OS error number. exadev.github_runner.cluster_supervisor_failed says which of these a result is. None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. ansible.builtin.assert: - that: not (github_runner_cluster_runtime_probe.rc in [127, 255] or (github_runner_cluster_runtime_probe.rc == 2 and 'Error executing command' in (github_runner_cluster_runtime_probe.msg | default('')))) + that: not (github_runner_cluster_runtime_probe | exadev.github_runner.cluster_supervisor_failed) fail_msg: >- files/run_bounded.pl could not run docker info on {{ inventory_hostname }} (exit {{ github_runner_cluster_runtime_probe.rc }}): {{ github_runner_cluster_runtime_probe.msg | default(github_runner_cluster_runtime_probe.stderr | default('')) | trim }}. This is not a verdict on the runtime. perl must be on the PATH the role uses (github_runner_cluster_runtime_path), and the command must exist. diff --git a/roles/github_runner_cluster/tasks/runtime_wedged.yml b/roles/github_runner_cluster/tasks/runtime_wedged.yml index 76b386fe..5b533939 100644 --- a/roles/github_runner_cluster/tasks/runtime_wedged.yml +++ b/roles/github_runner_cluster/tasks/runtime_wedged.yml @@ -49,14 +49,14 @@ failed_when: false check_mode: false # A supervisor that cannot run ends the loop at once: the check below fails the run. - until: github_runner_cluster_runtime_reprobe.rc == 0 or github_runner_cluster_runtime_reprobe.rc in [127, 255] or (github_runner_cluster_runtime_reprobe.rc == 2 and 'Error executing command' in (github_runner_cluster_runtime_reprobe.msg | default(''))) + until: github_runner_cluster_runtime_reprobe.rc == 0 or github_runner_cluster_runtime_reprobe | exadev.github_runner.cluster_supervisor_failed retries: "{{ github_runner_cluster_runtime_probe_attempts | int - 1 }}" delay: "{{ github_runner_cluster_runtime_probe_delay }}" - name: Check that the supervisor could run docker info - # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a missing perl as exit 2 with the message "Error executing command". None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. + # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a perl it cannot start (missing, not executable) with the message "Error executing command" and an rc that is an OS error number. exadev.github_runner.cluster_supervisor_failed says which of these a result is. None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. ansible.builtin.assert: - that: not (github_runner_cluster_runtime_reprobe.rc in [127, 255] or (github_runner_cluster_runtime_reprobe.rc == 2 and 'Error executing command' in (github_runner_cluster_runtime_reprobe.msg | default('')))) + that: not (github_runner_cluster_runtime_reprobe | exadev.github_runner.cluster_supervisor_failed) fail_msg: >- files/run_bounded.pl could not run docker info on {{ inventory_hostname }} (exit {{ github_runner_cluster_runtime_reprobe.rc }}): {{ github_runner_cluster_runtime_reprobe.msg | default(github_runner_cluster_runtime_reprobe.stderr | default('')) | trim }}. This is not a verdict on the runtime. perl must be on the PATH the role uses (github_runner_cluster_runtime_path), and the command must exist. @@ -91,9 +91,9 @@ check_mode: false - name: Check that the supervisor could run a command in the guest - # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a missing perl as exit 2 with the message "Error executing command". None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. + # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a perl it cannot start (missing, not executable) with the message "Error executing command" and an rc that is an OS error number. exadev.github_runner.cluster_supervisor_failed says which of these a result is. None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. ansible.builtin.assert: - that: not (github_runner_cluster_runtime_guest.rc in [127, 255] or (github_runner_cluster_runtime_guest.rc == 2 and 'Error executing command' in (github_runner_cluster_runtime_guest.msg | default('')))) + that: not (github_runner_cluster_runtime_guest | exadev.github_runner.cluster_supervisor_failed) fail_msg: >- files/run_bounded.pl could not run a command in the guest on {{ inventory_hostname }} (exit {{ github_runner_cluster_runtime_guest.rc }}): {{ github_runner_cluster_runtime_guest.msg | default(github_runner_cluster_runtime_guest.stderr | default('')) | trim }}. This is not a verdict on the runtime. perl must be on the PATH the role uses (github_runner_cluster_runtime_path), and the command must exist. @@ -202,9 +202,9 @@ when: not ansible_check_mode - name: Check that the supervisor could run colima stop - # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a missing perl as exit 2 with the message "Error executing command". None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. + # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a perl it cannot start (missing, not executable) with the message "Error executing command" and an rc that is an OS error number. exadev.github_runner.cluster_supervisor_failed says which of these a result is. None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. ansible.builtin.assert: - that: not (github_runner_cluster_runtime_clean_stop.rc in [127, 255] or (github_runner_cluster_runtime_clean_stop.rc == 2 and 'Error executing command' in (github_runner_cluster_runtime_clean_stop.msg | default('')))) + that: not (github_runner_cluster_runtime_clean_stop | exadev.github_runner.cluster_supervisor_failed) fail_msg: >- files/run_bounded.pl could not run colima stop on {{ inventory_hostname }} (exit {{ github_runner_cluster_runtime_clean_stop.rc }}): {{ github_runner_cluster_runtime_clean_stop.msg | default(github_runner_cluster_runtime_clean_stop.stderr | default('')) | trim }}. This is not a verdict on the runtime. perl must be on the PATH the role uses (github_runner_cluster_runtime_path), and the command must exist. diff --git a/tests/unit/test_container_runtime.py b/tests/unit/test_container_runtime.py index b55cdb53..8de6d3e7 100644 --- a/tests/unit/test_container_runtime.py +++ b/tests/unit/test_container_runtime.py @@ -14,6 +14,7 @@ runtime = container_runtime.cluster_container_runtime colima_home = container_runtime.cluster_colima_home +supervisor_failed = container_runtime.cluster_supervisor_failed AnsibleFilterError = container_runtime.AnsibleFilterError HOME = "/Users/example" @@ -89,6 +90,7 @@ def test_a_value_that_is_not_a_string_is_an_error(self) -> None: def test_the_filter_is_registered(self) -> None: self.assertIs(container_runtime.FilterModule().filters()["cluster_container_runtime"], runtime) self.assertIs(container_runtime.FilterModule().filters()["cluster_colima_home"], colima_home) + self.assertIs(container_runtime.FilterModule().filters()["cluster_supervisor_failed"], supervisor_failed) class ColimaHomeTest(unittest.TestCase): @@ -133,3 +135,29 @@ def test_a_value_that_is_not_a_string_is_refused(self) -> None: if __name__ == "__main__": unittest.main() + + +class SupervisorFailedTest(unittest.TestCase): + def test_the_supervisors_own_statuses_say_it_could_not_run_the_command(self) -> None: + for rc in (127, 255): + with self.subTest(rc=rc): + self.assertTrue(supervisor_failed({"rc": rc, "stdout": "", "stderr": ""})) + + def test_the_command_module_failing_to_start_perl_does_so_whatever_the_rc(self) -> None: + # rc is the OS error number (2 missing, 13 not executable) or 257 for another OS error; the message is what they share. + for rc in (2, 13, 257, 1): + with self.subTest(rc=rc): + self.assertTrue(supervisor_failed({"rc": rc, "msg": "Error executing command", "cmd": ["perl"]})) + + def test_a_supervised_command_that_failed_or_timed_out_is_not_a_supervisor_failure(self) -> None: + for rc in (0, 1, 2, 13, 124, 137, 143): + with self.subTest(rc=rc): + self.assertFalse(supervisor_failed({"rc": rc, "stdout": "", "stderr": "Cannot connect to the Docker daemon"})) + + def test_a_message_that_is_not_the_start_failure_does_not_count(self) -> None: + self.assertFalse(supervisor_failed({"rc": 1, "msg": "non-zero return code"})) + + def test_a_value_that_is_not_a_result_is_refused(self) -> None: + for value in (None, "rc", 127, ["rc"]): + with self.subTest(value=value), self.assertRaises(AnsibleFilterError): + supervisor_failed(value) diff --git a/tests/unit/test_runtime_autostart.py b/tests/unit/test_runtime_autostart.py index f97bb407..0c6b749c 100644 --- a/tests/unit/test_runtime_autostart.py +++ b/tests/unit/test_runtime_autostart.py @@ -554,10 +554,11 @@ def test_a_supervisor_that_is_signalled_ends_its_command_and_what_it_started(sel else: self.fail(f"process {pid} survived the {signal_name} sent to the supervisor") - def test_a_command_that_ends_just_as_the_deadline_passes_reports_its_own_status(self) -> None: - # The deadline is checked after one last look at the child, so a command that has already ended is not reported as timed out. - result, _ = self.run_bounded(0.2, 1, "sh", "-c", "sleep 0.1; exit 5") + def test_regression_guard_a_command_that_ends_before_the_deadline_reports_its_own_status(self) -> None: + # A guard only: it passes on a supervisor with or without the final look at the child before the timeout path, because making a command end in the last poll interval is a race against the scheduler, which a test on a loaded runner cannot win reliably. The final look is read from the script instead. + result, _ = self.run_bounded(1, 1, "sh", "-c", "sleep 0.1; exit 5") self.assertEqual(result.returncode, 5) + self.assertIn("one last look", self.SCRIPT.read_text().lower()) def test_the_deadline_is_measured_on_the_monotonic_clock(self) -> None: self.assertIn("CLOCK_MONOTONIC", self.SCRIPT.read_text()) @@ -855,6 +856,18 @@ def test_a_host_without_perl_fails_the_run_naming_the_supervisor(self) -> None: self.assertIn("run_bounded.pl", result.stdout) self.assertEqual(self.stops(), []) + def test_a_perl_that_is_not_executable_fails_the_run_naming_the_supervisor(self) -> None: + # The command module reports this with the OS error number as its rc (13), not 2: the message identifies it. + self.listed_running() + perl_dir = self.work / "no-perl" + perl_dir.mkdir() + (perl_dir / "perl").write_text("#!/bin/sh\n") + (perl_dir / "perl").chmod(0o644) + result = self.run_role(github_runner_cluster_runtime_path=f"{perl_dir}{os.pathsep}{self.bin}", **self.FAST) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("run_bounded.pl", result.stdout) + self.assertEqual(self.stops(), []) + def test_a_tool_that_fails_by_itself_still_counts_as_a_failed_probe(self) -> None: # docker exiting 1 (cannot connect) is the daemon's answer, not the supervisor's. self.listed_running()