diff --git a/plugins/filter/container_runtime.py b/plugins/filter/container_runtime.py index 11876ebb..3c3f4242 100644 --- a/plugins/filter/container_runtime.py +++ b/plugins/filter/container_runtime.py @@ -98,9 +98,28 @@ def cluster_colima_home(endpoint: Any, profile: Any, user_home: Any) -> str: return colima_home +SUPERVISOR_FAILURE_STATUSES = (127, 255) +SUPERVISOR_FAILURE_MESSAGE = "Error executing command" + + +def cluster_supervisor_failed(result: Any) -> bool: + """Say whether the registered result of a command run under ``files/run_bounded.pl`` shows that the supervisor itself could not run, as opposed to the supervised command failing or timing out. + + The supervisor exits 127 when it cannot start the command and 255 when it is misused (and the command module's own exit 127 is the same case for a missing interpreter). When the command module cannot start ``perl`` at all (not on the PATH, not executable) it reports the failure with the message ``Error executing command`` and an ``rc`` that is the OS error number (2 for a missing file, 13 for a permission error) or 257, so the message is what identifies it, whatever the ``rc``. + + :param result: The registered result of the command task: a mapping with ``rc`` and, when the module could not start the command, ``msg``. + :returns: True when the run says nothing about the command, so it must fail the play and never count as the command failing. + :raises AnsibleFilterError: the value is not a mapping. + """ + if not isinstance(result, dict): + raise AnsibleFilterError(f"The result of a supervised command must be a mapping, not {type(result).__name__}") + message = result.get("msg") + return result.get("rc") in SUPERVISOR_FAILURE_STATUSES or (isinstance(message, str) and SUPERVISOR_FAILURE_MESSAGE in message) + + class FilterModule: - """Registers the container runtime filter with Ansible.""" + """Registers the container runtime filters with Ansible.""" def filters(self) -> dict[str, Any]: """Return the filters this plugin provides.""" - return {"cluster_container_runtime": cluster_container_runtime, "cluster_colima_home": cluster_colima_home} + return {"cluster_container_runtime": cluster_container_runtime, "cluster_colima_home": cluster_colima_home, "cluster_supervisor_failed": cluster_supervisor_failed} diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index f4baba59..7172f3bc 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -166,7 +166,7 @@ On a Mac the container runtime is a Linux VM (Colima, or Docker Desktop) that st Which runtime a Mac runs is taken from the Docker context the role drives (see [Docker contexts](#docker-contexts)): `colima`, or `colima-` for another profile, is Colima, and `desktop-linux` is Docker Desktop. `github_runner_cluster_runtime` says so explicitly for a host whose context does not tell, for example one that uses `DOCKER_HOST`, and `github_runner_cluster_colima_profile` names the profile to start when the context does not. -**Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role starts the profile if it is stopped, with `colima start --profile --activate=false`, so the profile does not make its context the host's current one, and installs the launchd job that starts it again after a reboot. There are two kinds, chosen by `github_runner_cluster_runtime_autostart_at` (`login`, `boot` or `auto`, the default). `auto` is `boot` for the profile named by `github_runner_cluster_runtime_boot_profile` (default `node`, the profile the bridged k3s node runs in) and `login` for any other, such as a `default` profile a host keeps for other workloads. A profile has one kind at a time: installing either retires the other's property list, unloading it first, so the profile is not started twice. +**Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role starts the profile if it is stopped, with `colima start --profile --activate=false`, so the profile does not make its context the host's current one. A profile that `colima list` shows as running while its Docker runtime does not answer is not started again, because a start finds it running and does nothing. The role stops it, with `colima stop` and then `colima stop --force` only if that fails, and starts it again from the same disk, but only when `github_runner_cluster_runtime_force_stop_wedged` is on (the default) and all of these hold: `docker info` fails on `github_runner_cluster_runtime_probe_attempts` further tries (default 5), each given `github_runner_cluster_runtime_probe_timeout` seconds (default 20) and `github_runner_cluster_runtime_probe_delay` seconds apart (default 10); a command in the guest (`colima ssh -- true`) fails within the same timeout; the VM's host agent has been running for more than `github_runner_cluster_runtime_boot_grace` seconds (default 600), read from the elapsed time of the live process that the pid file in the profile's Lima instance directory names, and only when the program that process runs is Lima's host agent (named limactl, with the argument hostagent; a missing file, a dead pid, another program or an unreadable time leaves the age unknown, which never counts as old, whatever the file's date); and the Docker context the probes used can be read and points at that profile's own socket. A host that names its Colima home with `github_runner_cluster_colima_home` because it has no context that says (it uses `DOCKER_HOST`, say) runs as before, but its stop is never proven and the run fails with the signals it saw instead of stopping anything. Every probe, the guest command and the clean stop run under `files/run_bounded.pl`, which owns the deadline: it sends TERM to the command's process group when the time is up and KILL `github_runner_cluster_runtime_kill_grace` seconds later (default 5), so a program that ignores signals or never replies (docker and colima ignore `SIGALRM`) ends the try as failed and cannot hold the run; the first probe of the runtime is bounded the same way. If the supervisor itself is signalled (TERM, INT or HUP) it ends the command's process group the same way and exits 143; KILL of the supervisor cannot be handled and leaves the command running. Only the exit status 124 (deadline) or the tool's own non-zero status counts as a failed try: a supervisor that cannot start the command (127), is misused (255) or is missing (no `perl` on the PATH the role uses) fails the run and names `files/run_bounded.pl`, and is never evidence about the VM. A host that is only loaded, a Docker daemon that is restarting, a VM that is still booting and a context that points at another runtime each fail one of these, so nothing is stopped. A stop does not shut the guest down gracefully when the clean stop fails or times out (`github_runner_cluster_runtime_clean_stop_timeout`, default 120 seconds): every process in the guest, k3s and etcd included, ends, and the VM's disk, containers and volumes are kept, so on an etcd voter the member is out until the start completes. Switched off, or when a condition does not hold, the run fails with the signals it saw and the commands to recover by hand. With all of this in force the stop never follows a single failed or slow probe, never touches a VM whose guest answers or whose agent process is young or unknown, and never ends in a hang; that is why the switch is on by default. Under `--check` the reads are made, nothing is stopped and the report says what a real run would do. It installs the launchd job that starts it again after a reboot. There are two kinds, chosen by `github_runner_cluster_runtime_autostart_at` (`login`, `boot` or `auto`, the default). `auto` is `boot` for the profile named by `github_runner_cluster_runtime_boot_profile` (default `node`, the profile the bridged k3s node runs in) and `login` for any other, such as a `default` profile a host keeps for other workloads. A profile has one kind at a time: installing either retires the other's property list, unloading it first, so the profile is not started twice. **At login (an agent).** The role: @@ -491,6 +491,7 @@ The role controller holds the regrade Lease (`kube-system/fleet-regrade`, the Le - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)). - `github_runner_cluster_network_mode` (default `bridge`, or `host`): the network the node's container runs on, and with `host`, `github_runner_cluster_lan_cidr` (required: the LAN the nodes share) and `github_runner_cluster_network_probe_image` (default `busybox:1.37`, for reading the host's addresses) (see [Networking](#networking)). - `github_runner_cluster_runtime_autostart_at` (default `auto`; `login` or `boot`) and `github_runner_cluster_runtime_boot_profile` (default `node`): whether the Colima profile is started by a launchd agent at login or by a launchd daemon at boot, and which profile `auto` starts at boot (see [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). +- `github_runner_cluster_runtime_force_stop_wedged` (default true), `github_runner_cluster_runtime_probe_attempts` (default 5), `github_runner_cluster_runtime_probe_delay` (default 10), `github_runner_cluster_runtime_probe_timeout` (default 20), `github_runner_cluster_runtime_boot_grace` (default 600), `github_runner_cluster_runtime_kill_grace` (default 5) and `github_runner_cluster_runtime_clean_stop_timeout` (default 120): when a Colima profile listed as running with a silent runtime is stopped, see the Colima paragraph above. - `github_runner_cluster_runtime_autostart` (default true), `github_runner_cluster_runtime` (default `auto`; `colima`, `docker_desktop` or `other`), `github_runner_cluster_colima_profile` (default empty, taken from the Docker context), `github_runner_cluster_colima_home` (default empty: the `COLIMA_HOME` is read from the endpoint of the profile's Docker context, and is unset for Colima's own `~/.colima`; set it only for a context that does not say, and the role passes it to colima and to the launchd agent), `github_runner_cluster_runtime_path` (default both Homebrew prefixes and the system directories): on a macOS host, start Colima at login through a launchd agent and check the runtime (see [Container runtime at login](#container-runtime-after-a-reboot-macos)). - `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus` (default empty, keeping the profile's own): a Colima host's VM memory in GiB and its CPUs, each a positive whole number, applied one node at a time through the quorum guard when the VM differs (see Colima VM size under [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). `github_runner_cluster_colima_memory_reserved_gib` (default empty) derives the memory instead, or bounds one set explicitly, from the host's memory less it and `github_runner_cluster_colima_memory_host_reserve_gib` (default 10, macOS's own), never below `github_runner_cluster_colima_guest_baseline_gib` (default 3.4) plus `github_runner_cluster_colima_runner_memory_limit_gib` (default empty, required by a derivation) rounded up. `github_runner_cluster_colima_resize_lease_seconds`, `github_runner_cluster_colima_resize_lease_holder`, `github_runner_cluster_regrade_lease_name` and `github_runner_cluster_regrade_lease_namespace` are derived, never set directly. `github_runner_cluster_native_runner_wait_timeout` and `github_runner_cluster_native_ready_timeout` bound the wait for the node's runner pods and for the node to come back, as they do for a native node. - `github_runner_cluster_container_name` (default `github-runner-k3s`) and `github_runner_cluster_agent_container_name` (default `github-runner-k3s-agent`): the names of the node's server and agent containers, which a second cluster on the same Docker hosts sets to names of its own (see [Several clusters on one host](#several-clusters-on-one-host)). diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index 4ac8fd81..b881d742 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -237,7 +237,20 @@ github_runner_cluster_server_ready_timeout: 600 github_runner_cluster_runtime_autostart: true # When the role starts a Colima profile: login installs a per-user launchd agent (a property list in the connecting user's ~/Library/LaunchAgents, loaded into the user's login session), which runs only once that user has logged in; boot installs a launchd daemon (/Library/LaunchDaemons, owned by root, loaded into the system domain, run as the connecting user through UserName), which runs when the Mac starts, with nobody logged in, so a node recovers from a power loss or a reboot at the login window; auto is boot for the profile named by github_runner_cluster_runtime_boot_profile and login for any other. Only one of the two exists for a profile at a time: installing either removes the other's property list, and unloads it, so the profile is not started twice. A daemon takes root (become) to write and to load, which the agent never does. github_runner_cluster_runtime_autostart_at: auto -# The Colima profile that runs this collection's k3s node, which must come back with no one logged in: the one github_runner_cluster_runtime_autostart_at auto starts at boot. Any other profile on the host (the mini's default profile for its own workloads, say) keeps the login agent. +# Whether the role may stop a Colima profile that Colima lists as running while its container runtime does not answer, so that the start after it can boot it again. It does so only when tasks/runtime_wedged.yml has confirmed the failure (repeated probes, a failing guest command, a host agent older than the boot grace and a Docker context that points at that profile's socket). Off, such a profile fails the run with what was seen and the commands to recover by hand. Separate from github_runner_cluster_runtime_autostart, which starts a stopped profile and installs the launchd job. +github_runner_cluster_runtime_force_stop_wedged: true +# How many times docker info is tried, and the seconds between tries, before a profile listed as running with a silent runtime is looked into further. Together with the timeout they set the window in which a loaded host or a restarting Docker daemon can still answer. +github_runner_cluster_runtime_probe_attempts: 5 +github_runner_cluster_runtime_probe_delay: 10 +# The seconds each docker info, and the command run in the guest, is given before it counts as failed (a run that does not end by then is ended by files/run_bounded.pl). +github_runner_cluster_runtime_probe_timeout: 20 +# The seconds a VM's host agent process must have been running before a silent runtime counts as wedged rather than booting. +github_runner_cluster_runtime_boot_grace: 600 +# The seconds a probe, a guest command or a clean stop that has passed its deadline is given to end after TERM before it is killed. +github_runner_cluster_runtime_kill_grace: 5 +# The seconds colima stop is given before colima stop --force follows it. +github_runner_cluster_runtime_clean_stop_timeout: 120 +# The Colima profile that runs this collection's k3s node, which must come back with no one logged in: the one github_runner_cluster_runtime_autostart_at auto starts at boot. Any other profile on the host (a `default` profile a host keeps for other workloads, say) keeps the login agent. github_runner_cluster_runtime_boot_profile: node # Which container runtime a macOS host runs: auto takes it from the Docker context the role drives (colima or colima- is Colima, desktop-linux is Docker Desktop, anything else is other), colima, docker_desktop and other say so for a host whose context does not tell (DOCKER_HOST set, a context with another name). github_runner_cluster_runtime: auto diff --git a/roles/github_runner_cluster/files/run_bounded.pl b/roles/github_runner_cluster/files/run_bounded.pl new file mode 100644 index 00000000..82c84c74 --- /dev/null +++ b/roles/github_runner_cluster/files/run_bounded.pl @@ -0,0 +1,65 @@ +#!/usr/bin/perl +# Runs a command with a deadline that binds whatever the command does with signals. Usage: run_bounded.pl DEADLINE GRACE COMMAND [ARGUMENT...] (both in seconds). +# +# The command runs in a process group of its own. When DEADLINE passes, the group gets TERM; if any member is still there GRACE seconds later it gets KILL. A timer set with alarm before exec does not do this: a Go program such as docker or colima does not exit on SIGALRM, and a program may ignore TERM too. The deadline is measured on the monotonic clock, so a change of the system time does not move it. +# +# The exit status is the command's own, 128 plus the signal number when a signal ended it, 124 when the deadline passed (as timeout(1) reports it), 127 when the command could not be started and 255 when this script was misused. 143 means this supervisor itself was sent TERM, INT or HUP: the command's group is then ended the same way (TERM, then KILL after GRACE) so that nothing is left running on its own. KILL of the supervisor cannot be handled and leaves the command running. The command's standard input, output and error are the caller's. +use strict; +use warnings; +use POSIX qw(WNOHANG setsid); +use Time::HiRes qw(clock_gettime CLOCK_MONOTONIC sleep); + +my ($deadline, $grace, @command) = @ARGV; +die "usage: run_bounded.pl DEADLINE GRACE COMMAND [ARGUMENT...]\n" unless @command && defined $grace; + +sub now { return clock_gettime(CLOCK_MONOTONIC); } + +my $child = fork(); +die "fork: $!\n" unless defined $child; +if ($child == 0) { + setsid(); + exec { $command[0] } @command or exit 127; +} + +# The status of the child once it has ended, in the shell's convention. +sub status_of { + my ($status) = @_; + return $status & 127 ? 128 + ($status & 127) : $status >> 8; +} + +# True once the child has ended, with its status in $status. +our $status; +sub ended { + return 1 if defined $status; + my $done = waitpid($child, WNOHANG); + return 0 if $done == 0; + $status = status_of($?); + return 1; +} + +# Ends the child's group: TERM, then KILL when GRACE has passed or the leader is already gone. The group is killed whether or not its leader ended, so that nothing it started outlives it. +sub end_group { + kill 'TERM', -$child; + my $kill_at = now() + $grace; + while (now() < $kill_at) { + last if ended(); + sleep 0.05; + } + kill 'KILL', -$child; + waitpid($child, 0) unless defined $status; +} + +for my $name (qw(TERM INT HUP)) { + $SIG{$name} = sub { end_group(); exit 143; }; +} + +my $end = now() + $deadline; +while (now() < $end) { + exit $status if ended(); + sleep 0.05; +} +# One last look, so that a command that ended in the final poll interval is not reported as timed out. +exit $status if ended(); + +end_group(); +exit 124; diff --git a/roles/github_runner_cluster/tasks/runtime.yml b/roles/github_runner_cluster/tasks/runtime.yml index 9ef9c027..d0be7d30 100644 --- a/roles/github_runner_cluster/tasks/runtime.yml +++ b/roles/github_runner_cluster/tasks/runtime.yml @@ -31,7 +31,7 @@ | exadev.github_runner.cluster_container_runtime(github_runner_cluster_runtime, github_runner_cluster_colima_profile) }} - name: Read the Docker host of the Colima profile's context - # Where the profile lives: Colima points the context at //docker.sock. Needed only when github_runner_cluster_colima_home does not say. Read-only and answered by the context file alone, so it works with the VM stopped and under --check. It runs with the environment of every other docker call, so it reads the same context store. + # Where the profile lives: Colima points the context at //docker.sock. Needed only when github_runner_cluster_colima_home does not say, which is also why a host whose context has no such socket (one that uses DOCKER_HOST) sets the variable. Read-only and answered by the context file alone, so it works with the VM stopped and under --check. It runs with the environment of every other docker call, so it reads the same context store. ansible.builtin.command: argv: [docker, context, inspect, --format, "{{ '{{' }}.Endpoints.docker.Host{{ '}}' }}", "{{ github_runner_cluster_docker_cli_context }}"] environment: "{{ github_runner_cluster_docker_cli_environment | combine({'PATH': github_runner_cluster_runtime_path}) }}" @@ -72,6 +72,11 @@ - github_runner_cluster_effective_runtime.kind == 'colima' - github_runner_cluster_runtime_autostart | bool block: + - name: Deal with a profile Colima lists as running while its runtime does not answer + # A start would find such a profile running and do nothing. See runtime_wedged.yml for what is required before the VM is stopped. + ansible.builtin.include_tasks: runtime_wedged.yml + when: not github_runner_cluster_runtime_up | bool + - name: Start the Colima profile if it is stopped # --activate=false keeps the profile from making its Docker context the host's current one, which the role promises never to change (see Docker contexts in the README). Starting a profile that already runs would do nothing, but this only runs when the runtime did not answer. Skipped under --check, which reports the start as skipped, not as a failure. ansible.builtin.command: diff --git a/roles/github_runner_cluster/tasks/runtime_probe.yml b/roles/github_runner_cluster/tasks/runtime_probe.yml index 92ec41b5..2dd0226b 100644 --- a/roles/github_runner_cluster/tasks/runtime_probe.yml +++ b/roles/github_runner_cluster/tasks/runtime_probe.yml @@ -2,14 +2,33 @@ # Reads whether the container runtime this host's Docker context points at answers, and publishes github_runner_cluster_runtime_up. Read-only, so it runs under --check too. Needs github_runner_cluster_runtime_environment from runtime.yml. A daemon that is down is an answer here, not a failure: the callers decide what to do about it. - name: Read whether the container runtime answers + # Bounded by files/run_bounded.pl: a daemon that accepts the connection and never replies counts as not answering after github_runner_cluster_runtime_probe_timeout seconds, instead of holding the run. ansible.builtin.command: - argv: [docker, info, --format, "{{ '{{' }}.ServerVersion{{ '}}' }}"] + argv: + - perl + - -e + - "{{ lookup('ansible.builtin.file', 'run_bounded.pl') }}" + - "{{ github_runner_cluster_runtime_probe_timeout }}" + - "{{ github_runner_cluster_runtime_kill_grace }}" + - docker + - info + - --format + - "{{ '{{' }}.ServerVersion{{ '}}' }}" environment: "{{ github_runner_cluster_runtime_environment }}" register: github_runner_cluster_runtime_probe changed_when: false failed_when: false check_mode: false +- name: Check that the supervisor could run docker info + # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a perl it cannot start (missing, not executable) with the message "Error executing command" and an rc that is an OS error number. exadev.github_runner.cluster_supervisor_failed says which of these a result is. None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. + ansible.builtin.assert: + that: not (github_runner_cluster_runtime_probe | exadev.github_runner.cluster_supervisor_failed) + fail_msg: >- + files/run_bounded.pl could not run docker info on {{ inventory_hostname }} (exit {{ github_runner_cluster_runtime_probe.rc }}): {{ github_runner_cluster_runtime_probe.msg | default(github_runner_cluster_runtime_probe.stderr | default('')) | trim }}. + This is not a verdict on the runtime. perl must be on the PATH the role uses (github_runner_cluster_runtime_path), and the command must exist. + quiet: true + - name: Publish whether the container runtime answers ansible.builtin.set_fact: github_runner_cluster_runtime_up: "{{ github_runner_cluster_runtime_probe.rc == 0 }}" diff --git a/roles/github_runner_cluster/tasks/runtime_wedged.yml b/roles/github_runner_cluster/tasks/runtime_wedged.yml new file mode 100644 index 00000000..5b533939 --- /dev/null +++ b/roles/github_runner_cluster/tasks/runtime_wedged.yml @@ -0,0 +1,222 @@ +--- +# Reached on a Colima host whose container runtime did not answer. A profile that Colima lists as stopped is only started (runtime.yml). A profile it lists as running is either booting, slow, or stuck, and a start does nothing for it, so the VM could stay down for ever. This file ends such a VM with colima stop, and only when every one of these holds: +# +# 1. github_runner_cluster_runtime_force_stop_wedged is on; +# 2. docker info still fails on every one of github_runner_cluster_runtime_probe_attempts further tries, each given github_runner_cluster_runtime_probe_timeout seconds (files/run_bounded.pl ends one that has not finished, with TERM and then KILL after github_runner_cluster_runtime_kill_grace seconds, so a program that ignores signals cannot hold the run) and github_runner_cluster_runtime_probe_delay seconds apart, so a host that is only loaded or a dockerd that is restarting answers somewhere in that window and nothing is stopped; +# 3. a command run in the guest (colima ssh -- true) fails within github_runner_cluster_runtime_probe_timeout seconds, so a guest that works while its Docker daemon does not is left alone; +# 4. the VM's host agent has been running for more than github_runner_cluster_runtime_boot_grace seconds, read from the elapsed time of the live process the pid file in the profile's Lima instance directory names (and only if the program it runs is Lima's host agent, named by the basename limactl and the argument hostagent), so a VM that is still booting is left alone. A missing file, a dead pid, another program or an unreadable time means the agent's age is unknown and does not count as old, whatever the file's date; +# 5. the Docker context the probes used can be read and points at the socket of the profile that would be stopped (//docker.sock), so a failing probe of one runtime never stops another's VM and a context that cannot be read never proves anything. +# +# The stop is first a clean one (colima stop), given github_runner_cluster_runtime_clean_stop_timeout seconds, and a forced one (colima stop --force, which ends the VM without a graceful shutdown) only when that fails. Either way the VM's disk is kept and nothing is removed, so the start that follows boots it from the same disk. When any condition fails, or the switch is off, the run fails here and says which signals it saw and how to recover. Under --check every read is made and nothing is stopped; the report says what a real run would do. +# +# Needs github_runner_cluster_runtime_environment and github_runner_cluster_runtime_colima_home from runtime.yml; it reads the context's endpoint itself, because a host that names its Colima home may have no context to read. + +- name: Read whether Colima lists the profile as running + # colima list prints one JSON object per profile, whatever profile is asked for. + ansible.builtin.command: + argv: [colima, list, --json] + environment: "{{ github_runner_cluster_runtime_environment }}" + register: github_runner_cluster_runtime_colima_list + changed_when: false + check_mode: false + +- name: Note whether the profile is listed as running + ansible.builtin.set_fact: + github_runner_cluster_runtime_listed_running: >- + {{ github_runner_cluster_runtime_colima_list.stdout_lines | map('from_json') + | selectattr('name', 'equalto', github_runner_cluster_effective_runtime.colima_profile) + | selectattr('status', 'equalto', 'Running') | list | length > 0 }} + +- name: Look into a profile that is listed as running + when: github_runner_cluster_runtime_listed_running | bool + block: + - name: Ask the Docker daemon again, a bounded number of times + # files/run_bounded.pl bounds each try: macOS has no timeout command, docker info has no timeout of its own, and a timer set before exec does not stop a Go program (docker, colima), which ignores SIGALRM. + ansible.builtin.command: + argv: + - perl + - -e + - "{{ lookup('ansible.builtin.file', 'run_bounded.pl') }}" + - "{{ github_runner_cluster_runtime_probe_timeout }}" + - "{{ github_runner_cluster_runtime_kill_grace }}" + - docker + - info + - --format + - "{{ '{{' }}.ServerVersion{{ '}}' }}" + environment: "{{ github_runner_cluster_runtime_environment }}" + register: github_runner_cluster_runtime_reprobe + changed_when: false + failed_when: false + check_mode: false + # A supervisor that cannot run ends the loop at once: the check below fails the run. + until: github_runner_cluster_runtime_reprobe.rc == 0 or github_runner_cluster_runtime_reprobe | exadev.github_runner.cluster_supervisor_failed + retries: "{{ github_runner_cluster_runtime_probe_attempts | int - 1 }}" + delay: "{{ github_runner_cluster_runtime_probe_delay }}" + + - name: Check that the supervisor could run docker info + # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a perl it cannot start (missing, not executable) with the message "Error executing command" and an rc that is an OS error number. exadev.github_runner.cluster_supervisor_failed says which of these a result is. None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. + ansible.builtin.assert: + that: not (github_runner_cluster_runtime_reprobe | exadev.github_runner.cluster_supervisor_failed) + fail_msg: >- + files/run_bounded.pl could not run docker info on {{ inventory_hostname }} (exit {{ github_runner_cluster_runtime_reprobe.rc }}): {{ github_runner_cluster_runtime_reprobe.msg | default(github_runner_cluster_runtime_reprobe.stderr | default('')) | trim }}. + This is not a verdict on the runtime. perl must be on the PATH the role uses (github_runner_cluster_runtime_path), and the command must exist. + quiet: true + + - name: Note that the runtime answered after all + ansible.builtin.set_fact: + github_runner_cluster_runtime_up: true + when: github_runner_cluster_runtime_reprobe.rc == 0 + + - name: Gather the other signals while the runtime stays silent + when: github_runner_cluster_runtime_reprobe.rc != 0 + block: + - name: Run a command in the guest + ansible.builtin.command: + argv: + - perl + - -e + - "{{ lookup('ansible.builtin.file', 'run_bounded.pl') }}" + - "{{ github_runner_cluster_runtime_probe_timeout }}" + - "{{ github_runner_cluster_runtime_kill_grace }}" + - colima + - ssh + - --profile + - "{{ github_runner_cluster_effective_runtime.colima_profile }}" + - -- + - "true" + environment: "{{ github_runner_cluster_runtime_environment }}" + register: github_runner_cluster_runtime_guest + changed_when: false + failed_when: false + check_mode: false + + - name: Check that the supervisor could run a command in the guest + # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a perl it cannot start (missing, not executable) with the message "Error executing command" and an rc that is an OS error number. exadev.github_runner.cluster_supervisor_failed says which of these a result is. None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. + ansible.builtin.assert: + that: not (github_runner_cluster_runtime_guest | exadev.github_runner.cluster_supervisor_failed) + fail_msg: >- + files/run_bounded.pl could not run a command in the guest on {{ inventory_hostname }} (exit {{ github_runner_cluster_runtime_guest.rc }}): {{ github_runner_cluster_runtime_guest.msg | default(github_runner_cluster_runtime_guest.stderr | default('')) | trim }}. + This is not a verdict on the runtime. perl must be on the PATH the role uses (github_runner_cluster_runtime_path), and the command must exist. + quiet: true + + - name: Read the Docker host of the context the probes used + # On its own and allowed to fail: a host whose context says nothing about a Colima socket (DOCKER_HOST, an empty or unknown context) sets github_runner_cluster_colima_home and may have no context to read. An unreadable context means the socket is not proven to be this profile's, so nothing is stopped. + ansible.builtin.command: + argv: [docker, context, inspect, --format, "{{ '{{' }}.Endpoints.docker.Host{{ '}}' }}", "{{ github_runner_cluster_docker_cli_context }}"] + environment: "{{ github_runner_cluster_docker_cli_environment | combine({'PATH': github_runner_cluster_runtime_path}) }}" + register: github_runner_cluster_runtime_context_endpoint + changed_when: false + failed_when: false + check_mode: false + + - name: Find the VM's host agent pid file + # Colima keeps one Lima instance per profile under /_lima: colima for the default profile, colima- for another. + ansible.builtin.slurp: + src: >- + {{ (github_runner_cluster_runtime_colima_home if github_runner_cluster_runtime_colima_home | length > 0 else ansible_facts['env']['HOME'] ~ '/.colima') + ~ '/_lima/' ~ ('colima' if github_runner_cluster_effective_runtime.colima_profile == 'default' else 'colima-' ~ github_runner_cluster_effective_runtime.colima_profile) + ~ '/ha.pid' }} + register: github_runner_cluster_runtime_agent_pid + failed_when: false + + - name: Read the live process the pid file names + # A pid file can outlive the process (a reboot, a crash), so the age is the running process's own: its elapsed time, and only when its command is Lima's host agent. + ansible.builtin.command: + argv: [ps, -o, etime=, -o, command=, -p, "{{ (github_runner_cluster_runtime_agent_pid.content | b64decode | trim) if github_runner_cluster_runtime_agent_pid.content is defined else '0' }}"] + register: github_runner_cluster_runtime_agent_process + changed_when: false + failed_when: false + check_mode: false + + - name: Work out what the signals say + vars: + # ps prints [[days-]hours:]minutes:seconds, then the command. Anything else, a dead pid or another program leaves the age unknown (-1). + github_runner_cluster_runtime_agent_words: "{{ github_runner_cluster_runtime_agent_process.stdout | trim | split(None, 1) }}" + ansible.builtin.set_fact: + github_runner_cluster_runtime_agent_age: >- + {%- set words = github_runner_cluster_runtime_agent_words -%} + {%- if github_runner_cluster_runtime_agent_process.rc == 0 and words | length == 2 and words[1].split()[0].split('/')[-1] == 'limactl' and 'hostagent' in words[1].split()[1:] and words[0] is match('^([0-9]+-)?([0-9]+:)?[0-9]+:[0-9]+$') -%} + {%- set days_and_rest = words[0].split('-') -%} + {%- set days = days_and_rest[0] | int if days_and_rest | length > 1 else 0 -%} + {%- set parts = (days_and_rest[-1].split(':') | map('int') | list) -%} + {%- set clock = parts[-1] + 60 * parts[-2] + (3600 * parts[-3] if parts | length > 2 else 0) -%} + {{ days * 86400 + clock }} + {%- else -%}-1{%- endif -%} + github_runner_cluster_runtime_context_socket_ok: >- + {{ github_runner_cluster_runtime_context_endpoint.rc == 0 + and github_runner_cluster_runtime_context_endpoint.stdout | trim + == 'unix://' ~ (github_runner_cluster_runtime_colima_home if github_runner_cluster_runtime_colima_home | length > 0 else ansible_facts['env']['HOME'] ~ '/.colima') + ~ '/' ~ github_runner_cluster_effective_runtime.colima_profile ~ '/docker.sock' }} + + - name: Decide whether the VM is wedged + ansible.builtin.set_fact: + github_runner_cluster_runtime_wedged: >- + {{ github_runner_cluster_runtime_guest.rc != 0 + and github_runner_cluster_runtime_agent_age | int > github_runner_cluster_runtime_boot_grace | int + and github_runner_cluster_runtime_context_socket_ok | bool }} + github_runner_cluster_runtime_wedged_report: >- + Colima lists profile {{ github_runner_cluster_effective_runtime.colima_profile }} as running, but docker info failed on {{ github_runner_cluster_runtime_probe_attempts }} tries + {{ github_runner_cluster_runtime_probe_delay }} seconds apart; a command in the guest {{ 'failed' if github_runner_cluster_runtime_guest.rc != 0 else 'succeeded' }}; + the VM's host agent has been running {{ 'for more than ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds' if github_runner_cluster_runtime_agent_age | int > github_runner_cluster_runtime_boot_grace | int + else ('for less than ' ~ github_runner_cluster_runtime_boot_grace ~ ' seconds' if github_runner_cluster_runtime_agent_age | int >= 0 else 'for a time that could not be read') }}; + the Docker context {{ 'points at the socket of the profile' if github_runner_cluster_runtime_context_socket_ok | bool else ('could not be read, so its socket is not proven to belong to the profile' if github_runner_cluster_runtime_context_endpoint.rc != 0 else 'does not point at the socket of the profile') }}. + + - name: Say what a real run would do + ansible.builtin.debug: + msg: >- + {{ github_runner_cluster_runtime_wedged_report }} + {{ 'A real run would stop the profile (colima stop, then colima stop --force if that fails) and start it again.' + if github_runner_cluster_runtime_wedged | bool and github_runner_cluster_runtime_force_stop_wedged | bool + else 'A real run would stop nothing and fail here.' }} + changed_when: github_runner_cluster_runtime_wedged | bool and github_runner_cluster_runtime_force_stop_wedged | bool + when: ansible_check_mode + + - name: Fail where the VM is not shown to be wedged or the stop is off + ansible.builtin.fail: + msg: >- + {{ github_runner_cluster_runtime_wedged_report }} + {{ 'Nothing was stopped, because ' ~ ('github_runner_cluster_runtime_force_stop_wedged is off.' + if github_runner_cluster_runtime_wedged | bool + else 'the signals do not show a wedged VM (a VM that is still booting, or whose guest works, is left alone).') }} + To recover by hand: colima stop --profile {{ github_runner_cluster_effective_runtime.colima_profile }} --force, then colima start --profile {{ github_runner_cluster_effective_runtime.colima_profile }}. + when: + - not ansible_check_mode + - not (github_runner_cluster_runtime_wedged | bool and github_runner_cluster_runtime_force_stop_wedged | bool) + + - name: Stop the wedged profile cleanly + ansible.builtin.command: + argv: + - perl + - -e + - "{{ lookup('ansible.builtin.file', 'run_bounded.pl') }}" + - "{{ github_runner_cluster_runtime_clean_stop_timeout }}" + - "{{ github_runner_cluster_runtime_kill_grace }}" + - colima + - stop + - --profile + - "{{ github_runner_cluster_effective_runtime.colima_profile }}" + environment: "{{ github_runner_cluster_runtime_environment }}" + register: github_runner_cluster_runtime_clean_stop + changed_when: true + failed_when: false + when: not ansible_check_mode + + - name: Check that the supervisor could run colima stop + # files/run_bounded.pl exits 127 when it cannot start the command and 255 when it is misused, and the command module reports a perl it cannot start (missing, not executable) with the message "Error executing command" and an rc that is an OS error number. exadev.github_runner.cluster_supervisor_failed says which of these a result is. None of those is the command failing, so none counts as evidence about the VM: the run fails and names the supervisor. + ansible.builtin.assert: + that: not (github_runner_cluster_runtime_clean_stop | exadev.github_runner.cluster_supervisor_failed) + fail_msg: >- + files/run_bounded.pl could not run colima stop on {{ inventory_hostname }} (exit {{ github_runner_cluster_runtime_clean_stop.rc }}): {{ github_runner_cluster_runtime_clean_stop.msg | default(github_runner_cluster_runtime_clean_stop.stderr | default('')) | trim }}. + This is not a verdict on the runtime. perl must be on the PATH the role uses (github_runner_cluster_runtime_path), and the command must exist. + quiet: true + when: not ansible_check_mode + + - name: Stop the wedged profile without a graceful shutdown, since the clean stop failed + # colima stop --force ends the VM without a graceful shutdown (colima stop --help: "stop without graceful shutdown"): every process in the guest, k3s and etcd included, is killed. The VM's disk and everything on it are kept. + ansible.builtin.command: + argv: [colima, stop, --profile, "{{ github_runner_cluster_effective_runtime.colima_profile }}", --force] + environment: "{{ github_runner_cluster_runtime_environment }}" + changed_when: true + when: + - not ansible_check_mode + - github_runner_cluster_runtime_clean_stop.rc != 0 diff --git a/tests/unit/test_container_runtime.py b/tests/unit/test_container_runtime.py index b55cdb53..8de6d3e7 100644 --- a/tests/unit/test_container_runtime.py +++ b/tests/unit/test_container_runtime.py @@ -14,6 +14,7 @@ runtime = container_runtime.cluster_container_runtime colima_home = container_runtime.cluster_colima_home +supervisor_failed = container_runtime.cluster_supervisor_failed AnsibleFilterError = container_runtime.AnsibleFilterError HOME = "/Users/example" @@ -89,6 +90,7 @@ def test_a_value_that_is_not_a_string_is_an_error(self) -> None: def test_the_filter_is_registered(self) -> None: self.assertIs(container_runtime.FilterModule().filters()["cluster_container_runtime"], runtime) self.assertIs(container_runtime.FilterModule().filters()["cluster_colima_home"], colima_home) + self.assertIs(container_runtime.FilterModule().filters()["cluster_supervisor_failed"], supervisor_failed) class ColimaHomeTest(unittest.TestCase): @@ -133,3 +135,29 @@ def test_a_value_that_is_not_a_string_is_refused(self) -> None: if __name__ == "__main__": unittest.main() + + +class SupervisorFailedTest(unittest.TestCase): + def test_the_supervisors_own_statuses_say_it_could_not_run_the_command(self) -> None: + for rc in (127, 255): + with self.subTest(rc=rc): + self.assertTrue(supervisor_failed({"rc": rc, "stdout": "", "stderr": ""})) + + def test_the_command_module_failing_to_start_perl_does_so_whatever_the_rc(self) -> None: + # rc is the OS error number (2 missing, 13 not executable) or 257 for another OS error; the message is what they share. + for rc in (2, 13, 257, 1): + with self.subTest(rc=rc): + self.assertTrue(supervisor_failed({"rc": rc, "msg": "Error executing command", "cmd": ["perl"]})) + + def test_a_supervised_command_that_failed_or_timed_out_is_not_a_supervisor_failure(self) -> None: + for rc in (0, 1, 2, 13, 124, 137, 143): + with self.subTest(rc=rc): + self.assertFalse(supervisor_failed({"rc": rc, "stdout": "", "stderr": "Cannot connect to the Docker daemon"})) + + def test_a_message_that_is_not_the_start_failure_does_not_count(self) -> None: + self.assertFalse(supervisor_failed({"rc": 1, "msg": "non-zero return code"})) + + def test_a_value_that_is_not_a_result_is_refused(self) -> None: + for value in (None, "rc", 127, ["rc"]): + with self.subTest(value=value), self.assertRaises(AnsibleFilterError): + supervisor_failed(value) diff --git a/tests/unit/test_runtime_autostart.py b/tests/unit/test_runtime_autostart.py index 0b21059c..0c6b749c 100644 --- a/tests/unit/test_runtime_autostart.py +++ b/tests/unit/test_runtime_autostart.py @@ -10,9 +10,11 @@ import pwd import re import shutil +import signal import subprocess import sys import tempfile +import time import unittest from pathlib import Path from typing import Any @@ -80,9 +82,16 @@ def test_the_node_container_follows_the_node_s_role(self) -> None: self.assertIn("github_runner_cluster_container_name", template) self.assertIn("github_runner_cluster_agent_container_name", template) + def test_the_stop_of_a_silent_runtime_has_its_own_switch_and_bounded_probes(self) -> None: + self.assertIs(DEFAULTS["github_runner_cluster_runtime_force_stop_wedged"], True) + self.assertGreater(DEFAULTS["github_runner_cluster_runtime_probe_attempts"], 1) + self.assertGreater(DEFAULTS["github_runner_cluster_runtime_probe_timeout"], 0) + self.assertGreater(DEFAULTS["github_runner_cluster_runtime_kill_grace"], 0) + self.assertGreater(DEFAULTS["github_runner_cluster_runtime_boot_grace"], DEFAULTS["github_runner_cluster_runtime_probe_attempts"] * (DEFAULTS["github_runner_cluster_runtime_probe_timeout"] + DEFAULTS["github_runner_cluster_runtime_probe_delay"])) + def test_every_new_variable_is_in_the_readme(self) -> None: readme = (ROLE / "README.md").read_text() - for name in ("github_runner_cluster_runtime_autostart", "github_runner_cluster_runtime_autostart_at", "github_runner_cluster_runtime_boot_profile", "github_runner_cluster_runtime", "github_runner_cluster_colima_profile", "github_runner_cluster_runtime_path"): + for name in ("github_runner_cluster_runtime_force_stop_wedged", "github_runner_cluster_runtime_probe_attempts", "github_runner_cluster_runtime_probe_delay", "github_runner_cluster_runtime_probe_timeout", "github_runner_cluster_runtime_boot_grace", "github_runner_cluster_runtime_kill_grace", "github_runner_cluster_runtime_clean_stop_timeout", "github_runner_cluster_runtime_autostart", "github_runner_cluster_runtime_autostart_at", "github_runner_cluster_runtime_boot_profile", "github_runner_cluster_runtime", "github_runner_cluster_colima_profile", "github_runner_cluster_runtime_path"): with self.subTest(name=name): self.assertIn(f"`{name}`", readme) @@ -295,17 +304,40 @@ def test_the_comment_cannot_end_the_document_early(self) -> None: homes.write(name + " " + " ".join(args) + " " + os.environ.get("COLIMA_HOME", "-") + "\\n") def exists(file): return os.path.exists(os.path.join(stub, file)) +def hang(): + # A program that never answers and does not die on SIGALRM, as a Go binary does not; with ignore-term it ignores TERM as well, so only KILL ends it. + import signal, time + signal.signal(signal.SIGALRM, signal.SIG_IGN) + if exists("ignore-term"): + signal.signal(signal.SIGTERM, signal.SIG_IGN) + with open(os.path.join(stub, "hung.pids"), "a") as pids: + pids.write(str(os.getpid()) + "\\n") + time.sleep(300) def touch(file): open(os.path.join(stub, file), "w").close() def need_root(): if os.environ.get("RUNTIME_ROOT") != "1": sys.exit("launchctl: Operation not permitted while changing the system domain as a user") if name == "docker" and args[:2] == ["context", "inspect"]: - # Read from the context file alone, so it answers with the daemon down. + # Read from the context file alone, so it answers with the daemon down. An empty context name or an unknown one is an error, as docker's is. + if args[-1] == "" or exists("context-unknown"): + sys.exit("context " + repr(args[-1]) + ": context not found") if not exists("context-host"): sys.exit("context inspect: no context-host file") print(open(os.path.join(stub, "context-host")).read(), end="") elif name == "docker": + if args[:1] == ["info"] and exists("docker-hangs"): + hang() + if args[:1] == ["info"] and exists("docker-exit"): + sys.exit(int(open(os.path.join(stub, "docker-exit")).read())) + if not exists("runtime-up") and args[:1] == ["info"] and exists("docker-answers-after"): + # A daemon that fails its first tries and then answers, as one that is restarting or a loaded host does. + counter = os.path.join(stub, "docker-info-failures") + seen = int(open(counter).read()) if os.path.exists(counter) else 0 + if seen >= int(open(os.path.join(stub, "docker-answers-after")).read()): + touch("runtime-up") + else: + open(counter, "w").write(str(seen + 1)) if not exists("runtime-up"): sys.exit("Cannot connect to the Docker daemon") if args[0] == "info": @@ -317,6 +349,10 @@ def need_root(): else: sys.exit("unexpected docker call: " + " ".join(args)) elif name == "colima" and args[0] == "ssh": + if exists("guest-hangs") and args[args.index("--") + 1:] == ["true"]: + hang() + if exists("guest-answers") and args[args.index("--") + 1:] == ["true"]: + sys.exit(0) if not exists("runtime-up"): sys.exit("colima: the VM is not running") command = args[args.index("--") + 1:] @@ -380,12 +416,39 @@ def sysctl_file(key): print(content, end="") else: sys.exit("unexpected colima ssh call: " + " ".join(command)) +elif name == "colima" and args[0] == "list": + # One JSON object per profile, whatever profile is asked for: this host has the profile the test names and another one beside it, which is always running. + profile = os.environ.get("RUNTIME_PROFILE", "default") + print('{{"name":"other","status":"Running","runtime":"docker"}}') + print('{{"name":"' + profile + '","status":"' + ("Running" if exists("colima-listed-running") else "Stopped") + '","runtime":"docker"}}') +elif name == "colima" and args[0] == "stop": + if args[:3] != ["stop", "--profile", os.environ.get("RUNTIME_PROFILE", "default")] or args[3:] not in ([], ["--force"]): + sys.exit("unexpected colima stop call: " + " ".join(args)) + if args[3:] == [] and exists("clean-stop-hangs"): + hang() + if args[3:] == [] and exists("clean-stop-fails"): + sys.exit("colima: the VM does not stop") + for file in ("colima-listed-running", "runtime-up", "docker-hangs", "guest-hangs", "docker-exit"): + if exists(file): + os.remove(os.path.join(stub, file)) elif name == "colima": if args[0] != "start" or args[-1] != "--activate=false": sys.exit("unexpected colima call: " + " ".join(args)) if exists("colima-start-fails"): sys.exit("colima: the VM did not start") - touch("runtime-up") + # Starting a profile Colima already lists as running does nothing, so a wedged VM stays wedged. + if not exists("colima-listed-running"): + touch("colima-listed-running") + touch("runtime-up") + # A VM that is started again answers. + for file in ("docker-hangs", "guest-hangs", "docker-exit"): + if exists(file): + os.remove(os.path.join(stub, file)) +elif name == "ps": + # The host agent as ps shows it, from the file agent-ps (" "); a pid that is not running prints nothing and fails. + if not exists("agent-ps"): + sys.exit(1) + print(open(os.path.join(stub, "agent-ps")).read(), end="") elif name == "launchctl": system = args[1] == "system" or args[1].startswith("system/") if args[0] == "print": @@ -422,6 +485,89 @@ def failed_task(stdout: str) -> str | None: return None +class BoundedRunTest(unittest.TestCase): + """files/run_bounded.pl, which gives probes, the guest command and the clean stop a deadline that holds whatever the command does with signals.""" + + SCRIPT = ROLE / "files" / "run_bounded.pl" + + def run_bounded(self, deadline: float, grace: float, *command: str) -> tuple[subprocess.CompletedProcess[str], float]: + began = time.monotonic() + result = subprocess.run(["perl", str(self.SCRIPT), str(deadline), str(grace), *command], capture_output=True, text=True, check=False, timeout=60) + return result, time.monotonic() - began + + def test_the_commands_status_and_output_pass_through(self) -> None: + result, _ = self.run_bounded(10, 1, "sh", "-c", "echo out; echo err >&2; exit 3") + self.assertEqual((result.returncode, result.stdout, result.stderr), (3, "out\n", "err\n")) + + def test_a_signal_that_ends_the_command_is_reported_as_the_shell_does(self) -> None: + result, _ = self.run_bounded(10, 1, "sh", "-c", "kill -9 $$") + self.assertEqual(result.returncode, 137) + + def test_a_command_that_ignores_sigalrm_is_ended_at_the_deadline(self) -> None: + # What a Go program does with the timer a bare alarm would set. + result, elapsed = self.run_bounded(1, 1, sys.executable, "-c", "import signal, time; signal.signal(signal.SIGALRM, signal.SIG_IGN); time.sleep(60)") + self.assertEqual(result.returncode, 124) + self.assertLess(elapsed, 10) + + def test_a_command_that_ignores_term_is_killed_after_the_grace(self) -> None: + result, elapsed = self.run_bounded(1, 1, sys.executable, "-c", "import signal, time; signal.signal(signal.SIGTERM, signal.SIG_IGN); time.sleep(60)") + self.assertEqual(result.returncode, 124) + self.assertGreaterEqual(elapsed, 2) + self.assertLess(elapsed, 15) + + def test_what_the_command_started_does_not_outlive_the_deadline(self) -> None: + marker = Path(tempfile.mkdtemp(prefix="bounded-")) / "pid" + self.addCleanup(shutil.rmtree, marker.parent, True) + result, _ = self.run_bounded(1, 1, "sh", "-c", f"trap '' TERM; sleep 60 & echo $! > {marker}; wait") + self.assertEqual(result.returncode, 124) + pid = int(marker.read_text()) + for _ in range(50): + try: + os.kill(pid, 0) + except ProcessLookupError: + return + time.sleep(0.1) + self.fail("a process the command started is still running") + + def test_a_supervisor_that_is_signalled_ends_its_command_and_what_it_started(self) -> None: + # Ansible or a person killing the supervisor must not leave the hung command (a colima stop mid-shutdown, say) running on its own. + for signal_name in ("TERM", "INT", "HUP"): + with self.subTest(signal=signal_name): + marker = Path(tempfile.mkdtemp(prefix="bounded-")) / "pids" + self.addCleanup(shutil.rmtree, marker.parent, True) + script = f"trap '' TERM INT HUP; sleep 60 & echo $$ $! > {marker}; wait" + supervisor = subprocess.Popen(["perl", str(self.SCRIPT), "60", "1", "sh", "-c", script]) + for _ in range(100): + if marker.exists() and marker.read_text().strip(): + break + time.sleep(0.05) + pids = [int(word) for word in marker.read_text().split()] + os.kill(supervisor.pid, getattr(signal, "SIG" + signal_name)) + self.assertEqual(supervisor.wait(timeout=30), 143) + for pid in pids: + for _ in range(50): + try: + os.kill(pid, 0) + except ProcessLookupError: + break + time.sleep(0.1) + else: + self.fail(f"process {pid} survived the {signal_name} sent to the supervisor") + + def test_regression_guard_a_command_that_ends_before_the_deadline_reports_its_own_status(self) -> None: + # A guard only: it passes on a supervisor with or without the final look at the child before the timeout path, because making a command end in the last poll interval is a race against the scheduler, which a test on a loaded runner cannot win reliably. The final look is read from the script instead. + result, _ = self.run_bounded(1, 1, "sh", "-c", "sleep 0.1; exit 5") + self.assertEqual(result.returncode, 5) + self.assertIn("one last look", self.SCRIPT.read_text().lower()) + + def test_the_deadline_is_measured_on_the_monotonic_clock(self) -> None: + self.assertIn("CLOCK_MONOTONIC", self.SCRIPT.read_text()) + + def test_a_command_that_cannot_be_started_exits_127(self) -> None: + result, _ = self.run_bounded(5, 1, "/nonexistent/command") + self.assertEqual(result.returncode, 127) + + @unittest.skipIf(shutil.which(ANSIBLE_PLAYBOOK) is None, "ansible-playbook is not installed") class RoleTest(unittest.TestCase): def setUp(self) -> None: @@ -433,7 +579,7 @@ def setUp(self) -> None: self.home.mkdir() self.bin = self.work / "bin" self.bin.mkdir() - for name in ("docker", "colima", "launchctl", "plutil", "sudo"): + for name in ("docker", "colima", "launchctl", "plutil", "sudo", "ps"): tool = self.bin / name tool.write_text(STUB.format(python=sys.executable)) tool.chmod(0o755) @@ -471,7 +617,7 @@ def run_role(self, *extra: str, tasks: str = "runtime,end_checks", context: str inventory = self.work / "inventory.yml" inventory.write_text(yaml.safe_dump({"all": {"hosts": {"node1": host}}})) # The stand-ins come first on the playbook's own PATH as well as the tasks', so a tool a task runs without the role's PATH still cannot reach this machine's real launchd, Colima or Docker. - env = ansible_env() | {"RUNTIME_STUB": str(self.stub), "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}"} + env = ansible_env() | {"RUNTIME_STUB": str(self.stub), "RUNTIME_PROFILE": profile, "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}"} return subprocess.run([ANSIBLE_PLAYBOOK, "-i", str(inventory), str(RUN), *extra], capture_output=True, text=True, check=False, env=env, cwd=self.work) def calls(self) -> list[str]: @@ -524,6 +670,311 @@ def test_a_stopped_colima_is_started_and_the_agent_installed_and_loaded(self) -> self.assertEqual(self.plist.stat().st_mode & 0o777, 0o644) self.assertTrue((self.stub / "loaded").exists()) + # A profile Colima lists as running while its runtime does not answer. + + FAST = {"github_runner_cluster_runtime_probe_attempts": 3, "github_runner_cluster_runtime_probe_delay": 0, "github_runner_cluster_runtime_probe_timeout": 5} + + def forget(self) -> None: + """Start a sub-case from a clean stub directory and a clean home.""" + for entry in self.stub.iterdir(): + if entry.is_dir(): + shutil.rmtree(entry) + elif entry.name not in ("context-host", "github-runner-k3s.state"): + entry.unlink() + shutil.rmtree(self.home / ".colima", ignore_errors=True) + + def stops(self) -> list[str]: + return [call for call in self.calls() if re.match(r"\S+ colima stop", call)] + + def starts(self) -> list[str]: + return [call for call in self.calls() if re.match(r"\S+ colima start", call)] + + AGENT = "/usr/local/bin/limactl hostagent --pidfile /example/ha.pid --socket /example/ha.sock" + + @staticmethod + def etime(seconds: int) -> str: + """ps's elapsed time, [[days-]hours:]minutes:seconds.""" + days, rest = divmod(seconds, 86400) + hours, rest = divmod(rest, 3600) + minutes, secs = divmod(rest, 60) + return (f"{days}-" if days else "") + (f"{hours:02d}:" if days or hours else "") + f"{minutes:02d}:{secs:02d}" + + def listed_running(self, agent_age: int | None = 7200, command: str | None = None, stale_file: bool = False) -> None: + """A VM Colima lists as running whose Docker daemon is silent, with a live host agent that has run for `agent_age` seconds (None for no pid file at all). `stale_file` leaves a pid file that is old but names no running process, as one from before a reboot does; `command` is what the process runs.""" + self.state("session", "colima-listed-running") + if agent_age is None: + return + pid = self.home / ".colima" / "_lima" / "colima" / "ha.pid" + pid.parent.mkdir(parents=True, exist_ok=True) + pid.write_text("4242\n") + old = time.time() - 7 * 86400 + os.utime(pid, (old, old)) + if not stale_file: + (self.stub / "agent-ps").write_text(f"{self.etime(agent_age)} {command or self.AGENT}\n") + + def hung_pids(self) -> list[int]: + log = self.stub / "hung.pids" + return [int(line) for line in log.read_text().split()] if log.exists() else [] + + def assert_all_ended(self) -> None: + for pid in self.hung_pids(): + for _ in range(50): + try: + os.kill(pid, 0) + except ProcessLookupError: + break + time.sleep(0.1) + else: + self.fail(f"the hung stand-in {pid} is still running") + + def info_calls(self) -> list[str]: + return [call for call in self.calls() if re.match(r"\S+ docker info", call)] + + def test_a_vm_whose_runtime_stays_silent_and_whose_guest_and_agent_confirm_it_is_stopped_cleanly_and_started_again(self) -> None: + self.listed_running() + result = self.run_role(**self.FAST) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), ["colima colima stop --profile default"]) + self.assertEqual(self.starts(), ["colima colima start --profile default --activate=false"]) + self.assertLess(self.calls().index(self.stops()[0]), self.calls().index(self.starts()[0])) + # The first probe and every further try. + self.assertGreaterEqual(len(self.info_calls()), 4) + self.assertTrue((self.stub / "runtime-up").exists()) + + def test_a_clean_stop_that_fails_is_followed_by_a_forced_one(self) -> None: + self.listed_running() + self.state("clean-stop-fails") + result = self.run_role(**self.FAST) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), ["colima colima stop --profile default", "colima colima stop --profile default --force"]) + self.assertEqual(len(self.starts()), 1) + + def test_only_the_hosts_own_profile_is_stopped_while_another_listed_profile_runs(self) -> None: + self.listed_running() + self.run_role(**self.FAST) + self.assertEqual([call for call in self.stops() if "--profile other" in call], []) + self.assertTrue(self.stops()) + + def test_a_daemon_that_answers_on_a_later_try_is_never_stopped(self) -> None: + # The first probe and one further try fail, as on a loaded host or while dockerd restarts, and the next answers. + self.listed_running() + (self.stub / "docker-answers-after").write_text("2") + result = self.run_role(**self.FAST) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), []) + self.assertEqual(self.starts(), []) + + def assert_left_alone(self, result: subprocess.CompletedProcess[str], *words: str) -> None: + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertEqual(failed_task(result.stdout), "Fail where the VM is not shown to be wedged or the stop is off") + self.assertEqual(self.stops(), []) + self.assertEqual(self.starts(), []) + self.assertIn("colima stop --profile default --force", result.stdout, "the recovery by hand is named") + for word in words: + self.assertIn(word, result.stdout) + + def test_a_guest_that_answers_a_command_is_left_alone_although_docker_is_silent(self) -> None: + self.listed_running() + self.state("guest-answers") + self.assert_left_alone(self.run_role(**self.FAST), "a command in the guest succeeded") + + def test_a_vm_whose_agent_started_moments_ago_is_left_alone_however_old_its_pid_file_is(self) -> None: + # The pid file is a week old, from a boot long gone, and the process it names started thirty seconds ago. + self.listed_running(agent_age=30) + self.assert_left_alone(self.run_role(**self.FAST), "for less than 600 seconds") + + def test_a_pid_file_naming_no_running_process_leaves_the_age_unknown_and_nothing_is_stopped(self) -> None: + # An old pid file whose process is gone, as after a reboot: its date says the agent is old, and that must not count. + self.listed_running(stale_file=True) + self.assert_left_alone(self.run_role(**self.FAST), "could not be read") + + def test_a_pid_file_naming_another_program_leaves_the_age_unknown(self) -> None: + self.listed_running(command="/usr/sbin/cron -f") + self.assert_left_alone(self.run_role(**self.FAST), "could not be read") + + def test_a_pid_file_naming_a_program_whose_path_merely_contains_hostagent_leaves_the_age_unknown(self) -> None: + self.listed_running(command="/opt/hostagent/bin/cron -f") + self.assert_left_alone(self.run_role(**self.FAST), "could not be read") + + def test_a_report_of_an_old_agent_does_not_say_ago_twice(self) -> None: + self.listed_running(command="/usr/local/bin/limactl hostagent x", agent_age=7200) + self.state("guest-answers") + result = self.run_role(**self.FAST) + self.assertIn("for more than 600 seconds;", result.stdout) + self.assertNotIn("seconds ago", result.stdout) + + # The Docker context, which the stop reads for its socket check on its own. + + def test_a_host_that_names_its_colima_home_and_has_no_readable_context_still_runs(self) -> None: + # The README's route for a host whose context does not say where the profile is (DOCKER_HOST, say): the variable gives the home and the context cannot be read. Nothing is listed as running, so the play starts the profile as before. + for label, variables in (("an empty context", {"context": ""}), ("an unknown context", {})): + with self.subTest(label=label): + self.forget() + self.state("session") + if not variables: + self.state("context-unknown") + result = self.run_role(github_runner_cluster_colima_home="/elsewhere", github_runner_cluster_runtime="colima", **variables) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assert_colima_ran_in("/elsewhere") + self.assertEqual(self.stops(), []) + + def test_a_context_that_cannot_be_read_never_leads_to_a_stop_and_the_report_says_so(self) -> None: + for label, variables in (("an empty context", {"context": ""}), ("an unknown context", {"unknown": True})): + with self.subTest(label=label): + self.forget() + self.listed_running() + if variables.get("unknown"): + self.state("context-unknown") + result = self.run_role(github_runner_cluster_colima_home=f"{self.home}/.colima", github_runner_cluster_runtime="colima", **{k: v for k, v in variables.items() if k != "unknown"}, **self.FAST) + self.assert_left_alone(result, "so its socket is not proven to belong to the profile") + + def test_a_context_whose_endpoint_is_another_runtime_never_leads_to_a_stop(self) -> None: + for endpoint in ("tcp://remote.example.invalid:2375", "unix:///var/run/docker.sock"): + with self.subTest(endpoint=endpoint): + self.forget() + self.listed_running() + result = self.run_role(context_host=endpoint, github_runner_cluster_colima_home=f"{self.home}/.colima", **self.FAST) + self.assert_left_alone(result, "does not point at the socket of the profile") + + # The supervisor failing is not a failed probe. + + def test_a_supervisor_that_cannot_run_the_command_fails_the_run_naming_it_and_is_not_evidence(self) -> None: + for code in (127, 255): + with self.subTest(code=code): + self.forget() + self.listed_running() + (self.stub / "docker-exit").write_text(str(code)) + result = self.run_role(**self.FAST) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("run_bounded.pl", result.stdout) + self.assertEqual(self.stops(), []) + + def test_a_host_without_perl_fails_the_run_naming_the_supervisor(self) -> None: + self.listed_running() + result = self.run_role(github_runner_cluster_runtime_path=str(self.bin), **self.FAST) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("run_bounded.pl", result.stdout) + self.assertEqual(self.stops(), []) + + def test_a_perl_that_is_not_executable_fails_the_run_naming_the_supervisor(self) -> None: + # The command module reports this with the OS error number as its rc (13), not 2: the message identifies it. + self.listed_running() + perl_dir = self.work / "no-perl" + perl_dir.mkdir() + (perl_dir / "perl").write_text("#!/bin/sh\n") + (perl_dir / "perl").chmod(0o644) + result = self.run_role(github_runner_cluster_runtime_path=f"{perl_dir}{os.pathsep}{self.bin}", **self.FAST) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("run_bounded.pl", result.stdout) + self.assertEqual(self.stops(), []) + + def test_a_tool_that_fails_by_itself_still_counts_as_a_failed_probe(self) -> None: + # docker exiting 1 (cannot connect) is the daemon's answer, not the supervisor's. + self.listed_running() + (self.stub / "docker-exit").write_text("1") + result = self.run_role(**self.FAST) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(self.stops()), 1) + + def test_an_agent_that_has_run_for_days_counts_as_old(self) -> None: + self.listed_running(agent_age=3 * 86400 + 3600 + 61) + result = self.run_role(**self.FAST) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(self.stops()), 1) + + def test_an_agent_whose_age_cannot_be_read_does_not_count_as_old(self) -> None: + self.listed_running(agent_age=None) + self.assert_left_alone(self.run_role(**self.FAST), "could not be read") + + def test_a_colima_home_that_is_not_where_the_context_points_never_leads_to_a_stop(self) -> None: + # The variable names another Colima home than the one the Docker context's socket is in: the probes measured one runtime and a stop would act on another's profile. (A context whose endpoint is no Colima socket at all fails earlier, in the filter that reads the home from it.) + self.listed_running() + result = self.run_role(github_runner_cluster_colima_home="/elsewhere", **self.FAST) + self.assert_left_alone(result, "does not point at the socket of the profile") + + def test_with_the_stop_switched_off_a_confirmed_wedge_fails_naming_the_switch(self) -> None: + self.listed_running() + self.assert_left_alone(self.run_role(github_runner_cluster_runtime_force_stop_wedged=False, **self.FAST), "github_runner_cluster_runtime_force_stop_wedged is off") + + def test_the_stop_switch_is_independent_of_autostart(self) -> None: + self.listed_running() + result = self.run_role(github_runner_cluster_runtime_force_stop_wedged=False, github_runner_cluster_runtime_autostart=True, **self.FAST) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.stops(), []) + + def test_a_second_run_after_the_recovery_stops_and_starts_nothing(self) -> None: + self.listed_running() + self.run_role(**self.FAST) + before = len(self.calls()) + result = self.run_role(**self.FAST) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual([call for call in self.calls()[before:] if re.match(r"\S+ colima (stop|start)", call)], []) + + def test_a_stopped_profile_is_started_without_being_stopped_first(self) -> None: + self.state("session") + result = self.run_role() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), []) + + def test_a_running_runtime_is_neither_listed_nor_stopped(self) -> None: + self.state("session", "runtime-up", "colima-listed-running") + result = self.run_role() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), []) + self.assertEqual([call for call in self.calls() if re.match(r"\S+ colima list", call)], []) + + HANG = {"github_runner_cluster_runtime_probe_attempts": 2, "github_runner_cluster_runtime_probe_delay": 0, "github_runner_cluster_runtime_probe_timeout": 2, "github_runner_cluster_runtime_kill_grace": 1, "github_runner_cluster_runtime_clean_stop_timeout": 2} + + def test_probes_that_never_reply_and_ignore_sigalrm_are_ended_and_the_vm_is_stopped_and_started(self) -> None: + # docker info and the guest command never reply and ignore SIGALRM, as Go programs do; the clean stop never ends and ignores TERM too. + self.listed_running() + self.state("docker-hangs", "guest-hangs", "clean-stop-hangs", "ignore-term") + began = time.monotonic() + result = self.run_role(**self.HANG) + elapsed = time.monotonic() - began + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), ["colima colima stop --profile default", "colima colima stop --profile default --force"]) + self.assertEqual(len(self.starts()), 1) + # Unbounded, each stand-in sleeps for 300 seconds. + self.assertLess(elapsed, 120) + self.assertGreaterEqual(len(self.hung_pids()), 5) + self.assert_all_ended() + + def test_a_hung_docker_with_a_guest_that_answers_fails_the_run_within_the_bound_and_stops_nothing(self) -> None: + self.listed_running() + self.state("docker-hangs", "guest-answers") + began = time.monotonic() + result = self.run_role(**self.HANG) + self.assertLess(time.monotonic() - began, 120) + self.assert_left_alone(result, "a command in the guest succeeded") + self.assert_all_ended() + + def test_a_hung_first_probe_counts_as_the_runtime_not_answering(self) -> None: + # A stopped profile whose Docker socket accepts and never replies: the start follows, as for a socket that refuses. + self.state("session", "docker-hangs") + began = time.monotonic() + result = self.run_role(**self.HANG) + self.assertLess(time.monotonic() - began, 120) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(len(self.starts()), 1) + self.assert_all_ended() + + def test_check_mode_reads_everything_stops_nothing_and_says_what_a_real_run_would_do(self) -> None: + self.listed_running() + result = self.run_role("--check", **self.FAST) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), []) + self.assertTrue((self.stub / "colima-listed-running").exists()) + self.assertIn("A real run would stop the profile", result.stdout) + + def test_check_mode_on_a_vm_that_is_not_shown_to_be_wedged_says_nothing_would_be_stopped(self) -> None: + self.listed_running() + self.state("guest-answers") + result = self.run_role("--check", **self.FAST) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.stops(), []) + self.assertIn("A real run would stop nothing and fail here", result.stdout) + def test_every_docker_call_uses_the_hosts_docker_context(self) -> None: self.state("session") self.run_role(context="colima-example") @@ -723,6 +1174,7 @@ def test_autostart_off_leaves_an_installed_daemon_and_agent_as_they_are(self) -> self.assertEqual(self.changes(), []) def test_the_variable_wins_over_the_context_and_the_context_is_not_read(self) -> None: + # A profile that is stopped never reaches the stop's own read of the context, so the home comes from the variable alone. self.state("session") result = self.run_role(context="colima-node", github_runner_cluster_colima_home="/elsewhere") self.assertEqual(result.returncode, 0, result.stdout + result.stderr)