diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index f243662d..ae3c7af5 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -46,6 +46,7 @@ Helm 4 and server-side apply. The role reads `helm version --short` and, on Helm - `github_runner_arc_controller_replicas`: above 1, the chart turns on leader election and the role adds a preferred pod anti-affinity across nodes. `github_runner_arc_controller_extra_values` is merged over the role's controller values. - `github_runner_arc_metrics_enabled` (and the `_addr`/`_endpoint` settings): Prometheus metrics for the controller and every listener; the 0.14 chart only enables them together. - `github_runner_arc_listener_probes_enabled`: readiness and liveness probes on each listener's metrics endpoint, so a listener that starts and then fails its GitHub authentication is not counted ready during a rollout. Needs metrics on. +- `github_runner_arc_listener_resources` (default `{}`): resources for each listener's container, as a container's `resources` field. The chart gives the listener none, and the role's listener container replaces any a values file sets (Helm and the role's merge replace lists), so this is the only way to give it some. Under node memory pressure the kubelet evicts first the pods whose usage exceeds their request, and ranks by priority only after that ([Node-pressure eviction, "Pod selection for kubelet eviction"](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#pod-selection-for-kubelet-eviction)): a listener with no memory request is always over it, so it is evicted before idle runner pods within their requests whatever `github_runner_arc_control_pods_priority_enabled` gives it, and the scale set takes no job until it is rescheduled. A memory request above the listener's working set keeps it out of that first group. - `github_runner_arc_control_pods_priority_enabled`: runs the controller and every listener under the platform's PriorityClass (`github_runner_arc_platform_priority_class_name`, created by the role whether or not the platform is installed). A listener is one pod per scale set that ARC cannot replicate, and while it waits to be scheduled no new jobs are assigned; with the class a rescheduled one can take a full node's room from a runner pod, whose job ARC retries, instead of waiting behind it. Off by default, as it changes what a full node evicts. - `github_runner_arc_autoscaler_node_pressure_enabled`, `github_runner_arc_autoscaler_io_pressure_pct`, `github_runner_arc_autoscaler_cpu_pressure_pct`, `github_runner_arc_psi_reporter_seconds`, `github_runner_arc_node_pressure_configmap`: per-node pressure for the autoscaler, from each node's own `/proc/pressure` through a `psi-reporter` DaemonSet on the nodes runner pods may use (see the Autoscaler section of the repository README). Needs a profile with `autoscale: true`; with the option off, or no autoscaled profile, the role removes the DaemonSet and its ConfigMap and RBAC. - `github_runner_arc_node_label_key`, `github_runner_arc_node_label_value`: when the key is set, the controller, listeners and runner pods are restricted to nodes carrying the label, and the autoscaler reads memory pressure from those nodes only. The role labels the nodes named in `github_runner_arc_labelled_nodes`. diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index 68118927..0f098a5c 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -115,6 +115,9 @@ github_runner_arc_listener_metrics_endpoint: "/metrics" github_runner_arc_listener_probes_enabled: false github_runner_arc_listener_probe_port: 8080 +# Resources (requests and limits, as a container's resources field) for each listener's container. The chart gives the listener none, and the overlay's listener container replaces any a values file sets, so this is the only way to give it some. Under node memory pressure the kubelet evicts first the pods whose usage exceeds their request, and only then ranks by priority (https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#pod-selection-for-kubelet-eviction): a listener with no memory request is always over it, so it goes before idle runner pods within theirs whatever its PriorityClass, and the scale set takes no job until it is rescheduled. A memory request above the listener's working set keeps it out of that first group. Empty leaves the listener as the chart builds it. +github_runner_arc_listener_resources: {} + # Runs the controller and every listener under the platform's PriorityClass (github_runner_arc_platform_priority_class_name), which the role creates whether or not the platform is installed. A listener is one pod per scale set that ARC cannot replicate, and while it waits to be scheduled no new jobs are assigned, so one that was rescheduled on a full node waits behind runner pods at the default priority. With the class it can take a node's room from a runner pod, whose job ARC retries. An upgrade also reschedules every controller replica together. Off by default, as it changes what a full node evicts. github_runner_arc_control_pods_priority_enabled: false diff --git a/roles/github_runner_arc/meta/argument_specs.yml b/roles/github_runner_arc/meta/argument_specs.yml index ea221388..3215317e 100644 --- a/roles/github_runner_arc/meta/argument_specs.yml +++ b/roles/github_runner_arc/meta/argument_specs.yml @@ -209,6 +209,10 @@ argument_specs: type: int default: 8080 description: Port the listener probes poll. + github_runner_arc_listener_resources: + type: dict + default: {} + description: Resources (a container's resources field) for each listener's container, so a memory request keeps the listener from being the kubelet's first eviction under node memory pressure. Empty leaves the chart's listener without any. github_runner_arc_control_pods_priority_enabled: type: bool default: false diff --git a/roles/github_runner_arc/tasks/validate.yml b/roles/github_runner_arc/tasks/validate.yml index a523b196..cea042b4 100644 --- a/roles/github_runner_arc/tasks/validate.yml +++ b/roles/github_runner_arc/tasks/validate.yml @@ -131,6 +131,8 @@ }} - msg: github_runner_arc_controller_extra_values must be a mapping. failed: "{{ github_runner_arc_controller_extra_values is not mapping }}" + - msg: github_runner_arc_listener_resources must be a mapping (a container's resources field). + failed: "{{ github_runner_arc_listener_resources is not mapping }}" - msg: github_runner_arc_listener_probes_enabled needs github_runner_arc_metrics_enabled, since the probes poll the listener's metrics endpoint. failed: "{{ github_runner_arc_listener_probes_enabled | bool and not (github_runner_arc_metrics_enabled | bool) }}" - msg: github_runner_arc_listener_probe_port must match the port in github_runner_arc_listener_metrics_addr. diff --git a/roles/github_runner_arc/templates/scale-set-overlay.yaml.j2 b/roles/github_runner_arc/templates/scale-set-overlay.yaml.j2 index 8d7ed678..3ddd87d5 100644 --- a/roles/github_runner_arc/templates/scale-set-overlay.yaml.j2 +++ b/roles/github_runner_arc/templates/scale-set-overlay.yaml.j2 @@ -1,4 +1,4 @@ -{# The values the role itself owns on every scale-set release, rendered on the control node and merged over the profile's values (nested mappings merge, lists are replaced, as Helm does). With node eligibility, probes, max_runners, sizing and burst all off it holds only the profile's nodeSelector. A measured sizing's result is read from github_runner_arc_measured_sizing, which tasks/measure_sizing.yml fills in. `profile` is the expanded profile from plugins/filter/arc.py. #} +{# The values the role itself owns on every scale-set release, rendered on the control node and merged over the profile's values (nested mappings merge, lists are replaced, as Helm does). With node eligibility, probes, listener resources, max_runners, sizing and burst all off it holds only the profile's nodeSelector. A measured sizing's result is read from github_runner_arc_measured_sizing, which tasks/measure_sizing.yml fills in. `profile` is the expanded profile from plugins/filter/arc.py. #} {% set eligibility = {github_runner_arc_node_label_key: github_runner_arc_node_label_value | string} if github_runner_arc_node_label_key | length > 0 else {} %} {% if profile.burst is not none and profile.burst.exclusive %} # Burst nodes only: the runners the burst nodes hold, the profile's burst.max_runners. @@ -23,7 +23,8 @@ template: # The backstop for a pod that outlives its job: the kubelet stops it this long after it starts. activeDeadlineSeconds: {{ github_runner_arc_runner_active_deadline_seconds | int }} {% endif %} -{% if github_runner_arc_node_label_key | length > 0 or github_runner_arc_listener_probes_enabled | bool or github_runner_arc_control_pods_priority_enabled | bool %} +{% if github_runner_arc_node_label_key | length > 0 or github_runner_arc_listener_probes_enabled | bool or github_runner_arc_control_pods_priority_enabled | bool or github_runner_arc_listener_resources | length > 0 %} +{# The chart gives the listener no resources either; github_runner_arc_listener_resources alone builds the listener container when nothing else does, and this comment stays out of the render so the overlay is unchanged without it. #} # The chart puts no nodeSelector, no probe and no priority class on the listener. The AutoscalingRunnerSet needs containers whenever listenerTemplate.spec is set, and a container named listener is merged into the real listener rather than added as a sidecar. listenerTemplate: spec: @@ -35,6 +36,10 @@ listenerTemplate: {% endif %} containers: - name: listener +{% if github_runner_arc_listener_resources | length > 0 %} + # This list replaces the values file's listener containers, so the role's variable is the one place the listener's resources can come from. + resources: {{ github_runner_arc_listener_resources | to_json }} +{% endif %} {% if github_runner_arc_listener_probes_enabled | bool %} # An HTTP probe on the metrics endpoint: the listener image has no shell, so an exec probe can never succeed. A 200 means the process has got past its startup and authentication, not just that it has not exited. readinessProbe: diff --git a/tests/unit/test_scale_set_overlay.py b/tests/unit/test_scale_set_overlay.py index bdfabc75..08654258 100644 --- a/tests/unit/test_scale_set_overlay.py +++ b/tests/unit/test_scale_set_overlay.py @@ -17,6 +17,7 @@ PLUGIN = ROOT / "plugins" / "filter" / "arc.py" RENDER = ROOT / "tests" / "scale_set_overlay" / "render.yml" RENDER_CONTROLLER = ROOT / "tests" / "scale_set_overlay" / "render_controller.yml" +RENDER_VALUES = ROOT / "tests" / "capabilities" / "render.yml" _spec = importlib.util.spec_from_file_location("arc_filters", PLUGIN) assert _spec is not None and _spec.loader is not None arc = importlib.util.module_from_spec(_spec) @@ -48,6 +49,18 @@ def render(profile: dict[str, Any], **role_vars: Any) -> dict[str, Any]: return rendered +def render_values(profile: dict[str, Any], **role_vars: Any) -> dict[str, Any]: + """Render the release's values for profile through the role's own tasks/render_scale_set_values.yml, with no capabilities, and return them parsed.""" + with tempfile.TemporaryDirectory() as work: + output = Path(work) / "rendered.json" + extra = Path(work) / "extra.json" + extra.write_text(json.dumps({"profile": profile, "github_runner_arc_capabilities": {}, "render_output": str(output), **role_vars})) + env = {**os.environ, "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", "ANSIBLE_HOME": work, "ANSIBLE_LOCAL_TEMP": work} + subprocess.run([os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook"), str(RENDER_VALUES), "-e", f"@{extra}"], check=True, env=env, cwd=work, capture_output=True, text=True, stdin=subprocess.DEVNULL) + values: dict[str, Any] = json.loads(output.read_text())["values"] + return values + + def render_controller(**role_vars: Any) -> dict[str, Any]: """Render the controller values with ansible-playbook and return them parsed, an empty dict when nothing is set.""" with tempfile.TemporaryDirectory() as work: @@ -122,5 +135,49 @@ def test_set_adds_it_to_the_runner_pod_spec_and_nothing_else(self) -> None: self.assertEqual(overlay, without) + +class ListenerResourcesTest(unittest.TestCase): + RESOURCES = {"requests": {"memory": "32Mi", "cpu": "10m"}, "limits": {"memory": "64Mi"}} + # Each setting that makes the overlay build the listener container, and none of them. + CASES: dict[str, dict[str, Any]] = { + "nothing else": {}, + "the node label key": {"github_runner_arc_node_label_key": ELIGIBLE, "github_runner_arc_node_label_value": "true"}, + "the probes": {"github_runner_arc_listener_probes_enabled": True}, + "the priority class": {"github_runner_arc_control_pods_priority_enabled": True}, + } + + def test_empty_adds_no_listener_container_and_no_resources(self) -> None: + profile = expanded({"sizing": SIZING}) + self.assertNotIn("listenerTemplate", render(profile, github_runner_arc_listener_resources={})) + for case, role_vars in self.CASES.items(): + if not role_vars: + continue + with self.subTest(case): + containers = render(profile, github_runner_arc_listener_resources={}, **role_vars)["listenerTemplate"]["spec"]["containers"] + self.assertEqual([container["name"] for container in containers], ["listener"]) + self.assertNotIn("resources", containers[0]) + + def test_set_puts_them_on_the_listener_container_and_nothing_else(self) -> None: + profile = expanded({"sizing": SIZING}) + for case, role_vars in self.CASES.items(): + with self.subTest(case): + overlay = render(profile, github_runner_arc_listener_resources=self.RESOURCES, **role_vars) + containers = overlay["listenerTemplate"]["spec"]["containers"] + self.assertEqual([container["name"] for container in containers], ["listener"]) + self.assertEqual(containers[0].pop("resources"), self.RESOURCES) + without = render(profile, **role_vars) + # With no other listener setting the overlay has no listenerTemplate of its own, so the listener container is all it adds. + without.setdefault("listenerTemplate", {"spec": {"containers": [{"name": "listener"}]}}) + self.assertEqual(overlay, without) + + def test_set_reaches_the_release_values_over_a_values_file_listener(self) -> None: + # The overlay's containers list replaces the values file's, so the resources the variable sets are the ones the listener pod gets. + with tempfile.TemporaryDirectory() as work: + values_file = Path(work) / "values.yaml" + values_file.write_text(yaml.safe_dump({"listenerTemplate": {"spec": {"containers": [{"name": "listener", "resources": {"requests": {"memory": "1Mi"}}}]}}, "template": {"spec": {"containers": [{"name": "runner", "image": "ghcr.io/example/runner:1", "command": ["/home/runner/run.sh"]}]}}})) + values = render_values(expanded({"max_runners": 2, "values_file": str(values_file)}), github_runner_arc_listener_resources=self.RESOURCES) + self.assertEqual(values["listenerTemplate"]["spec"]["containers"], [{"name": "listener", "resources": self.RESOURCES}]) + + if __name__ == "__main__": unittest.main()