From 2f1347f9512a56ad14dcffcd2c5b40242fdd3edf Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 21:46:56 +0000 Subject: [PATCH 1/5] refactor(role): share the quorum guard between steps that restart a node The checks that refuse to recreate a node on the other network unless the cluster stays safe without it move to tasks/restart_guard.yml, so any step that restarts a running node uses the same reads and refusal. The step says what it does in the refusal, and whether runner pods on the node block it: a step that cordons the node and waits for them itself checks the cluster before that with the pods left out. --- .../tasks/host_network.yml | 79 +------------------ .../tasks/restart_guard.yml | 79 +++++++++++++++++++ 2 files changed, 83 insertions(+), 75 deletions(-) create mode 100644 roles/github_runner_cluster/tasks/restart_guard.yml diff --git a/roles/github_runner_cluster/tasks/host_network.yml b/roles/github_runner_cluster/tasks/host_network.yml index cbc478b0..c81590c5 100644 --- a/roles/github_runner_cluster/tasks/host_network.yml +++ b/roles/github_runner_cluster/tasks/host_network.yml @@ -136,80 +136,9 @@ recreating its node. Move one node at a time (--limit to one host). Nothing was changed. when: github_runner_cluster_network_movers | trim | split | length > 1 - - name: Read the cluster's nodes and this node's pods through each server of the inventory - # Through the image's kubectl in each server's own container (as running_version.yml reads the cluster), this one first, each on its own host with its pinned Docker context, so the read needs no kubeconfig or published port on the host and no particular server to be up; each read is bounded by github_runner_cluster_cluster_read_timeout as a whole and as kubectl's --request-timeout. The first server that answers is the one the checks below use; plugins/filter/cluster_guard.py's cluster_guard_state says why for each when none does. - ansible.builtin.shell: | - set -eu - nodes=$(docker exec "$container" kubectl get nodes --request-timeout="${timeout}s" -o json) - pods=$(docker exec "$container" kubectl get pods --all-namespaces --request-timeout="${timeout}s" --field-selector "spec.nodeName=${node}" -o json) - printf '{"nodes": %s, "pods": %s}\n' "$nodes" "$pods" - environment: >- - {{ ({'DOCKER_CONTEXT': github_runner_cluster_guard_context} if github_runner_cluster_guard_context | length > 0 else {}) - | combine({'container': github_runner_cluster_guard_container, 'node': github_runner_cluster_effective_node_name, - 'timeout': github_runner_cluster_cluster_read_timeout | string}) }} - delegate_to: "{{ item }}" - loop: >- - {{ ([inventory_hostname] if github_runner_cluster_effective_node_role == 'server' else []) - + (github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') - | map(attribute='key') | reject('equalto', inventory_hostname) | list) }} - vars: - github_runner_cluster_guard_container: >- - {{ github_runner_cluster_container_name if item == inventory_hostname - else hostvars[item].github_runner_cluster_container_name | default('github-runner-k3s') }} - github_runner_cluster_guard_context: >- - {{ (github_runner_cluster_docker_context | default('') if item == inventory_hostname - else hostvars[item].github_runner_cluster_docker_context | default('')) | trim }} - register: github_runner_cluster_guard_reads - changed_when: false - failed_when: false - ignore_unreachable: true - check_mode: false - timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" - - - name: Fail if no server of the inventory could be read - vars: - github_runner_cluster_guard_read: "{{ github_runner_cluster_guard_reads.results | exadev.github_runner.cluster_guard_state }}" - ansible.builtin.fail: - msg: >- - Not moving {{ github_runner_cluster_effective_node_name }} to the {{ github_runner_cluster_network_mode }} network, - which recreates its container: {{ github_runner_cluster_guard_read.problem }} Nothing was changed. - when: github_runner_cluster_guard_read.state is none - - - name: Read etcd's health through every other server's own API server - # kubectl inside each server's container asks that server's API server, whose /readyz/etcd answers "ok" only while its etcd serves; on that host, with its pinned Docker context. Bounded like the reads above by github_runner_cluster_cluster_read_timeout, as a whole and as the API call's --request-timeout. A host that cannot be reached, or a read that fails or runs out of time, counts as unhealthy rather than failing here, so the refusal below names it. - ansible.builtin.command: - argv: [docker, exec, "{{ hostvars[item.key].github_runner_cluster_container_name | default('github-runner-k3s') }}", kubectl, get, "--request-timeout={{ github_runner_cluster_cluster_read_timeout | int }}s", --raw, /readyz/etcd] - environment: >- - {{ {'DOCKER_CONTEXT': hostvars[item.key].github_runner_cluster_docker_context} - if hostvars[item.key].github_runner_cluster_docker_context | default('') | length > 0 else {} }} - delegate_to: "{{ item.key }}" - loop: >- - {{ github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') - | rejectattr('key', 'equalto', inventory_hostname) | list }} - loop_control: - label: "{{ item.value.node_name }}" - register: github_runner_cluster_guard_etcd - changed_when: false - failed_when: false - ignore_unreachable: true - check_mode: false - timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" - when: github_runner_cluster_effective_node_role == 'server' - - name: Refuse to recreate the node on the other network while the cluster needs it + ansible.builtin.include_tasks: restart_guard.yml vars: - github_runner_cluster_guard: "{{ (github_runner_cluster_guard_reads.results | exadev.github_runner.cluster_guard_state).state }}" - github_runner_cluster_guard_blockers: >- - {{ github_runner_cluster_guard.nodes - | exadev.github_runner.cluster_restart_blockers(github_runner_cluster_guard.pods, - github_runner_cluster_effective_node_name, - github_runner_cluster_topology.hosts | dict2items - | selectattr('value.role', 'equalto', 'server') - | map(attribute='value.node_name') | list, - github_runner_cluster_guard_etcd.results | default([])) }} - ansible.builtin.fail: - msg: >- - Not moving {{ github_runner_cluster_effective_node_name }} to the {{ github_runner_cluster_network_mode }} network, - which recreates its container: {{ github_runner_cluster_guard_blockers | join(' ') }} Nothing was changed; run again - once that has cleared. - when: github_runner_cluster_guard_blockers | length > 0 + github_runner_cluster_restart_guard_action: >- + moving {{ github_runner_cluster_effective_node_name }} to the {{ github_runner_cluster_network_mode }} network, which recreates its container + github_runner_cluster_restart_guard_runners: true diff --git a/roles/github_runner_cluster/tasks/restart_guard.yml b/roles/github_runner_cluster/tasks/restart_guard.yml new file mode 100644 index 00000000..00be5961 --- /dev/null +++ b/roles/github_runner_cluster/tasks/restart_guard.yml @@ -0,0 +1,79 @@ +--- +# Refuses a restart of this host's node unless the cluster stays safe without it (plugins/filter/cluster_guard.py's cluster_restart_blockers): the other servers with a Ready node and a healthy etcd must still be a majority of all servers, and, when github_runner_cluster_restart_guard_runners is true, no ARC runner pod that has not finished may be on the node. Whatever cannot be read refuses the restart. Shared by every step that restarts a running node: moving its container to the other network (tasks/host_network.yml) and resizing its Colima VM (tasks/colima_size_turn.yml). Takes github_runner_cluster_restart_guard_action, what the step does in words that follow "Not" in a refusal (for example "moving node1 to the host network, which recreates its container"), and github_runner_cluster_restart_guard_runners, false for a step that empties the node of its runner pods itself after this check. Every task only reads, and runs under --check too; a refusal changes nothing. + +- name: Read the cluster's nodes and this node's pods through each server of the inventory + # Through the image's kubectl in each server's own container (as running_version.yml reads the cluster), this one first, each on its own host with its pinned Docker context, so the read needs no kubeconfig or published port on the host and no particular server to be up; each read is bounded by github_runner_cluster_cluster_read_timeout as a whole and as kubectl's --request-timeout. The first server that answers is the one the checks below use; plugins/filter/cluster_guard.py's cluster_guard_state says why for each when none does. + ansible.builtin.shell: | + set -eu + nodes=$(docker exec "$container" kubectl get nodes --request-timeout="${timeout}s" -o json) + pods=$(docker exec "$container" kubectl get pods --all-namespaces --request-timeout="${timeout}s" --field-selector "spec.nodeName=${node}" -o json) + printf '{"nodes": %s, "pods": %s}\n' "$nodes" "$pods" + environment: >- + {{ ({'DOCKER_CONTEXT': github_runner_cluster_guard_context} if github_runner_cluster_guard_context | length > 0 else {}) + | combine({'container': github_runner_cluster_guard_container, 'node': github_runner_cluster_effective_node_name, + 'timeout': github_runner_cluster_cluster_read_timeout | string}) }} + delegate_to: "{{ item }}" + loop: >- + {{ ([inventory_hostname] if github_runner_cluster_effective_node_role == 'server' else []) + + (github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') + | map(attribute='key') | reject('equalto', inventory_hostname) | list) }} + vars: + github_runner_cluster_guard_container: >- + {{ github_runner_cluster_container_name if item == inventory_hostname + else hostvars[item].github_runner_cluster_container_name | default('github-runner-k3s') }} + github_runner_cluster_guard_context: >- + {{ (github_runner_cluster_docker_context | default('') if item == inventory_hostname + else hostvars[item].github_runner_cluster_docker_context | default('')) | trim }} + register: github_runner_cluster_guard_reads + changed_when: false + failed_when: false + ignore_unreachable: true + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + +- name: Fail if no server of the inventory could be read + vars: + github_runner_cluster_guard_read: "{{ github_runner_cluster_guard_reads.results | exadev.github_runner.cluster_guard_state }}" + ansible.builtin.fail: + msg: >- + Not {{ github_runner_cluster_restart_guard_action }}: {{ github_runner_cluster_guard_read.problem }} Nothing was changed. + when: github_runner_cluster_guard_read.state is none + +- name: Read etcd's health through every other server's own API server + # kubectl inside each server's container asks that server's API server, whose /readyz/etcd answers "ok" only while its etcd serves; on that host, with its pinned Docker context. Bounded like the reads above by github_runner_cluster_cluster_read_timeout, as a whole and as the API call's --request-timeout. A host that cannot be reached, or a read that fails or runs out of time, counts as unhealthy rather than failing here, so the refusal below names it. + ansible.builtin.command: + argv: [docker, exec, "{{ hostvars[item.key].github_runner_cluster_container_name | default('github-runner-k3s') }}", kubectl, get, "--request-timeout={{ github_runner_cluster_cluster_read_timeout | int }}s", --raw, /readyz/etcd] + environment: >- + {{ {'DOCKER_CONTEXT': hostvars[item.key].github_runner_cluster_docker_context} + if hostvars[item.key].github_runner_cluster_docker_context | default('') | length > 0 else {} }} + delegate_to: "{{ item.key }}" + loop: >- + {{ github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'server') + | rejectattr('key', 'equalto', inventory_hostname) | list }} + loop_control: + label: "{{ item.value.node_name }}" + register: github_runner_cluster_guard_etcd + changed_when: false + failed_when: false + ignore_unreachable: true + check_mode: false + timeout: "{{ github_runner_cluster_cluster_read_timeout | int }}" + when: github_runner_cluster_effective_node_role == 'server' + +- name: Refuse to restart the node while the cluster needs it + vars: + github_runner_cluster_guard: "{{ (github_runner_cluster_guard_reads.results | exadev.github_runner.cluster_guard_state).state }}" + # A step that cordons the node and waits for its runner pods itself checks the cluster before it does, with the pods left out, and again once they have gone. + github_runner_cluster_guard_blockers: >- + {{ github_runner_cluster_guard.nodes + | exadev.github_runner.cluster_restart_blockers(github_runner_cluster_guard.pods if github_runner_cluster_restart_guard_runners | bool else [], + github_runner_cluster_effective_node_name, + github_runner_cluster_topology.hosts | dict2items + | selectattr('value.role', 'equalto', 'server') + | map(attribute='value.node_name') | list, + github_runner_cluster_guard_etcd.results | default([])) }} + ansible.builtin.fail: + msg: >- + Not {{ github_runner_cluster_restart_guard_action }}: {{ github_runner_cluster_guard_blockers | join(' ') }} Nothing was + changed; run again once that has cleared. + when: github_runner_cluster_guard_blockers | length > 0 From 6416966a77648ff02c1f490d68f2e53d82a7478e Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 21:47:08 +0000 Subject: [PATCH 2/5] feat(role): size a Colima node VM from the inventory through the quorum guard github_runner_cluster_colima_memory_gib and github_runner_cluster_colima_cpus set a Colima host's VM size; empty keeps the profile's own. The role reads the VM Lima runs with colima list --json, and when a size differs it resizes one host at a time, each in its own turn: it checks the cluster stays safe without the node, takes the fleet-regrade Lease (refusing while another step holds it), cordons the node, waits for its runner pods and drains it, checks the cluster again, stops the profile, starts it with --memory and --cpus, waits for the node to be Ready (and an etcd voter on a server), uncordons it and gives the Lease back. A value that is not a positive whole number, or a size set on a host that does not run Colima, fails before anything is read. A VM of the size set is only read, so a rerun changes nothing, and --check reports the resize and changes nothing. The native role switch's drain and Ready wait are reused, the drain's refusal naming the step it serves. Closes #208 --- plugins/filter/cluster_colima.py | 72 +++ plugins/filter/cluster_guard.py | 85 ++- roles/github_runner_cluster/README.md | 9 +- roles/github_runner_cluster/defaults/main.yml | 16 +- .../tasks/colima_size.yml | 47 ++ .../tasks/colima_size_turn.yml | 218 +++++++ roles/github_runner_cluster/tasks/main.yml | 4 + .../tasks/native/drain.yml | 4 +- tests/colima_size/run.yml | 15 + tests/unit/test_cluster_colima_size.py | 562 ++++++++++++++++++ 10 files changed, 1024 insertions(+), 8 deletions(-) create mode 100644 plugins/filter/cluster_colima.py create mode 100644 roles/github_runner_cluster/tasks/colima_size.yml create mode 100644 roles/github_runner_cluster/tasks/colima_size_turn.yml create mode 100644 tests/colima_size/run.yml create mode 100644 tests/unit/test_cluster_colima_size.py diff --git a/plugins/filter/cluster_colima.py b/plugins/filter/cluster_colima.py new file mode 100644 index 00000000..0fdcb4b4 --- /dev/null +++ b/plugins/filter/cluster_colima.py @@ -0,0 +1,72 @@ +"""Filters the github_runner_cluster role uses to size a Colima node VM from the inventory (tasks/colima_size.yml): which sizes are set, and what `colima start` must be given to bring the profile's VM to them.""" + +from __future__ import annotations + +import json +import re +from typing import Any + +from ansible.errors import AnsibleFilterError + +# `colima start --memory` takes GiB (abiosoft/colima cmd/start.go, the "memory" flag), and `colima list --json` reports the VM's memory in bytes, as Lima does (environment/vm/lima/limautil/instance.go, InstanceInfo.Memory), so a GiB is 2^30 of them. +BYTES_PER_GIB = 1024**3 +WHOLE_NUMBER = re.compile(r"^[1-9][0-9]*$") +MEMORY_VARIABLE = "github_runner_cluster_colima_memory_gib" +CPUS_VARIABLE = "github_runner_cluster_colima_cpus" + + +def _size(name: str, value: Any) -> int | None: + """A positive whole number given as a number or as text, or None for an empty value, which keeps the profile's own.""" + if value is None or value == "": + return None + if isinstance(value, bool) or not isinstance(value, (int, str)) or not WHOLE_NUMBER.match(str(value)): + raise AnsibleFilterError(f"{name} must be a positive whole number, or empty to keep the profile's own, not {value!r}.") + return int(value) + + +def cluster_colima_size(memory_gib: Any, cpus: Any) -> dict[str, Any]: + """Check the sizes the inventory sets for a Colima node VM and give the `colima start` arguments that set them. + + :param memory_gib: github_runner_cluster_colima_memory_gib: the VM's memory in GiB, or empty. :param cpus: github_runner_cluster_colima_cpus: the VM's CPUs, or empty. + :returns: ``{"memory_gib": int | None, "cpus": int | None, "arguments": [...]}``, the arguments naming only the sizes that are set, as `colima start` keeps the profile's own value of a flag it is not given. + :raises AnsibleFilterError: if a value is set and is not a positive whole number. + """ + memory = _size(MEMORY_VARIABLE, memory_gib) + count = _size(CPUS_VARIABLE, cpus) + arguments = (["--memory", str(memory)] if memory is not None else []) + (["--cpus", str(count)] if count is not None else []) + return {"memory_gib": memory, "cpus": count, "arguments": arguments} + + +def _gib(memory: int) -> str: + return f"{memory / BYTES_PER_GIB:g}" + + +def cluster_colima_resize(listing: Any, profile: Any, memory_gib: Any, cpus: Any) -> dict[str, Any]: + """Compare the size of a profile's VM with the sizes the inventory sets. + + :param listing: what `colima list --profile --json` printed: one JSON document per profile and line, with ``name``, ``cpus`` and ``memory`` (bytes). It is the VM Lima runs, which a `colima start` given new sizes while the profile runs does not change (it writes them to the profile's configuration and leaves the VM as it is), so a size already written there but not applied still counts as a change. :param profile: the Colima profile. :param memory_gib: github_runner_cluster_colima_memory_gib, as cluster_colima_size takes it. :param cpus: github_runner_cluster_colima_cpus, likewise. :returns: ``{"changes": [...], "arguments": [...]}``: a sentence part for each size that differs, empty when the VM already has every size set, and the `colima start` arguments of cluster_colima_size. :raises AnsibleFilterError: if a size is not valid, the listing is not Colima's, or it does not name the profile. + """ + wanted = cluster_colima_size(memory_gib, cpus) + try: + records = [json.loads(line) for line in str(listing).splitlines() if line.strip()] + except ValueError as error: + raise AnsibleFilterError(f"`colima list --json` printed something that is not JSON: {error}") from error + record = next((item for item in records if isinstance(item, dict) and item.get("name") == profile), None) + if record is None: + raise AnsibleFilterError(f"Colima lists no profile named {profile}, so its size cannot be read (it listed {', '.join(str(item.get('name')) for item in records if isinstance(item, dict)) or 'none'}).") + if not isinstance(record.get("memory"), int) or not isinstance(record.get("cpus"), int): + raise AnsibleFilterError(f"Colima's listing of the profile {profile} gives no memory and CPU count: {record!r}.") + changes: list[str] = [] + if wanted["memory_gib"] is not None and record["memory"] != wanted["memory_gib"] * BYTES_PER_GIB: + changes.append(f"memory from {_gib(record['memory'])} GiB to {wanted['memory_gib']} GiB") + if wanted["cpus"] is not None and record["cpus"] != wanted["cpus"]: + changes.append(f"CPUs from {record['cpus']} to {wanted['cpus']}") + return {"changes": changes, "arguments": wanted["arguments"]} + + +class FilterModule: + """Registers the Colima sizing filters with Ansible.""" + + def filters(self) -> dict[str, Any]: + """Return the filters this plugin provides.""" + return {"cluster_colima_size": cluster_colima_size, "cluster_colima_resize": cluster_colima_resize} diff --git a/plugins/filter/cluster_guard.py b/plugins/filter/cluster_guard.py index 7cb2218f..e55f75f7 100644 --- a/plugins/filter/cluster_guard.py +++ b/plugins/filter/cluster_guard.py @@ -1,10 +1,12 @@ -"""Filters the github_runner_cluster role uses before it moves a node container between networks: whether recreating it would be safe for the cluster, and whether the Docker host has an address on the LAN.""" +"""Filters the github_runner_cluster role uses before it restarts a node (moving its container between networks, resizing its Colima VM): whether the restart would be safe for the cluster, whether the fleet-regrade Lease that serialises such steps may be taken and how it is given back, and whether the Docker host has an address on the LAN.""" from __future__ import annotations import ipaddress import json +import re from collections.abc import Mapping, Sequence +from datetime import datetime, timezone from typing import Any from ansible.errors import AnsibleFilterError @@ -170,9 +172,88 @@ def addresses_in_network(lines: Any, network: Any) -> list[str]: return found +def _micro_time(epoch: float) -> str: + """A Lease timestamp (RFC 3339 in UTC with microseconds, as the API server's MicroTime) for an epoch time, to the second as fleet-regrade's core writes it (micro_time in files/regrade/fleet-regrade-core.sh).""" + return datetime.fromtimestamp(int(epoch), tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.000000Z") + + +def _epoch(stamp: str) -> float: + return datetime.strptime(re.sub(r"\.[0-9]+Z$", "Z", stamp), "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc).timestamp() + + +def _lease_read(read: Any) -> dict[str, Any] | None | str: + """The Lease a `kubectl get lease --output json` result holds, None when the API server answered that there is none, or the reason the read failed.""" + if not isinstance(read, Mapping): + raise AnsibleFilterError("The Lease read must be a command result.") + if read.get("rc") == 0: + try: + current = json.loads(str(read.get("stdout", ""))) + except ValueError: + return "the API server's answer is not a Lease" + return current if isinstance(current, dict) else "the API server's answer is not a Lease" + detail = str(read.get("stderr", "") or read.get("msg", "")).strip() + if "(NotFound)" in detail: + return None + return detail or f"kubectl exited with status {read.get('rc')}" + + +def cluster_lease_claim(read: Any, holder: Any, now: Any, duration: Any, name: Any, namespace: Any) -> dict[str, Any]: + """Decide whether a holder may take a Lease, and give the Lease it would write: the protocol fleet-regrade's core follows (lease_state and acquire_lease in files/regrade/fleet-regrade-core.sh), so the two never step a node at the same time. + + A Lease nobody holds, whose holder's time has run out (renew time plus duration at or before now) or that the holder already holds is replaced, against the resourceVersion read so that the write fails if anyone changed it since; one that does not exist is created, which fails if anyone created it since; one another holds is refused. + + :param read: the result of `kubectl get lease --namespace --output json`, not failed by its own task. :param holder: the identity to hold it as. :param now: the epoch time to take and judge it at. :param duration: how many seconds it is held for if it is never given back. :param name: the Lease's name. :param namespace: its namespace. + :returns: ``{"state": "absent" | "free" | "expired" | "mine" | "held" | "unreadable", "holder": , "verb": "create" | "replace" | "" (refused), "manifest": , "problem": }``. + """ + current = _lease_read(read) + if isinstance(current, str): + return {"state": "unreadable", "holder": "", "verb": "", "manifest": {}, "problem": current} + stamp = _micro_time(float(now)) + manifest: dict[str, Any] = { + "apiVersion": "coordination.k8s.io/v1", + "kind": "Lease", + "metadata": {"name": str(name), "namespace": str(namespace)}, + "spec": {"holderIdentity": str(holder), "leaseDurationSeconds": int(duration), "acquireTime": stamp, "renewTime": stamp}, + } + if current is None: + return {"state": "absent", "holder": "", "verb": "create", "manifest": manifest, "problem": ""} + spec = current.get("spec") or {} + current_holder = str(spec.get("holderIdentity") or "") + renewed = spec.get("renewTime") or spec.get("acquireTime") + if not current_holder: + state = "free" + elif not renewed or _epoch(str(renewed)) + int(spec.get("leaseDurationSeconds") or 0) <= float(now): + state = "expired" + elif current_holder == holder: + state = "mine" + else: + return {"state": "held", "holder": current_holder, "verb": "", "manifest": {}, "problem": ""} + manifest["metadata"]["resourceVersion"] = str((current.get("metadata") or {}).get("resourceVersion", "")) + return {"state": state, "holder": current_holder, "verb": "replace", "manifest": manifest, "problem": ""} + + +def cluster_lease_release(read: Any, holder: Any) -> dict[str, Any]: + """Give a Lease back if the holder still holds it, as fleet-regrade's core does (release_lease): the holder is cleared, so the next one does not wait for it to expire, and a Lease that has meanwhile passed to another, or gone, is left alone. + + :param read: the result of `kubectl get lease ... --output json`, as cluster_lease_claim takes it. :param holder: the identity it was taken as. + :returns: ``{"verb": "replace" | "", "manifest": }``; an empty verb writes nothing. + """ + current = _lease_read(read) + if not isinstance(current, dict) or str((current.get("spec") or {}).get("holderIdentity") or "") != holder: + return {"verb": "", "manifest": {}} + released = {**current, "spec": {**current["spec"], "holderIdentity": ""}} + return {"verb": "replace", "manifest": released} + + class FilterModule: """Registers the cluster guard filters with Ansible.""" def filters(self) -> dict[str, Any]: """Return the filters this plugin provides.""" - return {"cluster_restart_blockers": cluster_restart_blockers, "cluster_guard_state": cluster_guard_state, "addresses_in_network": addresses_in_network} + return { + "cluster_restart_blockers": cluster_restart_blockers, + "cluster_guard_state": cluster_guard_state, + "addresses_in_network": addresses_in_network, + "cluster_lease_claim": cluster_lease_claim, + "cluster_lease_release": cluster_lease_release, + } diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 203b74ba..abc75e3f 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -178,6 +178,8 @@ A host that got the agent from release 2.19.0 moves over on its next run: the ro **Colima kernel.** On every Colima host, whether or not `github_runner_cluster_runtime_autostart` is on, once the VM answers (`tasks/colima_kernel.yml`), the role makes the VM's kernel pass bridged pod traffic through iptables, now and at every VM boot: it writes `/etc/modules-load.d/br_netfilter.conf` and `/etc/sysctl.d/90-github-runner-br-netfilter.conf` in the VM, runs `modprobe br_netfilter` when the VM has no `/proc/sys/net/bridge`, and sets `net.bridge.bridge-nf-call-iptables` and `net.bridge.bridge-nf-call-ip6tables` to 1 when they differ. Without the module a pod that talks to a Service whose only backend pod is on the same node gets no reply, because the Service's reverse NAT never applies to the bridged hairpin traffic; pods on other nodes keep working, so it stays latent until a single-backend Service such as CoreDNS lands on the node (the `br_netfilter` note under Non-obvious behaviour in the repository README has the symptoms). Everything runs in the VM through `colima ssh --profile `, as the VM's user with `sudo`, and no `become` on the host. A file is written only when its content differs, so a rerun changes nothing, and the module file takes the name the README's note told people to create by hand, so an existing one is replaced by the managed one. Under `--check` each change is reported and none is made, and nothing is read from a VM that a check run would only have started. Docker Desktop's VM is not reachable by the role and other runtimes are not touched. +**Colima VM size.** A Colima host can set its VM's memory in GiB and its number of CPUs, `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus`, each a positive whole number; empty, the default, keeps what the profile has (`tasks/colima_size.yml`). A value that is not a positive whole number, or either set on a host whose runtime is not Colima, fails the run before anything is read. The role reads the size of the VM Lima runs with `colima list --profile --json`, not the profile's configuration file, since `colima start` given a size while the profile runs writes it to the file and leaves the VM as it is ([`cmd/start.go`](https://github.com/abiosoft/colima/blob/v0.10.3/cmd/start.go)). A VM that already has every size set is left alone, so a rerun changes nothing. Otherwise the resize is a planned restart of the node, made one host at a time, each in its own turn, through the same quorum guard as moving a node between networks (`tasks/restart_guard.yml`): it refuses unless the other servers with a Ready node and a healthy etcd stay a majority of all servers; it takes the `fleet-regrade` Lease that regrading steps hold (`kube-system/fleet-regrade`, as `colima-resize/`), refusing while another step holds it, so a resize never overlaps a regrade step or another resize in this run or any other; it cordons the node, waits for its runner pods to finish and drains it (the role switch's `tasks/native/drain.yml`), checks the cluster again, runs `colima stop --profile ` and `colima start --profile --activate=false --memory --cpus ` (the flags of the values set), waits for the node to be Ready on a heartbeat later than the stop and, on a server, an etcd voter, uncordons it and gives the Lease back. Colima writes the new size to the profile's configuration, so the launchd job's later starts keep it. The node's container comes back with the VM (`restart: unless-stopped`). A node whose container is not running takes nothing from the cluster, so its VM is only stopped and started. When a step fails the node is uncordoned again and the run fails, saying whether the VM was stopped, and the Lease is given back, or expires after `github_runner_cluster_colima_resize_lease_seconds` when the API that holds it does not answer. Under `--check` the sizes, the Lease and the cluster are read, a refusal fails as it would in a real run, and the resize is reported and not made. Size the VM from what it runs (the autoscaler's measured per-job peaks plus the guest's own baseline) and record that beside the value in the inventory. + This is the role's whole scope: it never moves a host between Colima and Docker Desktop. ## Mesh providers @@ -333,7 +335,7 @@ On a Linux machine running Docker without a VM, the host network is the machine' Host mode needs the Docker host to have an address on the LAN the nodes share, given as `github_runner_cluster_lan_cidr`: before starting the node, the role reads the host's addresses from a throwaway container on the host network (`github_runner_cluster_network_probe_image`) and fails, changing nothing, when none lies in that network. Whenever the node's running container is on the other kind of network than the one asked for, Compose is about to recreate it, so the role first refuses unless the cluster stays safe without the node: the other servers whose node is Ready and whose own API server reports etcd healthy (`/readyz/etcd`, read through each server's container on its host) must still be a majority of all servers, and no ARC runner pod that has not finished may be on the node (cordon it and let its jobs finish). What cannot be told refuses the move: a cluster that cannot be read, servers the cluster does not label as servers, a server node that is missing, a server the inventory does not hold, or a server that cannot be reached. Only one node may move per run, since each host's checks see the others as healthy before any is recreated. All of this runs before the role changes anything, under `--check` too, and only reads (the address probe pulls its small image if the host lacks it). The check does not see which server leads etcd; the runbook below moves leadership off a node by hand. -On macOS the Docker host is a VM, which needs an address of its own on the LAN, and the role does not create or change VMs. With Colima that means a profile created bridged: a profile's network mode is fixed when it is created and its LAN address cannot be turned off afterwards, so an existing user-mode VM cannot be converted, and a node moves to a new profile instead. Docker Desktop has no bridged VM network. +On macOS the Docker host is a VM, which needs an address of its own on the LAN, and the role does not create VMs or change their network (it changes a Colima VM's memory and CPUs only; see [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). With Colima that means a profile created bridged: a profile's network mode is fixed when it is created and its LAN address cannot be turned off afterwards, so an existing user-mode VM cannot be converted, and a node moves to a new profile instead. Docker Desktop has no bridged VM network. ### Moving a node to a bridged Colima VM @@ -367,7 +369,7 @@ The new engine: 7. **Approve Colima's network helper once.** The first start of a profile with an address installs `socket_vmnet` and a sudoers entry for it, which an administrator approves once per host. 8. **Stop the node and keep its image.** Run `docker update --restart=no` and `docker stop` on the node's container, so nothing starts the old copy again. Then `docker save` the node's image under the name its container uses, so the new engine runs the same build rather than building one. 9. **Export the three volumes with the container stopped,** so the etcd data is consistent. Use a throwaway container that tars each volume to a file, and never print the node password volume's file: carrying it avoids a `NodePasswordValidationFailed` event when the node registers again. The image's `agent/containerd` directory can be left out of the k3s data; kubelet pulls its images again. -10. **Create the bridged profile**, sized for what the node runs: Colima's default VM is far smaller than a server that also runs runner pods needs. For example `colima start --profile --vm-type vz --cpus --memory --disk --network-address --network-mode bridged --network-interface --activate=false`. `--activate=false` leaves the host's current Docker context alone; without it Colima switches every other Docker client on the host to the new VM. Check that `colima status --profile --json` reports an `ip_address` on the LAN. On a memory-tight host, quit the old Docker engine first. +10. **Create the bridged profile**, sized for what the node runs: Colima's default VM is far smaller than a server that also runs runner pods needs. For example `colima start --profile --vm-type vz --cpus --memory --disk --network-address --network-mode bridged --network-interface --activate=false`. `--activate=false` leaves the host's current Docker context alone; without it Colima switches every other Docker client on the host to the new VM. Check that `colima status --profile --json` reports an `ip_address` on the LAN. On a memory-tight host, quit the old Docker engine first. Set `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus` for the host to the same sizes, so a later change of size goes through the role and its quorum guard. 11. **Import onto the new engine.** `docker load` the image, create the three volumes as Compose would, with its labels (`docker volume create --label com.docker.compose.project= --label com.docker.compose.volume= _`, for `k3s-data`, `k3s-tailscale-data` and `k3s-node`), and untar each export into its volume. Move the engine first and the network after, one change at a time, so that the second change goes through the role's guard: @@ -424,6 +426,7 @@ Each Compose project runs its own `docker-prune` service against the same daemon - `github_runner_cluster_network_mode` (default `bridge`, or `host`): the network the node's container runs on, and with `host`, `github_runner_cluster_lan_cidr` (required: the LAN the nodes share) and `github_runner_cluster_network_probe_image` (default `busybox:1.37`, for reading the host's addresses) (see [Networking](#networking)). - `github_runner_cluster_runtime_autostart_at` (default `auto`; `login` or `boot`) and `github_runner_cluster_runtime_boot_profile` (default `node`): whether the Colima profile is started by a launchd agent at login or by a launchd daemon at boot, and which profile `auto` starts at boot (see [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). - `github_runner_cluster_runtime_autostart` (default true), `github_runner_cluster_runtime` (default `auto`; `colima`, `docker_desktop` or `other`), `github_runner_cluster_colima_profile` (default empty, taken from the Docker context), `github_runner_cluster_colima_home` (default empty: the `COLIMA_HOME` is read from the endpoint of the profile's Docker context, and is unset for Colima's own `~/.colima`; set it only for a context that does not say, and the role passes it to colima and to the launchd agent), `github_runner_cluster_runtime_path` (default both Homebrew prefixes and the system directories): on a macOS host, start Colima at login through a launchd agent and check the runtime (see [Container runtime at login](#container-runtime-after-a-reboot-macos)). +- `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus` (default empty, keeping the profile's own): a Colima host's VM memory in GiB and its CPUs, each a positive whole number, applied one node at a time through the quorum guard when the VM differs (see Colima VM size under [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). `github_runner_cluster_colima_resize_lease_seconds`, `github_runner_cluster_colima_resize_lease_holder`, `github_runner_cluster_regrade_lease_name` and `github_runner_cluster_regrade_lease_namespace` are derived, never set directly. `github_runner_cluster_native_runner_wait_timeout` and `github_runner_cluster_native_ready_timeout` bound the wait for the node's runner pods and for the node to come back, as they do for a native node. - `github_runner_cluster_container_name` (default `github-runner-k3s`) and `github_runner_cluster_agent_container_name` (default `github-runner-k3s-agent`): the names of the node's server and agent containers, which a second cluster on the same Docker hosts sets to names of its own (see [Several clusters on one host](#several-clusters-on-one-host)). - `github_runner_cluster_compose_files`: extra Compose files for the host's k3s project, merged after `docker-compose.yml`, as paths relative to `github_runner_cluster_dir` or absolute. Several nodes on one Docker host each need their own `github_runner_cluster_dir`, `github_runner_cluster_compose_project` and container names; nodes of one cluster share its API port, so all but one of them on a host also need an extra file that publishes none (`ports: !reset []`, as `tests/mesh` does). - `github_runner_cluster_wait_for_nodes`: wait for every node of the inventory to be Ready afterwards, clearing the node-password registration of a replaced node that fails to rejoin (installs the kubernetes Python client through `github_runner_k8s_client`). The wait covers the nodes the inventory's hosts own, by their effective node names, and leaves out a server that memory-pressure regrading has stepped down and an inventory node that has not registered. A node the cluster holds that no inventory host owns (one flashed from an image and joined by hand, say) is never waited for: when it is not Ready the run prints one line with its name and status, and carries on. When no node of the inventory has registered, the run fails. @@ -484,6 +487,8 @@ or read from 1Password by adding `exadev.github_runner.github_runner_secrets_one `tests/unit/test_runtime_autostart.py` covers the [container runtime tasks](#container-runtime-after-a-reboot-macos) without a Mac: it renders the launchd agent and daemon through `tests/runtime_autostart/render_plist.yml`, and runs `tasks/runtime.yml` and `tasks/end_checks.yml` through the role against stand-ins for `docker`, `colima`, `launchctl`, `plutil` and `sudo` on `PATH`, the stand-in `launchctl` refusing to change the system domain unless it was run through the stand-in `sudo`. It checks the variable defaults, that Linux hosts are left alone, that a second run changes nothing, that `--check` writes and starts nothing, and that each way the runtime or the node's container can be down fails with its message. `tests/unit/test_container_runtime.py` covers the filter that works out the runtime from the Docker context. Nothing there touches a real Colima, launchd or Docker Desktop, so loading the agent into a real login session, and the daemon starting a Colima VM at a real boot with no one logged in, are the parts only a run on a Mac can show. +`tests/unit/test_cluster_colima_size.py` covers the Colima VM size: the filters that read the profile's size (`plugins/filter/cluster_colima.py`) and take and give back the `fleet-regrade` Lease (`plugins/filter/cluster_guard.py`), and `tasks/colima_size.yml` run through the role (`tests/colima_size/run.yml`) against stand-ins for `docker` and `colima` that keep each profile's VM, the node containers, the cluster's nodes and the Lease as files. It checks the `colima` arguments, that a VM of the size set is only read, that a second run changes nothing, the order of a resize (the Lease, the cordon, the runner pods, the drain, the stop, the start with the new size, the uncordon, the Lease given back), that a held Lease, a server that is not Ready and a server whose etcd is unhealthy each refuse before anything changes, that `--check` reports and changes nothing, that several hosts take their turns one at a time, and the refusals of a value that is not a whole number and of a host that does not run Colima. A real Colima VM's resize is shown only by its first run on a Mac. + `tests/unit/test_cluster_wait_nodes.py` runs the wait for nodes to be Ready and its rescue against a stand-in for `kubectl`: a node outside the inventory that is not Ready is reported and never waited for, and an inventory node that stays not Ready still fails the run. The mesh test does not cover it, since its nodes publish no API port for the host's `kubectl` to reach. `tests/unit/test_node_password.py` covers [the node password volume and the servers' turns](#node-passwords-and-recreating-servers) without Docker: the filters in `plugins/filter/cluster_recreate.py`, the Compose file's volumes, and `tasks/node_password.yml`, `tasks/server_recreate_check.yml`, `tasks/server_ready.yml` and `tasks/start_servers.yml` run through the role (`tests/node_password/run.yml`) against a stand-in for `docker` that moves a real tar stream between its `docker cp` processes and keeps containers, volumes and node readiness as files, and a stand-in for the Compose module that recreates a container whose rendered hash differs and takes its node out of Ready until the role has waited for it. It checks the seeding (mode, ownership, labels, no copy of the password anywhere on the host, a failed copy leaving no volume), that `--check` changes nothing and reports each recreate, that a server is refused while another is not Ready and that agents are not, and from the order of the calls that the turns do not overlap. The `hosted` and `in_cluster` mesh scenarios recreate a joined server in Docker (`recreate_a_joined_server` in `tests/mesh/run.sh`) and check that its node rejoins with the same password and Secret, that no other container restarts, and that the role's `--check` report names exactly that server. diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index 0b65d229..44693167 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -246,6 +246,16 @@ github_runner_cluster_colima_modules_load_file: /etc/modules-load.d/br_netfilter github_runner_cluster_colima_sysctl_file: /etc/sysctl.d/90-github-runner-br-netfilter.conf # Derived, never set directly: the command that runs another command in the profile's VM as the VM's user, which has passwordless sudo. github_runner_cluster_colima_vm_command: "{{ ['colima', 'ssh', '--profile', github_runner_cluster_effective_runtime.colima_profile, '--'] }}" +# The memory, in GiB, and the number of CPUs of this host's Colima VM: each a positive whole number, or empty (the default) to keep what the profile has. Colima hosts only: a host whose runtime is anything else and sets either fails before anything changes. When a value differs from what the profile's VM runs (`colima list --json`), the role resizes the VM through the quorum guard, one node at a time (tasks/colima_size.yml): it takes the fleet-regrade Lease, refusing while another step holds it, refuses unless the cluster stays safe without the node, cordons the node, waits for its runner pods to finish and drains it, checks the cluster again, stops the profile and starts it with the new size, waits for the node to be Ready again (and an etcd voter, on a server), uncordons it and gives the Lease back. `colima start` writes the size to the profile's configuration, so every later start, the launchd job's at boot or login included, keeps it. A node whose container is not running takes nothing from the cluster, and its VM is only stopped and started. Size the VM from what it runs, the autoscaler's measured per-job peaks plus the guest's own baseline, and record that beside the value in the inventory. +github_runner_cluster_colima_memory_gib: "" +github_runner_cluster_colima_cpus: "" +# Derived, never set directly: how long the Lease a resize takes lasts if its holder never gives it back. It covers the holder's long steps, each bounded by its own timeout: the wait for the runner pods and the drain (github_runner_cluster_native_runner_wait_timeout each), and the stop, the start and the wait for the node to come back (github_runner_cluster_native_ready_timeout each). A holder that is working never loses it; one that crashed blocks other steps for no longer than this. +github_runner_cluster_colima_resize_lease_seconds: "{{ 2 * (github_runner_cluster_native_runner_wait_timeout | int) + 3 * (github_runner_cluster_native_ready_timeout | int) }}" +# Derived, never set directly: who a resize holds the Lease as. It names the node but is never a node name, so that the node's own regrade service, which holds it by node name, reads it as held by another and does not step meanwhile. +github_runner_cluster_colima_resize_lease_holder: "colima-resize/{{ github_runner_cluster_effective_node_name }}" +# Not settings: the Lease that serialises every step that takes a node away, as fleet-regrade's core names it (LEASE_NAME and LEASE_NAMESPACE in files/regrade/fleet-regrade-core.sh). +github_runner_cluster_regrade_lease_name: fleet-regrade +github_runner_cluster_regrade_lease_namespace: kube-system # The directory on the target host the role manages as this host's Compose project: it copies the collection's docker-compose.yml and k3s/ build context there, templates .env there, and k3s writes the kubeconfig to its kubeconfig/ subdirectory. Needs no checkout of anything on the host. The default reads HOME from the target's gathered facts, so it is the connecting user's home on that host, not the control node's. github_runner_cluster_dir: "{{ ansible_facts['env']['HOME'] }}/{{ github_runner_cluster_dir_name }}" @@ -272,7 +282,7 @@ github_runner_cluster_node_mode: compose # Native mode only, from here on. Whether the role enables and starts the node's service (k3s-agent or k3s), restarts it when its binary or configuration changed, switches the host between the server and the agent unit when its role changed, and then waits for the node to report Ready as a server sees it. false installs and configures everything and leaves the services as it finds them (a first install stays disabled and stopped, and a host whose role changed keeps running its old unit), for staging a host before the cluster is joined or before its role is switched. github_runner_cluster_native_start: true -# The most seconds, about, the role waits for the started node to report Ready on a heartbeat from the k3s that was just started. The same bound as a recreated server's. +# The most seconds, about, the role waits for the started node to report Ready on a heartbeat from the k3s that was just started, a native node's or one whose Colima VM was resized, and the most each `colima stop` and `colima start` of a resize may take. The same bound as a recreated server's. github_runner_cluster_native_ready_timeout: "{{ github_runner_cluster_server_ready_timeout }}" # Where the k3s release binaries are downloaded from: //, as GitHub publishes them. The download is checked against the SHA-256 sum the same release lists for the asset, unless github_runner_cluster_native_k3s_sha256 is set. A variable so a mirror, or a test, can stand in. github_runner_cluster_native_k3s_download_url: https://github.com/k3s-io/k3s/releases/download @@ -299,8 +309,10 @@ github_runner_cluster_native_role_file: "{{ github_runner_cluster_native_data_di github_runner_cluster_native_demotion_min_voters: 3 # Derived, never set directly: the prefix of the file, one per unit (-k3s-agent, -k3s), that says the unit's files changed since it last started, so a restart that a run refused or did not reach is made by the next run. github_runner_cluster_native_restart_marker: "{{ github_runner_cluster_native_data_dir }}/fleet-restart-pending" -# How long, about, a role switch waits for the runner pods on the node to finish after it has cordoned it, in seconds, before it gives up, uncordons the node and fails, changing nothing else: a planned switch never cuts a job. Each poll lists the node's pods, so a job that finishes is seen on the next poll. +# How long, about, a planned restart of a node (a native node's role switch, a Colima VM resize) waits for the runner pods on the node to finish after it has cordoned it, in seconds, before it gives up, uncordons the node and fails, changing nothing else: a planned restart never cuts a job. Each poll lists the node's pods, so a job that finishes is seen on the next poll. The drain that follows is bounded by it too. github_runner_cluster_native_runner_wait_timeout: "{{ github_runner_cluster_native_ready_timeout }}" +# Derived, never set directly: what tasks/native/drain.yml says was not done when it cannot empty the node, in words that follow "Not". A role switch's by default; another caller of drain.yml passes its own. +github_runner_cluster_native_drain_for: "switching {{ inventory_hostname }} from {{ github_runner_cluster_native_current_role }} to {{ github_runner_cluster_native_role }}" # Not settings: where the native node's files live. Variables only so the tests can point them at a stand-in. The binary goes in bin_dir, the PATH wrapper and the helpers under lib_dir, the configurations and the two secrets (token, vpn-auth) in config_dir, the units in unit_dir, and data_dir is k3s's own data directory (a server's datastore is server/db under it, and the role file sits beside it); tailscale_cli is the Tailscale command the package installs. github_runner_cluster_native_bin_dir: /usr/local/bin github_runner_cluster_native_lib_dir: /usr/local/lib/github-runner-k3s diff --git a/roles/github_runner_cluster/tasks/colima_size.yml b/roles/github_runner_cluster/tasks/colima_size.yml new file mode 100644 index 00000000..1c550345 --- /dev/null +++ b/roles/github_runner_cluster/tasks/colima_size.yml @@ -0,0 +1,47 @@ +--- +# Sizes this host's Colima VM from github_runner_cluster_colima_memory_gib and github_runner_cluster_colima_cpus (see their defaults). With neither set it reads and changes nothing. With either set it checks the values and that the host's runtime is a Colima profile, before anything is read, then reads the size of the profile's VM with `colima list --json` and works out what differs; a VM that already has every size set is left as it is, so a second run changes nothing. The hosts whose VM differs are then resized one at a time, each in its own turn (tasks/colima_size_turn.yml), in the same way tasks/start_servers.yml gives each server its turn: Ansible's linear strategy runs every host's copy of a task together, except that the tasks one host includes dynamically are its own, so while one host's turn runs the others wait and no two nodes are ever away together. Needs runtime.yml (the runtime and its environment) and topology.yml to have run. Under --check the values are checked and the size, the Lease and the cluster are read, a resize is reported as a change and a refusal as a failure, and nothing is changed. + +- name: Check the sizes set for the Colima VM + # On every host, so that a value colima could not take fails here, before a VM is stopped for it. Empty values keep the profile's own size. + ansible.builtin.set_fact: + github_runner_cluster_colima_size_wanted: >- + {{ github_runner_cluster_colima_memory_gib | exadev.github_runner.cluster_colima_size(github_runner_cluster_colima_cpus) }} + +- name: Fail unless this host's runtime is a Colima profile + # The role sizes only a VM it can stop and start with colima; a size set for any other host would otherwise be ignored without a word. + ansible.builtin.fail: + msg: >- + {{ inventory_hostname }} sets github_runner_cluster_colima_memory_gib or github_runner_cluster_colima_cpus, but only a Colima profile can + be resized, and this host's container runtime is + {{ github_runner_cluster_effective_runtime.kind if ansible_facts['system'] == 'Darwin' else 'the Docker of a ' ~ ansible_facts['system'] ~ ' host' }}. + Remove both from this host's variables. Nothing was changed. + when: + - github_runner_cluster_colima_size_wanted.arguments | length > 0 + - ansible_facts['system'] != 'Darwin' or github_runner_cluster_effective_runtime.kind != 'colima' + +- name: Read the size of the profile's VM + # The VM Lima runs, not the profile's configuration file, since a `colima start` given sizes while the profile runs writes them to the file and leaves the VM as it is (RunE and PreRunE in abiosoft/colima cmd/start.go). Read-only, so it runs under --check too. + ansible.builtin.command: + argv: [colima, list, --profile, "{{ github_runner_cluster_effective_runtime.colima_profile }}", --json] + environment: "{{ github_runner_cluster_runtime_environment }}" + register: github_runner_cluster_colima_size_listing + changed_when: false + check_mode: false + when: github_runner_cluster_colima_size_wanted.arguments | length > 0 + +- name: Work out how the VM's size differs from the one set + ansible.builtin.set_fact: + github_runner_cluster_colima_size_plan: >- + {{ (github_runner_cluster_colima_size_listing.stdout + | exadev.github_runner.cluster_colima_resize(github_runner_cluster_effective_runtime.colima_profile, + github_runner_cluster_colima_memory_gib, github_runner_cluster_colima_cpus)) + if github_runner_cluster_colima_size_wanted.arguments | length > 0 else {'changes': [], 'arguments': []} }} + +- name: Resize each host's VM in its own turn + ansible.builtin.include_tasks: colima_size_turn.yml + loop: "{{ ansible_play_hosts_all }}" + loop_control: + loop_var: github_runner_cluster_colima_size_turn + when: + - github_runner_cluster_colima_size_turn == inventory_hostname + - github_runner_cluster_colima_size_plan.changes | length > 0 diff --git a/roles/github_runner_cluster/tasks/colima_size_turn.yml b/roles/github_runner_cluster/tasks/colima_size_turn.yml new file mode 100644 index 00000000..c2d93f44 --- /dev/null +++ b/roles/github_runner_cluster/tasks/colima_size_turn.yml @@ -0,0 +1,218 @@ +--- +# One host's turn of resizing its Colima VM (tasks/colima_size.yml gives each host whose VM differs from the size set its turn, one at a time), through the quorum guard, as any change that takes a running node away is made. The node's k3s container runs inside the VM, so stopping the VM stops the node, and Docker starts the container again with the VM (restart: unless-stopped in files/docker-compose.yml). In order: +# +# 1. checks that the cluster stays safe without the node (tasks/restart_guard.yml, with its runner pods left out, since step 3 waits for them); +# 2. takes the fleet-regrade Lease, the one every step that takes a node away holds (fleet-regrade's core takes it to step a server down), refusing while another holds it, so a resize never overlaps a regrade step or another resize, in this run or any other; +# 3. cordons the node, waits for its runner pods to finish and drains it (tasks/native/drain.yml, which a native node's role switch uses too); +# 4. checks the cluster again, runner pods included, since a peer may have gone away while the pods finished; +# 5. stops the profile and starts it with the new size, through the same `colima start --profile --activate=false` that runtime.yml and the launchd job run; +# 6. waits for the node to be Ready on a heartbeat later than the stop and, on a server, an etcd voter (tasks/native/server_ready.yml); +# 7. uncordons the node, and gives the Lease back whatever happened. +# +# A node whose container is not running (a first install, or a node stopped by hand) takes nothing from the cluster, so its VM is only stopped and started. When a step fails the node is uncordoned again and the run fails, saying whether the VM was stopped. Under --check the cluster and the Lease are read, a refusal fails as it would in a real run, and the resize is reported and not made. + +- name: Read whether this node's container runs + # docker ps rather than docker inspect, so that a container that does not exist reads as not running while a daemon that cannot be reached fails the run. + ansible.builtin.command: + argv: [docker, ps, --all, --filter, "name=^/{{ github_runner_cluster_node_container }}$", --format, "{{ '{{' }}.State{{ '}}' }}"] + environment: "{{ github_runner_cluster_runtime_environment }}" + register: github_runner_cluster_colima_size_container + changed_when: false + check_mode: false + +- name: Note what the resize changes and whether it takes a node away + ansible.builtin.set_fact: + github_runner_cluster_colima_size_live: "{{ github_runner_cluster_colima_size_container.stdout | trim == 'running' }}" + github_runner_cluster_colima_size_action: >- + resizing the Colima VM of {{ github_runner_cluster_effective_node_name }} ({{ github_runner_cluster_colima_size_plan.changes | join(', ') }}), which restarts its node + +- name: Check, before anything changes, that the cluster stays safe without this node + ansible.builtin.include_tasks: restart_guard.yml + vars: + github_runner_cluster_restart_guard_action: "{{ github_runner_cluster_colima_size_action }}" + github_runner_cluster_restart_guard_runners: false + when: github_runner_cluster_colima_size_live | bool + +- name: Work out which server's API the resize goes through + # This host's own when it is a server, since its API answers until the VM stops and again before the node is uncordoned; otherwise another server of the inventory's. Worked out whether or not the node runs, though only a running one is read through it, since Ansible templates a task's delegate_to before its condition, and api.yml reads nothing. + ansible.builtin.include_tasks: native/api.yml + vars: + github_runner_cluster_native_api_self: "{{ github_runner_cluster_effective_node_role == 'server' }}" + +- name: Read the fleet-regrade Lease + # A read that fails is recorded rather than failing here, so the refusal below says why. Bounded by github_runner_cluster_cluster_read_timeout, as the API call's --request-timeout. + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: >- + {{ github_runner_cluster_native_api_argv + ['get', 'lease', github_runner_cluster_regrade_lease_name, '--namespace', github_runner_cluster_regrade_lease_namespace, + '--output', 'json', '--request-timeout=' ~ github_runner_cluster_cluster_read_timeout ~ 's'] }} + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + register: github_runner_cluster_colima_size_lease + changed_when: false + failed_when: false + check_mode: false + when: github_runner_cluster_colima_size_live | bool + +- name: Work out whether the resize may take the Lease + ansible.builtin.set_fact: + github_runner_cluster_colima_size_claim: >- + {{ github_runner_cluster_colima_size_lease + | exadev.github_runner.cluster_lease_claim(github_runner_cluster_colima_resize_lease_holder, now().timestamp(), + github_runner_cluster_colima_resize_lease_seconds, github_runner_cluster_regrade_lease_name, + github_runner_cluster_regrade_lease_namespace) }} + when: github_runner_cluster_colima_size_live | bool + +- name: Refuse while another step holds the Lease, or when it cannot be read + ansible.builtin.fail: + msg: >- + Not {{ github_runner_cluster_colima_size_action }}: + {{ ('the fleet-regrade Lease is held by ' ~ github_runner_cluster_colima_size_claim.holder ~ ', which is taking a node away now') + if github_runner_cluster_colima_size_claim.state == 'held' + else ('the fleet-regrade Lease cannot be read (' ~ github_runner_cluster_colima_size_claim.problem ~ ')') }}. + Only one node is taken away at a time. Nothing was changed; run again once the Lease is free. + when: + - github_runner_cluster_colima_size_live | bool + - github_runner_cluster_colima_size_claim.verb | length == 0 + +- name: Report the resize a real run would make + ansible.builtin.debug: + msg: >- + Would resize the Colima VM of {{ github_runner_cluster_effective_node_name }} ({{ github_runner_cluster_colima_size_plan.changes | join(', ') }}): + {{ 'take the fleet-regrade Lease, cordon the node, wait for its runner pods and drain it, ' if github_runner_cluster_colima_size_live | bool else '' }}stop + the profile {{ github_runner_cluster_effective_runtime.colima_profile }} and start it with + {{ github_runner_cluster_colima_size_plan.arguments | join(' ') }}{{ ', wait for the node to be Ready again, uncordon it and give the Lease back' + if github_runner_cluster_colima_size_live | bool else ' (its node container is not running, so the cluster is not involved)' }}. + changed_when: true + when: ansible_check_mode + +- name: Resize the VM + when: not ansible_check_mode + block: + - name: Take the fleet-regrade Lease + # The write fails if the Lease changed, or was created, since it was read, so two steps can never both take it. The Lease is piped to `kubectl create -f -` or `replace -f -`, and docker exec passes its standard input on only with --interactive; a native server's `k3s kubectl` reads it directly. + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: >- + {{ ((github_runner_cluster_native_api_argv[:2] + ['--interactive'] + github_runner_cluster_native_api_argv[2:]) + if github_runner_cluster_native_api_argv[0] == 'docker' else github_runner_cluster_native_api_argv) + + [github_runner_cluster_colima_size_claim.verb, '-f', '-'] }} + stdin: "{{ github_runner_cluster_colima_size_claim.manifest | to_json }}" + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + changed_when: true + when: github_runner_cluster_colima_size_live | bool + + - name: Empty the node + ansible.builtin.include_tasks: native/drain.yml + vars: + github_runner_cluster_native_drain_for: "{{ github_runner_cluster_colima_size_action }}" + when: github_runner_cluster_colima_size_live | bool + + - name: Check again, now the node is empty, that the cluster stays safe without it + ansible.builtin.include_tasks: restart_guard.yml + vars: + github_runner_cluster_restart_guard_action: "{{ github_runner_cluster_colima_size_action }}" + github_runner_cluster_restart_guard_runners: true + when: github_runner_cluster_colima_size_live | bool + + - name: Stop the Colima profile + ansible.builtin.command: + argv: [colima, stop, --profile, "{{ github_runner_cluster_effective_runtime.colima_profile }}"] + environment: "{{ github_runner_cluster_runtime_environment }}" + changed_when: true + timeout: "{{ github_runner_cluster_native_ready_timeout | int }}" + + - name: Read when the VM had stopped + # On the host, whose clock Lima keeps the VM's in step with: any renewal of the node's kubelet lease later than this is the restarted kubelet's, since the old one stopped with the VM. + ansible.builtin.command: + argv: [date, -u, "+%Y-%m-%dT%H:%M:%S"] + register: github_runner_cluster_colima_size_stopped + changed_when: false + + - name: Start the Colima profile with the new size + # --memory and --cpus take effect only on a start of a stopped profile, and are written to the profile's configuration, so later starts keep them (PreRunE and prepareConfig in abiosoft/colima cmd/start.go). + ansible.builtin.command: + argv: >- + {{ ['colima', 'start', '--profile', github_runner_cluster_effective_runtime.colima_profile, '--activate=false'] + + github_runner_cluster_colima_size_plan.arguments }} + environment: "{{ github_runner_cluster_runtime_environment }}" + changed_when: true + timeout: "{{ github_runner_cluster_native_ready_timeout | int }}" + + - name: Wait for the node to come back + ansible.builtin.include_tasks: native/server_ready.yml + vars: + github_runner_cluster_native_role: "{{ github_runner_cluster_effective_node_role }}" + github_runner_cluster_native_started: "{{ github_runner_cluster_colima_size_stopped }}" + when: github_runner_cluster_colima_size_live | bool + + - name: Uncordon the node + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: "{{ github_runner_cluster_native_api_argv + ['uncordon', github_runner_cluster_effective_node_name] }}" + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + changed_when: true + when: github_runner_cluster_colima_size_live | bool + rescue: + - name: Uncordon the node again, since the resize did not finish + # Through the same API, which answers unless this host is a server whose VM did not come back; then the node stays cordoned and the failure below says so. + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: "{{ github_runner_cluster_native_api_argv + ['uncordon', github_runner_cluster_effective_node_name] }}" + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + register: github_runner_cluster_colima_size_uncordoned + changed_when: github_runner_cluster_colima_size_uncordoned.rc == 0 + failed_when: false + when: github_runner_cluster_colima_size_live | bool + + - name: Fail, saying where the resize stopped + vars: + github_runner_cluster_colima_size_failure: >- + {{ ansible_failed_task.name }} failed: {{ ansible_failed_result.msg | default('') | trim }}{{ ' ' ~ (ansible_failed_result.stderr | default('') | trim) if ansible_failed_result.stderr | default('') | trim | length > 0 else '' }} + github_runner_cluster_colima_size_cordon: >- + {{ '' if not github_runner_cluster_colima_size_live | bool + else ('The node was uncordoned again.' if github_runner_cluster_colima_size_uncordoned.rc | default(1) == 0 + else 'The node could not be uncordoned and is still cordoned: uncordon it once it is back (kubectl uncordon ' ~ github_runner_cluster_effective_node_name ~ ').') }} + ansible.builtin.fail: + msg: >- + {% if github_runner_cluster_colima_size_stopped is defined or ansible_failed_task.name == 'Stop the Colima profile' %}The Colima VM of + {{ github_runner_cluster_effective_node_name }} was {{ 'stopped' if github_runner_cluster_colima_size_stopped is defined else 'being stopped' }} to resize it + ({{ github_runner_cluster_colima_size_plan.changes | join(', ') }}), and {{ github_runner_cluster_colima_size_failure }}. + If the VM is not running, start it with `colima start --profile {{ github_runner_cluster_effective_runtime.colima_profile }} --activate=false` + and check its node before running again.{% else %}Not {{ github_runner_cluster_colima_size_action }}: {{ github_runner_cluster_colima_size_failure }}. + The VM was not touched.{% endif %} {{ github_runner_cluster_colima_size_cordon }} + always: + - name: Read the Lease again, to give it back + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: >- + {{ github_runner_cluster_native_api_argv + ['get', 'lease', github_runner_cluster_regrade_lease_name, '--namespace', github_runner_cluster_regrade_lease_namespace, + '--output', 'json', '--request-timeout=' ~ github_runner_cluster_cluster_read_timeout ~ 's'] }} + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + register: github_runner_cluster_colima_size_lease_held + changed_when: false + failed_when: false + when: github_runner_cluster_colima_size_live | bool + + - name: Give the Lease back, if this resize still holds it + # A Lease that cannot be given back now (this server's API did not come back) expires on its own after github_runner_cluster_colima_resize_lease_seconds. + vars: + github_runner_cluster_colima_size_release: >- + {{ github_runner_cluster_colima_size_lease_held | exadev.github_runner.cluster_lease_release(github_runner_cluster_colima_resize_lease_holder) }} + become: "{{ github_runner_cluster_native_api_become }}" + ansible.builtin.command: + argv: >- + {{ ((github_runner_cluster_native_api_argv[:2] + ['--interactive'] + github_runner_cluster_native_api_argv[2:]) + if github_runner_cluster_native_api_argv[0] == 'docker' else github_runner_cluster_native_api_argv) + + [github_runner_cluster_colima_size_release.verb, '-f', '-'] }} + stdin: "{{ github_runner_cluster_colima_size_release.manifest | to_json }}" + environment: "{{ github_runner_cluster_native_api_environment }}" + delegate_to: "{{ github_runner_cluster_native_api_host }}" + changed_when: true + when: + - github_runner_cluster_colima_size_live | bool + - github_runner_cluster_colima_size_release.verb | length > 0 diff --git a/roles/github_runner_cluster/tasks/main.yml b/roles/github_runner_cluster/tasks/main.yml index 54e91fd2..f60fdbb4 100644 --- a/roles/github_runner_cluster/tasks/main.yml +++ b/roles/github_runner_cluster/tasks/main.yml @@ -78,6 +78,10 @@ # Before anything changes on the host, and so before the mesh step, which starts the bootstrap server's Compose project itself with the headscale_in_cluster provider. ansible.builtin.include_tasks: host_network.yml +- name: Size this host's Colima VM from the inventory, one node at a time through the quorum guard + # After the runtime is up and the topology decided, both of which the resize reads. See tasks/colima_size.yml. + ansible.builtin.include_tasks: colima_size.yml + - name: Check the Compose files the node's .env names ansible.builtin.include_tasks: compose_files.yml diff --git a/roles/github_runner_cluster/tasks/native/drain.yml b/roles/github_runner_cluster/tasks/native/drain.yml index d0f84d04..889c45ac 100644 --- a/roles/github_runner_cluster/tasks/native/drain.yml +++ b/roles/github_runner_cluster/tasks/native/drain.yml @@ -51,7 +51,7 @@ - name: Fail, with the node as it was ansible.builtin.fail: msg: >- - Not switching {{ inventory_hostname }} from {{ github_runner_cluster_native_current_role }} to {{ github_runner_cluster_native_role }}: + Not {{ github_runner_cluster_native_drain_for }}: its node could not be emptied ({{ ansible_failed_result.msg | default('') | trim }}{{ ' ' ~ (ansible_failed_result.stdout | default('') | trim) if ansible_failed_result.stdout | default('') | trim | length > 0 else '' }}). Runner pods still running after about {{ github_runner_cluster_native_runner_wait_timeout }} seconds are left to finish. The node - is uncordoned and its units are as they were; run the role again once its jobs have finished. + is uncordoned and nothing else was changed; run the role again once its jobs have finished. diff --git a/tests/colima_size/run.yml b/tests/colima_size/run.yml new file mode 100644 index 00000000..60821f7d --- /dev/null +++ b/tests/colima_size/run.yml @@ -0,0 +1,15 @@ +# Runs the cluster role's tasks/colima_size.yml through the role itself, as tasks/main.yml includes it, for tests/unit/test_cluster_colima_size.py, on the hosts of an inventory the test writes (all on a local connection, so a task delegated to another server runs here too). The host's operating system is stood in for by runtime_system, and the docker and colima the tasks call are the test's stand-ins on PATH. +- name: Run the cluster role's Colima VM size tasks + hosts: all + connection: local + gather_facts: false + tasks: + - name: Stand in for the host's facts + ansible.builtin.set_fact: + ansible_facts: + system: "{{ runtime_system | default('Darwin') }}" + + - name: Include the role's tasks + ansible.builtin.include_role: + name: exadev.github_runner.github_runner_cluster + tasks_from: colima_size diff --git a/tests/unit/test_cluster_colima_size.py b/tests/unit/test_cluster_colima_size.py new file mode 100644 index 00000000..83427389 --- /dev/null +++ b/tests/unit/test_cluster_colima_size.py @@ -0,0 +1,562 @@ +"""Tests for how the cluster role sizes a Colima node VM from the inventory (tasks/colima_size.yml and tasks/colima_size_turn.yml): the filters that read the profile's size and the fleet-regrade Lease (plugins/filter/cluster_colima.py, plugins/filter/cluster_guard.py), and the tasks run through the role (tests/colima_size/run.yml) against stand-ins for docker and colima, so no real VM, Docker daemon or cluster is reached.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from types import ModuleType +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ROLE = ROOT / "roles" / "github_runner_cluster" +RUN = ROOT / "tests" / "colima_size" / "run.yml" +ANSIBLE_PLAYBOOK = os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook") +DEFAULTS: dict[str, Any] = yaml.safe_load((ROLE / "defaults" / "main.yml").read_text()) +GIB = 1024**3 + + +def load_filter(name: str) -> ModuleType: + spec = importlib.util.spec_from_file_location(f"{name}_filters", ROOT / "plugins" / "filter" / f"{name}.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +colima = load_filter("cluster_colima") +guard = load_filter("cluster_guard") + + +def listing(name: str = "node", cpus: int = 2, memory_gib: int = 4, status: str = "Running") -> str: + """What `colima list --profile --json` prints: one JSON document per line, memory and disk in bytes.""" + return json.dumps({"name": name, "status": status, "arch": "aarch64", "cpus": cpus, "memory": memory_gib * GIB, "disk": 100 * GIB, "runtime": "docker"}) + "\n" + + +class ColimaSizeFilterTest(unittest.TestCase): + def test_a_profile_of_another_size_is_resized_to_the_values_set(self) -> None: + plan = colima.cluster_colima_resize(listing(cpus=2, memory_gib=4), "node", 8, 4) + self.assertEqual(plan["arguments"], ["--memory", "8", "--cpus", "4"]) + self.assertEqual(plan["changes"], ["memory from 4 GiB to 8 GiB", "CPUs from 2 to 4"]) + + def test_a_profile_of_the_size_set_needs_nothing(self) -> None: + self.assertEqual(colima.cluster_colima_resize(listing(cpus=4, memory_gib=8), "node", "8", "4")["changes"], []) + + def test_an_empty_value_keeps_that_part_of_the_size(self) -> None: + plan = colima.cluster_colima_resize(listing(cpus=2, memory_gib=4), "node", "", 4) + self.assertEqual((plan["arguments"], plan["changes"]), (["--cpus", "4"], ["CPUs from 2 to 4"])) + self.assertEqual(colima.cluster_colima_resize(listing(), "node", "", "")["arguments"], []) + + def test_memory_that_is_not_a_whole_number_of_gib_is_reported_as_it_is(self) -> None: + line = json.dumps({"name": "node", "cpus": 2, "memory": 6 * GIB + GIB // 2}) + "\n" + self.assertEqual(colima.cluster_colima_resize(line, "node", 6, "")["changes"], ["memory from 6.5 GiB to 6 GiB"]) + + def test_the_profile_asked_for_is_the_one_read(self) -> None: + plan = colima.cluster_colima_resize(listing("other", cpus=8, memory_gib=16) + listing("node", cpus=4, memory_gib=8), "node", 8, 4) + self.assertEqual(plan["changes"], []) + + def test_a_profile_colima_does_not_list_is_an_error(self) -> None: + with self.assertRaisesRegex(colima.AnsibleFilterError, "Colima lists no profile named node"): + colima.cluster_colima_resize(listing("other"), "node", 8, 4) + + def test_a_value_that_is_not_a_positive_whole_number_is_refused_with_its_name(self) -> None: + for memory, cpus, name in ((8.5, "", "github_runner_cluster_colima_memory_gib"), ("8.5", "", "github_runner_cluster_colima_memory_gib"), ("", "four", "github_runner_cluster_colima_cpus"), (0, "", "github_runner_cluster_colima_memory_gib"), ("", -2, "github_runner_cluster_colima_cpus"), (True, "", "github_runner_cluster_colima_memory_gib"), ("", "4 ", "github_runner_cluster_colima_cpus")): + with self.subTest(memory=memory, cpus=cpus), self.assertRaisesRegex(colima.AnsibleFilterError, name): + colima.cluster_colima_size(memory, cpus) + + def test_whole_numbers_given_as_text_or_numbers_are_taken(self) -> None: + self.assertEqual(colima.cluster_colima_size("12", 6), {"memory_gib": 12, "cpus": 6, "arguments": ["--memory", "12", "--cpus", "6"]}) + self.assertEqual(colima.cluster_colima_size("", None), {"memory_gib": None, "cpus": None, "arguments": []}) + + def test_the_filters_are_registered(self) -> None: + filters = colima.FilterModule().filters() + self.assertIs(filters["cluster_colima_size"], colima.cluster_colima_size) + self.assertIs(filters["cluster_colima_resize"], colima.cluster_colima_resize) + + +def lease(holder: str, renewed: str = "2026-01-01T00:00:00.000000Z", duration: int = 60, version: str = "7") -> dict[str, Any]: + return {"rc": 0, "stdout": json.dumps({"metadata": {"name": "fleet-regrade", "namespace": "kube-system", "resourceVersion": version}, "spec": {"holderIdentity": holder, "renewTime": renewed, "leaseDurationSeconds": duration}}), "stderr": ""} + + +NOT_FOUND = {"rc": 1, "stdout": "", "stderr": 'Error from server (NotFound): leases.coordination.k8s.io "fleet-regrade" not found'} +# 2026-01-01T00:00:30Z as an epoch time: thirty seconds into a sixty-second Lease renewed at midnight. +DURING = 1767225630.0 +AFTER = DURING + 60 + + +class LeaseFilterTest(unittest.TestCase): + def claim(self, read: dict[str, Any], now: float = DURING) -> dict[str, Any]: + result: dict[str, Any] = guard.cluster_lease_claim(read, "colima-resize/node1", now, 900, "fleet-regrade", "kube-system") + return result + + def test_no_lease_is_created(self) -> None: + claim = self.claim(NOT_FOUND) + self.assertEqual((claim["state"], claim["verb"]), ("absent", "create")) + self.assertNotIn("resourceVersion", claim["manifest"]["metadata"]) + self.assertEqual(claim["manifest"]["spec"], {"holderIdentity": "colima-resize/node1", "leaseDurationSeconds": 900, "acquireTime": "2026-01-01T00:00:30.000000Z", "renewTime": "2026-01-01T00:00:30.000000Z"}) + self.assertEqual((claim["manifest"]["apiVersion"], claim["manifest"]["kind"]), ("coordination.k8s.io/v1", "Lease")) + + def test_a_free_expired_or_own_lease_is_replaced_against_the_version_read(self) -> None: + for read, now, state in ((lease(""), DURING, "free"), (lease("node2"), AFTER, "expired"), (lease("colima-resize/node1"), DURING, "mine")): + with self.subTest(state=state): + claim = self.claim(read, now) + self.assertEqual((claim["state"], claim["verb"]), (state, "replace")) + self.assertEqual(claim["manifest"]["metadata"]["resourceVersion"], "7") + self.assertEqual(claim["manifest"]["spec"]["holderIdentity"], "colima-resize/node1") + + def test_a_lease_another_holds_is_refused_and_names_the_holder(self) -> None: + claim = self.claim(lease("node2")) + self.assertEqual((claim["state"], claim["holder"], claim["verb"]), ("held", "node2", "")) + + def test_a_lease_that_cannot_be_read_says_why(self) -> None: + claim = self.claim({"rc": 1, "stdout": "", "stderr": "The connection to the server was refused"}) + self.assertEqual(claim["state"], "unreadable") + self.assertIn("The connection to the server was refused", claim["problem"]) + + def test_the_release_clears_only_our_own_holder(self) -> None: + release = guard.cluster_lease_release(lease("colima-resize/node1"), "colima-resize/node1") + self.assertEqual(release["verb"], "replace") + self.assertEqual((release["manifest"]["spec"]["holderIdentity"], release["manifest"]["metadata"]["resourceVersion"]), ("", "7")) + self.assertEqual(guard.cluster_lease_release(lease("node2"), "colima-resize/node1")["verb"], "") + self.assertEqual(guard.cluster_lease_release(NOT_FOUND, "colima-resize/node1")["verb"], "") + + def test_the_lease_is_the_one_fleet_regrade_takes(self) -> None: + core = (ROLE / "files" / "regrade" / "fleet-regrade-core.sh").read_text() + self.assertIn(f"LEASE_NAME=${{LEASE_NAME:-{DEFAULTS['github_runner_cluster_regrade_lease_name']}}}", core) + self.assertIn(f"LEASE_NAMESPACE=${{LEASE_NAMESPACE:-{DEFAULTS['github_runner_cluster_regrade_lease_namespace']}}}", core) + + def test_the_filters_are_registered(self) -> None: + filters = guard.FilterModule().filters() + self.assertIs(filters["cluster_lease_claim"], guard.cluster_lease_claim) + self.assertIs(filters["cluster_lease_release"], guard.cluster_lease_release) + + +# Stands in for docker and colima, one script under two names, with state in files under $STUB: vm/.json is the profile's VM as `colima list --json` prints it, and while its status is not Running the Docker daemon of that profile does not answer and its containers do not run; containers/ holds what `docker ps` prints for a container (each host's node container is -k3s, in the profile named for the host, as its node is); nodes.json and pods.json are what kubectl lists, etcd- exists while that server's etcd reports ok, lease.json is the fleet-regrade Lease (no file: NotFound), runners holds how many polls still find a runner pod on the node, restarted- exists once the VM was started again after a stop, which is when its node's kubelet lease is renewed later than the stop, and colima-start-fails makes a start fail. Every call is appended to calls.log as " ". +STUB = """#!{python} +import json, os, sys +stub = os.environ["STUB"] +tool = os.path.basename(sys.argv[0]) +args = sys.argv[1:] +def path(*parts): + return os.path.join(stub, *parts) +def read(*parts): + return open(path(*parts)).read() if os.path.exists(path(*parts)) else None +def write(text, *parts): + os.makedirs(os.path.dirname(path(*parts)), exist_ok=True) + open(path(*parts), "w").write(text) +def die(message, code=1): + sys.stderr.write(message + "\\n") + sys.exit(code) +with open(path("calls.log"), "a") as log: + log.write(os.environ.get("DOCKER_CONTEXT", "-") + " " + tool + " " + " ".join(args) + "\\n") +def vm(profile): + return json.loads(read("vm", profile + ".json")) +def running(profile): + return vm(profile)["status"] == "Running" +if tool == "colima": + profile = args[args.index("--profile") + 1] + state = vm(profile) + if args[0] == "list": + print(json.dumps(state)) + elif args[0] == "stop": + state["status"] = "Stopped" + write(json.dumps(state), "vm", profile + ".json") + elif args[0] == "start": + if read("colima-start-fails") is not None: + die("colima: the VM did not start") + if "--activate=false" not in args: + die("colima start without --activate=false would change the host's Docker context") + if "--memory" in args: + state["memory"] = int(args[args.index("--memory") + 1]) * 1024 ** 3 + if "--cpus" in args: + state["cpus"] = int(args[args.index("--cpus") + 1]) + state["status"] = "Running" + write(json.dumps(state), "vm", profile + ".json") + write("", "restarted-" + profile) + else: + die("unexpected colima call: " + " ".join(args)) + sys.exit(0) +command = args[0] +if command == "ps": + container = args[args.index("--filter") + 1].removeprefix("name=^/").removesuffix("$") + if not running(container.removesuffix("-k3s")): + die("Cannot connect to the Docker daemon") + print(read("containers", container) or "", end="") + sys.exit(0) +if command != "exec": + die("unexpected docker call: " + " ".join(args)) +rest = args[1:] +interactive = rest[0] == "--interactive" +if interactive: + rest = rest[1:] +container, kubectl = rest[0], rest[1:] +if not running(container.removesuffix("-k3s")): + die("Cannot connect to the Docker daemon") +if (read("containers", container) or "") != "running": + die("Error response from daemon: container " + container + " is not running") +if kubectl[0] != "kubectl": + die("unexpected docker exec: " + " ".join(rest)) +verb, words = kubectl[1], kubectl[2:] +if verb in ("cordon", "uncordon", "drain"): + pass +elif verb in ("create", "replace"): + if not interactive: + die("kubectl " + verb + " -f - needs docker exec --interactive to read the manifest") + manifest = json.loads(sys.stdin.read()) + current = read("lease.json") + if verb == "create" and current is not None: + die('Error from server (AlreadyExists): leases.coordination.k8s.io "fleet-regrade" already exists') + if verb == "replace": + if current is None: + die("Error from server (NotFound)") + if json.loads(current)["metadata"]["resourceVersion"] != manifest["metadata"].get("resourceVersion"): + die("Error from server (Conflict): the object has been modified") + version = int(json.loads(current)["metadata"]["resourceVersion"]) + 1 if current else 1 + manifest["metadata"]["resourceVersion"] = str(version) + write(json.dumps(manifest), "lease.json") +elif verb == "get" and words[0] == "nodes": + print(read("nodes.json")) +elif verb == "get" and words[0] == "node": + print("True," + ("True" if container.removesuffix("-k3s") in json.loads(read("servers")) else ""), end="") +elif verb == "get" and "--raw" in words: + etcd = read("etcd-" + container) + if etcd is None: + die("etcd failed: reason withheld") + print(etcd, end="") +elif verb == "get" and words[0] == "lease" and words[1] == "fleet-regrade": + current = read("lease.json") + if current is None: + die('Error from server (NotFound): leases.coordination.k8s.io "fleet-regrade" not found') + print(current) +elif verb == "get" and words[0] == "lease": + node = words[words.index("kube-node-lease") + 1] + print("2999-01-01T00:00:00.000000Z" if read("restarted-" + node) is not None else "2000-01-01T00:00:00.000000Z", end="") +elif verb == "get" and words[0] == "pods" and "--selector" in words: + left = int(read("runners") or 0) + if left > 0: + write(str(left - 1), "runners") + print("pod/runner-1") +elif verb == "get" and words[0] == "pods": + print(read("pods.json")) +else: + die("unexpected kubectl call: " + " ".join(kubectl)) +""" + +# The calls that change something: a node's schedulability, the Lease, or a VM. +MUTATING = re.compile(r"\S+ (docker exec (--interactive )?\S+ kubectl (cordon|uncordon|drain|create|replace)|colima (stop|start))\b") + + +def node(name: str, ready: bool = True) -> dict[str, Any]: + labels = {"node-role.kubernetes.io/control-plane": "true", "node-role.kubernetes.io/etcd": "true"} if name.startswith("server") else {} + return {"metadata": {"name": name, "labels": labels}, "status": {"conditions": [{"type": "Ready", "status": "True" if ready else "Unknown"}]}} + + +def failed_task(stdout: str) -> str | None: + """The name of the task whose header last precedes a "fatal:" line in ansible-playbook's output, or None when nothing failed.""" + task = None + for line in stdout.splitlines(): + header = re.match(r"TASK \[(?:[^:\]]+ : )?(.*)\] \*+$", line) + if header: + task = header.group(1) + elif line.startswith("fatal: "): + return task + return None + + +@unittest.skipIf(shutil.which(ANSIBLE_PLAYBOOK) is None, "ansible-playbook is not installed") +class ColimaSizeRoleTest(unittest.TestCase): + SERVERS = ("server1", "server2", "server3") + + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.work) + self.stub = self.work / "stub" + self.bin = self.work / "bin" + self.stub.mkdir() + self.bin.mkdir() + for tool in ("docker", "colima"): + (self.bin / tool).write_text(STUB.format(python=sys.executable)) + (self.bin / tool).chmod(0o755) + self.write("servers", json.dumps(self.SERVERS)) + for host in (*self.SERVERS, "agent1"): + self.vm(host) + self.write(f"containers/{host}-k3s", "running") + for server in self.SERVERS: + self.write(f"etcd-{server}-k3s", "ok") + self.nodes() + self.write("pods.json", json.dumps({"items": []})) + + def write(self, name: str, text: str) -> None: + path = self.stub / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + + def vm(self, profile: str, cpus: int = 2, memory_gib: int = 4, status: str = "Running") -> None: + self.write(f"vm/{profile}.json", listing(profile, cpus, memory_gib, status).strip()) + + def size(self, profile: str) -> tuple[int, int, str]: + state = json.loads((self.stub / "vm" / f"{profile}.json").read_text()) + return state["cpus"], state["memory"] // GIB, state["status"] + + def nodes(self, *not_ready: str) -> None: + self.write("nodes.json", json.dumps({"items": [node(name, name not in not_ready) for name in (*self.SERVERS, "agent1")]})) + + def run_role(self, *extra: str, limit: str | None = None, **variables: Any) -> subprocess.CompletedProcess[str]: + topology = {"servers": list(self.SERVERS), "hosts": {name: {"role": "server" if name in self.SERVERS else "agent", "bootstrap": name == self.SERVERS[0], "node_name": name, "native": False} for name in (*self.SERVERS, "agent1")}} + hosts = { + name: { + "github_runner_cluster_effective_node_role": "server" if name in self.SERVERS else "agent", + "github_runner_cluster_effective_node_name": name, + "github_runner_cluster_container_name": f"{name}-k3s", + "github_runner_cluster_agent_container_name": f"{name}-k3s", + "github_runner_cluster_docker_context": f"colima-{name}", + "github_runner_cluster_docker_cli_environment": {"DOCKER_CONTEXT": f"colima-{name}"}, + "github_runner_cluster_effective_runtime": {"kind": "colima", "colima_profile": name}, + "github_runner_cluster_runtime_environment": {"DOCKER_CONTEXT": f"colima-{name}", "PATH": f"{self.bin}{os.pathsep}/usr/bin{os.pathsep}/bin"}, + } + for name in (*self.SERVERS, "agent1") + } + inventory = self.work / "inventory.yml" + inventory.write_text(yaml.safe_dump({"all": {"vars": {"github_runner_cluster_topology": topology, "github_runner_cluster_native_ready_timeout": 20, "github_runner_cluster_native_runner_wait_timeout": 20}, "hosts": hosts}})) + # As extra variables, which win over the inventory's, so a test can override what each host sets. + extra_file = self.work / "extra.json" + extra_file.write_text(json.dumps(variables)) + env = { + **os.environ, + "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}", + "STUB": str(self.stub), + "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), + "ANSIBLE_LOCALHOST_WARNING": "false", + "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", + "ANSIBLE_RETRY_FILES_ENABLED": "false", + } + command = [ANSIBLE_PLAYBOOK, "-i", str(inventory), str(RUN), "-e", f"@{extra_file}", *(["--limit", limit] if limit else []), *extra] + return subprocess.run(command, capture_output=True, text=True, check=False, env=env, cwd=self.work, timeout=600) + + def calls(self) -> list[str]: + log = self.stub / "calls.log" + return log.read_text().splitlines() if log.exists() else [] + + def mutating(self) -> list[str]: + """The calls that changed something, without the Docker context and with the kubectl container and the colima arguments that only repeat the profile shortened.""" + return [re.sub(r"^\S+ ", "", call).replace("docker exec --interactive ", "").replace("docker exec ", "") for call in self.calls() if MUTATING.match(call)] + + def lease(self) -> dict[str, Any]: + return json.loads((self.stub / "lease.json").read_text()) + + def recap(self, result: subprocess.CompletedProcess[str], host: str) -> dict[str, int]: + match = re.search(rf"{host}\s+: ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)", result.stdout) + assert match is not None, result.stdout + result.stderr + return dict(zip(("ok", "changed", "unreachable", "failed"), map(int, match.groups()))) + + def test_nothing_set_reads_and_changes_nothing(self) -> None: + result = self.run_role() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.calls(), []) + + def test_a_profile_of_the_size_set_is_left_running_and_only_read(self) -> None: + result = self.run_role(limit="server1", github_runner_cluster_colima_memory_gib=4, github_runner_cluster_colima_cpus=2) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.mutating(), []) + self.assertEqual(self.calls(), ["colima-server1 colima list --profile server1 --json"]) + self.assertEqual(self.recap(result, "server1")["changed"], 0) + + def test_a_server_is_drained_resized_through_colima_and_uncordoned_holding_the_lease(self) -> None: + self.write("runners", "2") + result = self.run_role(limit="server1", github_runner_cluster_colima_memory_gib=8, github_runner_cluster_colima_cpus=4) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual( + self.mutating(), + [ + "server1-k3s kubectl create -f -", + "server1-k3s kubectl cordon server1", + "server1-k3s kubectl drain server1 --ignore-daemonsets --delete-emptydir-data --timeout=20s", + "colima stop --profile server1", + "colima start --profile server1 --activate=false --memory 8 --cpus 4", + "server1-k3s kubectl uncordon server1", + "server1-k3s kubectl replace -f -", + ], + ) + self.assertEqual(self.size("server1"), (4, 8, "Running")) + # The runner pods are waited for after the cordon and before the drain, until none is left. + calls = self.calls() + polls = [index for index, call in enumerate(calls) if "--selector actions-ephemeral-runner=True" in call] + self.assertEqual(len(polls), 3) + self.assertLess(next(index for index, call in enumerate(calls) if " cordon " in call), polls[0]) + self.assertLess(polls[-1], next(index for index, call in enumerate(calls) if " drain " in call)) + # The node came back Ready and an etcd voter on a heartbeat later than the stop before it was uncordoned. + ready = next(index for index, call in enumerate(calls) if "EtcdIsVoter" in call) + self.assertLess(next(index for index, call in enumerate(calls) if "colima start" in call), ready) + self.assertLess(ready, next(index for index, call in enumerate(calls) if " uncordon " in call)) + # The Lease was the fleet-regrade one, given back with nobody holding it. + self.assertEqual((self.lease()["metadata"]["name"], self.lease()["metadata"]["namespace"]), ("fleet-regrade", "kube-system")) + self.assertEqual(self.lease()["spec"]["holderIdentity"], "") + + def test_the_cluster_is_checked_before_the_cordon_and_again_before_the_stop(self) -> None: + self.run_role(limit="server1", github_runner_cluster_colima_memory_gib=8) + calls = self.calls() + # Each check reads the cluster through every server of the inventory; this server's own container is read first. + reads = [index for index, call in enumerate(calls) if "exec server1-k3s kubectl get nodes" in call] + etcd = [index for index, call in enumerate(calls) if "/readyz/etcd" in call] + cordon = next(index for index, call in enumerate(calls) if " cordon " in call) + stop = next(index for index, call in enumerate(calls) if "colima stop" in call) + self.assertEqual(len(reads), 2) + self.assertLess(reads[0], cordon) + self.assertLess(cordon, reads[1]) + self.assertLess(reads[1], stop) + self.assertEqual(sorted({re.search(r"exec (\S+)", calls[index]).group(1) for index in etcd}), ["server2-k3s", "server3-k3s"]) + + def test_a_second_run_changes_nothing(self) -> None: + self.run_role(limit="server1", github_runner_cluster_colima_memory_gib=8, github_runner_cluster_colima_cpus=4) + before = len(self.mutating()) + result = self.run_role(limit="server1", github_runner_cluster_colima_memory_gib=8, github_runner_cluster_colima_cpus=4) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.mutating()[before:], []) + self.assertEqual(self.recap(result, "server1")["changed"], 0) + + def test_a_held_lease_refuses_before_anything_changes(self) -> None: + self.write("lease.json", json.dumps({"metadata": {"name": "fleet-regrade", "namespace": "kube-system", "resourceVersion": "3"}, "spec": {"holderIdentity": "server2", "renewTime": "2999-01-01T00:00:00.000000Z", "leaseDurationSeconds": 900}})) + result = self.run_role(limit="server1", github_runner_cluster_colima_memory_gib=8) + self.assertNotEqual(result.returncode, 0) + self.assertIn("the fleet-regrade Lease is held by server2", result.stdout) + self.assertEqual(self.mutating(), []) + self.assertEqual(self.size("server1"), (2, 4, "Running")) + self.assertEqual(self.lease()["spec"]["holderIdentity"], "server2") + + def test_another_server_that_is_not_ready_refuses_before_anything_changes(self) -> None: + self.nodes("server3") + result = self.run_role(limit="server1", github_runner_cluster_colima_memory_gib=8) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Not resizing the Colima VM of server1", result.stdout) + self.assertIn("server3 (not Ready)", result.stdout) + self.assertEqual(self.mutating(), []) + + def test_another_server_whose_etcd_is_unhealthy_refuses_before_anything_changes(self) -> None: + (self.stub / "etcd-server2-k3s").unlink() + result = self.run_role(limit="server1", github_runner_cluster_colima_memory_gib=8) + self.assertNotEqual(result.returncode, 0) + self.assertIn("server2 (etcd not reported healthy)", result.stdout) + self.assertEqual(self.mutating(), []) + + def test_check_mode_reports_the_resize_and_changes_nothing(self) -> None: + result = self.run_role("--check", limit="server1", github_runner_cluster_colima_memory_gib=8, github_runner_cluster_colima_cpus=4) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("Would resize the Colima VM of server1 (memory from 4 GiB to 8 GiB, CPUs from 2 to 4)", result.stdout) + self.assertEqual(self.mutating(), []) + self.assertFalse((self.stub / "lease.json").exists()) + self.assertGreater(self.recap(result, "server1")["changed"], 0) + + def test_check_mode_still_refuses_what_a_real_run_would_refuse(self) -> None: + self.nodes("server2") + result = self.run_role("--check", limit="server1", github_runner_cluster_colima_memory_gib=8) + self.assertNotEqual(result.returncode, 0) + self.assertIn("server2 (not Ready)", result.stdout) + + def test_a_value_that_is_not_a_whole_number_fails_before_anything_is_read(self) -> None: + result = self.run_role(limit="server1", github_runner_cluster_colima_memory_gib="7.5") + self.assertNotEqual(result.returncode, 0) + self.assertIn("github_runner_cluster_colima_memory_gib must be a positive whole number", result.stdout) + self.assertEqual(self.calls(), []) + + def test_a_host_whose_runtime_is_not_colima_fails_before_anything_is_read(self) -> None: + for variables in ({"github_runner_cluster_effective_runtime": {"kind": "docker_desktop", "colima_profile": ""}}, {"runtime_system": "Linux"}): + with self.subTest(variables=variables): + (self.stub / "calls.log").unlink(missing_ok=True) + result = self.run_role(limit="server1", github_runner_cluster_colima_cpus=4, **variables) + self.assertNotEqual(result.returncode, 0) + self.assertIn("only a Colima profile can be resized", result.stdout) + self.assertEqual(self.calls(), []) + + def test_an_agent_is_drained_through_a_server_and_waits_for_ready_without_a_vote(self) -> None: + result = self.run_role(limit="agent1", github_runner_cluster_colima_cpus=4) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual( + self.mutating(), + [ + "server1-k3s kubectl create -f -", + "server1-k3s kubectl cordon agent1", + "server1-k3s kubectl drain agent1 --ignore-daemonsets --delete-emptydir-data --timeout=20s", + "colima stop --profile agent1", + "colima start --profile agent1 --activate=false --cpus 4", + "server1-k3s kubectl uncordon agent1", + "server1-k3s kubectl replace -f -", + ], + ) + # An agent holds no etcd member, so no server's etcd decides whether it may restart. + self.assertEqual([call for call in self.calls() if "/readyz/etcd" in call], []) + + def test_a_node_container_that_is_not_running_is_resized_without_the_cluster(self) -> None: + self.write("containers/server1-k3s", "exited") + result = self.run_role(limit="server1", github_runner_cluster_colima_memory_gib=8) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.mutating(), ["colima stop --profile server1", "colima start --profile server1 --activate=false --memory 8"]) + self.assertEqual([call for call in self.calls() if " kubectl " in call], []) + + def test_several_hosts_are_resized_one_at_a_time(self) -> None: + result = self.run_role(github_runner_cluster_colima_memory_gib=8) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + steps = [call for call in self.mutating() if re.search(r"(cordon|colima (stop|start))", call)] + hosts = [re.search(r"(server\d|agent\d)\s*(--|$)", step).group(1) for step in steps] + # Each host's cordon, stop, start and uncordon come together, before the next host's cordon. + self.assertEqual(len(hosts), 16) + for turn in range(4): + self.assertEqual(len(set(hosts[turn * 4 : turn * 4 + 4])), 1, steps) + self.assertEqual(len(set(hosts)), 4) + for host in (*self.SERVERS, "agent1"): + self.assertEqual(self.size(host), (2, 8, "Running")) + + def test_a_start_that_fails_names_the_vm_and_gives_the_lease_back(self) -> None: + self.write("colima-start-fails", "") + result = self.run_role(limit="agent1", github_runner_cluster_colima_memory_gib=8) + self.assertNotEqual(result.returncode, 0) + self.assertIn("The Colima VM of agent1 was stopped to resize it", result.stdout) + self.assertEqual(self.mutating()[-1], "server1-k3s kubectl replace -f -") + self.assertEqual(self.lease()["spec"]["holderIdentity"], "") + + +class StaticTest(unittest.TestCase): + def test_the_sizes_keep_the_profile_as_it_is_by_default(self) -> None: + self.assertEqual((DEFAULTS["github_runner_cluster_colima_memory_gib"], DEFAULTS["github_runner_cluster_colima_cpus"]), ("", "")) + + def test_the_resize_is_included_after_the_topology_and_the_runtime_and_before_the_node_starts(self) -> None: + tasks = yaml.safe_load((ROLE / "tasks" / "main.yml").read_text()) + order = [task.get("ansible.builtin.include_tasks") for task in tasks] + self.assertLess(order.index("runtime.yml"), order.index("colima_size.yml")) + self.assertLess(order.index("topology.yml"), order.index("colima_size.yml")) + first_start = next(index for index, task in enumerate(tasks) if "block" in task) + self.assertLess(order.index("colima_size.yml"), first_start) + + def test_no_variable_the_tasks_register_or_set_is_a_role_default(self) -> None: + def written(tasks: list[dict[str, Any]]) -> set[str]: + names: set[str] = set() + for task in tasks: + if "register" in task: + names.add(task["register"]) + if "ansible.builtin.set_fact" in task: + names.update(name for name in task["ansible.builtin.set_fact"] if name != "cacheable") + for key in ("block", "rescue", "always"): + names.update(written(task.get(key, []))) + return names + + names = set().union(*(written(yaml.safe_load((ROLE / "tasks" / name).read_text())) for name in ("colima_size.yml", "colima_size_turn.yml", "restart_guard.yml"))) + self.assertTrue(names) + self.assertEqual(sorted(names & DEFAULTS.keys()), []) + + def test_every_new_variable_is_in_the_readme(self) -> None: + readme = (ROLE / "README.md").read_text() + for name in ("github_runner_cluster_colima_memory_gib", "github_runner_cluster_colima_cpus"): + with self.subTest(name=name): + self.assertIn(f"`{name}`", readme) + + +if __name__ == "__main__": + unittest.main() From a5d075fc38e61ed09b89990e10540e51d5abdd78 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 22:25:42 +0000 Subject: [PATCH 3/5] feat(role): derive a Colima node VM's memory from what its host must keep github_runner_cluster_colima_memory_reserved_gib, the memory a host's other workloads need, derives the VM's memory when no memory is set: the host's memory (memtotal_mb) less it and github_runner_cluster_colima_memory_host_reserve_gib, macOS's own, rounded down to a whole GiB, since colima takes whole GiB and rounding up would take memory a reserve keeps. The VM is never smaller than the node floor: the guest's baseline, github_runner_cluster_colima_guest_baseline_gib, plus one runner pod's memory limit, github_runner_cluster_colima_runner_memory_limit_gib, rounded up to a whole GiB. The cluster role does not install the runners, so it is told that limit, and a derivation without it fails before the VM is read. A derivation below the floor is sized at the floor, and the run prints a warning that the host keeps less than its reserves. A memory set explicitly wins, and with neither declared the profile is left as it is. A host whose runtime is not Colima is now left alone instead of failing, so the sizes and reserves can be set for a group that mixes runtimes. The host's memory is read from the hardware facts only when a derivation needs it and the caller did not gather them. --- plugins/filter/cluster_colima.py | 59 +++++++++++- roles/github_runner_cluster/README.md | 6 +- roles/github_runner_cluster/defaults/main.yml | 10 ++- .../tasks/colima_size.yml | 61 +++++++++---- tests/colima_size/run.yml | 7 +- tests/unit/test_cluster_colima_size.py | 90 +++++++++++++++++-- 6 files changed, 203 insertions(+), 30 deletions(-) diff --git a/plugins/filter/cluster_colima.py b/plugins/filter/cluster_colima.py index 0fdcb4b4..293981b2 100644 --- a/plugins/filter/cluster_colima.py +++ b/plugins/filter/cluster_colima.py @@ -1,9 +1,10 @@ -"""Filters the github_runner_cluster role uses to size a Colima node VM from the inventory (tasks/colima_size.yml): which sizes are set, and what `colima start` must be given to bring the profile's VM to them.""" +"""Filters the github_runner_cluster role uses to size a Colima node VM from the inventory (tasks/colima_size.yml): the memory it is to have, set or derived from what the host must keep, which sizes are wanted, and what `colima start` must be given to bring the profile's VM to them.""" from __future__ import annotations import json import re +from decimal import ROUND_CEILING, ROUND_FLOOR, Decimal from typing import Any from ansible.errors import AnsibleFilterError @@ -13,6 +14,14 @@ WHOLE_NUMBER = re.compile(r"^[1-9][0-9]*$") MEMORY_VARIABLE = "github_runner_cluster_colima_memory_gib" CPUS_VARIABLE = "github_runner_cluster_colima_cpus" +RESERVED_VARIABLE = "github_runner_cluster_colima_memory_reserved_gib" +HOST_RESERVE_VARIABLE = "github_runner_cluster_colima_memory_host_reserve_gib" +BASELINE_VARIABLE = "github_runner_cluster_colima_guest_baseline_gib" +RUNNER_VARIABLE = "github_runner_cluster_colima_runner_memory_limit_gib" +HOST_MEMORY_FACT = "ansible_facts['memtotal_mb']" +# ansible_facts['memtotal_mb'] is in MiB (the hardware facts divide the bytes macOS's hw.memsize reports by 2^20), so a GiB is 2^10 of them. +MIB_PER_GIB = 1024 +AMOUNT = re.compile(r"^[0-9]+(\.[0-9]+)?$") def _size(name: str, value: Any) -> int | None: @@ -37,6 +46,52 @@ def cluster_colima_size(memory_gib: Any, cpus: Any) -> dict[str, Any]: return {"memory_gib": memory, "cpus": count, "arguments": arguments} +def _amount(name: str, value: Any) -> Decimal | None: + """A non-negative number of GiB given as a number or as text, or None for an empty value.""" + if value is None or value == "": + return None + if isinstance(value, bool) or not isinstance(value, (int, float, str)) or not AMOUNT.match(str(value)): + raise AnsibleFilterError(f"{name} must be a number of GiB, zero or more, not {value!r}.") + return Decimal(str(value)) + + +def _needed(name: str, value: Any) -> Decimal: + amount = _amount(name, value) + if amount is None: + raise AnsibleFilterError(f"{name} must be set to derive the Colima VM's memory from {RESERVED_VARIABLE}.") + return amount + + +def _number(amount: Decimal) -> int | float: + return int(amount) if amount == amount.to_integral_value() else float(amount) + + +def cluster_colima_memory(memory_gib: Any, host_memory_mb: Any, reserved_gib: Any, host_reserve_gib: Any, guest_baseline_gib: Any, runner_limit_gib: Any) -> dict[str, Any]: + """Work out the memory a Colima node VM is to have: the value set, or what the host can spare, never less than a node needs to run one runner. + + :param memory_gib: github_runner_cluster_colima_memory_gib: the VM's memory in GiB, which wins when set. :param host_memory_mb: ansible_facts['memtotal_mb'], the host's memory in MiB, needed only to derive. :param reserved_gib: github_runner_cluster_colima_memory_reserved_gib: the GiB the host's other workloads need; empty derives nothing. :param host_reserve_gib: github_runner_cluster_colima_memory_host_reserve_gib: the GiB kept for the host's operating system. :param guest_baseline_gib: github_runner_cluster_colima_guest_baseline_gib: the GiB the node VM uses with no runner pod on it. :param runner_limit_gib: github_runner_cluster_colima_runner_memory_limit_gib: one runner pod's memory limit in GiB. + :returns: ``{"memory_gib": int | None, "source": "set" | "derived" | "none", "available_gib": , "floor_gib": , "clamped": bool}``. A derived memory is the available memory rounded down to a whole GiB, since `colima start --memory` takes whole GiB and rounding up would take memory the reserves keep; below the floor it is the floor, with ``clamped`` true. ``memory_gib`` None keeps the profile's own. + :raises AnsibleFilterError: if a value is set and is not a number of the kind its variable takes, or a derivation lacks the runner's limit or the host's memory. + """ + explicit = _size(MEMORY_VARIABLE, memory_gib) + reserved = _amount(RESERVED_VARIABLE, reserved_gib) + if explicit is not None: + return {"memory_gib": explicit, "source": "set", "available_gib": None, "floor_gib": None, "clamped": False} + if reserved is None: + return {"memory_gib": None, "source": "none", "available_gib": None, "floor_gib": None, "clamped": False} + host_reserve = _needed(HOST_RESERVE_VARIABLE, host_reserve_gib) + baseline = _needed(BASELINE_VARIABLE, guest_baseline_gib) + runner = _needed(RUNNER_VARIABLE, runner_limit_gib) + if runner == 0: + raise AnsibleFilterError(f"{RUNNER_VARIABLE} must be more than zero: a node VM is sized to run at least one runner pod.") + if isinstance(host_memory_mb, bool) or not isinstance(host_memory_mb, (int, str)) or not WHOLE_NUMBER.match(str(host_memory_mb)): + raise AnsibleFilterError(f"{HOST_MEMORY_FACT} must give the host's memory in MiB to derive the Colima VM's memory, not {host_memory_mb!r}.") + available = Decimal(int(host_memory_mb)) / MIB_PER_GIB - reserved - host_reserve + floor = int((baseline + runner).to_integral_value(rounding=ROUND_CEILING)) + derived = int(available.to_integral_value(rounding=ROUND_FLOOR)) + return {"memory_gib": max(derived, floor), "source": "derived", "available_gib": _number(available), "floor_gib": floor, "clamped": derived < floor} + + def _gib(memory: int) -> str: return f"{memory / BYTES_PER_GIB:g}" @@ -69,4 +124,4 @@ class FilterModule: def filters(self) -> dict[str, Any]: """Return the filters this plugin provides.""" - return {"cluster_colima_size": cluster_colima_size, "cluster_colima_resize": cluster_colima_resize} + return {"cluster_colima_memory": cluster_colima_memory, "cluster_colima_size": cluster_colima_size, "cluster_colima_resize": cluster_colima_resize} diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index abc75e3f..323c3d3a 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -178,7 +178,7 @@ A host that got the agent from release 2.19.0 moves over on its next run: the ro **Colima kernel.** On every Colima host, whether or not `github_runner_cluster_runtime_autostart` is on, once the VM answers (`tasks/colima_kernel.yml`), the role makes the VM's kernel pass bridged pod traffic through iptables, now and at every VM boot: it writes `/etc/modules-load.d/br_netfilter.conf` and `/etc/sysctl.d/90-github-runner-br-netfilter.conf` in the VM, runs `modprobe br_netfilter` when the VM has no `/proc/sys/net/bridge`, and sets `net.bridge.bridge-nf-call-iptables` and `net.bridge.bridge-nf-call-ip6tables` to 1 when they differ. Without the module a pod that talks to a Service whose only backend pod is on the same node gets no reply, because the Service's reverse NAT never applies to the bridged hairpin traffic; pods on other nodes keep working, so it stays latent until a single-backend Service such as CoreDNS lands on the node (the `br_netfilter` note under Non-obvious behaviour in the repository README has the symptoms). Everything runs in the VM through `colima ssh --profile `, as the VM's user with `sudo`, and no `become` on the host. A file is written only when its content differs, so a rerun changes nothing, and the module file takes the name the README's note told people to create by hand, so an existing one is replaced by the managed one. Under `--check` each change is reported and none is made, and nothing is read from a VM that a check run would only have started. Docker Desktop's VM is not reachable by the role and other runtimes are not touched. -**Colima VM size.** A Colima host can set its VM's memory in GiB and its number of CPUs, `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus`, each a positive whole number; empty, the default, keeps what the profile has (`tasks/colima_size.yml`). A value that is not a positive whole number, or either set on a host whose runtime is not Colima, fails the run before anything is read. The role reads the size of the VM Lima runs with `colima list --profile --json`, not the profile's configuration file, since `colima start` given a size while the profile runs writes it to the file and leaves the VM as it is ([`cmd/start.go`](https://github.com/abiosoft/colima/blob/v0.10.3/cmd/start.go)). A VM that already has every size set is left alone, so a rerun changes nothing. Otherwise the resize is a planned restart of the node, made one host at a time, each in its own turn, through the same quorum guard as moving a node between networks (`tasks/restart_guard.yml`): it refuses unless the other servers with a Ready node and a healthy etcd stay a majority of all servers; it takes the `fleet-regrade` Lease that regrading steps hold (`kube-system/fleet-regrade`, as `colima-resize/`), refusing while another step holds it, so a resize never overlaps a regrade step or another resize in this run or any other; it cordons the node, waits for its runner pods to finish and drains it (the role switch's `tasks/native/drain.yml`), checks the cluster again, runs `colima stop --profile ` and `colima start --profile --activate=false --memory --cpus ` (the flags of the values set), waits for the node to be Ready on a heartbeat later than the stop and, on a server, an etcd voter, uncordons it and gives the Lease back. Colima writes the new size to the profile's configuration, so the launchd job's later starts keep it. The node's container comes back with the VM (`restart: unless-stopped`). A node whose container is not running takes nothing from the cluster, so its VM is only stopped and started. When a step fails the node is uncordoned again and the run fails, saying whether the VM was stopped, and the Lease is given back, or expires after `github_runner_cluster_colima_resize_lease_seconds` when the API that holds it does not answer. Under `--check` the sizes, the Lease and the cluster are read, a refusal fails as it would in a real run, and the resize is reported and not made. Size the VM from what it runs (the autoscaler's measured per-job peaks plus the guest's own baseline) and record that beside the value in the inventory. +**Colima VM size.** A Colima host can set its VM's memory in GiB and its number of CPUs, `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus`, each a positive whole number; empty, the default, keeps what the profile has (`tasks/colima_size.yml`). Instead of a memory, a host can say what it must keep: `github_runner_cluster_colima_memory_reserved_gib`, the memory its other workloads need, and `github_runner_cluster_colima_memory_host_reserve_gib`, the memory kept for macOS itself (the resident memory of the processes macOS runs as root and its service accounts, rounded up to a whole GiB, by default). The VM then gets the host's memory (`ansible_facts['memtotal_mb']`) less both, rounded down to a whole GiB, but never less than the node floor: the guest's own use with no runner pod on it, `github_runner_cluster_colima_guest_baseline_gib`, plus one runner pod's memory limit, `github_runner_cluster_colima_runner_memory_limit_gib` (which the scale set's values set, so the cluster role is told it, and which a derivation requires), rounded up to a whole GiB. A derivation below the floor sizes the VM at the floor and prints a warning that the host keeps less than its reserves. A memory set explicitly wins over the derivation, and with neither the profile's memory is left as it is. A host whose runtime is not Colima ignores all of these, so they can be set for a group that mixes runtimes. A value of the wrong kind, or a derivation without the runner's limit, fails the run before anything is read from the VM. The role reads the size of the VM Lima runs with `colima list --profile --json`, not the profile's configuration file, since `colima start` given a size while the profile runs writes it to the file and leaves the VM as it is ([`cmd/start.go`](https://github.com/abiosoft/colima/blob/v0.10.3/cmd/start.go)). A VM that already has every size set is left alone, so a rerun changes nothing. Otherwise the resize is a planned restart of the node, made one host at a time, each in its own turn, through the same quorum guard as moving a node between networks (`tasks/restart_guard.yml`): it refuses unless the other servers with a Ready node and a healthy etcd stay a majority of all servers; it takes the `fleet-regrade` Lease that regrading steps hold (`kube-system/fleet-regrade`, as `colima-resize/`), refusing while another step holds it, so a resize never overlaps a regrade step or another resize in this run or any other; it cordons the node, waits for its runner pods to finish and drains it (the role switch's `tasks/native/drain.yml`), checks the cluster again, runs `colima stop --profile ` and `colima start --profile --activate=false --memory --cpus ` (the flags of the values set), waits for the node to be Ready on a heartbeat later than the stop and, on a server, an etcd voter, uncordons it and gives the Lease back. Colima writes the new size to the profile's configuration, so the launchd job's later starts keep it. The node's container comes back with the VM (`restart: unless-stopped`). A node whose container is not running takes nothing from the cluster, so its VM is only stopped and started. When a step fails the node is uncordoned again and the run fails, saying whether the VM was stopped, and the Lease is given back, or expires after `github_runner_cluster_colima_resize_lease_seconds` when the API that holds it does not answer. Under `--check` the host's memory, the sizes, the Lease and the cluster are read, a refusal fails as it would in a real run, and the resize is reported and not made. This is the role's whole scope: it never moves a host between Colima and Docker Desktop. @@ -426,7 +426,7 @@ Each Compose project runs its own `docker-prune` service against the same daemon - `github_runner_cluster_network_mode` (default `bridge`, or `host`): the network the node's container runs on, and with `host`, `github_runner_cluster_lan_cidr` (required: the LAN the nodes share) and `github_runner_cluster_network_probe_image` (default `busybox:1.37`, for reading the host's addresses) (see [Networking](#networking)). - `github_runner_cluster_runtime_autostart_at` (default `auto`; `login` or `boot`) and `github_runner_cluster_runtime_boot_profile` (default `node`): whether the Colima profile is started by a launchd agent at login or by a launchd daemon at boot, and which profile `auto` starts at boot (see [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). - `github_runner_cluster_runtime_autostart` (default true), `github_runner_cluster_runtime` (default `auto`; `colima`, `docker_desktop` or `other`), `github_runner_cluster_colima_profile` (default empty, taken from the Docker context), `github_runner_cluster_colima_home` (default empty: the `COLIMA_HOME` is read from the endpoint of the profile's Docker context, and is unset for Colima's own `~/.colima`; set it only for a context that does not say, and the role passes it to colima and to the launchd agent), `github_runner_cluster_runtime_path` (default both Homebrew prefixes and the system directories): on a macOS host, start Colima at login through a launchd agent and check the runtime (see [Container runtime at login](#container-runtime-after-a-reboot-macos)). -- `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus` (default empty, keeping the profile's own): a Colima host's VM memory in GiB and its CPUs, each a positive whole number, applied one node at a time through the quorum guard when the VM differs (see Colima VM size under [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). `github_runner_cluster_colima_resize_lease_seconds`, `github_runner_cluster_colima_resize_lease_holder`, `github_runner_cluster_regrade_lease_name` and `github_runner_cluster_regrade_lease_namespace` are derived, never set directly. `github_runner_cluster_native_runner_wait_timeout` and `github_runner_cluster_native_ready_timeout` bound the wait for the node's runner pods and for the node to come back, as they do for a native node. +- `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus` (default empty, keeping the profile's own): a Colima host's VM memory in GiB and its CPUs, each a positive whole number, applied one node at a time through the quorum guard when the VM differs (see Colima VM size under [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). `github_runner_cluster_colima_memory_reserved_gib` (default empty) derives the memory instead, from the host's memory less it and `github_runner_cluster_colima_memory_host_reserve_gib` (default 4, macOS's own), never below `github_runner_cluster_colima_guest_baseline_gib` (default 3.4) plus `github_runner_cluster_colima_runner_memory_limit_gib` (default empty, required by a derivation) rounded up. `github_runner_cluster_colima_resize_lease_seconds`, `github_runner_cluster_colima_resize_lease_holder`, `github_runner_cluster_regrade_lease_name` and `github_runner_cluster_regrade_lease_namespace` are derived, never set directly. `github_runner_cluster_native_runner_wait_timeout` and `github_runner_cluster_native_ready_timeout` bound the wait for the node's runner pods and for the node to come back, as they do for a native node. - `github_runner_cluster_container_name` (default `github-runner-k3s`) and `github_runner_cluster_agent_container_name` (default `github-runner-k3s-agent`): the names of the node's server and agent containers, which a second cluster on the same Docker hosts sets to names of its own (see [Several clusters on one host](#several-clusters-on-one-host)). - `github_runner_cluster_compose_files`: extra Compose files for the host's k3s project, merged after `docker-compose.yml`, as paths relative to `github_runner_cluster_dir` or absolute. Several nodes on one Docker host each need their own `github_runner_cluster_dir`, `github_runner_cluster_compose_project` and container names; nodes of one cluster share its API port, so all but one of them on a host also need an extra file that publishes none (`ports: !reset []`, as `tests/mesh` does). - `github_runner_cluster_wait_for_nodes`: wait for every node of the inventory to be Ready afterwards, clearing the node-password registration of a replaced node that fails to rejoin (installs the kubernetes Python client through `github_runner_k8s_client`). The wait covers the nodes the inventory's hosts own, by their effective node names, and leaves out a server that memory-pressure regrading has stepped down and an inventory node that has not registered. A node the cluster holds that no inventory host owns (one flashed from an image and joined by hand, say) is never waited for: when it is not Ready the run prints one line with its name and status, and carries on. When no node of the inventory has registered, the run fails. @@ -487,7 +487,7 @@ or read from 1Password by adding `exadev.github_runner.github_runner_secrets_one `tests/unit/test_runtime_autostart.py` covers the [container runtime tasks](#container-runtime-after-a-reboot-macos) without a Mac: it renders the launchd agent and daemon through `tests/runtime_autostart/render_plist.yml`, and runs `tasks/runtime.yml` and `tasks/end_checks.yml` through the role against stand-ins for `docker`, `colima`, `launchctl`, `plutil` and `sudo` on `PATH`, the stand-in `launchctl` refusing to change the system domain unless it was run through the stand-in `sudo`. It checks the variable defaults, that Linux hosts are left alone, that a second run changes nothing, that `--check` writes and starts nothing, and that each way the runtime or the node's container can be down fails with its message. `tests/unit/test_container_runtime.py` covers the filter that works out the runtime from the Docker context. Nothing there touches a real Colima, launchd or Docker Desktop, so loading the agent into a real login session, and the daemon starting a Colima VM at a real boot with no one logged in, are the parts only a run on a Mac can show. -`tests/unit/test_cluster_colima_size.py` covers the Colima VM size: the filters that read the profile's size (`plugins/filter/cluster_colima.py`) and take and give back the `fleet-regrade` Lease (`plugins/filter/cluster_guard.py`), and `tasks/colima_size.yml` run through the role (`tests/colima_size/run.yml`) against stand-ins for `docker` and `colima` that keep each profile's VM, the node containers, the cluster's nodes and the Lease as files. It checks the `colima` arguments, that a VM of the size set is only read, that a second run changes nothing, the order of a resize (the Lease, the cordon, the runner pods, the drain, the stop, the start with the new size, the uncordon, the Lease given back), that a held Lease, a server that is not Ready and a server whose etcd is unhealthy each refuse before anything changes, that `--check` reports and changes nothing, that several hosts take their turns one at a time, and the refusals of a value that is not a whole number and of a host that does not run Colima. A real Colima VM's resize is shown only by its first run on a Mac. +`tests/unit/test_cluster_colima_size.py` covers the Colima VM size: the filters that read the profile's size (`plugins/filter/cluster_colima.py`) and take and give back the `fleet-regrade` Lease (`plugins/filter/cluster_guard.py`), and `tasks/colima_size.yml` run through the role (`tests/colima_size/run.yml`) against stand-ins for `docker` and `colima` that keep each profile's VM, the node containers, the cluster's nodes and the Lease as files. It checks the `colima` arguments, the derived memory (the host's less both reserves, a larger reserve giving a smaller VM, a derivation below the floor sized at the floor with a warning, an explicit memory winning), that a VM of the size set is only read, that a second run changes nothing, the order of a resize (the Lease, the cordon, the runner pods, the drain, the stop, the start with the new size, the uncordon, the Lease given back), that a held Lease, a server that is not Ready and a server whose etcd is unhealthy each refuse before anything changes, that `--check` reports and changes nothing, that several hosts take their turns one at a time, the refusal of a value that is not a whole number, and that a host that does not run Colima is left alone. A real Colima VM's resize is shown only by its first run on a Mac. `tests/unit/test_cluster_wait_nodes.py` runs the wait for nodes to be Ready and its rescue against a stand-in for `kubectl`: a node outside the inventory that is not Ready is reported and never waited for, and an inventory node that stays not Ready still fails the run. The mesh test does not cover it, since its nodes publish no API port for the host's `kubectl` to reach. diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index 44693167..c7f81021 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -246,9 +246,17 @@ github_runner_cluster_colima_modules_load_file: /etc/modules-load.d/br_netfilter github_runner_cluster_colima_sysctl_file: /etc/sysctl.d/90-github-runner-br-netfilter.conf # Derived, never set directly: the command that runs another command in the profile's VM as the VM's user, which has passwordless sudo. github_runner_cluster_colima_vm_command: "{{ ['colima', 'ssh', '--profile', github_runner_cluster_effective_runtime.colima_profile, '--'] }}" -# The memory, in GiB, and the number of CPUs of this host's Colima VM: each a positive whole number, or empty (the default) to keep what the profile has. Colima hosts only: a host whose runtime is anything else and sets either fails before anything changes. When a value differs from what the profile's VM runs (`colima list --json`), the role resizes the VM through the quorum guard, one node at a time (tasks/colima_size.yml): it takes the fleet-regrade Lease, refusing while another step holds it, refuses unless the cluster stays safe without the node, cordons the node, waits for its runner pods to finish and drains it, checks the cluster again, stops the profile and starts it with the new size, waits for the node to be Ready again (and an etcd voter, on a server), uncordons it and gives the Lease back. `colima start` writes the size to the profile's configuration, so every later start, the launchd job's at boot or login included, keeps it. A node whose container is not running takes nothing from the cluster, and its VM is only stopped and started. Size the VM from what it runs, the autoscaler's measured per-job peaks plus the guest's own baseline, and record that beside the value in the inventory. +# The memory, in GiB, and the number of CPUs of this host's Colima VM: each a positive whole number, or empty (the default) to keep what the profile has, unless the memory is derived from github_runner_cluster_colima_memory_reserved_gib (below); a memory set here wins over the derivation. Colima hosts only: on a host whose runtime is anything else these and the reserves are ignored, so they can be set for a group that mixes runtimes. When a size differs from what the profile's VM runs (`colima list --json`), the role resizes the VM through the quorum guard, one node at a time (tasks/colima_size.yml): it refuses unless the cluster stays safe without the node, takes the fleet-regrade Lease, refusing while another step holds it, cordons the node, waits for its runner pods to finish and drains it, checks the cluster again, stops the profile and starts it with the new size, waits for the node to be Ready again (and an etcd voter, on a server), uncordons it and gives the Lease back. `colima start` writes the size to the profile's configuration, so every later start, the launchd job's at boot or login included, keeps it. A node whose container is not running takes nothing from the cluster, and its VM is only stopped and started. github_runner_cluster_colima_memory_gib: "" github_runner_cluster_colima_cpus: "" +# The memory, in GiB, this host's other workloads (everything on the Mac but macOS itself and the node VM) need, a number zero or more. Set, and github_runner_cluster_colima_memory_gib empty, the VM's memory is derived: the host's memory (ansible_facts['memtotal_mb']) less this and github_runner_cluster_colima_memory_host_reserve_gib, rounded down to a whole GiB since colima takes whole GiB and rounding up would take memory a reserve keeps. It is never less than the node floor, github_runner_cluster_colima_guest_baseline_gib plus github_runner_cluster_colima_runner_memory_limit_gib rounded up to a whole GiB, since a node that cannot hold its own baseline and one runner pod is offered jobs it cannot finish; a derivation below the floor is sized at the floor and the run prints a warning that the host keeps less than its reserves. Empty, the default, derives nothing. +github_runner_cluster_colima_memory_reserved_gib: "" +# The memory, in GiB, a derivation keeps for macOS itself: the resident memory of the processes macOS runs as root and its service accounts (`ps -axo user,rss` over every user but the connecting one), about 3.8 GiB on an Apple silicon Mac with no user workload, rounded up to a whole GiB. A host whose own reading differs sets its own. +github_runner_cluster_colima_memory_host_reserve_gib: 4 +# The memory, in GiB, a node VM uses with no runner pod on it (the guest kernel, containerd, k3s with its datastore on a server, and the cluster's system pods), as measured on a Colima server node VM: the first part of the node floor. +github_runner_cluster_colima_guest_baseline_gib: 3.4 +# One runner pod's memory limit, in GiB, as the scale set's values give it (the cluster role does not install the runners, so it cannot read it): the second part of the node floor. Required when the memory is derived; empty, the default, otherwise. +github_runner_cluster_colima_runner_memory_limit_gib: "" # Derived, never set directly: how long the Lease a resize takes lasts if its holder never gives it back. It covers the holder's long steps, each bounded by its own timeout: the wait for the runner pods and the drain (github_runner_cluster_native_runner_wait_timeout each), and the stop, the start and the wait for the node to come back (github_runner_cluster_native_ready_timeout each). A holder that is working never loses it; one that crashed blocks other steps for no longer than this. github_runner_cluster_colima_resize_lease_seconds: "{{ 2 * (github_runner_cluster_native_runner_wait_timeout | int) + 3 * (github_runner_cluster_native_ready_timeout | int) }}" # Derived, never set directly: who a resize holds the Lease as. It names the node but is never a node name, so that the node's own regrade service, which holds it by node name, reads it as held by another and does not step meanwhile. diff --git a/roles/github_runner_cluster/tasks/colima_size.yml b/roles/github_runner_cluster/tasks/colima_size.yml index 1c550345..7a72a0b2 100644 --- a/roles/github_runner_cluster/tasks/colima_size.yml +++ b/roles/github_runner_cluster/tasks/colima_size.yml @@ -1,23 +1,52 @@ --- -# Sizes this host's Colima VM from github_runner_cluster_colima_memory_gib and github_runner_cluster_colima_cpus (see their defaults). With neither set it reads and changes nothing. With either set it checks the values and that the host's runtime is a Colima profile, before anything is read, then reads the size of the profile's VM with `colima list --json` and works out what differs; a VM that already has every size set is left as it is, so a second run changes nothing. The hosts whose VM differs are then resized one at a time, each in its own turn (tasks/colima_size_turn.yml), in the same way tasks/start_servers.yml gives each server its turn: Ansible's linear strategy runs every host's copy of a task together, except that the tasks one host includes dynamically are its own, so while one host's turn runs the others wait and no two nodes are ever away together. Needs runtime.yml (the runtime and its environment) and topology.yml to have run. Under --check the values are checked and the size, the Lease and the cluster are read, a resize is reported as a change and a refusal as a failure, and nothing is changed. +# Sizes this host's Colima VM from github_runner_cluster_colima_memory_gib, or the memory derived from github_runner_cluster_colima_memory_reserved_gib, and github_runner_cluster_colima_cpus (see their defaults). A host whose runtime is not a Colima profile is left alone, so the sizes and the reserve can be set for a group that also holds Docker Desktop and Linux hosts; with nothing declared a Colima host is left alone too. Otherwise it checks the values before anything is read from the VM, works out the memory, then reads the size of the profile's VM with `colima list --json` and works out what differs; a VM that already has every size wanted is left as it is, so a second run changes nothing. The hosts whose VM differs are then resized one at a time, each in its own turn (tasks/colima_size_turn.yml), in the same way tasks/start_servers.yml gives each server its turn: Ansible's linear strategy runs every host's copy of a task together, except that the tasks one host includes dynamically are its own, so while one host's turn runs the others wait and no two nodes are ever away together. Needs runtime.yml (the runtime and its environment) and topology.yml to have run. Under --check the values are checked and the host's memory, the VM's size, the Lease and the cluster are read, a resize is reported as a change and a refusal as a failure, and nothing is changed. -- name: Check the sizes set for the Colima VM - # On every host, so that a value colima could not take fails here, before a VM is stopped for it. Empty values keep the profile's own size. +- name: Note whether this host's runtime is a Colima profile + # Only a VM the role can stop and start with colima is sized. github_runner_cluster_effective_runtime is published by runtime.yml, which runs on macOS only, so the operating system is tested first. ansible.builtin.set_fact: - github_runner_cluster_colima_size_wanted: >- - {{ github_runner_cluster_colima_memory_gib | exadev.github_runner.cluster_colima_size(github_runner_cluster_colima_cpus) }} + github_runner_cluster_colima_size_applies: "{{ ansible_facts['system'] == 'Darwin' and github_runner_cluster_effective_runtime.kind == 'colima' }}" + +- name: Read this host's memory, to derive the VM's from it + # Only the hardware subset, which holds memtotal_mb, and only when a derivation needs it and the caller did not gather it. Read-only, so it runs under --check too. + ansible.builtin.setup: + gather_subset: + - "!all" + - "!min" + - hardware + when: + - github_runner_cluster_colima_size_applies | bool + - github_runner_cluster_colima_memory_gib | string | length == 0 + - github_runner_cluster_colima_memory_reserved_gib | string | length > 0 + - "'memtotal_mb' not in ansible_facts" + +- name: Work out the memory the Colima VM is to have + # Before anything is read from the VM, so that a value colima could not take, or a derivation missing what it needs, fails here rather than after a VM is stopped for it. + ansible.builtin.set_fact: + github_runner_cluster_colima_memory: >- + {{ github_runner_cluster_colima_memory_gib + | exadev.github_runner.cluster_colima_memory(ansible_facts['memtotal_mb'] | default(''), github_runner_cluster_colima_memory_reserved_gib, + github_runner_cluster_colima_memory_host_reserve_gib, github_runner_cluster_colima_guest_baseline_gib, + github_runner_cluster_colima_runner_memory_limit_gib) }} + when: github_runner_cluster_colima_size_applies | bool -- name: Fail unless this host's runtime is a Colima profile - # The role sizes only a VM it can stop and start with colima; a size set for any other host would otherwise be ignored without a word. - ansible.builtin.fail: +- name: Warn that the derived memory is below the node floor + # The VM is never sized below the floor, since a node that cannot hold its own baseline and one runner pod takes runner jobs it cannot finish; the host then keeps less than its reserves ask for, which this says. + ansible.builtin.debug: msg: >- - {{ inventory_hostname }} sets github_runner_cluster_colima_memory_gib or github_runner_cluster_colima_cpus, but only a Colima profile can - be resized, and this host's container runtime is - {{ github_runner_cluster_effective_runtime.kind if ansible_facts['system'] == 'Darwin' else 'the Docker of a ' ~ ansible_facts['system'] ~ ' host' }}. - Remove both from this host's variables. Nothing was changed. + Warning: the Colima VM of {{ github_runner_cluster_effective_node_name }} would get the host's + {{ (ansible_facts['memtotal_mb'] | int / 1024) | round(1) }} GiB less {{ github_runner_cluster_colima_memory_reserved_gib }} GiB reserved for + the host's other workloads and {{ github_runner_cluster_colima_memory_host_reserve_gib }} GiB for its operating system: + {{ github_runner_cluster_colima_memory.available_gib }} GiB is below the node floor of {{ github_runner_cluster_colima_memory.floor_gib }} GiB + (the guest's baseline and one runner pod's memory limit), so the VM is sized at the floor and the host keeps less than its reserves. when: - - github_runner_cluster_colima_size_wanted.arguments | length > 0 - - ansible_facts['system'] != 'Darwin' or github_runner_cluster_effective_runtime.kind != 'colima' + - github_runner_cluster_colima_size_applies | bool + - github_runner_cluster_colima_memory.clamped + +- name: Check the sizes wanted for the Colima VM + ansible.builtin.set_fact: + github_runner_cluster_colima_size_wanted: >- + {{ (github_runner_cluster_colima_memory.memory_gib | exadev.github_runner.cluster_colima_size(github_runner_cluster_colima_cpus)) + if github_runner_cluster_colima_size_applies | bool else {'memory_gib': none, 'cpus': none, 'arguments': []} }} - name: Read the size of the profile's VM # The VM Lima runs, not the profile's configuration file, since a `colima start` given sizes while the profile runs writes them to the file and leaves the VM as it is (RunE and PreRunE in abiosoft/colima cmd/start.go). Read-only, so it runs under --check too. @@ -29,12 +58,12 @@ check_mode: false when: github_runner_cluster_colima_size_wanted.arguments | length > 0 -- name: Work out how the VM's size differs from the one set +- name: Work out how the VM's size differs from the one wanted ansible.builtin.set_fact: github_runner_cluster_colima_size_plan: >- {{ (github_runner_cluster_colima_size_listing.stdout | exadev.github_runner.cluster_colima_resize(github_runner_cluster_effective_runtime.colima_profile, - github_runner_cluster_colima_memory_gib, github_runner_cluster_colima_cpus)) + github_runner_cluster_colima_size_wanted.memory_gib, github_runner_cluster_colima_cpus)) if github_runner_cluster_colima_size_wanted.arguments | length > 0 else {'changes': [], 'arguments': []} }} - name: Resize each host's VM in its own turn diff --git a/tests/colima_size/run.yml b/tests/colima_size/run.yml index 60821f7d..e26852ea 100644 --- a/tests/colima_size/run.yml +++ b/tests/colima_size/run.yml @@ -1,4 +1,4 @@ -# Runs the cluster role's tasks/colima_size.yml through the role itself, as tasks/main.yml includes it, for tests/unit/test_cluster_colima_size.py, on the hosts of an inventory the test writes (all on a local connection, so a task delegated to another server runs here too). The host's operating system is stood in for by runtime_system, and the docker and colima the tasks call are the test's stand-ins on PATH. +# Runs the cluster role's tasks/colima_size.yml through the role itself, as tasks/main.yml includes it, for tests/unit/test_cluster_colima_size.py, on the hosts of an inventory the test writes (all on a local connection, so a task delegated to another server runs here too). The host's operating system is stood in for by runtime_system and its memory in MiB by runtime_memtotal_mb, and the docker and colima the tasks call are the test's stand-ins on PATH. - name: Run the cluster role's Colima VM size tasks hosts: all connection: local @@ -6,8 +6,9 @@ tasks: - name: Stand in for the host's facts ansible.builtin.set_fact: - ansible_facts: - system: "{{ runtime_system | default('Darwin') }}" + ansible_facts: >- + {{ {'system': runtime_system | default('Darwin')} + | combine({'memtotal_mb': runtime_memtotal_mb | int} if runtime_memtotal_mb is defined else {}) }} - name: Include the role's tasks ansible.builtin.include_role: diff --git a/tests/unit/test_cluster_colima_size.py b/tests/unit/test_cluster_colima_size.py index 83427389..c8136c8f 100644 --- a/tests/unit/test_cluster_colima_size.py +++ b/tests/unit/test_cluster_colima_size.py @@ -83,6 +83,60 @@ def test_the_filters_are_registered(self) -> None: self.assertIs(filters["cluster_colima_resize"], colima.cluster_colima_resize) +# A 32 GiB host as ansible_facts['memtotal_mb'] gives it (MiB). With a 3.4 GiB guest baseline and a 7 GiB runner limit the node floor is ceil(3.4 + 7) = 11 GiB. +HOST_MIB = 32 * 1024 + + +def memory(explicit: Any = "", reserved: Any = "", host_reserve: Any = 2, runner: Any = 7, host_mib: Any = HOST_MIB, baseline: Any = 3.4) -> dict[str, Any]: + result: dict[str, Any] = colima.cluster_colima_memory(explicit, host_mib, reserved, host_reserve, baseline, runner) + return result + + +class ColimaMemoryFilterTest(unittest.TestCase): + def test_the_derived_memory_is_the_host_less_what_its_other_workloads_and_its_os_need(self) -> None: + plan = memory(reserved=6, host_reserve=2) + self.assertEqual((plan["memory_gib"], plan["source"], plan["clamped"]), (24, "derived", False)) + + def test_changing_the_reserved_memory_changes_the_derived_cap(self) -> None: + self.assertEqual([memory(reserved=reserved)["memory_gib"] for reserved in (4, 10, "12.5")], [26, 20, 17]) + + def test_a_part_of_a_gib_left_over_is_kept_by_the_host(self) -> None: + # 32 - 6.5 - 2 = 23.5 GiB available: the VM gets the whole GiB below it, never more than the host can spare. + self.assertEqual(memory(reserved="6.5")["memory_gib"], 23) + + def test_the_floor_is_the_guest_baseline_and_one_runner_rounded_up_to_a_whole_gib(self) -> None: + self.assertEqual(memory(reserved=6)["floor_gib"], 11) + self.assertEqual(memory(reserved=6, runner=8, baseline=3)["floor_gib"], 11) + + def test_a_derivation_below_the_floor_is_clamped_to_it(self) -> None: + plan = memory(reserved=20, host_reserve=4) + self.assertEqual((plan["memory_gib"], plan["available_gib"], plan["clamped"]), (11, 8, True)) + + def test_a_memory_set_explicitly_wins_over_the_derivation(self) -> None: + plan = memory(explicit=16, reserved=6) + self.assertEqual((plan["memory_gib"], plan["source"], plan["clamped"]), (16, "set", False)) + + def test_nothing_declared_keeps_the_profile_as_it_is(self) -> None: + plan = memory() + self.assertEqual((plan["memory_gib"], plan["source"]), (None, "none")) + # Without a derivation the host's memory and the runner's limit are not needed. + self.assertEqual(memory(runner="", host_mib="")["memory_gib"], None) + + def test_a_derivation_needs_the_runner_limit_and_the_host_memory(self) -> None: + with self.assertRaisesRegex(colima.AnsibleFilterError, "github_runner_cluster_colima_runner_memory_limit_gib"): + memory(reserved=6, runner="") + with self.assertRaisesRegex(colima.AnsibleFilterError, "memtotal_mb"): + memory(reserved=6, host_mib="") + + def test_a_reserve_that_is_not_a_number_is_refused_with_its_name(self) -> None: + for arguments, name in (({"reserved": "six"}, "github_runner_cluster_colima_memory_reserved_gib"), ({"reserved": -1}, "github_runner_cluster_colima_memory_reserved_gib"), ({"reserved": 6, "host_reserve": "two"}, "github_runner_cluster_colima_memory_host_reserve_gib"), ({"reserved": True}, "github_runner_cluster_colima_memory_reserved_gib")): + with self.subTest(arguments=arguments), self.assertRaisesRegex(colima.AnsibleFilterError, name): + memory(**arguments) + + def test_the_filter_is_registered(self) -> None: + self.assertIs(colima.FilterModule().filters()["cluster_colima_memory"], colima.cluster_colima_memory) + + def lease(holder: str, renewed: str = "2026-01-01T00:00:00.000000Z", duration: int = 60, version: str = "7") -> dict[str, Any]: return {"rc": 0, "stdout": json.dumps({"metadata": {"name": "fleet-regrade", "namespace": "kube-system", "resourceVersion": version}, "spec": {"holderIdentity": holder, "renewTime": renewed, "leaseDurationSeconds": duration}}), "stderr": ""} @@ -467,14 +521,38 @@ def test_a_value_that_is_not_a_whole_number_fails_before_anything_is_read(self) self.assertIn("github_runner_cluster_colima_memory_gib must be a positive whole number", result.stdout) self.assertEqual(self.calls(), []) - def test_a_host_whose_runtime_is_not_colima_fails_before_anything_is_read(self) -> None: + def test_a_host_whose_runtime_is_not_colima_is_left_alone(self) -> None: + # The sizes and the reserve are typically set for a whole group, which can hold Docker Desktop and Linux hosts as well. for variables in ({"github_runner_cluster_effective_runtime": {"kind": "docker_desktop", "colima_profile": ""}}, {"runtime_system": "Linux"}): with self.subTest(variables=variables): (self.stub / "calls.log").unlink(missing_ok=True) - result = self.run_role(limit="server1", github_runner_cluster_colima_cpus=4, **variables) - self.assertNotEqual(result.returncode, 0) - self.assertIn("only a Colima profile can be resized", result.stdout) + result = self.run_role(limit="server1", github_runner_cluster_colima_cpus=4, github_runner_cluster_colima_memory_reserved_gib=6, **variables) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(self.calls(), []) + self.assertEqual(self.recap(result, "server1")["changed"], 0) + + def test_the_memory_derived_from_the_host_is_what_colima_starts_with(self) -> None: + result = self.run_role(limit="agent1", runtime_memtotal_mb=HOST_MIB, github_runner_cluster_colima_memory_reserved_gib=6, github_runner_cluster_colima_memory_host_reserve_gib=2, github_runner_cluster_colima_runner_memory_limit_gib=7) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("colima start --profile agent1 --activate=false --memory 24", self.mutating()) + # The warning task runs only for a clamp; its header is printed whether or not it runs, so its message is what is looked for. + self.assertNotIn("GiB is below the node floor of", result.stdout) + + def test_a_larger_reserve_gives_a_smaller_vm(self) -> None: + result = self.run_role(limit="agent1", runtime_memtotal_mb=HOST_MIB, github_runner_cluster_colima_memory_reserved_gib=12, github_runner_cluster_colima_memory_host_reserve_gib=2, github_runner_cluster_colima_runner_memory_limit_gib=7) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("colima start --profile agent1 --activate=false --memory 18", self.mutating()) + + def test_a_derivation_below_the_floor_starts_at_the_floor_and_warns(self) -> None: + result = self.run_role(limit="agent1", runtime_memtotal_mb=HOST_MIB, github_runner_cluster_colima_memory_reserved_gib=24, github_runner_cluster_colima_memory_host_reserve_gib=2, github_runner_cluster_colima_runner_memory_limit_gib=7) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("colima start --profile agent1 --activate=false --memory 11", self.mutating()) + self.assertIn("6 GiB is below the node floor of 11 GiB", result.stdout) + + def test_an_explicit_memory_wins_over_the_reserve(self) -> None: + result = self.run_role(limit="agent1", runtime_memtotal_mb=HOST_MIB, github_runner_cluster_colima_memory_gib=9, github_runner_cluster_colima_memory_reserved_gib=6, github_runner_cluster_colima_runner_memory_limit_gib=7) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("colima start --profile agent1 --activate=false --memory 9", self.mutating()) def test_an_agent_is_drained_through_a_server_and_waits_for_ready_without_a_vote(self) -> None: result = self.run_role(limit="agent1", github_runner_cluster_colima_cpus=4) @@ -526,6 +604,8 @@ def test_a_start_that_fails_names_the_vm_and_gives_the_lease_back(self) -> None: class StaticTest(unittest.TestCase): def test_the_sizes_keep_the_profile_as_it_is_by_default(self) -> None: self.assertEqual((DEFAULTS["github_runner_cluster_colima_memory_gib"], DEFAULTS["github_runner_cluster_colima_cpus"]), ("", "")) + # No reserve declared, so nothing is derived until a host says what its other workloads need. + self.assertEqual((DEFAULTS["github_runner_cluster_colima_memory_reserved_gib"], DEFAULTS["github_runner_cluster_colima_runner_memory_limit_gib"]), ("", "")) def test_the_resize_is_included_after_the_topology_and_the_runtime_and_before_the_node_starts(self) -> None: tasks = yaml.safe_load((ROLE / "tasks" / "main.yml").read_text()) @@ -553,7 +633,7 @@ def written(tasks: list[dict[str, Any]]) -> set[str]: def test_every_new_variable_is_in_the_readme(self) -> None: readme = (ROLE / "README.md").read_text() - for name in ("github_runner_cluster_colima_memory_gib", "github_runner_cluster_colima_cpus"): + for name in ("github_runner_cluster_colima_memory_gib", "github_runner_cluster_colima_cpus", "github_runner_cluster_colima_memory_reserved_gib", "github_runner_cluster_colima_memory_host_reserve_gib", "github_runner_cluster_colima_guest_baseline_gib", "github_runner_cluster_colima_runner_memory_limit_gib"): with self.subTest(name=name): self.assertIn(f"`{name}`", readme) From 71d941a5339293b099eea0efe3345b3a720aa7a7 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 22:47:22 +0000 Subject: [PATCH 4/5] test(role): count the drain's refusal wording among the native role's derived defaults github_runner_cluster_native_drain_for is what tasks/native/drain.yml says was not done when it cannot empty the node, set by each caller and never by the inventory, so it is derived like the native units' names: the defaults test now lists it, and the README describes it with them. --- roles/github_runner_cluster/README.md | 2 +- tests/unit/test_native_node.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 323c3d3a..fd034b7e 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -441,7 +441,7 @@ Each Compose project runs its own `docker-prune` service against the same daemon - `github_runner_cluster_native_tailscale_install` (default `true`): whether the role installs the pinned Tailscale package from Tailscale's apt repository (Debian and Ubuntu). `false` leaves the host's client as it is and only checks its login. - `github_runner_cluster_native_tailscale_repository_url` (default `https://pkgs.tailscale.com/stable`): where that repository's signing key and list file are fetched from, when the host has none. - `github_runner_cluster_native_tailscale_leave_other_tailnet` (default `false`): whether the role logs a client out of another tailnet so that k3s can join the cluster's; `false` stops the run instead. -- `github_runner_cluster_native_agent_service`, `github_runner_cluster_native_server_service`, `github_runner_cluster_native_service`, `github_runner_cluster_native_join_url`, `github_runner_cluster_native_join_host`, `github_runner_cluster_native_role_file` and `github_runner_cluster_native_restart_marker` are derived: the units' names (`k3s-agent` and `k3s`), the unit of the host's role, the URL the node joins and the host in it, which the Tailscale wrapper waits for, the role file both units' conditions read, and the prefix of the per-unit file that says a restart is waiting; `github_runner_cluster_native_regrade_level_file`, `github_runner_cluster_native_regrade_started_file`, `github_runner_cluster_native_regrade_dir`, `github_runner_cluster_native_regrade_service`, `github_runner_cluster_native_regrade_commands`, `github_runner_cluster_native_agent_runtime_dir` and `github_runner_cluster_native_server_runtime_dir` are derived too: a native server's requested and started regrading levels, where its regrade scripts and settings go, the unit that runs them, the commands regrading needs with the package of each, and each unit's own runtime directory under `/run` (one per unit, since systemd removes a unit's runtime directory when its condition skips a start, which on a shared one would take the running unit's resolver file). `github_runner_cluster_native_bin_dir`, `github_runner_cluster_native_lib_dir`, `github_runner_cluster_native_config_dir`, `github_runner_cluster_native_unit_dir`, `github_runner_cluster_native_data_dir`, `github_runner_cluster_native_tailscale_cli`, `github_runner_cluster_native_apt_keyring`, `github_runner_cluster_native_apt_source`, `github_runner_cluster_native_run_dir` and `github_runner_cluster_native_psi_file` are where the native node's files live, which are not settings, and are variables only so the tests can point them at a stand-in. +- `github_runner_cluster_native_agent_service`, `github_runner_cluster_native_server_service`, `github_runner_cluster_native_service`, `github_runner_cluster_native_join_url`, `github_runner_cluster_native_join_host`, `github_runner_cluster_native_role_file` and `github_runner_cluster_native_restart_marker` are derived: the units' names (`k3s-agent` and `k3s`), the unit of the host's role, the URL the node joins and the host in it, which the Tailscale wrapper waits for, the role file both units' conditions read, and the prefix of the per-unit file that says a restart is waiting; `github_runner_cluster_native_regrade_level_file`, `github_runner_cluster_native_regrade_started_file`, `github_runner_cluster_native_regrade_dir`, `github_runner_cluster_native_regrade_service`, `github_runner_cluster_native_regrade_commands`, `github_runner_cluster_native_agent_runtime_dir` and `github_runner_cluster_native_server_runtime_dir` are derived too: a native server's requested and started regrading levels, where its regrade scripts and settings go, the unit that runs them, the commands regrading needs with the package of each, and each unit's own runtime directory under `/run` (one per unit, since systemd removes a unit's runtime directory when its condition skips a start, which on a shared one would take the running unit's resolver file). `github_runner_cluster_native_drain_for` is derived as well: what `tasks/native/drain.yml` says was not done when it cannot empty the node, a role switch's by default, which a Colima VM resize passes as its own. `github_runner_cluster_native_bin_dir`, `github_runner_cluster_native_lib_dir`, `github_runner_cluster_native_config_dir`, `github_runner_cluster_native_unit_dir`, `github_runner_cluster_native_data_dir`, `github_runner_cluster_native_tailscale_cli`, `github_runner_cluster_native_apt_keyring`, `github_runner_cluster_native_apt_source`, `github_runner_cluster_native_run_dir` and `github_runner_cluster_native_psi_file` are where the native node's files live, which are not settings, and are variables only so the tests can point them at a stand-in. ## Examples diff --git a/tests/unit/test_native_node.py b/tests/unit/test_native_node.py index fc5d1953..7fe0695e 100644 --- a/tests/unit/test_native_node.py +++ b/tests/unit/test_native_node.py @@ -339,6 +339,7 @@ def test_every_native_default_is_declared_with_the_same_default_and_described_in "github_runner_cluster_native_regrade_commands", "github_runner_cluster_native_agent_runtime_dir", "github_runner_cluster_native_server_runtime_dir", + "github_runner_cluster_native_drain_for", } self.assertEqual(set(SPECS), native - derived) for name, spec in SPECS.items(): From e40622d5f8719d320ca95193dd82246d0c98dd80 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 23:10:12 +0000 Subject: [PATCH 5/5] docs(role): name the host memory fact without reading it from the defaults The runtime tasks' guard test holds every ansible_facts read in the defaults to the subsets runtime.yml gathers. The reserve's comment named the memory fact in that form, though only tasks/colima_size.yml reads it, after gathering the hardware subset itself, so the comment now names the fact and the task that gathers it. --- roles/github_runner_cluster/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index c7f81021..920ac2c5 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -249,7 +249,7 @@ github_runner_cluster_colima_vm_command: "{{ ['colima', 'ssh', '--profile', gith # The memory, in GiB, and the number of CPUs of this host's Colima VM: each a positive whole number, or empty (the default) to keep what the profile has, unless the memory is derived from github_runner_cluster_colima_memory_reserved_gib (below); a memory set here wins over the derivation. Colima hosts only: on a host whose runtime is anything else these and the reserves are ignored, so they can be set for a group that mixes runtimes. When a size differs from what the profile's VM runs (`colima list --json`), the role resizes the VM through the quorum guard, one node at a time (tasks/colima_size.yml): it refuses unless the cluster stays safe without the node, takes the fleet-regrade Lease, refusing while another step holds it, cordons the node, waits for its runner pods to finish and drains it, checks the cluster again, stops the profile and starts it with the new size, waits for the node to be Ready again (and an etcd voter, on a server), uncordons it and gives the Lease back. `colima start` writes the size to the profile's configuration, so every later start, the launchd job's at boot or login included, keeps it. A node whose container is not running takes nothing from the cluster, and its VM is only stopped and started. github_runner_cluster_colima_memory_gib: "" github_runner_cluster_colima_cpus: "" -# The memory, in GiB, this host's other workloads (everything on the Mac but macOS itself and the node VM) need, a number zero or more. Set, and github_runner_cluster_colima_memory_gib empty, the VM's memory is derived: the host's memory (ansible_facts['memtotal_mb']) less this and github_runner_cluster_colima_memory_host_reserve_gib, rounded down to a whole GiB since colima takes whole GiB and rounding up would take memory a reserve keeps. It is never less than the node floor, github_runner_cluster_colima_guest_baseline_gib plus github_runner_cluster_colima_runner_memory_limit_gib rounded up to a whole GiB, since a node that cannot hold its own baseline and one runner pod is offered jobs it cannot finish; a derivation below the floor is sized at the floor and the run prints a warning that the host keeps less than its reserves. Empty, the default, derives nothing. +# The memory, in GiB, this host's other workloads (everything on the Mac but macOS itself and the node VM) need, a number zero or more. Set, and github_runner_cluster_colima_memory_gib empty, the VM's memory is derived: the host's memory (memtotal_mb of its hardware facts, which tasks/colima_size.yml gathers) less this and github_runner_cluster_colima_memory_host_reserve_gib, rounded down to a whole GiB since colima takes whole GiB and rounding up would take memory a reserve keeps. It is never less than the node floor, github_runner_cluster_colima_guest_baseline_gib plus github_runner_cluster_colima_runner_memory_limit_gib rounded up to a whole GiB, since a node that cannot hold its own baseline and one runner pod is offered jobs it cannot finish; a derivation below the floor is sized at the floor and the run prints a warning that the host keeps less than its reserves. Empty, the default, derives nothing. github_runner_cluster_colima_memory_reserved_gib: "" # The memory, in GiB, a derivation keeps for macOS itself: the resident memory of the processes macOS runs as root and its service accounts (`ps -axo user,rss` over every user but the connecting one), about 3.8 GiB on an Apple silicon Mac with no user workload, rounded up to a whole GiB. A host whose own reading differs sets its own. github_runner_cluster_colima_memory_host_reserve_gib: 4