diff --git a/plugins/filter/cluster_colima.py b/plugins/filter/cluster_colima.py index 293981b2..0f707056 100644 --- a/plugins/filter/cluster_colima.py +++ b/plugins/filter/cluster_colima.py @@ -67,26 +67,34 @@ def _number(amount: Decimal) -> int | float: def cluster_colima_memory(memory_gib: Any, host_memory_mb: Any, reserved_gib: Any, host_reserve_gib: Any, guest_baseline_gib: Any, runner_limit_gib: Any) -> dict[str, Any]: - """Work out the memory a Colima node VM is to have: the value set, or what the host can spare, never less than a node needs to run one runner. + """Work out the memory a Colima node VM is to have: the value set, within what the host can spare when a reserve is set, or what the host can spare, never less than a node needs to run one runner. - :param memory_gib: github_runner_cluster_colima_memory_gib: the VM's memory in GiB, which wins when set. :param host_memory_mb: ansible_facts['memtotal_mb'], the host's memory in MiB, needed only to derive. :param reserved_gib: github_runner_cluster_colima_memory_reserved_gib: the GiB the host's other workloads need; empty derives nothing. :param host_reserve_gib: github_runner_cluster_colima_memory_host_reserve_gib: the GiB kept for the host's operating system. :param guest_baseline_gib: github_runner_cluster_colima_guest_baseline_gib: the GiB the node VM uses with no runner pod on it. :param runner_limit_gib: github_runner_cluster_colima_runner_memory_limit_gib: one runner pod's memory limit in GiB. - :returns: ``{"memory_gib": int | None, "source": "set" | "derived" | "none", "available_gib": , "floor_gib": , "clamped": bool}``. A derived memory is the available memory rounded down to a whole GiB, since `colima start --memory` takes whole GiB and rounding up would take memory the reserves keep; below the floor it is the floor, with ``clamped`` true. ``memory_gib`` None keeps the profile's own. - :raises AnsibleFilterError: if a value is set and is not a number of the kind its variable takes, or a derivation lacks the runner's limit or the host's memory. + :param memory_gib: github_runner_cluster_colima_memory_gib: the VM's memory in GiB, which wins when set. :param host_memory_mb: ansible_facts['memtotal_mb'], the host's memory in MiB, needed only when a reserve is set. :param reserved_gib: github_runner_cluster_colima_memory_reserved_gib: the GiB the host's other workloads need; set, it bounds a memory set explicitly and derives one otherwise; empty, it does neither. :param host_reserve_gib: github_runner_cluster_colima_memory_host_reserve_gib: the GiB kept for the host's operating system. :param guest_baseline_gib: github_runner_cluster_colima_guest_baseline_gib: the GiB the node VM uses with no runner pod on it. :param runner_limit_gib: github_runner_cluster_colima_runner_memory_limit_gib: one runner pod's memory limit in GiB. + :returns: ``{"memory_gib": int | None, "source": "set" | "derived" | "none", "available_gib": , "floor_gib": , "clamped": bool}``. A derived memory is the available memory rounded down to a whole GiB, since `colima start --memory` takes whole GiB and rounding up would take memory the reserves keep; below the floor it is the floor, with ``clamped`` true. ``memory_gib`` None keeps the profile's own. + :raises AnsibleFilterError: if a value is set and is not a number of the kind its variable takes, a reserve is set without the host's memory, a derivation lacks the runner's limit, or a memory set explicitly is more than the host can spare (the budget is a safety limit, so a workload whose reserve grows can never leave the VM larger than the host affords). """ explicit = _size(MEMORY_VARIABLE, memory_gib) reserved = _amount(RESERVED_VARIABLE, reserved_gib) - if explicit is not None: - return {"memory_gib": explicit, "source": "set", "available_gib": None, "floor_gib": None, "clamped": False} if reserved is None: - return {"memory_gib": None, "source": "none", "available_gib": None, "floor_gib": None, "clamped": False} + source = "none" if explicit is None else "set" + return {"memory_gib": explicit, "source": source, "available_gib": None, "floor_gib": None, "clamped": False} host_reserve = _needed(HOST_RESERVE_VARIABLE, host_reserve_gib) + if isinstance(host_memory_mb, bool) or not isinstance(host_memory_mb, (int, str)) or not WHOLE_NUMBER.match(str(host_memory_mb)): + raise AnsibleFilterError(f"{HOST_MEMORY_FACT} must give the host's memory in MiB to size the Colima VM within {RESERVED_VARIABLE}, not {host_memory_mb!r}.") + host = Decimal(int(host_memory_mb)) / MIB_PER_GIB + available = host - reserved - host_reserve + if explicit is not None: + if explicit > available: + raise AnsibleFilterError( + f"{MEMORY_VARIABLE} {explicit} is more than the {_number(available)} GiB this host can spare: its {_number(host)} GiB less " + f"{_number(reserved)} GiB for its other workloads ({RESERVED_VARIABLE}) and {_number(host_reserve)} GiB for its operating system " + f"({HOST_RESERVE_VARIABLE}). Lower the memory, or the reserve if the workloads need less." + ) + return {"memory_gib": explicit, "source": "set", "available_gib": _number(available), "floor_gib": None, "clamped": False} baseline = _needed(BASELINE_VARIABLE, guest_baseline_gib) runner = _needed(RUNNER_VARIABLE, runner_limit_gib) if runner == 0: raise AnsibleFilterError(f"{RUNNER_VARIABLE} must be more than zero: a node VM is sized to run at least one runner pod.") - if isinstance(host_memory_mb, bool) or not isinstance(host_memory_mb, (int, str)) or not WHOLE_NUMBER.match(str(host_memory_mb)): - raise AnsibleFilterError(f"{HOST_MEMORY_FACT} must give the host's memory in MiB to derive the Colima VM's memory, not {host_memory_mb!r}.") - available = Decimal(int(host_memory_mb)) / MIB_PER_GIB - reserved - host_reserve floor = int((baseline + runner).to_integral_value(rounding=ROUND_CEILING)) derived = int(available.to_integral_value(rounding=ROUND_FLOOR)) return {"memory_gib": max(derived, floor), "source": "derived", "available_gib": _number(available), "floor_gib": floor, "clamped": derived < floor} diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 0fca170a..31f076d9 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -178,7 +178,7 @@ A host that got the agent from release 2.19.0 moves over on its next run: the ro **Colima kernel.** On every Colima host, whether or not `github_runner_cluster_runtime_autostart` is on, once the VM answers (`tasks/colima_kernel.yml`), the role makes the VM's kernel pass bridged pod traffic through iptables, now and at every VM boot: it writes `/etc/modules-load.d/br_netfilter.conf` and `/etc/sysctl.d/90-github-runner-br-netfilter.conf` in the VM, runs `modprobe br_netfilter` when the VM has no `/proc/sys/net/bridge`, and sets `net.bridge.bridge-nf-call-iptables` and `net.bridge.bridge-nf-call-ip6tables` to 1 when they differ. Without the module a pod that talks to a Service whose only backend pod is on the same node gets no reply, because the Service's reverse NAT never applies to the bridged hairpin traffic; pods on other nodes keep working, so it stays latent until a single-backend Service such as CoreDNS lands on the node (the `br_netfilter` note under Non-obvious behaviour in the repository README has the symptoms). Everything runs in the VM through `colima ssh --profile `, as the VM's user with `sudo`, and no `become` on the host. A file is written only when its content differs, so a rerun changes nothing, and the module file takes the name the README's note told people to create by hand, so an existing one is replaced by the managed one. Under `--check` each change is reported and none is made, and nothing is read from a VM that a check run would only have started. Docker Desktop's VM is not reachable by the role and other runtimes are not touched. -**Colima VM size.** A Colima host can set its VM's memory in GiB and its number of CPUs, `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus`, each a positive whole number; empty, the default, keeps what the profile has (`tasks/colima_size.yml`). Instead of a memory, a host can say what it must keep: `github_runner_cluster_colima_memory_reserved_gib`, the memory its other workloads need, and `github_runner_cluster_colima_memory_host_reserve_gib`, the memory kept for macOS itself (the resident memory of the processes macOS runs as root and its service accounts, rounded up to a whole GiB, by default). The VM then gets the host's memory (`ansible_facts['memtotal_mb']`) less both, rounded down to a whole GiB, but never less than the node floor: the guest's own use with no runner pod on it, `github_runner_cluster_colima_guest_baseline_gib`, plus one runner pod's memory limit, `github_runner_cluster_colima_runner_memory_limit_gib` (which the scale set's values set, so the cluster role is told it, and which a derivation requires), rounded up to a whole GiB. A derivation below the floor sizes the VM at the floor and prints a warning that the host keeps less than its reserves. A memory set explicitly wins over the derivation, and with neither the profile's memory is left as it is. A host whose runtime is not Colima ignores all of these, so they can be set for a group that mixes runtimes. A value of the wrong kind, or a derivation without the runner's limit, fails the run before anything is read from the VM. The role reads the size of the VM Lima runs with `colima list --profile --json`, not the profile's configuration file, since `colima start` given a size while the profile runs writes it to the file and leaves the VM as it is ([`cmd/start.go`](https://github.com/abiosoft/colima/blob/v0.10.3/cmd/start.go)). A VM that already has every size set is left alone, so a rerun changes nothing. Otherwise the resize is a planned restart of the node, made one host at a time, each in its own turn, through the same quorum guard as moving a node between networks (`tasks/restart_guard.yml`): it refuses unless the other servers with a Ready node and a healthy etcd stay a majority of all servers; it takes the `fleet-regrade` Lease that regrading steps hold (`kube-system/fleet-regrade`, as `colima-resize/`), refusing while another step holds it, so a resize never overlaps a regrade step or another resize in this run or any other; it cordons the node, waits for its runner pods to finish and drains it (the role switch's `tasks/native/drain.yml`), checks the cluster again, runs `colima stop --profile ` and `colima start --profile --activate=false --memory --cpus ` (the flags of the values set), waits for the node to be Ready on a heartbeat later than the stop and, on a server, an etcd voter, uncordons it and gives the Lease back. Colima writes the new size to the profile's configuration, so the launchd job's later starts keep it. The node's container comes back with the VM (`restart: unless-stopped`). A node whose container is not running takes nothing from the cluster, so its VM is only stopped and started. When a step fails the node is uncordoned again and the run fails, saying whether the VM was stopped, and the Lease is given back, or expires after `github_runner_cluster_colima_resize_lease_seconds` when the API that holds it does not answer. Under `--check` the host's memory, the sizes, the Lease and the cluster are read, a refusal fails as it would in a real run, and the resize is reported and not made. +**Colima VM size.** A Colima host can set its VM's memory in GiB and its number of CPUs, `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus`, each a positive whole number; empty, the default, keeps what the profile has (`tasks/colima_size.yml`). Instead of a memory, a host can say what it must keep: `github_runner_cluster_colima_memory_reserved_gib`, the memory its other workloads need, and `github_runner_cluster_colima_memory_host_reserve_gib`, the memory kept for macOS itself (by default macOS's wired memory plus the resident memory of every process but the node VM's, measured on node hosts and rounded to a whole GiB). The VM then gets the host's memory (`ansible_facts['memtotal_mb']`) less both, rounded down to a whole GiB, but never less than the node floor: the guest's own use with no runner pod on it, `github_runner_cluster_colima_guest_baseline_gib`, plus one runner pod's memory limit, `github_runner_cluster_colima_runner_memory_limit_gib` (which the scale set's values set, so the cluster role is told it, and which a derivation requires), rounded up to a whole GiB. A derivation below the floor sizes the VM at the floor and prints a warning that the host keeps less than its reserves. A memory set explicitly wins over the derivation, but with a reserve set it must fit the same budget: a memory above the host's memory less both reserves fails the run, naming the host's memory and both reserves, before the VM is read or the node cordoned, so a workload whose reserve grows can never leave the VM larger than the host affords. With neither a memory nor a reserve the profile's memory is left as it is. A host whose runtime is not Colima ignores all of these, so they can be set for a group that mixes runtimes, and says so under `-v`, naming the values it skips. A value of the wrong kind, or a derivation without the runner's limit, fails the run before anything is read from the VM. The role reads the size of the VM Lima runs with `colima list --profile --json`, not the profile's configuration file, since `colima start` given a size while the profile runs writes it to the file and leaves the VM as it is ([`cmd/start.go`](https://github.com/abiosoft/colima/blob/v0.10.3/cmd/start.go)). A VM that already has every size set is left alone, so a rerun changes nothing. Otherwise the resize is a planned restart of the node, made one host at a time, each in its own turn, through the same quorum guard as moving a node between networks (`tasks/restart_guard.yml`): it refuses unless the other servers with a Ready node and a healthy etcd stay a majority of all servers; it takes the `fleet-regrade` Lease that regrading steps hold (`kube-system/fleet-regrade`, as `colima-resize/`), refusing while another step holds it, so a resize never overlaps a regrade step or another resize in this run or any other; it cordons the node, waits for its runner pods to finish and drains it (the role switch's `tasks/native/drain.yml`), checks the cluster again, runs `colima stop --profile ` and `colima start --profile --activate=false --memory --cpus ` (the flags of the values set), waits for the node to be Ready on a heartbeat later than the stop and, on a server, an etcd voter, uncordons it and gives the Lease back. Colima writes the new size to the profile's configuration, so the launchd job's later starts keep it. The node's container comes back with the VM (`restart: unless-stopped`). A node whose container is not running takes nothing from the cluster, so its VM is only stopped and started. When a step fails the node is uncordoned again and the run fails, saying whether the VM was stopped, and the Lease is given back, or expires after `github_runner_cluster_colima_resize_lease_seconds` when the API that holds it does not answer. Under `--check` the host's memory, the sizes, the Lease and the cluster are read, a refusal fails as it would in a real run, and the resize is reported and not made. This is the role's whole scope: it never moves a host between Colima and Docker Desktop. @@ -438,7 +438,7 @@ Each Compose project runs its own `docker-prune` service against the same daemon - `github_runner_cluster_network_mode` (default `bridge`, or `host`): the network the node's container runs on, and with `host`, `github_runner_cluster_lan_cidr` (required: the LAN the nodes share) and `github_runner_cluster_network_probe_image` (default `busybox:1.37`, for reading the host's addresses) (see [Networking](#networking)). - `github_runner_cluster_runtime_autostart_at` (default `auto`; `login` or `boot`) and `github_runner_cluster_runtime_boot_profile` (default `node`): whether the Colima profile is started by a launchd agent at login or by a launchd daemon at boot, and which profile `auto` starts at boot (see [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). - `github_runner_cluster_runtime_autostart` (default true), `github_runner_cluster_runtime` (default `auto`; `colima`, `docker_desktop` or `other`), `github_runner_cluster_colima_profile` (default empty, taken from the Docker context), `github_runner_cluster_colima_home` (default empty: the `COLIMA_HOME` is read from the endpoint of the profile's Docker context, and is unset for Colima's own `~/.colima`; set it only for a context that does not say, and the role passes it to colima and to the launchd agent), `github_runner_cluster_runtime_path` (default both Homebrew prefixes and the system directories): on a macOS host, start Colima at login through a launchd agent and check the runtime (see [Container runtime at login](#container-runtime-after-a-reboot-macos)). -- `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus` (default empty, keeping the profile's own): a Colima host's VM memory in GiB and its CPUs, each a positive whole number, applied one node at a time through the quorum guard when the VM differs (see Colima VM size under [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). `github_runner_cluster_colima_memory_reserved_gib` (default empty) derives the memory instead, from the host's memory less it and `github_runner_cluster_colima_memory_host_reserve_gib` (default 4, macOS's own), never below `github_runner_cluster_colima_guest_baseline_gib` (default 3.4) plus `github_runner_cluster_colima_runner_memory_limit_gib` (default empty, required by a derivation) rounded up. `github_runner_cluster_colima_resize_lease_seconds`, `github_runner_cluster_colima_resize_lease_holder`, `github_runner_cluster_regrade_lease_name` and `github_runner_cluster_regrade_lease_namespace` are derived, never set directly. `github_runner_cluster_native_runner_wait_timeout` and `github_runner_cluster_native_ready_timeout` bound the wait for the node's runner pods and for the node to come back, as they do for a native node. +- `github_runner_cluster_colima_memory_gib` and `github_runner_cluster_colima_cpus` (default empty, keeping the profile's own): a Colima host's VM memory in GiB and its CPUs, each a positive whole number, applied one node at a time through the quorum guard when the VM differs (see Colima VM size under [Container runtime after a reboot](#container-runtime-after-a-reboot-macos)). `github_runner_cluster_colima_memory_reserved_gib` (default empty) derives the memory instead, or bounds one set explicitly, from the host's memory less it and `github_runner_cluster_colima_memory_host_reserve_gib` (default 10, macOS's own), never below `github_runner_cluster_colima_guest_baseline_gib` (default 3.4) plus `github_runner_cluster_colima_runner_memory_limit_gib` (default empty, required by a derivation) rounded up. `github_runner_cluster_colima_resize_lease_seconds`, `github_runner_cluster_colima_resize_lease_holder`, `github_runner_cluster_regrade_lease_name` and `github_runner_cluster_regrade_lease_namespace` are derived, never set directly. `github_runner_cluster_native_runner_wait_timeout` and `github_runner_cluster_native_ready_timeout` bound the wait for the node's runner pods and for the node to come back, as they do for a native node. - `github_runner_cluster_container_name` (default `github-runner-k3s`) and `github_runner_cluster_agent_container_name` (default `github-runner-k3s-agent`): the names of the node's server and agent containers, which a second cluster on the same Docker hosts sets to names of its own (see [Several clusters on one host](#several-clusters-on-one-host)). - `github_runner_cluster_compose_files`: extra Compose files for the host's k3s project, merged after `docker-compose.yml`, as paths relative to `github_runner_cluster_dir` or absolute. Several nodes on one Docker host each need their own `github_runner_cluster_dir`, `github_runner_cluster_compose_project` and container names; nodes of one cluster share its API port, so all but one of them on a host also need an extra file that publishes none (`ports: !reset []`, as `tests/mesh` does). - `github_runner_cluster_wait_for_nodes`: wait for every node of the inventory to be Ready afterwards, clearing the node-password registration of a replaced node that fails to rejoin (installs the kubernetes Python client through `github_runner_k8s_client`). The wait covers the nodes the inventory's hosts own, by their effective node names, and leaves out a server that memory-pressure regrading has stepped down and an inventory node that has not registered. A node the cluster holds that no inventory host owns (one flashed from an image and joined by hand, say) is never waited for: when it is not Ready the run prints one line with its name and status, and carries on. When no node of the inventory has registered, the run fails. @@ -499,7 +499,7 @@ or read from 1Password by adding `exadev.github_runner.github_runner_secrets_one `tests/unit/test_runtime_autostart.py` covers the [container runtime tasks](#container-runtime-after-a-reboot-macos) without a Mac: it renders the launchd agent and daemon through `tests/runtime_autostart/render_plist.yml`, and runs `tasks/runtime.yml` and `tasks/end_checks.yml` through the role against stand-ins for `docker`, `colima`, `launchctl`, `plutil` and `sudo` on `PATH`, the stand-in `launchctl` refusing to change the system domain unless it was run through the stand-in `sudo`. It checks the variable defaults, that Linux hosts are left alone, that a second run changes nothing, that `--check` writes and starts nothing, and that each way the runtime or the node's container can be down fails with its message. `tests/unit/test_container_runtime.py` covers the filter that works out the runtime from the Docker context. Nothing there touches a real Colima, launchd or Docker Desktop, so loading the agent into a real login session, and the daemon starting a Colima VM at a real boot with no one logged in, are the parts only a run on a Mac can show. -`tests/unit/test_cluster_colima_size.py` covers the Colima VM size: the filters that read the profile's size (`plugins/filter/cluster_colima.py`) and take and give back the `fleet-regrade` Lease (`plugins/filter/cluster_guard.py`), and `tasks/colima_size.yml` run through the role (`tests/colima_size/run.yml`) against stand-ins for `docker` and `colima` that keep each profile's VM, the node containers, the cluster's nodes and the Lease as files. It checks the `colima` arguments, the derived memory (the host's less both reserves, a larger reserve giving a smaller VM, a derivation below the floor sized at the floor with a warning, an explicit memory winning), that a VM of the size set is only read, that a second run changes nothing, the order of a resize (the Lease, the cordon, the runner pods, the drain, the stop, the start with the new size, the uncordon, the Lease given back), that a held Lease, a server that is not Ready and a server whose etcd is unhealthy each refuse before anything changes, that `--check` reports and changes nothing, that several hosts take their turns one at a time, the refusal of a value that is not a whole number, and that a host that does not run Colima is left alone. A real Colima VM's resize is shown only by its first run on a Mac. +`tests/unit/test_cluster_colima_size.py` covers the Colima VM size: the filters that read the profile's size (`plugins/filter/cluster_colima.py`) and take and give back the `fleet-regrade` Lease (`plugins/filter/cluster_guard.py`), and `tasks/colima_size.yml` run through the role (`tests/colima_size/run.yml`) against stand-ins for `docker` and `colima` that keep each profile's VM, the node containers, the cluster's nodes and the Lease as files. It checks the `colima` arguments, the derived memory (the host's less both reserves, a larger reserve giving a smaller VM, a derivation below the floor sized at the floor with a warning, an explicit memory winning within the budget and refused above it before anything is read from the VM), that a VM of the size set is only read, that a second run changes nothing, the order of a resize (the Lease, the cordon, the runner pods, the drain, the stop, the start with the new size, the uncordon, the Lease given back), that a held Lease, a server that is not Ready and a server whose etcd is unhealthy each refuse before anything changes, that `--check` reports and changes nothing, that several hosts take their turns one at a time, the refusal of a value that is not a whole number, and that a host that does not run Colima is left alone and says so under `-v`. A real Colima VM's resize is shown only by its first run on a Mac. `tests/unit/test_cluster_wait_nodes.py` runs the wait for nodes to be Ready and its rescue against a stand-in for `kubectl`: a node outside the inventory that is not Ready is reported and never waited for, and an inventory node that stays not Ready still fails the run. The mesh test does not cover it, since its nodes publish no API port for the host's `kubectl` to reach. diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index 5ab09fb0..fc636e72 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -251,11 +251,11 @@ github_runner_cluster_colima_vm_command: "{{ ['colima', 'ssh', '--profile', gith # The memory, in GiB, and the number of CPUs of this host's Colima VM: each a positive whole number, or empty (the default) to keep what the profile has, unless the memory is derived from github_runner_cluster_colima_memory_reserved_gib (below); a memory set here wins over the derivation. Colima hosts only: on a host whose runtime is anything else these and the reserves are ignored, so they can be set for a group that mixes runtimes. When a size differs from what the profile's VM runs (`colima list --json`), the role resizes the VM through the quorum guard, one node at a time (tasks/colima_size.yml): it refuses unless the cluster stays safe without the node, takes the fleet-regrade Lease, refusing while another step holds it, cordons the node, waits for its runner pods to finish and drains it, checks the cluster again, stops the profile and starts it with the new size, waits for the node to be Ready again (and an etcd voter, on a server), uncordons it and gives the Lease back. `colima start` writes the size to the profile's configuration, so every later start, the launchd job's at boot or login included, keeps it. A node whose container is not running takes nothing from the cluster, and its VM is only stopped and started. github_runner_cluster_colima_memory_gib: "" github_runner_cluster_colima_cpus: "" -# The memory, in GiB, this host's other workloads (everything on the Mac but macOS itself and the node VM) need, a number zero or more. Set, and github_runner_cluster_colima_memory_gib empty, the VM's memory is derived: the host's memory (memtotal_mb of its hardware facts, which tasks/colima_size.yml gathers) less this and github_runner_cluster_colima_memory_host_reserve_gib, rounded down to a whole GiB since colima takes whole GiB and rounding up would take memory a reserve keeps. It is never less than the node floor, github_runner_cluster_colima_guest_baseline_gib plus github_runner_cluster_colima_runner_memory_limit_gib rounded up to a whole GiB, since a node that cannot hold its own baseline and one runner pod is offered jobs it cannot finish; a derivation below the floor is sized at the floor and the run prints a warning that the host keeps less than its reserves. Empty, the default, derives nothing. +# The memory, in GiB, this host's other workloads (everything on the Mac but macOS itself and the node VM) need, a number zero or more. Set, and github_runner_cluster_colima_memory_gib empty, the VM's memory is derived: the host's memory (memtotal_mb of its hardware facts, which tasks/colima_size.yml gathers) less this and github_runner_cluster_colima_memory_host_reserve_gib, rounded down to a whole GiB since colima takes whole GiB and rounding up would take memory a reserve keeps. Set alongside github_runner_cluster_colima_memory_gib, it is the budget that memory must fit: a memory above the host's memory less both reserves fails the run, naming the figures, before the VM is read or the node cordoned, so a workload whose reserve grows can never leave the VM larger than the host affords. A derived memory is never less than the node floor, github_runner_cluster_colima_guest_baseline_gib plus github_runner_cluster_colima_runner_memory_limit_gib rounded up to a whole GiB, since a node that cannot hold its own baseline and one runner pod is offered jobs it cannot finish; a derivation below the floor is sized at the floor and the run prints a warning that the host keeps less than its reserves. Empty, the default, derives and bounds nothing. github_runner_cluster_colima_memory_reserved_gib: "" -# The memory, in GiB, a derivation keeps for macOS itself: the resident memory of the processes macOS runs as root and its service accounts (`ps -axo user,rss` over every user but the connecting one), about 3.8 GiB on an Apple silicon Mac with no user workload, rounded up to a whole GiB. A host whose own reading differs sets its own. -github_runner_cluster_colima_memory_host_reserve_gib: 4 -# The memory, in GiB, a node VM uses with no runner pod on it (the guest kernel, containerd, k3s with its datastore on a server, and the cluster's system pods), as measured on a Colima server node VM: the first part of the node floor. +# The memory, in GiB, a reserve keeps for macOS itself: the memory macOS has wired plus the resident memory of every process but the node VM's, as measured on two node hosts: 3.68 + 6.28 = 9.96 GiB on one of 64 GiB and 2.52 + 7.85 = 10.4 GiB on one of 32 GiB, within half a GiB of each other, so 10, the whole GiB nearest both. The compressor is left out: what it holds follows from the memory pressure at the time it is read, not from what macOS needs. A host whose own reading differs sets its own. +github_runner_cluster_colima_memory_host_reserve_gib: 10 +# The memory, in GiB, a node VM uses with no runner pod on it (the guest kernel, containerd, k3s with its datastore on a server, and the cluster's system pods), as measured inside a Colima server node VM with `free -m` in its k3s container: 3396 MiB used by k3s, etcd, the system pods and two idle runner pods (https://github.com/ExaDev/github-runner/issues/208). The first part of the node floor. github_runner_cluster_colima_guest_baseline_gib: 3.4 # One runner pod's memory limit, in GiB, as the scale set's values give it (the cluster role does not install the runners, so it cannot read it): the second part of the node floor. Required when the memory is derived; empty, the default, otherwise. github_runner_cluster_colima_runner_memory_limit_gib: "" diff --git a/roles/github_runner_cluster/tasks/colima_size.yml b/roles/github_runner_cluster/tasks/colima_size.yml index 7a72a0b2..21db9114 100644 --- a/roles/github_runner_cluster/tasks/colima_size.yml +++ b/roles/github_runner_cluster/tasks/colima_size.yml @@ -1,13 +1,27 @@ --- -# Sizes this host's Colima VM from github_runner_cluster_colima_memory_gib, or the memory derived from github_runner_cluster_colima_memory_reserved_gib, and github_runner_cluster_colima_cpus (see their defaults). A host whose runtime is not a Colima profile is left alone, so the sizes and the reserve can be set for a group that also holds Docker Desktop and Linux hosts; with nothing declared a Colima host is left alone too. Otherwise it checks the values before anything is read from the VM, works out the memory, then reads the size of the profile's VM with `colima list --json` and works out what differs; a VM that already has every size wanted is left as it is, so a second run changes nothing. The hosts whose VM differs are then resized one at a time, each in its own turn (tasks/colima_size_turn.yml), in the same way tasks/start_servers.yml gives each server its turn: Ansible's linear strategy runs every host's copy of a task together, except that the tasks one host includes dynamically are its own, so while one host's turn runs the others wait and no two nodes are ever away together. Needs runtime.yml (the runtime and its environment) and topology.yml to have run. Under --check the values are checked and the host's memory, the VM's size, the Lease and the cluster are read, a resize is reported as a change and a refusal as a failure, and nothing is changed. +# Sizes this host's Colima VM from github_runner_cluster_colima_memory_gib, or the memory derived from github_runner_cluster_colima_memory_reserved_gib, and github_runner_cluster_colima_cpus (see their defaults); with the reserve set, a memory set explicitly must fit within what the host can spare. A host whose runtime is not a Colima profile is left alone, saying so under -v, so the sizes and the reserve can be set for a group that also holds Docker Desktop and Linux hosts; with nothing declared a Colima host is left alone too. Otherwise it checks the values before anything is read from the VM, works out the memory, then reads the size of the profile's VM with `colima list --json` and works out what differs; a VM that already has every size wanted is left as it is, so a second run changes nothing. The hosts whose VM differs are then resized one at a time, each in its own turn (tasks/colima_size_turn.yml), in the same way tasks/start_servers.yml gives each server its turn: Ansible's linear strategy runs every host's copy of a task together, except that the tasks one host includes dynamically are its own, so while one host's turn runs the others wait and no two nodes are ever away together. Needs runtime.yml (the runtime and its environment) and topology.yml to have run. Under --check the values are checked and the host's memory, the VM's size, the Lease and the cluster are read, a resize is reported as a change and a refusal as a failure, and nothing is changed. - name: Note whether this host's runtime is a Colima profile # Only a VM the role can stop and start with colima is sized. github_runner_cluster_effective_runtime is published by runtime.yml, which runs on macOS only, so the operating system is tested first. ansible.builtin.set_fact: github_runner_cluster_colima_size_applies: "{{ ansible_facts['system'] == 'Darwin' and github_runner_cluster_effective_runtime.kind == 'colima' }}" -- name: Read this host's memory, to derive the VM's from it - # Only the hardware subset, which holds memtotal_mb, and only when a derivation needs it and the caller did not gather it. Read-only, so it runs under --check too. +- name: Say that a host whose runtime is not Colima keeps its VM sizes unapplied + # Shown under -v, so a size set for the wrong host is visible rather than silently skipped, without a line on every host of a group that mixes runtimes. + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }} is not a Colima profile, so its VM sizes are not applied + ({{ [('memory ' ~ github_runner_cluster_colima_memory_gib ~ ' GiB') if github_runner_cluster_colima_memory_gib | string | length > 0 else none, + ('cpus ' ~ github_runner_cluster_colima_cpus) if github_runner_cluster_colima_cpus | string | length > 0 else none, + ('memory reserved ' ~ github_runner_cluster_colima_memory_reserved_gib ~ ' GiB') if github_runner_cluster_colima_memory_reserved_gib | string | length > 0 else none] + | reject('none') | join(', ') }}). + verbosity: 1 + when: + - not github_runner_cluster_colima_size_applies | bool + - (github_runner_cluster_colima_memory_gib | string ~ github_runner_cluster_colima_cpus | string ~ github_runner_cluster_colima_memory_reserved_gib | string) | length > 0 + +- name: Read this host's memory, to size the VM within what the host can spare + # Only the hardware subset, which holds memtotal_mb, and only when a reserve is set (to derive the memory, or to check one set explicitly against it) and the caller did not gather it. Read-only, so it runs under --check too. ansible.builtin.setup: gather_subset: - "!all" @@ -15,12 +29,11 @@ - hardware when: - github_runner_cluster_colima_size_applies | bool - - github_runner_cluster_colima_memory_gib | string | length == 0 - github_runner_cluster_colima_memory_reserved_gib | string | length > 0 - "'memtotal_mb' not in ansible_facts" - name: Work out the memory the Colima VM is to have - # Before anything is read from the VM, so that a value colima could not take, or a derivation missing what it needs, fails here rather than after a VM is stopped for it. + # Before anything is read from the VM or the node is cordoned, so that a value colima could not take, a memory set above what the host can spare, or a derivation missing what it needs, fails here rather than after a VM is stopped for it. ansible.builtin.set_fact: github_runner_cluster_colima_memory: >- {{ github_runner_cluster_colima_memory_gib diff --git a/tests/unit/test_cluster_colima_size.py b/tests/unit/test_cluster_colima_size.py index c8136c8f..d44f740f 100644 --- a/tests/unit/test_cluster_colima_size.py +++ b/tests/unit/test_cluster_colima_size.py @@ -136,6 +136,28 @@ def test_a_reserve_that_is_not_a_number_is_refused_with_its_name(self) -> None: def test_the_filter_is_registered(self) -> None: self.assertIs(colima.FilterModule().filters()["cluster_colima_memory"], colima.cluster_colima_memory) + def test_an_explicit_memory_within_the_budget_is_kept(self) -> None: + # 32 - 6 - 2 = 24 GiB the host can spare: an explicit 20, or exactly 24, fits. + for explicit in (20, 24): + with self.subTest(explicit=explicit): + plan = memory(explicit=explicit, reserved=6, host_reserve=2) + self.assertEqual((plan["memory_gib"], plan["source"], plan["available_gib"]), (explicit, "set", 24)) + + def test_an_explicit_memory_above_the_budget_is_refused_naming_the_figures(self) -> None: + with self.assertRaises(colima.AnsibleFilterError) as raised: + memory(explicit=25, reserved=6, host_reserve=2) + message = str(raised.exception) + for figure in ("github_runner_cluster_colima_memory_gib 25", "32 GiB", "6 GiB", "2 GiB", "24 GiB"): + self.assertIn(figure, message) + + def test_the_budget_needs_neither_the_runner_limit_nor_the_baseline(self) -> None: + self.assertEqual(memory(explicit=20, reserved=6, host_reserve=2, runner="", baseline="")["memory_gib"], 20) + + def test_without_a_reserve_an_explicit_memory_has_no_budget(self) -> None: + # Larger than the host itself, and the host's memory is not even known: nothing to check it against. + plan = memory(explicit=40, host_mib="") + self.assertEqual((plan["memory_gib"], plan["available_gib"]), (40, None)) + def lease(holder: str, renewed: str = "2026-01-01T00:00:00.000000Z", duration: int = 60, version: str = "7") -> dict[str, Any]: return {"rc": 0, "stdout": json.dumps({"metadata": {"name": "fleet-regrade", "namespace": "kube-system", "resourceVersion": version}, "spec": {"holderIdentity": holder, "renewTime": renewed, "leaseDurationSeconds": duration}}), "stderr": ""} @@ -530,6 +552,9 @@ def test_a_host_whose_runtime_is_not_colima_is_left_alone(self) -> None: self.assertEqual(result.returncode, 0, result.stdout + result.stderr) self.assertEqual(self.calls(), []) self.assertEqual(self.recap(result, "server1")["changed"], 0) + # Said under -v, so a size set for the wrong host is visible rather than silently skipped. + verbose = self.run_role("-v", limit="server1", github_runner_cluster_colima_cpus=4, github_runner_cluster_colima_memory_reserved_gib=6, **variables) + self.assertIn("not a Colima profile, so its VM sizes are not applied (cpus 4, memory reserved 6 GiB)", verbose.stdout) def test_the_memory_derived_from_the_host_is_what_colima_starts_with(self) -> None: result = self.run_role(limit="agent1", runtime_memtotal_mb=HOST_MIB, github_runner_cluster_colima_memory_reserved_gib=6, github_runner_cluster_colima_memory_host_reserve_gib=2, github_runner_cluster_colima_runner_memory_limit_gib=7) @@ -549,6 +574,17 @@ def test_a_derivation_below_the_floor_starts_at_the_floor_and_warns(self) -> Non self.assertIn("colima start --profile agent1 --activate=false --memory 11", self.mutating()) self.assertIn("6 GiB is below the node floor of 11 GiB", result.stdout) + def test_an_explicit_memory_above_the_budget_fails_before_anything_is_read_from_the_vm(self) -> None: + result = self.run_role(limit="agent1", runtime_memtotal_mb=HOST_MIB, github_runner_cluster_colima_memory_gib=25, github_runner_cluster_colima_memory_reserved_gib=6, github_runner_cluster_colima_memory_host_reserve_gib=2) + self.assertNotEqual(result.returncode, 0) + self.assertIn("is more than the 24 GiB this host can spare", result.stdout) + self.assertEqual(self.calls(), []) + + def test_an_explicit_memory_within_the_budget_is_what_colima_starts_with(self) -> None: + result = self.run_role(limit="agent1", runtime_memtotal_mb=HOST_MIB, github_runner_cluster_colima_memory_gib=24, github_runner_cluster_colima_memory_reserved_gib=6, github_runner_cluster_colima_memory_host_reserve_gib=2) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("colima start --profile agent1 --activate=false --memory 24", self.mutating()) + def test_an_explicit_memory_wins_over_the_reserve(self) -> None: result = self.run_role(limit="agent1", runtime_memtotal_mb=HOST_MIB, github_runner_cluster_colima_memory_gib=9, github_runner_cluster_colima_memory_reserved_gib=6, github_runner_cluster_colima_runner_memory_limit_gib=7) self.assertEqual(result.returncode, 0, result.stdout + result.stderr) @@ -602,11 +638,22 @@ def test_a_start_that_fails_names_the_vm_and_gives_the_lease_back(self) -> None: class StaticTest(unittest.TestCase): + def test_the_host_memory_is_gathered_whenever_a_reserve_is_set(self) -> None: + # Whether the memory is set or derived: a set memory is checked against the budget, which needs the host's memory too. The role tests stand the fact in, so this reads the condition itself. + tasks = yaml.safe_load((ROOT / "roles" / "github_runner_cluster" / "tasks" / "colima_size.yml").read_text()) + [gather] = [task for task in tasks if "ansible.builtin.setup" in task] + self.assertEqual(gather["ansible.builtin.setup"]["gather_subset"], ["!all", "!min", "hardware"]) + self.assertEqual(gather["when"], ["github_runner_cluster_colima_size_applies | bool", "github_runner_cluster_colima_memory_reserved_gib | string | length > 0", "'memtotal_mb' not in ansible_facts"]) + def test_the_sizes_keep_the_profile_as_it_is_by_default(self) -> None: self.assertEqual((DEFAULTS["github_runner_cluster_colima_memory_gib"], DEFAULTS["github_runner_cluster_colima_cpus"]), ("", "")) # No reserve declared, so nothing is derived until a host says what its other workloads need. self.assertEqual((DEFAULTS["github_runner_cluster_colima_memory_reserved_gib"], DEFAULTS["github_runner_cluster_colima_runner_memory_limit_gib"]), ("", "")) + def test_the_measured_reserves_are_the_defaults(self) -> None: + # The figures each default's comment derives: macOS's own memory on a node host and the guest's baseline in a node VM. + self.assertEqual((DEFAULTS["github_runner_cluster_colima_memory_host_reserve_gib"], DEFAULTS["github_runner_cluster_colima_guest_baseline_gib"]), (10, 3.4)) + def test_the_resize_is_included_after_the_topology_and_the_runtime_and_before_the_node_starts(self) -> None: tasks = yaml.safe_load((ROLE / "tasks" / "main.yml").read_text()) order = [task.get("ansible.builtin.include_tasks") for task in tasks]