diff --git a/README.md b/README.md index 9b646734..aec28661 100644 --- a/README.md +++ b/README.md @@ -197,13 +197,13 @@ Its image, `github_runner_arc_node_recovery_image`, is pulled without a pull Sec `fleet-role-probe` and `fleet-role` (an optional DaemonSet and Deployment in the platform namespace, see `roles/github_runner_arc/templates/fleet-role.yaml.j2`, `scripts/fleet-role-probe.sh` and `scripts/fleet-role.sh`; `github_runner_cluster_role_controller`, off by default, which `github_runner_arc_role_controller_enabled` takes as its default) measure which hosts are fit to hold an etcd vote and report the swap of an unfit voter for a fit standby that the controller would make. They only report: the pods' ClusterRoles can `get` and `list` nodes and nothing else, and the only objects they write are their own two ConfigMaps. The controller refuses to start with any setting other than `FLEET_ROLE_DRY_RUN=true`, and runs unchanged when no host is marked eligible, when it reports the voters' fitness alone. -The probe runs on every Linux node that is an etcd voter (k3s's `node-role.kubernetes.io/etcd` label) or is marked eligible and has a kubelet (a server regraded to etcd-only has none, and its side is read from its peers), on the node's own network, through a node affinity on those two labels, so a node that is neither (a Unraid host, an image-flashed agent) has no pod and nothing is asked of its kernel. A pod stopped on a node that has lost both labels, or is gone, removes the node's entry as it ends, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the control plane and not with the fleet. Every `github_runner_arc_role_controller_sample_seconds` (15) it appends one sample to its node's key of the `fleet-role-probe` ConfigMap, which keeps the last window of them so that a replaced pod loses nothing. A sample holds the TCP connect time to every voter's etcd peer port (a connect that does not finish within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what its etcd metrics (the `127.0.0.1:2381` listener) gained since the last sample: the WAL fsync and backend commit histograms, the round trip to each peer, leader changes, and whether it has a leader, is a learner and knows as many members as there are voters (a learner or a member that has not started shows as an extra known member); on a node that is not a voter, an fdatasync reading from fio with the write size etcd's own disk check uses, once per window; the memory pressure (`/proc/pressure/memory` of the pod's own procfs, which serves the whole node's, so nothing is mounted) and `MemAvailable` the regrade core reads, with the core's own functions, where a node on a kernel without pressure stall information gives `MemAvailable` and `MemTotal` and no pressure figure, which the controller counts as not over the memory limit; and the medium of the node's link, `wired`, `wifi` or `unknown`: the value of the node's `github-runner.exadev.com/link` label (`github_runner_arc_role_controller_link_label`), which the cluster role reads on the host itself (see the cluster role's README) since a node in a container or a VM sees only its own virtual interface, and otherwise the medium of the interface the pod's default route leaves by (an interface that is virtual, such as the k3s container's veth, hides the link beneath it), each sample saying which (`link_source`, `label` or `interface`). The pod's own reading is taken from the interface's `/sys/class/net` entry and never by walking `/sys`, whose symlinks loop. `scripts/fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of these readings alone (the link as the pod's own interface gives it), with no cluster access, to compare with a reading taken by hand on the host. +The probe runs on every Linux node that is an etcd voter (k3s's `node-role.kubernetes.io/etcd` label) or is marked eligible and has a kubelet (a server regraded to etcd-only has none, and its side is read from its peers), on the node's own network, through a node affinity on those two labels, so a node that is neither (a Unraid host, an image-flashed agent) has no pod and nothing is asked of its kernel. A pod stopped on a node that has lost both labels, or is gone, removes the node's entry as it ends, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the control plane and not with the fleet. Every `github_runner_arc_role_controller_sample_seconds` (15) it appends one sample to its node's key of the `fleet-role-probe` ConfigMap, which keeps the last window of them so that a replaced pod loses nothing. A sample holds the TCP connect time to every voter's etcd peer port (a connect that does not finish within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what its etcd metrics (the `127.0.0.1:2381` listener) gained since the last sample: the WAL fsync and backend commit histograms, the round trip to each peer, leader changes, and whether it has a leader, is a learner and knows as many members as there are voters (a learner or a member that has not started shows as an extra known member); on a node that is not a voter, an fdatasync reading from fio with the write size etcd's own disk check uses, once per window; the memory pressure (`/proc/pressure/memory` of the pod's own procfs, which serves the whole node's, so nothing is mounted) and `MemAvailable` the regrade core reads, with the core's own functions, where a node on a kernel without pressure stall information (built or booted without it, which has no `/proc/pressure` or, on older kernels, has the files and refuses to read them) gives `MemAvailable` and `MemTotal` and no pressure figure; and the medium of the node's link, `wired`, `wifi` or `unknown`: the value of the node's `github-runner.exadev.com/link` label (`github_runner_arc_role_controller_link_label`), which the cluster role reads on the host itself (see the cluster role's README) since a node in a container or a VM sees only its own virtual interface, and otherwise the medium of the interface the pod's default route leaves by (an interface that is virtual, such as the k3s container's veth, hides the link beneath it), each sample saying which (`link_source`, `label` or `interface`). The pod's own reading is taken from the interface's `/sys/class/net` entry and never by walking `/sys`, whose symlinks loop. `scripts/fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of these readings alone (the link as the pod's own interface gives it), with no cluster access, to compare with a reading taken by hand on the host. The controller reads the Nodes and the probes' samples every sample period and publishes `fleet-role-status`, which the heartbeat republishes in its gist beside the autoscaler's status. For every voter and every eligible host it gives a verdict, `fit`, `unfit` or `unknown`, with each reason and the numbers behind it. Every limit derives from etcd's guidance and the cluster's own timing, not chosen numbers: - A voter is unfit when its tail (99th percentile) round trip to any other voter exceeds a tenth of the election timeout, etcd's rule that the election timeout is at least ten round trips. A standby is fit only at or below a twentieth of it, the 0.5 end of etcd's heartbeat band; the gap between the two is the hysteresis. The election timeout is `github_runner_cluster_etcd_election_timeout`, or k3s's 1000 ms when that is empty. A pair's figure is the worse of the TCP connect tail and etcd's own round trip tail. - A voter's WAL fsync tail must be under 10 ms and its backend commit tail under 25 ms (etcd's FAQ); a standby's fdatasync tail under 10 ms. -- Memory is never over the regrade limit (`github_runner_cluster_regrade_pressure_threshold` and `github_runner_cluster_regrade_available_floor_percent`) in any sample of the window, and `MemAvailable` is at least etcd's storage quota (what the voters report, 2 GiB by default). +- Memory is never over the regrade limit (`github_runner_cluster_regrade_pressure_threshold` and `github_runner_cluster_regrade_available_floor_percent`) in any sample of the window, and `MemAvailable` is at least etcd's storage quota (what the voters report, 2 GiB by default). A host none of whose samples has a pressure reading cannot be over the regrade limit, which needs both, so it is judged by `MemAvailable` against the quota alone, and its status says so: `unknown_signals` lists `memory pressure` (and the log adds `memory pressure unknown` to its verdict), beside `figures.psi_avg300_max`, the worst pressure of the window, which is then `null`. - Stability: no `NotReady` spell in the window and no k3s restart, seen on a voter as etcd's counters going backwards. On a node that is not a voter the only sign of a restart is its probe starting again, so a new probe pod counts as one. - A window is 100 samples, the fewest that give the 99th percentile a sample of its own, so one outlier never decides; a host with fewer is `unknown`, as is one whose probe has not reported for three sample periods. Readings from before the last change of the voter set are not counted. - Hosts that pass are ranked wired before Wi-Fi before unknown, then by round trip, then by disk. The link ranks hosts and is never a gate; the status gives each host's link and where it was read (`link` and `link_source`), and the log names it beside the verdict. diff --git a/scripts/fleet-role-probe.sh b/scripts/fleet-role-probe.sh index f1ffc96f..67bf6142 100755 --- a/scripts/fleet-role-probe.sh +++ b/scripts/fleet-role-probe.sh @@ -21,7 +21,7 @@ # - ETCD_PEER_PORT: etcd's peer port (default 2380). # - FSYNC_DIR: a directory on the filesystem k3s keeps its data on, which the fdatasync probe writes in (default /var/lib/rancher/k3s/fleet-role-probe). # - NODE_LINK: the link reader the cluster role runs on the host too (roles/github_runner_cluster/files/node-link.sh), whose default_route_interface and link_type read this pod's own (default /app/node-link.sh). SYS_CLASS_NET and PROC_NET_ROUTE, which it reads, are where the host's /sys/class/net and /proc/net/route are (the defaults). -# - FLEET_REGRADE_CORE: the regrade core (roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh), whose own psi_avg300 and meminfo_kb read the memory signals (default /app/fleet-regrade-core.sh). PSI_FILE and MEMINFO_FILE, which it reads, default here to /proc/pressure/memory and /proc/meminfo, which a pod's own procfs serves for the whole node as it does /proc/meminfo, so nothing is mounted for them. A kernel without pressure stall information has no /proc/pressure, and the sample then has no pressure reading and the memory figures from /proc/meminfo alone, which the controller counts as not over the limit. +# - FLEET_REGRADE_CORE: the regrade core (roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh), whose own psi_avg300 and meminfo_kb read the memory signals (default /app/fleet-regrade-core.sh). PSI_FILE and MEMINFO_FILE, which it reads, default here to /proc/pressure/memory and /proc/meminfo, which a pod's own procfs serves for the whole node as it does /proc/meminfo, so nothing is mounted for them. A kernel without pressure stall information (built or booted without it) has no /proc/pressure or, on older kernels, has the files and refuses to read them (EOPNOTSUPP); the sample then has no pressure reading and the memory figures from /proc/meminfo alone, which the controller judges by MemAvailable alone and reports with the pressure unknown. set -euo pipefail PROBE_CONFIGMAP="${PROBE_CONFIGMAP:-fleet-role-probe}" @@ -53,6 +53,12 @@ NODE_LINK_SYSTEM=Linux log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" >&2; } +# The memory pressure the regrade core reads, or nothing on a kernel without pressure stall information: one built or booted without it has no PSI_FILE or, on older kernels, has it and refuses to read it (EOPNOTSUPP), so a read that fails gives the same answer as a file that is not there. +pressure_reading() { + [ -e "$PSI_FILE" ] || return 0 + psi_avg300 2> /dev/null || true +} + # The tail percentile and the samples a window holds are defined once, in the module the controller reads them from too. read -r tail_percentile window_samples < <(jq -nr -L "$MODULES" 'include "fleet-role-stats"; "\(tail_percentile) \(window_samples)"') window_seconds=$((REPORT_SECONDS * window_samples)) @@ -172,10 +178,7 @@ report() { fsync_at="$now" fi - # A kernel without pressure stall information has no such file, and then the sample has no pressure reading. - if [ -r "$PSI_FILE" ]; then - psi="$(psi_avg300)" - fi + psi="$(pressure_reading)" # The entry goes in on standard input, never as an argument: it holds a window of samples, and one argument is limited to a fraction of the argument space (128 KiB on Linux). new_entry="$(jq -c -L "$MODULES" --argjson now "$now" --argjson tcp "$tcp" --argjson etcd "$etcd" \ @@ -205,8 +208,7 @@ if [ "${1:-}" = check ]; then connect) connect_ms "${3:?usage: check connect
}" ;; memory) # No pressure reading on a kernel without pressure stall information, as in a sample. - psi='' - if [ -r "$PSI_FILE" ]; then psi="$(psi_avg300)"; fi + psi="$(pressure_reading)" jq -nc --arg psi "$psi" --arg avail "$(meminfo_kb MemAvailable)" --arg total "$(meminfo_kb MemTotal)" '{psi_avg300: (if $psi == "" then null else $psi | tonumber end), mem_available_kb: ($avail | tonumber), mem_total_kb: ($total | tonumber)}' ;; etcd) curl --silent --show-error --fail --max-time "$REPORT_SECONDS" "$ETCD_METRICS_URL" | jq -Rsc -L "$MODULES" 'include "fleet-role-stats"; etcd_reading | {id, has_leader, is_learner, known_members, active_peers, quota_bytes, leader_changes, wal_fsync_p99_ms: (.wal | per_bucket | histogram_quantile(tail_percentile) * 1000), backend_commit_p99_ms: (.commit | per_bucket | histogram_quantile(tail_percentile) * 1000)}' ;; diff --git a/scripts/fleet-role.jq b/scripts/fleet-role.jq index 6f3a3d0c..58f7e60e 100644 --- a/scripts/fleet-role.jq +++ b/scripts/fleet-role.jq @@ -95,6 +95,7 @@ $config as $c | ([$etcd[].commit] | sum_buckets | histogram_quantile(tail_percentile)) as $commit | ([$w[] | .fsync_ms | select(. != null)] | max) as $fsync | ($w | map(select(.psi != null and over_regrade_limit($c))) | length) as $over + | ($w | map(.psi | select(. != null))) as $pressures | ($w | map(.avail_kb | select(. != null)) | min) as $min_avail_kb | ($w | map(select(.restarted == true)) | length) as $restarts | ($node.ready | not) as $not_ready @@ -115,6 +116,8 @@ $config as $c commit_p99_ms: (if $commit == null then null else $commit * 1000 end), fsync_p99_ms: $fsync, over_limit_samples: $over, + pressure_max: ($pressures | max), + pressure_samples: ($pressures | length), min_available_bytes: (if $min_avail_kb == null then null else $min_avail_kb * kib end), restarts: $restarts, not_ready: $not_ready, @@ -149,6 +152,9 @@ $config as $c def faults($m): rtt_faults($m) + own_faults($m); + # The signals no sample of the window could read but that do not withhold a verdict, by name: memory pressure, on a kernel without pressure stall information, where the regrade limit (which needs both pressure and a low MemAvailable) cannot be reached and MemAvailable is judged against the storage quota alone. + def unknown_signals($m): if $m.samples > 0 and $m.pressure_samples == 0 then ["memory pressure"] else [] end; + # What is missing for a verdict, as sentences; empty when the readings are all there. def gaps($m): $m.node as $node @@ -186,8 +192,10 @@ $config as $c link: $m.link, link_source: $m.link_source, samples: $m.samples, + unknown_signals: unknown_signals($m), figures: {rtt_p99_ms: ($m.rtt_ms | round2), rtt_peer: $m.rtt_peer, wal_fsync_p99_ms: ($m.wal_p99_ms | round2), backend_commit_p99_ms: ($m.commit_p99_ms | round2), fdatasync_p99_ms: ($m.fsync_p99_ms | round2), + psi_avg300_max: ($m.pressure_max | round2), min_available_mib: (if $m.min_available_bytes == null then null else ($m.min_available_bytes / 1048576 | round) end), leader_changes: $m.leader_changes, restarts: $m.restarts}, rank: rank($m), diff --git a/scripts/fleet-role.sh b/scripts/fleet-role.sh index 6f8b64de..75bc1baa 100755 --- a/scripts/fleet-role.sh +++ b/scripts/fleet-role.sh @@ -80,7 +80,7 @@ kubectl patch configmap "$status_configmap" -n "$namespace" --type merge --patch # What each host's verdict rests on, then the decision, in the log. while IFS= read -r line; do log "$line"; done < <(jq -r ' .hosts | to_entries[] - | "\(.key) (\(.value.role)\(if .value.pinned then ", pinned" else "" end)\(if .value.eligible then "" else ", not eligible" end), link \(.value.link)): \(.value.verdict), \(.value.reasons | if length == 0 then "all readings within their limits" else join("; ") end)\(if (.value.attributed_to | length) > 0 then " (attributed to \(.value.attributed_to | join(" and ")))" else "" end)"' <<< "$result") + | "\(.key) (\(.value.role)\(if .value.pinned then ", pinned" else "" end)\(if .value.eligible then "" else ", not eligible" end), link \(.value.link)): \(.value.verdict), \(.value.reasons | if length == 0 then "all readings within their limits" else join("; ") end)\(if (.value.attributed_to | length) > 0 then " (attributed to \(.value.attributed_to | join(" and ")))" else "" end)\(.value.unknown_signals | map("; \(.) unknown") | join(""))"' <<< "$result") while IFS= read -r line; do log "$line"; done < <(jq -r ' if .decision.action == "swap" then "DRY RUN: would swap out \(.decision.swap.out) for \(.decision.swap.in): \(.decision.because | join("; "))" else "DRY RUN: would not act: \(.decision.because | join("; "))\(if .decision.wanted == null and (.blocks | length) > 0 then " (also: \(.blocks | join("; ")))" else "" end)" end' <<< "$result") diff --git a/tests/unit/test_fleet_role.py b/tests/unit/test_fleet_role.py index 7da78b35..ec1cea7b 100644 --- a/tests/unit/test_fleet_role.py +++ b/tests/unit/test_fleet_role.py @@ -65,7 +65,7 @@ class Host: wal: Series | list[dict[str, int]] = 4.0 commit: Series | list[dict[str, int]] = 8.0 fsync_ms: float | None = 3.0 - psi: float = 0.0 + psi: float | None = 0.0 avail_kb: int = AVAILABLE_KB total_kb: int = TOTAL_KB samples: int = WINDOW_SAMPLES @@ -282,6 +282,21 @@ def test_memory_over_the_regrade_limit_needs_both_pressure_and_low_available(sel status = self.pass_(healthy(a={"psi": psi, "avail_kb": avail, "total_kb": total, **quota}, b=quota, c=quota)) self.assertEqual(status["hosts"]["a"]["verdict"], expected) + def test_a_host_without_pressure_readings_is_judged_by_available_memory_and_says_its_pressure_is_unknown(self) -> None: + # A kernel without pressure stall information: every sample has MemAvailable and no pressure. + status = self.pass_(healthy(a={"psi": None, "avail_kb": TOTAL_KB * AVAILABLE_FLOOR_PERCENT // 100 - 1})) + host = status["hosts"]["a"] + self.assertEqual((host["verdict"], host["unknown_signals"], host["figures"]["psi_avg300_max"]), ("fit", ["memory pressure"], None)) + self.assertIn("a (voter, link wired): fit, all readings within their limits; memory pressure unknown", self.log) + self.assertEqual(status["decision"]["action"], "none") + status = self.pass_(healthy(a={"psi": None, "avail_kb": ETCD_QUOTA_BYTES // 1024 - 1, "total_kb": TOTAL_KB})) + self.assertEqual(status["hosts"]["a"]["verdict"], "unfit", "MemAvailable under etcd's quota still fails a host without pressure readings") + + def test_a_host_with_pressure_readings_has_no_unknown_signal_and_reports_its_worst(self) -> None: + status = self.pass_(healthy(a={"psi": 2.5})) + self.assertEqual((status["hosts"]["a"]["unknown_signals"], status["hosts"]["a"]["figures"]["psi_avg300_max"]), ([], 2.5)) + self.assertNotIn("unknown", self.log.split("a (voter, link wired)")[1].splitlines()[0]) + def test_available_memory_must_be_at_least_etcds_storage_quota(self) -> None: quota_kb = ETCD_QUOTA_BYTES // 1024 for avail, expected in ((quota_kb, "fit"), (quota_kb - 1, "unfit")): diff --git a/tests/unit/test_fleet_role_probe.py b/tests/unit/test_fleet_role_probe.py index 44369d23..266d3018 100644 --- a/tests/unit/test_fleet_role_probe.py +++ b/tests/unit/test_fleet_role_probe.py @@ -652,6 +652,16 @@ def test_a_node_without_pressure_stall_information_still_records_memory_and_publ self.assertEqual((latest["psi"], latest["avail_kb"], latest["total_kb"]), (None, 8192000, 16384000)) + @unittest.skipUnless(os.access("/proc/self/mem", os.R_OK), "needs Linux's /proc/self/mem") + def test_a_pressure_file_the_kernel_refuses_to_read_gives_a_sample_without_a_pressure_reading(self) -> None: + # An older kernel booted with psi=0 keeps /proc/pressure/memory and refuses to read it (EOPNOTSUPP). /proc/self/mem stands in for it: it exists and is readable by its mode, and a read at its start fails (EIO). + completed = self.probe(PSI_FILE="/proc/self/mem") + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(completed.stderr, "") + latest = self.newest() + self.assertEqual((latest["psi"], latest["avail_kb"], latest["total_kb"]), (None, 8192000, 16384000)) + + class RingTest(ProbeCase): def setUp(self) -> None: super().setUp() @@ -908,6 +918,13 @@ def test_memory_without_pressure_stall_information_prints_the_memory_figures_and self.assertEqual(completed.returncode, 0, completed.stderr) self.assertEqual(json.loads(completed.stdout), {"psi_avg300": None, "mem_available_kb": 4096000, "mem_total_kb": 32768000}) + @unittest.skipUnless(os.access("/proc/self/mem", os.R_OK), "needs Linux's /proc/self/mem") + def test_memory_with_a_pressure_file_the_kernel_refuses_to_read_prints_no_pressure(self) -> None: + # As in MemoryTest: /proc/self/mem stands in for the pressure file of an older kernel booted with psi=0. + completed = self.check("memory", PSI_FILE="/proc/self/mem") + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(json.loads(completed.stdout), {"psi_avg300": None, "mem_available_kb": 8192000, "mem_total_kb": 16384000}) + def test_connect_prints_the_milliseconds_to_a_listener(self) -> None: with listener() as port: completed = self.check("connect", "127.0.0.1", ETCD_PEER_PORT=str(port))