diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72ffa850..9a9eb2ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,12 +157,12 @@ jobs: shard: [0, 1, 2, 3, 4] steps: - uses: actions/checkout@v7 - - name: Install Ansible and zsh - # zsh is one of the two shells the host tools role writes startup lines for, and its tests read those lines back with a real zsh. + - name: Install Ansible, zsh and fio + # zsh is one of the two shells the host tools role writes startup lines for, and its tests read those lines back with a real zsh. fio is the disk check the role controller's probe takes its fdatasync reading with, which its tests run for real. run: | set -euo pipefail sudo apt-get update -qq - sudo apt-get install -y -qq ansible pkg-config zsh + sudo apt-get install -y -qq ansible pkg-config zsh fio - name: Install mise # The host tools tests read the mise configuration the role renders back with a real mise, to check that every value reaches the variable as given. A release more than a week old, pinned to its published checksum so the download is the file that was tested; the version is the one the tests were written against, raised by hand with its checksum from the release's SHASUMS256.txt. env: @@ -176,6 +176,20 @@ jobs: echo "${MISE_SHA256} ${directory}/mise.tar.gz" | sha256sum --check --strict tar -xzf "${directory}/mise.tar.gz" -C "${directory}" echo "${directory}/mise/bin" >> "$GITHUB_PATH" + - name: Install etcd for the probe's live test + # The probe's tests start a real three-member etcd to read its metrics from. Ubuntu's own etcd-server is 3.4, which has none of the 3.6 metrics the probe reads, so the official release is used: the exact version, and the SHA-256 of its linux-amd64 tarball as the release publishes it in its SHA256SUMS asset (https://github.com/etcd-io/etcd/releases/download/v3.6.15/SHA256SUMS), checked before anything is extracted. A mismatch fails the job. + env: + ETCD_VERSION: v3.6.15 + ETCD_LINUX_AMD64_SHA256: b51d86e6bff2168de5ba8cd631c0fe6c785c7a1b32b083079f5677d6b7e2772c + run: | + set -euo pipefail + [ "$(uname -m)" = x86_64 ] || { echo "the pinned checksum is for linux-amd64, not $(uname -m)" >&2; exit 1; } + dir="${RUNNER_TEMP}/etcd" + mkdir -p "$dir" + curl -fsSL -o "${dir}/etcd.tar.gz" "https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz" + echo "${ETCD_LINUX_AMD64_SHA256} ${dir}/etcd.tar.gz" | sha256sum --check --strict + tar -xzf "${dir}/etcd.tar.gz" -C "$dir" --strip-components=1 "etcd-${ETCD_VERSION}-linux-amd64/etcd" "etcd-${ETCD_VERSION}-linux-amd64/etcdctl" + echo "ETCD_BINARY=${dir}/etcd" >> "$GITHUB_ENV" - name: Unit-test the collection # Ubuntu's ansible package puts ansible-core on the system Python the filter plugins import from. The runner image can carry another Ansible earlier on PATH, so the tests run with /usr/bin first and ANSIBLE_PLAYBOOK pointing at Ubuntu's, which makes every playbook a test runs use the same ansible-core as the filters it imports; both versions are printed for the record. A module is a shard's when its position in the list of modules, largest first, leaves this shard's index modulo the number of shards. env: @@ -271,9 +285,30 @@ jobs: PR_TITLE: ${{ github.event.pull_request.title }} run: printf '%s\n' "$PR_TITLE" | npx --no-install commitlint --verbose + fleet-role-image: + name: Role controller image build + needs: verified + if: ${{ !cancelled() && needs.verified.outputs.skip != 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: docker/setup-buildx-action@v3 + # Built and not pushed, from the same Dockerfile and context the release workflow builds it from, so a missing COPY source or a package apk cannot install fails the pull request and not the release. + - uses: docker/build-push-action@v6 + with: + context: . + file: fleet-role/Dockerfile + push: false + load: true + tags: fleet-role:ci + - name: Run what the pods run + # The tools and files the pods need, the jq modules under the image's own jq, the probe's check mode through the regrade core the image carries, and one pass of the controller against a kubectl that answers an empty cluster. Nothing reaches a cluster. + run: tests/fleet_role/image-smoke.sh fleet-role:ci + required-checks: name: Required Checks - needs: [shellcheck, actionlint, yamllint, hadolint, ansible-syntax-check, unit-tests, ansible-lint, collection-build, commitlint] + needs: [shellcheck, actionlint, yamllint, hadolint, ansible-syntax-check, unit-tests, ansible-lint, collection-build, commitlint, fleet-role-image] # Always runs, even if a dependency failed or was skipped, so it can report that failure itself rather than being skipped - a required check that's silently absent blocks a PR indefinitely instead of failing it visibly. if: always() runs-on: ubuntu-latest diff --git a/.github/workflows/mesh-integration.yml b/.github/workflows/mesh-integration.yml index d0525494..9da80d1f 100644 --- a/.github/workflows/mesh-integration.yml +++ b/.github/workflows/mesh-integration.yml @@ -1,6 +1,6 @@ name: Mesh integration -# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. Only on changes that can affect how nodes join the mesh. +# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. native_server joins a native host (a systemd container) as an agent beside three Docker servers, then promotes it to a server in place: it must join etcd as a learner, be promoted to a voter, and keep the cluster writable throughout. role_controller adds a standby agent, runs the role controller's probe and controller from the image built from this checkout, makes a voter slow with tc netem and pauses another, and checks the controller reports the voter unfit with the swap named, reports that it would not act with a voter paused, and changes nothing. Only on changes that can affect how nodes join the mesh. # # Runs on GitHub-hosted ubuntu-latest, whose Docker daemon runs the privileged containers the test needs (k3s, and Tailscale inside it). The work directory sits under runner.temp, which the job always has write access to. @@ -14,6 +14,10 @@ on: - playbooks/recover_in_cluster_mesh.yml - tests/mesh/** - tests/lib/** + - tests/fleet_role/** + - fleet-role/** + - scripts/fleet-role* + - roles/github_runner_arc/templates/fleet-role.yaml.j2 - .github/workflows/mesh-integration.yml workflow_dispatch: @@ -39,6 +43,7 @@ jobs: - {scenario: in_cluster, compose: v5.5.1} - {scenario: regrade, compose: v5.5.1} - {scenario: native_server, compose: v5.5.1} + - {scenario: role_controller, compose: v5.5.1} - {scenario: hosted, compose: v2.39.0} - {scenario: compose_faulty, compose: v2.38.2} - {scenario: in_cluster, compose: v5.5.1, pin: true} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b869aa77..5527cfc1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,6 @@ name: Release -# Runs semantic-release against every push to main. Almost every conventional-commit type triggers at least a patch release (see .releaserc.json's commit-analyzer releaseRules), so in practice this fires on nearly every push, stamping the collection version into galaxy.yml and the arc role's heartbeat, autoscaler, pull-secret-renewer and node-recovery image defaults, building the collection tarball, and, if a release was actually published, building and pushing all five container images (runner, heartbeat, autoscaler, pull-secret-renewer, node-recovery) tagged with the exact new version alongside floating major and latest tags on the same manifest. +# Runs semantic-release against every push to main. Almost every conventional-commit type triggers at least a patch release (see .releaserc.json's commit-analyzer releaseRules), so in practice this fires on nearly every push, stamping the collection version into galaxy.yml and the arc role's heartbeat, autoscaler, pull-secret-renewer, node-recovery and role controller image defaults, building the collection tarball, and, if a release was actually published, building and pushing every container image (runner, heartbeat, autoscaler, pull-secret-renewer, node-recovery, fleet-role) tagged with the exact new version alongside floating major and latest tags on the same manifest. # # The release commit and tag are pushed over SSH with a write-enabled deploy key (secret RELEASE_DEPLOY_KEY): actions/checkout loads it for every later git command, and .releaserc.json's repositoryUrl is the SSH remote, so @semantic-release/git pushes with it. A deploy key is scoped to this one repository and can be listed as a ruleset bypass actor. It can't call the GitHub API, so the semantic-release process uses the ordinary secrets.GITHUB_TOKEN for creating the Release and uploading the collection tarball, which need no bypass. # @@ -79,6 +79,9 @@ jobs: - name: node-recovery dockerfile: node-recovery/Dockerfile image: ghcr.io/exadev/github-runner-node-recovery + - name: fleet-role + dockerfile: fleet-role/Dockerfile + image: ghcr.io/exadev/github-runner-fleet-role steps: - uses: actions/checkout@v7 with: diff --git a/README.md b/README.md index 4a5fdbe4..1ce15f23 100644 --- a/README.md +++ b/README.md @@ -57,8 +57,8 @@ Each Docker-in-Docker runner pod starts with an empty Docker daemon, so every jo ## Build, test, and smoke-test -- **Build the runner image locally (the only working path right now):** `docker buildx build --platform linux/arm64,linux/amd64 --push -t ghcr.io/exadev/github-runner:latest .` from a machine with buildx/QEMU cross-platform support (Docker Desktop has this built in). A plain `docker build` with no `--platform` only produces an image for the building machine's own architecture, which silently breaks scheduling on the fleet's other architecture. The fleet-health platform's own two images need the identical treatment, for the identical reason (they're genuinely unpinned pods now too - see [Heartbeat](#heartbeat)/[Autoscaler](#autoscaler)): `docker buildx build --platform linux/arm64,linux/amd64 --push -f heartbeat/Dockerfile -t ghcr.io/exadev/github-runner-heartbeat:latest .` and the same with `autoscaler/Dockerfile` → `ghcr.io/exadev/github-runner-autoscaler:latest`, `pull-secret-renewer/Dockerfile` → `ghcr.io/exadev/github-runner-pull-secret-renewer:latest` (the image of the ARC role's App-sourced pull Secret renewal), and `node-recovery/Dockerfile` → `ghcr.io/exadev/github-runner-node-recovery:latest` (see [Node recovery](#node-recovery)). -- **Build and push via CI:** `.github/workflows/release.yml`'s `build-images` job builds and pushes all five images (this one, `heartbeat`, `autoscaler`, `pull-secret-renewer` and `node-recovery`) multi-arch on GitHub-hosted `ubuntu-latest` whenever a release is published - see [Releases](#releases) below. +- **Build the runner image locally (the only working path right now):** `docker buildx build --platform linux/arm64,linux/amd64 --push -t ghcr.io/exadev/github-runner:latest .` from a machine with buildx/QEMU cross-platform support (Docker Desktop has this built in). A plain `docker build` with no `--platform` only produces an image for the building machine's own architecture, which silently breaks scheduling on the fleet's other architecture. The fleet-health platform's own two images need the identical treatment, for the identical reason (they're genuinely unpinned pods now too - see [Heartbeat](#heartbeat)/[Autoscaler](#autoscaler)): `docker buildx build --platform linux/arm64,linux/amd64 --push -f heartbeat/Dockerfile -t ghcr.io/exadev/github-runner-heartbeat:latest .` and the same with `autoscaler/Dockerfile` → `ghcr.io/exadev/github-runner-autoscaler:latest`, `pull-secret-renewer/Dockerfile` → `ghcr.io/exadev/github-runner-pull-secret-renewer:latest` (the image of the ARC role's App-sourced pull Secret renewal), `node-recovery/Dockerfile` → `ghcr.io/exadev/github-runner-node-recovery:latest` (see [Node recovery](#node-recovery)), and `fleet-role/Dockerfile` → `ghcr.io/exadev/github-runner-fleet-role:latest` (see [Role controller](#role-controller)). +- **Build and push via CI:** `.github/workflows/release.yml`'s `build-images` job builds and pushes every image (this one, `heartbeat`, `autoscaler`, `pull-secret-renewer`, `node-recovery` and `fleet-role`) multi-arch on GitHub-hosted `ubuntu-latest` whenever a release is published - see [Releases](#releases) below. - **Smoke-test the ARC scale set** (confirms a job gets a real ephemeral pod): `gh workflow run test-arc-runner.yml`, then watch `kubectl get pods -n arc-runners- -w` (`arc-runners---` for a repository entry). A pod must appear only once the job is queued, run to completion, and be deleted within seconds. - **Smoke-test `ubuntu-latest` routing** (confirms `runner-fallback-action` still reaches GitHub-hosted runners when needed): `gh workflow run test-ubuntu-latest.yml`. - **Generate load for the autoscaler** (see [Autoscaler](#autoscaler) below): `gh workflow run test-autoscaler.yml`. @@ -85,7 +85,7 @@ A release: 1. Stamps the new version into `galaxy.yml`'s `version:` field (which stays `0.0.0` in source between releases) and into the published-image tag defaults in `roles/github_runner_arc/defaults/main.yml` (`github_runner_arc_heartbeat_image`, `github_runner_arc_autoscaler_image`, `github_runner_arc_image_pull_secret_renewer_image`, `github_runner_arc_node_recovery_image`), and into each role's `*_collection_version` default, which the deployed-state stamps record - `scripts/release/stamp_version.py` does this by matching each variable's own name, not a line number, so it stays correct even while those files are edited concurrently by unrelated work. 2. Builds the collection tarball (`ansible-galaxy collection build --force`) and attaches it to the GitHub Release. 3. Updates `CHANGELOG.md` and commits it, `galaxy.yml`, and the stamped defaults file back to `main` as `chore(release): [skip ci]`. -4. Builds and pushes all five container images (`Dockerfile`, `heartbeat/Dockerfile`, `autoscaler/Dockerfile`, `pull-secret-renewer/Dockerfile`, `node-recovery/Dockerfile`) multi-arch (`linux/arm64,linux/amd64`) to their `ghcr.io/exadev/github-runner*` repositories, each tagged with the exact version, the floating major version, and `latest` - all three tags on the same manifest, via `docker/metadata-action`'s `type=semver` patterns. To pin a host to a specific release rather than floating on `latest`, set `roles/github_runner_arc/defaults/main.yml`'s image variables (or a host's own override) to an exact version tag instead. +4. Builds and pushes every container image (`Dockerfile`, `heartbeat/Dockerfile`, `autoscaler/Dockerfile`, `pull-secret-renewer/Dockerfile`, `node-recovery/Dockerfile`, `fleet-role/Dockerfile`) multi-arch (`linux/arm64,linux/amd64`) to their `ghcr.io/exadev/github-runner*` repositories, each tagged with the exact version, the floating major version, and `latest` - all three tags on the same manifest, via `docker/metadata-action`'s `type=semver` patterns. To pin a host to a specific release rather than floating on `latest`, set `roles/github_runner_arc/defaults/main.yml`'s image variables (or a host's own override) to an exact version tag instead. 5. Optionally publishes the collection to Ansible Galaxy, only if a `GALAXY_API_KEY` secret is present - the `[skip ci]` release commit and the image build both happen unconditionally, in this same run, regardless of whether Galaxy publishing is configured. The `[skip ci]` in the release commit message is deliberate: the image build already happens in this same run using the exact version semantic-release just decided, so there is nothing useful for a second, separately-triggered run of this workflow (or of `ci.yml`) to do against a commit that only changed a version stamp and a changelog - `ci.yml`'s own lint jobs already ran against every commit before it reached `main`. @@ -193,6 +193,25 @@ Pruning is opt-in (`github_runner_arc_node_recovery_prune_after_seconds`, empty Its image, `github_runner_arc_node_recovery_image`, is pulled without a pull Secret, so the published package must be public. +### Role controller + +`fleet-role-probe` and `fleet-role` (an optional DaemonSet and Deployment in the platform namespace, see `roles/github_runner_arc/templates/fleet-role.yaml.j2`, `scripts/fleet-role-probe.sh` and `scripts/fleet-role.sh`; `github_runner_cluster_role_controller`, off by default, which `github_runner_arc_role_controller_enabled` takes as its default) measure which hosts are fit to hold an etcd vote and report the swap of an unfit voter for a fit standby that the controller would make. They only report: the pods' ClusterRoles can `get` and `list` nodes and nothing else, and the only objects they write are their own two ConfigMaps. The controller refuses to start with any setting other than `FLEET_ROLE_DRY_RUN=true`, and runs unchanged when no host is marked eligible, when it reports the voters' fitness alone. + +The probe runs on every Linux node that has a kubelet (a server regraded to etcd-only has none, and its side is read from its peers), on the node's own network, but records only on a voter or a host marked eligible (a node that is neither has no entry, and loses one it had), so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the control plane and not with the fleet. Every `github_runner_arc_role_controller_sample_seconds` (15) it appends one sample to its node's key of the `fleet-role-probe` ConfigMap, which keeps the last window of them so that a replaced pod loses nothing. A sample holds the TCP connect time to every voter's etcd peer port (a connect that does not finish within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what its etcd metrics (the `127.0.0.1:2381` listener) gained since the last sample: the WAL fsync and backend commit histograms, the round trip to each peer, leader changes, and whether it has a leader, is a learner and knows as many members as there are voters (a learner or a member that has not started shows as an extra known member); on a node that is not a voter, an fdatasync reading from fio with the write size etcd's own disk check uses, once per window; the memory pressure and `MemAvailable` the regrade core reads, with the core's own functions; and whether the interface the default route leaves by is `wired`, `wifi` or `unknown` (an interface that is virtual, such as the k3s container's veth, hides the link beneath it). The link is read from the interface's own `/sys/class/net` entry and never by walking `/sys`, whose symlinks loop. `scripts/fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of these readings alone, with no cluster access, to compare with a reading taken by hand on the host. + +The controller reads the Nodes and the probes' samples every sample period and publishes `fleet-role-status`, which the heartbeat republishes in its gist beside the autoscaler's status. For every voter and every eligible host it gives a verdict, `fit`, `unfit` or `unknown`, with each reason and the numbers behind it. Every limit derives from etcd's guidance and the cluster's own timing, not chosen numbers: + +- A voter is unfit when its tail (99th percentile) round trip to any other voter exceeds a tenth of the election timeout, etcd's rule that the election timeout is at least ten round trips. A standby is fit only at or below a twentieth of it, the 0.5 end of etcd's heartbeat band; the gap between the two is the hysteresis. The election timeout is `github_runner_cluster_etcd_election_timeout`, or k3s's 1000 ms when that is empty. A pair's figure is the worse of the TCP connect tail and etcd's own round trip tail. +- A voter's WAL fsync tail must be under 10 ms and its backend commit tail under 25 ms (etcd's FAQ); a standby's fdatasync tail under 10 ms. +- Memory is never over the regrade limit (`github_runner_cluster_regrade_pressure_threshold` and `github_runner_cluster_regrade_available_floor_percent`) in any sample of the window, and `MemAvailable` is at least etcd's storage quota (what the voters report, 2 GiB by default). +- Stability: no `NotReady` spell in the window and no k3s restart, seen on a voter as etcd's counters going backwards. On a node that is not a voter the only sign of a restart is its probe starting again, so a new probe pod counts as one. +- A window is 100 samples, the fewest that give the 99th percentile a sample of its own, so one outlier never decides; a host with fewer is `unknown`, as is one whose probe has not reported for three sample periods. Readings from before the last change of the voter set are not counted. +- Hosts that pass are ranked wired before Wi-Fi before unknown, then by round trip, then by disk. The link ranks hosts and is never a gate. + +A slow link makes both of its ends unfit, since a voter is unfit when its round trip to any other voter is, so the voter to replace is the one at fault: the unfit voter on the most slow links (a link is slow when either end measures it above the limit), the worst-ranked among those on as many. A swap is reported only when that voter is eligible and not pinned (when it is not, replacing a voter that shares a slow link with it would fix nothing, so nothing is swapped) and a fit eligible standby ranks above it. A candidate is judged against the voters that would remain, since a slow link to the very voter it would replace says nothing against it. It is held back, with the reason in the status and the log, while the voter count is not `github_runner_arc_role_controller_voters`, any voter is below `full`, is not Ready, has no leader, is a learner or knows a member that is not a voter, is not connected to every other voter, fewer hosts are eligible than the target, or the cooldown after a change of the voter set has not passed (k3s's five minutes to remove a stalled learner plus one window). The target must be odd, at least 3, and at most `github_runner_cluster_max_servers` minus one, which leaves a swap one place of headroom; the role refuses anything else. A host is eligible, and a voter pinned, by a Node label with the value `true` (`github_runner_arc_role_controller_eligible_label` and `_pinned_label`), which the cluster role publishes from `github_runner_cluster_voter_eligible` and `github_runner_cluster_voter_pinned` per host (and the target from `github_runner_cluster_voters`, refused unless odd, at least 3 and below `github_runner_cluster_max_servers`; see the cluster role's README); a voter not marked eligible is never swapped out, since nothing has prepared it to come back. + +The controller and the probe run one image, `github_runner_arc_role_controller_image`, pulled without a pull Secret, so the published package must be public. + ## Conventions Adding a second org, or a repository, is additive, never a change to anything existing: diff --git a/fleet-role/Dockerfile b/fleet-role/Dockerfile new file mode 100644 index 00000000..02b0e8cc --- /dev/null +++ b/fleet-role/Dockerfile @@ -0,0 +1,16 @@ +# Image for the role controller (see roles/github_runner_arc/templates/fleet-role.yaml.j2): the `fleet-role` Deployment runs scripts/fleet-role.sh through loop.sh, and the `fleet-role-probe` DaemonSet runs scripts/fleet-role-probe.sh from the same image. bash, curl, jq, kubectl and fio, all from Alpine's apk (fio is the disk check etcd's own documentation uses), so it is small and builds for either architecture. Build context is the repository root so it can COPY the scripts and the regrade core, whose own functions the probe reads memory with. The Deployment and the DaemonSet pull it without a pull Secret, so the published package must be public. +FROM alpine:3.20 + +RUN apk add --no-cache bash curl jq kubectl fio + +COPY fleet-role/loop.sh /app/loop.sh +COPY scripts/fleet-role.sh scripts/fleet-role-probe.sh /app/scripts/ +# The jq programs the scripts run from their own directory, and the module they share. +COPY scripts/fleet-role.jq scripts/fleet-role-stats.jq /app/scripts/ +COPY roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh /app/fleet-regrade-core.sh +RUN chmod +x /app/loop.sh /app/scripts/fleet-role.sh /app/scripts/fleet-role-probe.sh + +# A non-root user by number, so the Deployment's runAsNonRoot check can verify it without reading /etc/passwd. The DaemonSet runs the same image as root, for the directory it measures the disk in. +USER 65532:65532 +WORKDIR /app +ENTRYPOINT ["/app/loop.sh"] diff --git a/fleet-role/loop.sh b/fleet-role/loop.sh new file mode 100755 index 00000000..eda9f9d5 --- /dev/null +++ b/fleet-role/loop.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Container entrypoint for the role controller Deployment (see roles/github_runner_arc/templates/fleet-role.yaml.j2). Runs scripts/fleet-role.sh every FLEET_ROLE_SAMPLE_SECONDS forever, the period the probes sample at; a failed pass is logged and retried on the next one. kubectl finds the in-cluster config from the pod's mounted ServiceAccount token. +set -euo pipefail + +FLEET_ROLE_SAMPLE_SECONDS="${FLEET_ROLE_SAMPLE_SECONDS:?FLEET_ROLE_SAMPLE_SECONDS must be set}" + +while true; do + /app/scripts/fleet-role.sh || true + sleep "$FLEET_ROLE_SAMPLE_SECONDS" +done diff --git a/galaxy.yml b/galaxy.yml index 92fd9f26..a39c976a 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -40,6 +40,7 @@ build_ignore: - heartbeat - pull-secret-renewer - node-recovery + - fleet-role - tool-images - scripts - Dockerfile diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 700a358b..8a3c59bb 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -56,6 +56,7 @@ Helm 4 and server-side apply. The role reads `helm version --short` and, on Helm - `github_runner_arc_heartbeat_gist_id`: install the fleet-health platform from this host, refreshing this gist. `github_runner_arc_heartbeat_bootstrap_gist`, `github_runner_arc_heartbeat_gist_description` and `github_runner_arc_heartbeat_gist_consumer` control the bootstrap described above. - `github_runner_arc_autoscaler_usable_budget_gi`, `github_runner_arc_autoscaler_floor`: required when a profile sets `autoscale: true` and the platform is installed. The floor must equal the autoscaled profile's `maxRunners`, which every Helm upgrade reverts to. The autoscaler works its ceiling out from the nodes on every poll (see the project README's Autoscaler section); `github_runner_arc_autoscaler_max_ceiling`, optional, is an upper bound on it and must be no lower than the floor. The other `github_runner_arc_autoscaler_*` and `github_runner_arc_heartbeat_*` settings have defaults; see `defaults/main.yml`. - `github_runner_arc_node_recovery_enabled` (default `true`), `github_runner_arc_node_recovery_after_seconds` (default 120), `github_runner_arc_node_recovery_poll_seconds` (default 15), `github_runner_arc_node_recovery_prune_after_seconds` (default empty, which leaves pruning off), `github_runner_arc_node_recovery_prune_label` (default `github-runner.exadev.io/removable`), `github_runner_arc_node_recovery_image`: the node recovery watcher (see [Node recovery](#node-recovery)). +- `github_runner_arc_role_controller_enabled` (default the cluster role's `github_runner_cluster_role_controller`, itself `false`), `github_runner_arc_role_controller_image`, `github_runner_arc_role_controller_sample_seconds` (default 15), `github_runner_arc_role_controller_voters` (default 3), `github_runner_arc_role_controller_max_servers`, `github_runner_arc_role_controller_election_timeout_ms`, `github_runner_arc_role_controller_pressure_threshold`, `github_runner_arc_role_controller_available_floor_percent`, `github_runner_arc_role_controller_eligible_label`, `github_runner_arc_role_controller_pinned_label`, `github_runner_arc_role_controller_level_label`, `github_runner_arc_role_controller_probe_configmap`, `github_runner_arc_role_controller_status_configmap`, `github_runner_arc_role_controller_data_dir`: the role controller, which reports which hosts are fit to hold an etcd vote (see [Role controller](#role-controller)). The five that mirror the cluster role (`max_servers`, `election_timeout_ms`, `pressure_threshold`, `available_floor_percent`, `level_label`) take the cluster role's own variable when it is defined. - `github_runner_arc_runner_reaper_enabled` (default `false`), `github_runner_arc_runner_reaper_max_job_seconds` (default 432000, five days), `github_runner_arc_runner_reaper_completed_grace_seconds` (default 300), `github_runner_arc_runner_reaper_schedule` (default every 5 minutes), `github_runner_arc_runner_reaper_dry_run` (default `false`), `github_runner_arc_runner_reaper_image` (default the renewal's image): the runner reaper (see [Runner reaper](#runner-reaper)). - `github_runner_arc_runner_active_deadline_seconds` (default unset): `activeDeadlineSeconds` on every runner pod (see [Runner reaper](#runner-reaper)). - `github_runner_arc_app_setup_*`, `github_runner_arc_app_manifest_code`, `github_runner_arc_app_private_key_path`: inputs to `playbooks/github_app_setup.yml` (see below), including `github_runner_arc_app_setup_write_secret`, `github_runner_arc_app_setup_secret_namespaces`, `github_runner_arc_app_setup_secret_name`, `github_runner_arc_app_setup_keep_private_key` and `github_runner_arc_app_setup_command`. @@ -478,6 +479,14 @@ Turning pruning on adds `delete` on nodes to the watcher's ClusterRole. RBAC can A pruned node returns only when its k3s agent process restarts. A running k3s agent does not register itself again when its Node object disappears: its kubelet registers at start-up and then logs `node "..." not found` for as long as it runs, however long the Node has been missing and whether or not the network came back. After a reboot or a power cut the agent starts, registers a new Node with its labels and becomes `Ready` without any step by hand. A node whose agent keeps running while it is unreachable (a network outage, a suspended machine) and is pruned in the meantime stays out of the cluster until its agent is restarted. Pruning is therefore safe only for nodes that restart their k3s agent when their own Node is gone, or that reboot; arranging that is the node's responsibility, not this role's. Do not label a node you cannot restart. +## Role controller + +With `github_runner_cluster_role_controller: true` (or `github_runner_arc_role_controller_enabled: true`, for a host that does not run the cluster role) the role installs `fleet-role-probe`, a DaemonSet on every Linux node that has a kubelet, and `fleet-role`, a Deployment, both in `github_runner_arc_platform_namespace` and both from `github_runner_arc_role_controller_image`. Together they report, for every etcd voter and every host marked eligible, whether it is fit to hold an etcd vote and why, and the swap of an unfit voter for a fit standby that the controller would make. They change nothing in the cluster: each has a ClusterRole that can `get` and `list` nodes and nothing else, and a Role that reaches only the two ConfigMaps it uses by name (`get` and `patch`, never `create`; the role creates both empty and keeps them). `false` removes the workloads, their RBAC and both ConfigMaps. The repository README's Role controller section explains each signal, how it is read, the thresholds and when a swap is held back. + +The probe runs with the node's network and as root with every capability dropped, because the directory it measures the disk in (`github_runner_arc_role_controller_data_dir`, on the filesystem k3s keeps its data on) is created by the kubelet and owned by root. It mounts the host's `/proc/pressure` read-only. The controller runs as an unprivileged user with a read-only root filesystem. The heartbeat, when it is installed, reads the controller's status ConfigMap too and republishes it in its gist as `fleet-role-status.json`. + +The check in `tasks/validate.yml` refuses a target (`github_runner_arc_role_controller_voters`) that is even, under 3 or without a place of headroom below `github_runner_arc_role_controller_max_servers`, a period, election timeout or maximum that is not a whole number of at least 1, and two labels that are not label keys or are the same. Nothing is checked while the controller is off. + ## Runner reaper A runner pod can outlive its job: a step hangs and the run is cancelled but the runner never exits, or the run ends while the pod keeps its slot. With `github_runner_arc_runner_reaper_enabled: true` each scale-set profile's namespace gets a `runner-reaper` CronJob that finds such runners and deletes their EphemeralRunner, which the scale set then replaces. It is off by default. diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index d770abd9..7b20b038 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -189,6 +189,32 @@ github_runner_arc_node_recovery_prune_label: github-runner.exadev.io/removable # Image the watcher runs. Pulled without a pull Secret, so it must be public. github_runner_arc_node_recovery_image: "ghcr.io/exadev/github-runner-node-recovery:2.25.0" +# The role controller (see tasks/install_fleet_role.yml, scripts/fleet-role.sh and scripts/fleet-role-probe.sh): a `fleet-role-probe` DaemonSet that measures, on every node with a kubelet, what decides whether a host is fit to hold an etcd vote, and a `fleet-role` Deployment that reads the measurements and reports which eligible hosts are fit and which swap of an unfit voter for a fit standby it would make. It reports and changes nothing: its only writes are its own two ConfigMaps. Off by default, so a consumer that upgrades changes nothing. +# The cluster role's github_runner_cluster_role_controller when it is defined, so one inventory setting turns it on; set this one to turn it on from a host that does not run the cluster role (playbooks/arc.yml against a kubeconfig). +github_runner_arc_role_controller_enabled: "{{ github_runner_cluster_role_controller | default(false) }}" +# Image the controller and the probe run, one image whose entrypoint is the controller. Pulled without a pull Secret, so it must be public. +github_runner_arc_role_controller_image: "ghcr.io/exadev/github-runner-fleet-role:2.22.0" +# Seconds between probe samples and between controller passes. A figure is read over a window of 100 samples (the fewest that give the 99th percentile a sample of its own), so this sets how long a window is: 1500 seconds at the default, node recovery's own poll. +github_runner_arc_role_controller_sample_seconds: 15 +# The number of etcd voters the cluster is held at: odd, at least 3, and at most github_runner_arc_role_controller_max_servers minus one, which leaves a swap (the new voter joins before the old one leaves) one place of headroom. +github_runner_arc_role_controller_voters: "{{ github_runner_cluster_voters | default(3) }}" +# The cluster role's github_runner_cluster_max_servers, which the controller must agree with. Read from the cluster role's variable when it is defined, so the two cannot differ; the fallback is that variable's default (tests/unit/test_fleet_role_arc.py holds the two together). +github_runner_arc_role_controller_max_servers: "{{ github_runner_cluster_max_servers | default(5) }}" +# The cluster's etcd election timeout in milliseconds, which sets the round trip a voter may have (a tenth of it) and a standby must (a twentieth). The cluster role's github_runner_cluster_etcd_election_timeout when that is set; k3s's own default, 1000, when it is empty. +github_runner_arc_role_controller_election_timeout_ms: "{{ github_runner_cluster_etcd_election_timeout | default('', true) | string or 1000 }}" +# The regrade core's memory limit, which a host over for any sample of a window is unfit for: the cluster role's github_runner_cluster_regrade_pressure_threshold and github_runner_cluster_regrade_available_floor_percent when defined, else their defaults (held together by the same test). +github_runner_arc_role_controller_pressure_threshold: "{{ github_runner_cluster_regrade_pressure_threshold | default(10) }}" +github_runner_arc_role_controller_available_floor_percent: "{{ github_runner_cluster_regrade_available_floor_percent | default(15) }}" +# The Node labels, with the value true, that mark a host the controller may promote or demote and one it must never demote, and the label regrading puts on a server below full. The cluster role publishes the first two from github_runner_cluster_voter_eligible and github_runner_cluster_voter_pinned, and these take its label names and its github_runner_cluster_voters when those are defined; with none marked eligible the controller only reports the voters. +github_runner_arc_role_controller_eligible_label: "{{ github_runner_cluster_voter_eligible_label | default('github-runner.exadev.com/voter-eligible') }}" +github_runner_arc_role_controller_pinned_label: "{{ github_runner_cluster_voter_pinned_label | default('github-runner.exadev.com/voter-pinned') }}" +github_runner_arc_role_controller_level_label: "{{ github_runner_cluster_regrade_level_label | default('github-runner.exadev.com/regrade-level') }}" +# The ConfigMaps in the platform namespace that the probes publish to (a key per node) and that the controller publishes its status to, which the heartbeat republishes beside the autoscaler's. +github_runner_arc_role_controller_probe_configmap: fleet-role-probe +github_runner_arc_role_controller_status_configmap: fleet-role-status +# Where the probe writes its fdatasync reading on a node that is not a voter: a directory on the filesystem k3s keeps its data on, which the pod mounts from the node, created by the kubelet and owned by root. +github_runner_arc_role_controller_data_dir: /var/lib/rancher/k3s/fleet-role-probe + # Runner reaper (see the role README's Runner reaper section, tasks/install_runner_reaper.yml and files/reap-runners.sh): a CronJob in each runner namespace that deletes an EphemeralRunner whose job's workflow run is completed or cancelled, whose own job is completed, or whose job has been running for longer than github_runner_arc_runner_reaper_max_job_seconds; the scale set replaces it. It reads the runs with an installation token from the org's GitHub App restricted to actions: read, so the App needs the Actions repository permission (read) on every repository whose jobs the runners take, approved on its installation. It never deletes a runner that has not been assigned a job, and deletes nothing when GitHub or the token is unavailable. false removes the CronJob, its RBAC and its ConfigMap. github_runner_arc_runner_reaper_enabled: false # A job running for longer than this has its runner deleted. The default is GitHub's own limit on a job's execution time on a self-hosted runner, five days, past which GitHub ends the job itself, so no job can legitimately run longer; a fleet whose jobs are all shorter sets it a little above its longest job's timeout-minutes. diff --git a/roles/github_runner_arc/meta/argument_specs.yml b/roles/github_runner_arc/meta/argument_specs.yml index 060aafad..2edb9fb7 100644 --- a/roles/github_runner_arc/meta/argument_specs.yml +++ b/roles/github_runner_arc/meta/argument_specs.yml @@ -374,6 +374,55 @@ argument_specs: github_runner_arc_node_recovery_image: type: str description: Image the node recovery watcher runs, pulled without a pull Secret. + github_runner_arc_role_controller_enabled: + type: bool + description: + - Whether to install the role controller. Defaults to the cluster role's github_runner_cluster_role_controller when that is defined, else false. + - Install the role controller, a `fleet-role-probe` DaemonSet that measures on every node with a kubelet what decides whether a host is fit to hold an etcd vote, and a `fleet-role` Deployment that reports which eligible hosts are fit and which swap of an unfit voter for a fit standby it would make. + - It only reports. Its writes are its own two ConfigMaps; its ClusterRoles can get and list nodes and nothing else. false removes both, their RBAC and their ConfigMaps. + github_runner_arc_role_controller_image: + type: str + description: Image the controller and the probe run, pulled without a pull Secret. + github_runner_arc_role_controller_sample_seconds: + type: int + default: 15 + description: Seconds between probe samples and between controller passes. A figure is read over a window of 100 samples. + github_runner_arc_role_controller_voters: + type: int + description: The number of etcd voters the cluster is held at, from github_runner_cluster_voters when that is defined, else 3. Odd, at least 3, and at most github_runner_arc_role_controller_max_servers minus one. + github_runner_arc_role_controller_max_servers: + type: int + description: The cluster role's github_runner_cluster_max_servers when it is defined, else 5. + github_runner_arc_role_controller_election_timeout_ms: + type: int + description: The cluster's etcd election timeout in milliseconds, from github_runner_cluster_etcd_election_timeout when that is set, else k3s's own 1000. + github_runner_arc_role_controller_pressure_threshold: + type: float + description: The regrade core's pressure threshold, from github_runner_cluster_regrade_pressure_threshold when defined, else its default. + github_runner_arc_role_controller_available_floor_percent: + type: float + description: The regrade core's MemAvailable floor in percent of MemTotal, from github_runner_cluster_regrade_available_floor_percent when defined, else its default. + github_runner_arc_role_controller_eligible_label: + type: str + description: The Node label, with the value true, that marks a host the controller may promote or demote, from github_runner_cluster_voter_eligible_label when that is defined, else github-runner.exadev.com/voter-eligible. + github_runner_arc_role_controller_pinned_label: + type: str + description: The Node label, with the value true, that marks a voter the controller must never demote, from github_runner_cluster_voter_pinned_label when that is defined, else github-runner.exadev.com/voter-pinned. + github_runner_arc_role_controller_level_label: + type: str + description: The label regrading puts on a server below full, from github_runner_cluster_regrade_level_label when defined, else github-runner.exadev.com/regrade-level. + github_runner_arc_role_controller_probe_configmap: + type: str + default: fleet-role-probe + description: The ConfigMap in the platform namespace that each probe patches its own key of. + github_runner_arc_role_controller_status_configmap: + type: str + default: fleet-role-status + description: The ConfigMap in the platform namespace that the controller publishes its status to. + github_runner_arc_role_controller_data_dir: + type: str + default: /var/lib/rancher/k3s/fleet-role-probe + description: The directory on the node, on the filesystem k3s keeps its data on, where the probe takes its fdatasync reading. github_runner_arc_runner_reaper_enabled: type: bool default: false diff --git a/roles/github_runner_arc/tasks/install_fleet_role.yml b/roles/github_runner_arc/tasks/install_fleet_role.yml new file mode 100644 index 00000000..b8b48597 --- /dev/null +++ b/roles/github_runner_arc/tasks/install_fleet_role.yml @@ -0,0 +1,76 @@ +--- +# Installs the role controller (templates/fleet-role.yaml.j2, running scripts/fleet-role.sh and scripts/fleet-role-probe.sh) when github_runner_arc_role_controller_enabled is on, and otherwise removes it along with the two ConfigMaps it publishes to. Included from main.yml on any host that installs something. + +- name: "Role controller: describe its resources" + ansible.builtin.set_fact: + github_runner_arc_role_controller_resources: "{{ lookup('ansible.builtin.template', 'fleet-role.yaml.j2') | from_yaml_all | select | list }}" + +- name: "Role controller: create the namespace it runs in" + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ github_runner_arc_platform_namespace }}" + when: github_runner_arc_role_controller_enabled | bool + +# Pre-created empty rather than left for the controller and the probes to create: neither is granted `create` on configmaps, only get and patch of the two names. Created only when absent: applying an empty definition again would wipe what they have published. +- name: "Role controller: check whether its ConfigMaps exist" + kubernetes.core.k8s_info: + kind: ConfigMap + namespace: "{{ github_runner_arc_platform_namespace }}" + name: "{{ item }}" + loop: + - "{{ github_runner_arc_role_controller_probe_configmap }}" + - "{{ github_runner_arc_role_controller_status_configmap }}" + register: github_runner_arc_role_controller_configmaps + when: github_runner_arc_role_controller_enabled | bool + +- name: "Role controller: pre-create its ConfigMaps" + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: ConfigMap + metadata: + name: "{{ item.item }}" + namespace: "{{ github_runner_arc_platform_namespace }}" + data: "{{ {'status.json': ''} if item.item == github_runner_arc_role_controller_status_configmap else {} }}" + loop: "{{ github_runner_arc_role_controller_configmaps.results }}" + loop_control: + label: "{{ item.item }}" + when: + - github_runner_arc_role_controller_enabled | bool + - item.resources | length == 0 + +- name: "Role controller: install the controller and the probe" + kubernetes.core.k8s: + definition: "{{ item }}" + loop: "{{ github_runner_arc_role_controller_resources }}" + loop_control: + label: "{{ item.kind }}/{{ item.metadata.name }}" + when: github_runner_arc_role_controller_enabled | bool + +# The workloads go before the permissions they run with, so nothing is left running that can no longer reach its ConfigMap; the ConfigMaps go last. +- name: "Role controller: remove the controller, the probe and their ConfigMaps" + kubernetes.core.k8s: + state: absent + api_version: "{{ item.apiVersion }}" + kind: "{{ item.kind }}" + namespace: "{{ item.metadata.namespace | default(omit) }}" + name: "{{ item.metadata.name }}" + loop: "{{ (github_runner_arc_role_controller_resources | reverse | list) + github_runner_arc_role_controller_configmap_definitions }}" + loop_control: + label: "{{ item.kind }}/{{ item.metadata.name }}" + vars: + github_runner_arc_role_controller_configmap_definitions: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: "{{ github_runner_arc_role_controller_probe_configmap }}" + namespace: "{{ github_runner_arc_platform_namespace }}" + - apiVersion: v1 + kind: ConfigMap + metadata: + name: "{{ github_runner_arc_role_controller_status_configmap }}" + namespace: "{{ github_runner_arc_platform_namespace }}" + when: not (github_runner_arc_role_controller_enabled | bool) diff --git a/roles/github_runner_arc/tasks/install_platform.yml b/roles/github_runner_arc/tasks/install_platform.yml index fde9f194..22cf9dba 100644 --- a/roles/github_runner_arc/tasks/install_platform.yml +++ b/roles/github_runner_arc/tasks/install_platform.yml @@ -139,7 +139,7 @@ name: github-runner-heartbeat-controller-reader apiGroup: rbac.authorization.k8s.io -# heartbeat: read the autoscaler's own shared status ConfigMap and the deployed-state ConfigMap, scoped by resourceNames to the two objects it actually touches. +# heartbeat: read the autoscaler's own shared status ConfigMap, the deployed-state ConfigMap and, with the role controller installed, its status ConfigMap, scoped by resourceNames to the objects it actually touches. - name: "Platform: heartbeat RBAC - read the autoscaler status ConfigMap" kubernetes.core.k8s: definition: @@ -151,7 +151,7 @@ rules: - apiGroups: [""] resources: ["configmaps"] - resourceNames: ["autoscaler-status", "{{ github_runner_arc_deployed_state_configmap }}"] + resourceNames: "{{ ['autoscaler-status', github_runner_arc_deployed_state_configmap] + ([github_runner_arc_role_controller_status_configmap] if github_runner_arc_role_controller_enabled | bool else []) }}" verbs: ["get"] - name: "Platform: heartbeat RBAC - bind status-reader" @@ -406,7 +406,7 @@ containers: - name: heartbeat image: "{{ github_runner_arc_heartbeat_image }}" - env: "{{ github_runner_arc_heartbeat_env + github_runner_arc_heartbeat_controller_env }}" + env: "{{ github_runner_arc_heartbeat_env + github_runner_arc_heartbeat_controller_env + github_runner_arc_heartbeat_role_env }}" resources: requests: cpu: "50m" @@ -434,6 +434,12 @@ name: heartbeat-gh-token key: token # Where the heartbeat looks for the controller's Deployment. Passed only when it differs from scripts/heartbeat.sh's own default, so that a heartbeat Deployment installed before this setting existed is not rolled by an unchanged configuration. + # Passed only with the role controller installed, for the same reason: the status the controller publishes is then republished in the gist beside the autoscaler's. + github_runner_arc_heartbeat_role_env: >- + {{ + [{'name': 'FLEET_ROLE_STATUS_CONFIGMAP', 'value': github_runner_arc_role_controller_status_configmap}] + if github_runner_arc_role_controller_enabled | bool else [] + }} github_runner_arc_heartbeat_controller_env: >- {{ [{'name': 'HEARTBEAT_CONTROLLER_NAMESPACE', 'value': github_runner_arc_controller_namespace}] diff --git a/roles/github_runner_arc/tasks/main.yml b/roles/github_runner_arc/tasks/main.yml index 95c240ce..59a441a0 100644 --- a/roles/github_runner_arc/tasks/main.yml +++ b/roles/github_runner_arc/tasks/main.yml @@ -64,6 +64,9 @@ - name: Install the node recovery watcher, or remove it when turned off ansible.builtin.include_tasks: install_node_recovery.yml + - name: Install the role controller, or remove it when turned off + ansible.builtin.include_tasks: install_fleet_role.yml + - name: Ensure the fleet-health platform (heartbeat and autoscaler) is installed ansible.builtin.include_tasks: install_platform.yml when: github_runner_arc_heartbeat_gist_id | length > 0 diff --git a/roles/github_runner_arc/tasks/validate.yml b/roles/github_runner_arc/tasks/validate.yml index 0fa48c4e..4d346fb4 100644 --- a/roles/github_runner_arc/tasks/validate.yml +++ b/roles/github_runner_arc/tasks/validate.yml @@ -171,6 +171,30 @@ and github_runner_arc_node_recovery_prune_after_seconds | string | length > 0 and github_runner_arc_node_recovery_prune_label | string is not match('^([a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?/)?[A-Za-z0-9]([-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?$') }} + - msg: github_runner_arc_role_controller_sample_seconds, github_runner_arc_role_controller_election_timeout_ms and github_runner_arc_role_controller_max_servers must be whole numbers, at least 1. + failed: >- + {{ + github_runner_arc_role_controller_enabled | bool + and ([github_runner_arc_role_controller_sample_seconds, github_runner_arc_role_controller_election_timeout_ms, github_runner_arc_role_controller_max_servers] + | map('string') | reject('match', '^[1-9][0-9]*$') | list | length > 0) + }} + - msg: github_runner_arc_role_controller_voters must be odd, at least 3, and at most github_runner_arc_role_controller_max_servers minus one, which leaves a swap one place of headroom. + failed: >- + {{ + github_runner_arc_role_controller_enabled | bool + and (github_runner_arc_role_controller_voters | string is not match('^[1-9][0-9]*$') + or github_runner_arc_role_controller_voters | int is even + or github_runner_arc_role_controller_voters | int < 3 + or github_runner_arc_role_controller_voters | int > (github_runner_arc_role_controller_max_servers | int) - 1) + }} + - msg: github_runner_arc_role_controller_eligible_label and github_runner_arc_role_controller_pinned_label must be Kubernetes label keys, such as example.com/voter-eligible, and differ. + failed: >- + {{ + github_runner_arc_role_controller_enabled | bool + and ([github_runner_arc_role_controller_eligible_label, github_runner_arc_role_controller_pinned_label] + | map('string') | reject('match', '^([a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?/)?[A-Za-z0-9]([-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?$') | list | length > 0 + or github_runner_arc_role_controller_eligible_label == github_runner_arc_role_controller_pinned_label) + }} - msg: github_runner_arc_runner_reaper_max_job_seconds must be a whole number of seconds, at least 1, and github_runner_arc_runner_reaper_completed_grace_seconds a whole number of seconds. failed: >- {{ diff --git a/roles/github_runner_arc/templates/fleet-role.yaml.j2 b/roles/github_runner_arc/templates/fleet-role.yaml.j2 new file mode 100644 index 00000000..fc312b91 --- /dev/null +++ b/roles/github_runner_arc/templates/fleet-role.yaml.j2 @@ -0,0 +1,243 @@ +{# The role controller (see tasks/install_fleet_role.yml, scripts/fleet-role.sh and scripts/fleet-role-probe.sh): the controller's ServiceAccount, ClusterRole, binding, ConfigMap Role and Deployment, and the probe's own, with its DaemonSet. The namespace is ensured, and the two ConfigMaps are created, separately, since applying an empty ConfigMap again would wipe what the controller and the probes have published. #} +{% set controller = 'fleet-role' %} +{% set probe = 'fleet-role-probe' %} +{% set labels = {'app.kubernetes.io/part-of': 'github-runner'} %} +{% set namespace = github_runner_arc_platform_namespace %} +{% set probe_configmap = github_runner_arc_role_controller_probe_configmap %} +{% set status_configmap = github_runner_arc_role_controller_status_configmap %} +{# A ClusterRole can only name nodes cluster-wide; the controller and the probe both list them to find the voters and the standbys, and neither can write one. #} +{% set node_rules = [{'apiGroups': [''], 'resources': ['nodes'], 'verbs': ['get', 'list']}] %} +{% for name in [controller, probe] %} +{% set own_labels = labels | combine({'app.kubernetes.io/name': name}) %} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ name | to_json }} + namespace: {{ namespace | to_json }} + labels: {{ own_labels | to_json }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ ('github-runner-' ~ name) | to_json }} + labels: {{ own_labels | to_json }} +rules: {{ node_rules | to_json }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ ('github-runner-' ~ name) | to_json }} + labels: {{ own_labels | to_json }} +subjects: + - kind: ServiceAccount + name: {{ name | to_json }} + namespace: {{ namespace | to_json }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ ('github-runner-' ~ name) | to_json }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ name | to_json }} + namespace: {{ namespace | to_json }} + labels: {{ own_labels | to_json }} +{# Scoped by resourceNames to the objects each touches. Neither is granted create, so a deleted ConfigMap fails loudly and the next run of the role makes it again, as the autoscaler's status ConfigMap does. #} +rules: +{% if name == controller %} + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: {{ [probe_configmap] | to_json }} + verbs: ["get"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: {{ [status_configmap] | to_json }} + verbs: ["get", "patch"] +{% else %} + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: {{ [probe_configmap] | to_json }} + verbs: ["get", "patch"] +{% endif %} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ name | to_json }} + namespace: {{ namespace | to_json }} + labels: {{ own_labels | to_json }} +subjects: + - kind: ServiceAccount + name: {{ name | to_json }} + namespace: {{ namespace | to_json }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ name | to_json }} +--- +{% endfor %} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ controller | to_json }} + namespace: {{ namespace | to_json }} + labels: {{ (labels | combine({'app.kubernetes.io/name': controller})) | to_json }} +spec: + replicas: 1 + selector: + matchLabels: {{ {'app.kubernetes.io/name': controller} | to_json }} + template: + metadata: + labels: {{ (labels | combine({'app.kubernetes.io/name': controller})) | to_json }} + spec: + serviceAccountName: {{ controller | to_json }} + priorityClassName: {{ github_runner_arc_platform_priority_class_name | to_json }} + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: {{ controller | to_json }} + image: {{ github_runner_arc_role_controller_image | to_json }} + env: + # The controller reports and changes nothing; the script refuses any other value. + - name: FLEET_ROLE_DRY_RUN + value: "true" + - name: FLEET_ROLE_VOTERS + value: {{ github_runner_arc_role_controller_voters | string | to_json }} + - name: FLEET_ROLE_MAX_SERVERS + value: {{ github_runner_arc_role_controller_max_servers | string | to_json }} + - name: FLEET_ROLE_ELECTION_TIMEOUT_MS + value: {{ github_runner_arc_role_controller_election_timeout_ms | string | to_json }} + - name: FLEET_ROLE_SAMPLE_SECONDS + value: {{ github_runner_arc_role_controller_sample_seconds | string | to_json }} + - name: FLEET_ROLE_PRESSURE_THRESHOLD + value: {{ github_runner_arc_role_controller_pressure_threshold | string | to_json }} + - name: FLEET_ROLE_AVAILABLE_FLOOR_PERCENT + value: {{ github_runner_arc_role_controller_available_floor_percent | string | to_json }} + - name: FLEET_ROLE_ELIGIBLE_LABEL + value: {{ github_runner_arc_role_controller_eligible_label | to_json }} + - name: FLEET_ROLE_PINNED_LABEL + value: {{ github_runner_arc_role_controller_pinned_label | to_json }} + - name: FLEET_ROLE_LEVEL_LABEL + value: {{ github_runner_arc_role_controller_level_label | to_json }} + - name: FLEET_ROLE_STATE_NAMESPACE + value: {{ namespace | to_json }} + - name: FLEET_ROLE_PROBE_CONFIGMAP + value: {{ probe_configmap | to_json }} + - name: FLEET_ROLE_STATUS_CONFIGMAP + value: {{ status_configmap | to_json }} + # kubectl's cache goes to the in-memory volume, since the root filesystem is read-only. + - name: HOME + value: /tmp + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 32Mi + ephemeral-storage: {{ github_runner_arc_platform_ephemeral_storage_request | to_json }} + limits: + cpu: 200m + memory: 128Mi + ephemeral-storage: {{ github_runner_arc_platform_ephemeral_storage_limit | to_json }} + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 8Mi +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ probe | to_json }} + namespace: {{ namespace | to_json }} + labels: {{ (labels | combine({'app.kubernetes.io/name': probe})) | to_json }} +spec: + selector: + matchLabels: {{ {'app.kubernetes.io/name': probe} | to_json }} + template: + metadata: + labels: {{ (labels | combine({'app.kubernetes.io/name': probe})) | to_json }} + spec: + serviceAccountName: {{ probe | to_json }} + priorityClassName: {{ github_runner_arc_platform_priority_class_name | to_json }} + # The host's network, so the probe measures the path the node's own etcd and kubelet use: on a node that runs in Docker, that is the k3s container's, where the node's Tailscale client lives. On a Docker node the k3s container also has the interfaces of its own network, which the probe reports as a link it cannot see. + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: {{ {'kubernetes.io/os': 'linux'} | to_json }} + # Every node with a kubelet, whatever it is tainted for; a server regraded to etcd-only has no kubelet, so no pod, and its side is read from its peers. + tolerations: + - operator: Exists + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: {{ probe | to_json }} + image: {{ github_runner_arc_role_controller_image | to_json }} + command: ["/app/scripts/fleet-role-probe.sh"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NAMESPACE + value: {{ namespace | to_json }} + - name: ELECTION_TIMEOUT_MS + value: {{ github_runner_arc_role_controller_election_timeout_ms | string | to_json }} + - name: ELIGIBLE_LABEL + value: {{ github_runner_arc_role_controller_eligible_label | to_json }} + - name: PROBE_CONFIGMAP + value: {{ probe_configmap | to_json }} + - name: REPORT_SECONDS + value: {{ github_runner_arc_role_controller_sample_seconds | string | to_json }} + - name: FSYNC_DIR + value: /fsync + # kubectl's cache goes to the in-memory volume, since the root filesystem is read-only. + - name: HOME + value: /tmp + # Root, with every capability dropped, because the directory the kubelet creates on the node for the fdatasync reading belongs to root and a root without CAP_DAC_OVERRIDE writes only what it owns. + securityContext: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 32Mi + ephemeral-storage: {{ github_runner_arc_platform_ephemeral_storage_request | to_json }} + limits: + cpu: 200m + memory: 128Mi + ephemeral-storage: {{ github_runner_arc_platform_ephemeral_storage_limit | to_json }} + volumeMounts: + - name: tmp + mountPath: /tmp + - name: pressure + mountPath: /host-pressure + readOnly: true + - name: fsync + mountPath: /fsync + volumes: + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 8Mi + - name: pressure + hostPath: + path: /proc/pressure + - name: fsync + hostPath: + path: {{ github_runner_arc_role_controller_data_dir | to_json }} + type: DirectoryOrCreate diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 358ec719..b82e22d3 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -300,6 +300,18 @@ The service mounts the Docker socket, which is a permission cost: whatever runs The decision logic lives in `files/regrade/fleet-regrade-core.sh` and calls only three functions of an actuator: `levels`, `current_level` and `apply_level NAME`. The Docker actuator is `files/regrade/fleet-regrade-docker.sh`. Another platform supplies its own three functions and uses the core unchanged; a NixOS image can take the core from a pinned release of this collection. +## Role controller + +`github_runner_cluster_role_controller: true` turns on the ARC role's role controller, which measures which hosts are fit to hold an etcd vote and reports the swap of an unfit voter for a fit standby that it would make (see the ARC role's README, Role controller). It reports and changes nothing. It is off by default. This role holds the controller's inventory: + +- `github_runner_cluster_voter_eligible` (per host, default false) marks a host the controller may promote to a voter or demote from one. The role prepares no host for promotion by marking it, so mark only a host that is, or has been staged as, a server. +- `github_runner_cluster_voter_pinned` (per host, default false) marks a voter the controller must never demote. Set each pin in the inventory with a comment saying why. +- `github_runner_cluster_voters` (for the group, default 3) is the number of voters the cluster is held at. + +While the controller is on, the role refuses a target that is even, under 3, or without a place of headroom below `github_runner_cluster_max_servers` (a swap adds the new voter before it removes the old one, so the target is at most the maximum minus one), a datastore that is not etcd, and fewer eligible hosts than the target once any host is marked. With no host marked the controller only reports the voters. + +The role publishes the two host settings as Node labels, `github-runner.exadev.com/voter-eligible` and `github-runner.exadev.com/voter-pinned` (`github_runner_cluster_voter_eligible_label` and `github_runner_cluster_voter_pinned_label`), by passing them to k3s as `--node-label` on every runtime: a Docker server or agent, and a native one. The kubelet writes its labels to its Node each time it starts and never removes one, so while the controller is on, or a host is marked eligible or pinned, a host carries both labels with the value `true` or `false`, and turning a setting off writes `false` over the `true`. A host that uses none of this gets no label and no extra Compose file, so a fleet that does not use the controller keeps every node's configuration as it is and no container is recreated by an upgrade. A host's labels change with its next run of the role and the recreation or restart that follows it. + ## Networking Every node's k3s uses its mesh address for everything: its node IP, etcd's peer URLs, the API server and pod traffic. Which physical path the mesh takes between two nodes is Tailscale's choice, among the endpoints each node's Tailscale client can offer its peers. On Compose's default bridge network (`github_runner_cluster_network_mode: bridge`, the default) the client sits behind the Docker host's NAT and, inside a Docker VM, behind the VM's too, so it has no address its peers can reach and every pair of nodes is relayed, even on one LAN. @@ -396,6 +408,7 @@ Each Compose project runs its own `docker-prune` service against the same daemon - `github_runner_cluster_node_name`: the k3s node name and mesh hostname, defaulting to the inventory name. - `github_runner_cluster_node_taints`: Kubernetes taints (`key=value:Effect`, no spaces) the host's node registers with, empty by default. `NoSchedule` keeps pods without a matching toleration off it and `NoExecute` also evicts the ones already there. k3s applies them on every registration, so they come back if the node object is deleted, which a taint set with `kubectl` does not. - `github_runner_cluster_etcd_heartbeat_interval`, `github_runner_cluster_etcd_election_timeout`: etcd's timing in milliseconds for every server, empty by default (k3s's own). Set together, the same on every server, the election timeout at least five times the heartbeat interval; see [etcd timing](#etcd-timing). +- `github_runner_cluster_role_controller` (default false), `github_runner_cluster_voters` (default 3), `github_runner_cluster_voter_eligible` and `github_runner_cluster_voter_pinned` (per host, default false): the role controller's inventory (see [Role controller](#role-controller)). `github_runner_cluster_voter_eligible_label` and `github_runner_cluster_voter_pinned_label` are the Node labels they are published under, derived, never set directly. - `github_runner_cluster_regrade` (default false): run the `regrade` service, which steps this server down under memory pressure while it keeps its etcd vote (see [Memory-pressure regrading](#memory-pressure-regrading)). Servers only. `github_runner_cluster_regrade_pressure_threshold` (percent, default 10) and `github_runner_cluster_regrade_available_floor_percent` (default 15) are the limit; `github_runner_cluster_regrade_step_up` (`automatic`, the default, or `manual`), `github_runner_cluster_regrade_sample_seconds`, `github_runner_cluster_regrade_sustain_seconds`, `github_runner_cluster_regrade_cooldown_seconds`, `github_runner_cluster_regrade_operation_timeout_seconds` and `github_runner_cluster_regrade_lease_duration_seconds` are the timing, all derived from `github_runner_cluster_regrade_pressure_window_seconds` (the kernel's `avg300`, 300 seconds, not a setting) unless set. `github_runner_cluster_regrade_level_label` and `github_runner_cluster_regrade_taint` are derived, never set directly. - Per-host overrides: `github_runner_cluster_node_role` (`server` or `agent`), `github_runner_cluster_bootstrap` (true picks the bootstrap host, false excludes a host), `github_runner_cluster_server_url` (join this server instead, for a cluster whose servers are outside the inventory), `github_runner_cluster_tls_sans`, `github_runner_cluster_node_address`, and `github_runner_cluster_voter_eligible` (true stages a native host as a standby server, its server configuration written and its name on every server's certificate, without making it a server; see [Native servers](#native-servers)). The run fails if the result has an even number of servers, no servers, or more than one bootstrap host. - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)). diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index c03802a7..4ae28d38 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -53,6 +53,21 @@ github_runner_cluster_regrade_level_label: "github-runner.exadev.com/regrade-lev # Derived, never set directly: the standard control-plane taint a server carries while it runs no workloads. The core applies it with the cordon on the first step down and removes it on the way back to full; the NoSchedule effect is what it removes. github_runner_cluster_regrade_taint: "node-role.kubernetes.io/control-plane=true:NoSchedule" +# The role controller (see the ARC role's Role controller section): a probe on every node and a controller that report which hosts are fit to hold an etcd vote and the swap they would make, changing nothing. false, the default, installs nothing. The ARC role, which installs it, reads this variable (github_runner_arc_role_controller_enabled takes it as its default), so one setting in the inventory turns it on for the group. +github_runner_cluster_role_controller: false +# Which hosts the role controller may move an etcd vote between. github_runner_cluster_voter_eligible, per host, marks a host the controller may promote to a voter or demote from one, and github_runner_cluster_voter_pinned a voter it must never demote; neither is set by default, so no host is eligible or pinned and the controller only reports. github_runner_cluster_voters, for the group, is the number of voters the cluster is held at: odd, at least 3, and at most github_runner_cluster_max_servers minus one, which leaves a swap (the new voter joins before the old one leaves) one place of headroom. tasks/voters.yml refuses anything else while github_runner_cluster_role_controller is true, and refuses fewer eligible hosts than that once any host is marked eligible. A host that is eligible must be able to hold a vote: the role prepares no host for promotion by being marked. +github_runner_cluster_voter_eligible: false +github_runner_cluster_voter_pinned: false +github_runner_cluster_voters: 3 +# Derived, never set directly: the Node labels the role publishes the two host settings under, with the value true or false on every node, so that turning a setting off writes false over the true a node registered with (a kubelet adds and overwrites the labels it is given and never removes one). The ARC role's role controller reads them (github_runner_arc_role_controller_eligible_label and github_runner_arc_role_controller_pinned_label default to these), as a label with the value true. +github_runner_cluster_voter_eligible_label: "github-runner.exadev.com/voter-eligible" +github_runner_cluster_voter_pinned_label: "github-runner.exadev.com/voter-pinned" +# Derived, never set directly: this host's labels, as the key=value entries every runtime passes to k3s as --node-label, empty unless the role controller is on or this host is marked eligible or pinned, so that a fleet that uses none of them keeps its nodes' configuration as it is (a Docker node is not recreated by an upgrade). +github_runner_cluster_node_labels: >- + {{ [github_runner_cluster_voter_eligible_label ~ '=' ~ ('true' if github_runner_cluster_voter_eligible | bool else 'false'), + github_runner_cluster_voter_pinned_label ~ '=' ~ ('true' if github_runner_cluster_voter_pinned | bool else 'false')] + if github_runner_cluster_role_controller | bool or github_runner_cluster_voter_eligible | bool or github_runner_cluster_voter_pinned | bool else [] }} + # The images every node's k3s container is built from: the rancher/k3s tag (k3s's version with "+" written as "-") and the tailscale/tailscale tag whose tailscale and tailscaled binaries are copied in. Pinned, never latest: Compose builds a node's image only when it is missing, so a floating tag would put whatever version is newest on the day a node is first built, rebuilt or moved into a running cluster, unannounced. Set both once for the whole cluster group, in inventory or group_vars (the check below reads every host's through hostvars). Before it changes anything on a host the role fails if either is not a pinned tag, if the group's hosts name different k3s minor versions, or if the k3s version would break k3s's upgrade path or Kubernetes' version skew policy from what this node and the cluster run now (see the role README's Versions section). Changing them changes no running node: its image is kept until it is rebuilt (see the role README's Versions section for upgrading). github_runner_cluster_k3s_version: "{{ github_runner_cluster_k3s_tested_version }}" github_runner_cluster_tailscale_version: "v1.102.4" @@ -155,11 +170,12 @@ github_runner_cluster_network_probe_image: "busybox:1.37" # Extra Compose files merged after docker-compose.yml for this host's k3s project, as paths relative to github_runner_cluster_dir or absolute. Nothing needs them in normal use; the mesh integration tests use one to put several nodes on one Docker host. github_runner_cluster_compose_files: [] -# Derived, never set directly: every Compose file for this host's k3s project, in merge order: docker-compose.yml, then the override the headscale_in_cluster provider writes for the bootstrap server, then docker-compose.api-port.yml when github_runner_cluster_api_port is not the default, then github_runner_cluster_compose_files, then, with github_runner_cluster_network_mode host, docker-compose.host-network.yml, last so that its emptied port list wins over any file that publishes one. With no files beyond docker-compose.yml the role passes none at all and Compose reads it by itself, exactly as a plain `docker compose up` in github_runner_cluster_dir would. +# Derived, never set directly: every Compose file for this host's k3s project, in merge order: docker-compose.yml, then the override the headscale_in_cluster provider writes for the bootstrap server, then docker-compose.api-port.yml when github_runner_cluster_api_port is not the default, then docker-compose.node-labels.yml when the host has node labels, then github_runner_cluster_compose_files, then, with github_runner_cluster_network_mode host, docker-compose.host-network.yml, last so that its emptied port list wins over any file that publishes one. With no files beyond docker-compose.yml the role passes none at all and Compose reads it by itself, exactly as a plain `docker compose up` in github_runner_cluster_dir would. github_runner_cluster_compose_file_list: >- {{ ['docker-compose.yml'] + (['docker-compose.mesh.yml'] if github_runner_cluster_mesh == 'headscale_in_cluster' and github_runner_cluster_effective_bootstrap | default(false) | bool else []) + (['docker-compose.api-port.yml'] if github_runner_cluster_api_port | int != github_runner_cluster_k3s_default_api_port | int else []) + + (['docker-compose.node-labels.yml'] if github_runner_cluster_node_labels | length > 0 else []) + github_runner_cluster_compose_files + (['docker-compose.host-network.yml'] if github_runner_cluster_network_mode == 'host' else []) }} # The Docker context this host's node runs in, pinned by name (for example "colima"). Empty, the default, follows the host's current Docker context, or DOCKER_HOST when that is set. When set, every community.docker module call in the role passes it as cli_context and every docker CLI command runs with DOCKER_CONTEXT set to it, so both always reach the same daemon, whatever the host's current context is at the time. The role fails before touching the host if the context does not exist there, or if DOCKER_HOST is set as well, and it never changes the host's current context. Set it on any host with more than one Docker engine installed: on a Mac running its node in Colima with Docker Desktop also installed, merely launching Docker Desktop makes desktop-linux the current context, and an unpinned run would then build a second, empty node in Docker Desktop instead of managing the real one. Per host, like every value here that names something on the host; the Headscale tasks read it from the inventory of the host they act on. diff --git a/roles/github_runner_cluster/files/docker-compose.node-labels.yml b/roles/github_runner_cluster/files/docker-compose.node-labels.yml new file mode 100644 index 00000000..fd04c339 --- /dev/null +++ b/roles/github_runner_cluster/files/docker-compose.node-labels.yml @@ -0,0 +1,8 @@ +# Merged after docker-compose.yml by the github_runner_cluster role when this host has node labels to publish (github_runner_cluster_node_labels in defaults/main.yml: the role controller is on, or this host is marked a voter candidate or pinned): gives both services the labels the entrypoint passes to k3s as --node-label. Its own file, as docker-compose.api-port.yml is, so that a host that uses neither setting keeps the Compose configuration it has and its container is not recreated by an upgrade. +services: + k3s: + environment: + - K3S_NODE_LABELS=${K3S_NODE_LABELS:-} + k3s-agent: + environment: + - K3S_NODE_LABELS=${K3S_NODE_LABELS:-} diff --git a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh index f5a3ddd5..47702c1e 100755 --- a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh +++ b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh @@ -16,6 +16,15 @@ for node_taint in ${K3S_NODE_TAINTS:-}; do node_taint_flags="$node_taint_flags --node-taint $node_taint" done +# One --node-label flag per entry of K3S_NODE_LABELS (key=value entries separated by spaces, none containing a space), for the server and the agent alike. The kubelet writes them to its Node each time it starts, overwriting a value a label already has, so the role publishes both of its voter labels with true or false and a setting turned off is written over. Nothing removes a label the role stops passing. +set_node_label_flags() { + node_label_flags="" + for node_label in ${K3S_NODE_LABELS:-}; do + node_label_flags="$node_label_flags --node-label $node_label" + done +} +set_node_label_flags + # Sets etcd_timing_flags from etcd's timing, for the server only (the agent runs no etcd): K3S_ETCD_HEARTBEAT_INTERVAL and K3S_ETCD_ELECTION_TIMEOUT, in milliseconds, both set or both empty (the role checks that the election timeout is at least five times the heartbeat interval, and that every server has the same values). Empty leaves etcd on k3s's own timing. Each is passed as an --etcd-arg, which k3s also hands to the temporary etcd it starts on every restart to reconcile with an initialised datastore; these two flags do not change what that instance binds to, unlike the listen-peer-urls override that once broke it (see the README's known issues). set_etcd_timing_flags() { etcd_timing_flags="" @@ -212,8 +221,8 @@ supervise_self_hosted() { fi echo "Starting k3s server ($mode)" rm -f /output/kubeconfig.yaml - # shellcheck disable=SC2086 # Word splitting of server_args and node_taint_flags is deliberate: every argument is a flag or a value without spaces. - /bin/k3s server $server_args $node_taint_flags & + # shellcheck disable=SC2086 # Word splitting of server_args, node_taint_flags and node_label_flags is deliberate: every argument is a flag or a value without spaces. + /bin/k3s server $server_args $node_taint_flags $node_label_flags & k3s_pid=$! watcher_pid="" if [ "$mode" = direct ] && [ -n "$join_key" ]; then @@ -239,13 +248,13 @@ case "$role" in set_regrade_flags mesh_server_args # shellcheck disable=SC2086 # As above. - PATH="$tailscale_up_wait_dir:$PATH" exec /bin/k3s server $server_args $node_taint_flags + PATH="$tailscale_up_wait_dir:$PATH" exec /bin/k3s server $server_args $node_taint_flags $node_label_flags ;; agent) agent_join_host=$(join_host_of "${K3S_URL:-}") install_tailscale_up_wait "$agent_join_host" "$tailscale_up_wait_dir" /usr/local/bin/tailscale # shellcheck disable=SC2086 # As above. - PATH="$tailscale_up_wait_dir:$PATH" exec /bin/k3s agent --vpn-auth="$vpn_auth" --resolv-conf=/etc/k3s-resolv.conf $node_taint_flags + PATH="$tailscale_up_wait_dir:$PATH" exec /bin/k3s agent --vpn-auth="$vpn_auth" --resolv-conf=/etc/k3s-resolv.conf $node_taint_flags $node_label_flags ;; *) echo "usage: node-entrypoint.sh server|agent" >&2 diff --git a/roles/github_runner_cluster/tasks/main.yml b/roles/github_runner_cluster/tasks/main.yml index bc92c524..02f79ec2 100644 --- a/roles/github_runner_cluster/tasks/main.yml +++ b/roles/github_runner_cluster/tasks/main.yml @@ -61,6 +61,9 @@ - name: Check etcd's heartbeat interval and election timeout ansible.builtin.include_tasks: etcd_timing.yml +- name: Check the role controller's settings + ansible.builtin.include_tasks: voters.yml + - name: Check the memory-pressure regrading settings ansible.builtin.include_tasks: regrade.yml @@ -100,6 +103,7 @@ loop: - {src: docker-compose.yml, mode: "0644"} - {src: docker-compose.api-port.yml, mode: "0644"} + - {src: docker-compose.node-labels.yml, mode: "0644"} - {src: docker-compose.host-network.yml, mode: "0644"} - {src: k3s/Dockerfile, mode: "0644"} - {src: k3s/node-entrypoint.sh, mode: "0755"} diff --git a/roles/github_runner_cluster/tasks/native/main.yml b/roles/github_runner_cluster/tasks/native/main.yml index bb1ef1b5..edaac43a 100644 --- a/roles/github_runner_cluster/tasks/native/main.yml +++ b/roles/github_runner_cluster/tasks/native/main.yml @@ -69,6 +69,9 @@ - name: Check etcd's heartbeat interval and election timeout, which a native server passes to its etcd as every server does ansible.builtin.include_tasks: etcd_timing.yml +- name: Check the role controller's settings + ansible.builtin.include_tasks: voters.yml + - name: Check the versions the node is pinned to ansible.builtin.include_tasks: versions.yml diff --git a/roles/github_runner_cluster/tasks/voters.yml b/roles/github_runner_cluster/tasks/voters.yml new file mode 100644 index 00000000..a48a436c --- /dev/null +++ b/roles/github_runner_cluster/tasks/voters.yml @@ -0,0 +1,65 @@ +--- +# Checks the settings of the role controller (github_runner_cluster_role_controller, github_runner_cluster_voters, github_runner_cluster_voter_eligible) before the role changes anything on the host. Run after the topology is known, because the target is held against github_runner_cluster_max_servers. Nothing is checked while the controller is off. + +- name: Check the role controller's settings + when: github_runner_cluster_role_controller | bool + vars: + github_runner_cluster_eligible_hosts: >- + {{ groups[github_runner_cluster_group] | map('extract', hostvars) + | selectattr('github_runner_cluster_voter_eligible', 'defined') | selectattr('github_runner_cluster_voter_eligible', 'equalto', true) + | map(attribute='inventory_hostname') | list }} + block: + - name: Fail if the cluster has no etcd for the controller to move votes in + ansible.builtin.assert: + that: github_runner_cluster_datastore == 'etcd' + quiet: true + fail_msg: >- + github_runner_cluster_role_controller is true, but github_runner_cluster_datastore is {{ github_runner_cluster_datastore }}, which has + no etcd votes to move. Nothing was changed. + + - name: Fail unless the voter count is a whole number + ansible.builtin.assert: + that: github_runner_cluster_voters | string is match('^[1-9][0-9]*\\Z') + quiet: true + fail_msg: >- + github_runner_cluster_voters is '{{ github_runner_cluster_voters }}'; it must be a whole number, the number of etcd voters the + cluster is held at. Nothing was changed. + + - name: Fail if the voter count is even + # An even member count tolerates no more failures than the odd count below it, and a tie cannot be broken. + ansible.builtin.assert: + that: github_runner_cluster_voters | int is odd + quiet: true + fail_msg: >- + github_runner_cluster_voters is {{ github_runner_cluster_voters }}; it must be odd, since an even count of etcd voters tolerates + no more failures than the odd count below it. Nothing was changed. + + - name: Fail if the voter count is under three + # With fewer, losing one voter loses the majority. + ansible.builtin.assert: + that: github_runner_cluster_voters | int >= 3 + quiet: true + fail_msg: >- + github_runner_cluster_voters is {{ github_runner_cluster_voters }}; it must be at least 3, since with fewer the loss of one voter + loses etcd's majority. Nothing was changed. + + - name: Fail if the voter count leaves a swap no place of headroom + # A swap adds the new voter before it removes the old one, so the cluster briefly has one voter more than the target and needs a place for it. + ansible.builtin.assert: + that: github_runner_cluster_voters | int <= github_runner_cluster_max_servers | int - 1 + quiet: true + fail_msg: >- + github_runner_cluster_voters is {{ github_runner_cluster_voters }} and github_runner_cluster_max_servers is + {{ github_runner_cluster_max_servers }}. A swap adds the new voter before it removes the old one, so the target can be at most + one less than the most servers allowed ({{ github_runner_cluster_max_servers | int - 1 }}). Lower the target or raise + github_runner_cluster_max_servers. Nothing was changed. + + - name: Fail if some hosts are eligible but fewer than the voter count + # With no host eligible the controller only reports, so that a cluster can run it before choosing any host; once some are chosen, fewer than the target could never be held at it. + ansible.builtin.assert: + that: github_runner_cluster_eligible_hosts | length == 0 or github_runner_cluster_eligible_hosts | length >= github_runner_cluster_voters | int + quiet: true + fail_msg: >- + {{ github_runner_cluster_eligible_hosts | length }} host(s) of {{ github_runner_cluster_group }} set github_runner_cluster_voter_eligible + ({{ github_runner_cluster_eligible_hosts | join(', ') }}), fewer than github_runner_cluster_voters ({{ github_runner_cluster_voters }}). + Mark at least that many, or none to have the controller only report. Nothing was changed. diff --git a/roles/github_runner_cluster/templates/env-agent.j2 b/roles/github_runner_cluster/templates/env-agent.j2 index 1a5ea852..b9b2626d 100644 --- a/roles/github_runner_cluster/templates/env-agent.j2 +++ b/roles/github_runner_cluster/templates/env-agent.j2 @@ -17,6 +17,9 @@ K3S_VPN_AUTH_CONTROL_SERVER_URL={{ github_runner_cluster_mesh_control_server_url {% endif %} # Unused by the k3s-agent service itself, but docker compose validates every service's env interpolation up front regardless of which --profile is active - the k3s service requires K3S_TLS_SAN_LIST, and an agent host's .env needs to set SOMETHING for that check to pass under `docker compose --profile agent`. K3S_TLS_SAN_LIST={{ github_runner_cluster_effective_tls_sans | join(' ') }} +{% if github_runner_cluster_node_labels | length > 0 %} +K3S_NODE_LABELS={{ github_runner_cluster_node_labels | join(' ') }} +{% endif %} {% if github_runner_cluster_node_taints | length > 0 %} K3S_NODE_TAINTS={{ github_runner_cluster_node_taints | join(' ') }} {% endif %} diff --git a/roles/github_runner_cluster/templates/env.j2 b/roles/github_runner_cluster/templates/env.j2 index d636243c..dffffbf3 100644 --- a/roles/github_runner_cluster/templates/env.j2 +++ b/roles/github_runner_cluster/templates/env.j2 @@ -17,6 +17,9 @@ K3S_JOIN_SERVER_URL={{ github_runner_cluster_effective_server_url }} K3S_DATASTORE=sqlite {% endif %} K3S_NODE_NAME={{ github_runner_cluster_effective_node_name }} +{% if github_runner_cluster_node_labels | length > 0 %} +K3S_NODE_LABELS={{ github_runner_cluster_node_labels | join(' ') }} +{% endif %} {% if github_runner_cluster_node_taints | length > 0 %} K3S_NODE_TAINTS={{ github_runner_cluster_node_taints | join(' ') }} {% endif %} diff --git a/roles/github_runner_cluster/templates/native/agent.yaml.j2 b/roles/github_runner_cluster/templates/native/agent.yaml.j2 index a1f55ad9..b36ccc9e 100644 --- a/roles/github_runner_cluster/templates/native/agent.yaml.j2 +++ b/roles/github_runner_cluster/templates/native/agent.yaml.j2 @@ -11,3 +11,9 @@ node-taint: - {{ taint | to_json }} {% endfor %} {% endif %} +{% if github_runner_cluster_node_labels | length > 0 %} +node-label: +{% for label in github_runner_cluster_node_labels %} + - {{ label | to_json }} +{% endfor %} +{% endif %} diff --git a/roles/github_runner_cluster/templates/native/server.yaml.j2 b/roles/github_runner_cluster/templates/native/server.yaml.j2 index e05ec3c0..f32613e9 100644 --- a/roles/github_runner_cluster/templates/native/server.yaml.j2 +++ b/roles/github_runner_cluster/templates/native/server.yaml.j2 @@ -29,3 +29,9 @@ node-taint: - {{ taint | to_json }} {% endfor %} {% endif %} +{% if github_runner_cluster_node_labels | length > 0 %} +node-label: +{% for label in github_runner_cluster_node_labels %} + - {{ label | to_json }} +{% endfor %} +{% endif %} diff --git a/scripts/fleet-role-probe.sh b/scripts/fleet-role-probe.sh new file mode 100755 index 00000000..d405a03d --- /dev/null +++ b/scripts/fleet-role-probe.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +# The role controller's per-node probe (see roles/github_runner_arc/templates/fleet-role.yaml.j2): the `fleet-role-probe` DaemonSet runs this on every node that has a kubelet, with host networking, so it measures the path the node's own etcd and kubelet use. Every REPORT_SECONDS it takes one sample and appends it to this node's entry in the shared probe ConfigMap, which scripts/fleet-role.sh reads. An entry keeps the last window of samples, so it survives this pod being replaced. +# +# Only a node the controller judges is measured: a voter, or a host the owner marks eligible (ELIGIBLE_LABEL, with the value true). Any other node has nothing recorded, and an entry it has from when it was one is removed, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the cluster's control plane and not with the fleet. +# +# A sample holds, per node: the TCP connect time to every voter's etcd peer port (a connect that does not complete within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what etcd's own metrics gained since the last sample (the WAL fsync and backend commit histograms, the round trip to each peer, leader changes) and whether it has a leader, is a learner and is connected to its peers; on a node that is not a voter, an fdatasync probe in the k3s data directory once per window, with the write size etcd's own fio check uses; the memory pressure and MemAvailable the regrade core reads; and the link the default route leaves by. +# +# `fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of those readings alone, prints it and exits, with no cluster access (connect needs ELECTION_TIMEOUT_MS; the others need nothing but the settings below), so that each can be compared with a reading taken by hand on the host it runs on. +# +# Env: +# - NODE_NAME: the node this pod runs on (the downward API's spec.nodeName). Required. +# - NAMESPACE: the namespace of the ConfigMap. Required. +# - ELECTION_TIMEOUT_MS: the cluster's etcd election timeout, which is how long a peer connect may take before it counts as lost. Required. +# - ELIGIBLE_LABEL: the Node label, with the value true, that marks a host the controller may promote or demote. Required. +# - PROBE_CONFIGMAP: the shared ConfigMap, created by the role, that every probe patches its own key of (default fleet-role-probe). +# - REPORT_SECONDS: seconds between samples (default 15, the node recovery watcher's poll). +# - REPORT_ONCE: when "true", take one sample and exit, for tests. +# - VOTER_LABEL: the Node label that marks an etcd member (default node-role.kubernetes.io/etcd=true). +# - ETCD_METRICS_URL: where a voter's etcd serves its metrics (default http://127.0.0.1:2381/metrics, the listener k3s keeps). +# - ETCD_PEER_PORT: etcd's peer port (default 2380). +# - FSYNC_DIR: a directory on the filesystem k3s keeps its data on, which the fdatasync probe writes in (default /var/lib/rancher/k3s/fleet-role-probe). +# - SYS_CLASS_NET, PROC_NET_ROUTE: where the host's /sys/class/net and /proc/net/route are (the defaults). +# - FLEET_REGRADE_CORE: the regrade core (roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh), whose own psi_avg300 and meminfo_kb read the memory signals (default /app/fleet-regrade-core.sh). PSI_FILE and MEMINFO_FILE, which it reads, default here to the host's /proc/pressure mounted at /host-pressure and the node's /proc/meminfo. +set -euo pipefail + +PROBE_CONFIGMAP="${PROBE_CONFIGMAP:-fleet-role-probe}" +REPORT_SECONDS="${REPORT_SECONDS:-15}" +REPORT_ONCE="${REPORT_ONCE:-false}" +VOTER_LABEL="${VOTER_LABEL:-node-role.kubernetes.io/etcd=true}" +ETCD_METRICS_URL="${ETCD_METRICS_URL:-http://127.0.0.1:2381/metrics}" +ETCD_PEER_PORT="${ETCD_PEER_PORT:-2380}" +FSYNC_DIR="${FSYNC_DIR:-/var/lib/rancher/k3s/fleet-role-probe}" +SYS_CLASS_NET="${SYS_CLASS_NET:-/sys/class/net}" +PROC_NET_ROUTE="${PROC_NET_ROUTE:-/proc/net/route}" +FLEET_REGRADE_CORE="${FLEET_REGRADE_CORE:-/app/fleet-regrade-core.sh}" +PSI_FILE="${PSI_FILE:-/host-pressure/memory}" +MEMINFO_FILE="${MEMINFO_FILE:-/proc/meminfo}" +# The core reads its settings from an environment file when there is one; the probe has none, and must not pick up another by the core's default path. +# shellcheck disable=SC2034 # read by the sourced core +FLEET_REGRADE_ENV=/nonexistent +MODULES="$(dirname "$0")" + +# The size of one write of etcd's own disk check (fio --bs=2300), the size of a typical WAL entry. +readonly etcd_fio_write_bytes=2300 + +# shellcheck source=/dev/null +. "$FLEET_REGRADE_CORE" + +log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*" >&2; } + +# The tail percentile and the samples a window holds are defined once, in the module the controller reads them from too. +read -r tail_percentile window_samples < <(jq -nr -L "$MODULES" 'include "fleet-role-stats"; "\(tail_percentile) \(window_samples)"') +window_seconds=$((REPORT_SECONDS * window_samples)) + +# The interface the default route leaves by, the one with the lowest metric among the routes that are up and have a gateway; empty when there is none. /proc/net/route is hexadecimal, tab separated, with a header line. +default_route_interface() { + local iface destination _gateway flags _refcnt _use metric mask best='' best_metric='' + while read -r iface destination _gateway flags _refcnt _use metric mask _; do + [ "$destination" = 00000000 ] && [ "$mask" = 00000000 ] || continue + (( (0x$flags & 3) == 3 )) || continue + if [ -z "$best" ] || [ "$metric" -lt "$best_metric" ]; then + best=$iface + best_metric=$metric + fi + done < "$PROC_NET_ROUTE" + echo "$best" +} + +# wired, wifi or unknown for an interface: wifi when the kernel gives it a wireless phy, wired when it is backed by a device, unknown when it is virtual (a veth in a container, a bridge, a tunnel) or absent, since then the link beneath it cannot be seen from here. Reads the interface's own directory and resolves it with cd -P, never a find over /sys, whose symlinks (device, subsystem, phy80211) loop back on themselves. +link_type() { + local iface=$1 dir="${SYS_CLASS_NET}/$1" real + if [ -z "$iface" ] || [ ! -d "$dir" ]; then + echo unknown + return + fi + if [ -e "${dir}/phy80211" ] || [ -d "${dir}/wireless" ] || grep -qx 'DEVTYPE=wlan' "${dir}/uevent" 2> /dev/null; then + echo wifi + return + fi + real="$(cd -P "$dir" && pwd -P)" + case "$real" in + */devices/virtual/*) echo unknown ;; + *) echo wired ;; + esac +} + +# The time curl took to connect to a peer's etcd peer port, in milliseconds. The request itself is not the point (the port speaks TLS, and curl's HTTP is refused), only the connection: curl reports a connect time of zero when it never completed, and then the reading is the election timeout. +connect_ms() { + local host=$1 seconds + : "${ELECTION_TIMEOUT_MS:?ELECTION_TIMEOUT_MS is required}" + [[ "$host" != *:* ]] || host="[${host}]" + seconds="$(curl --silent --output /dev/null --connect-timeout "$(awk -v ms="$ELECTION_TIMEOUT_MS" 'BEGIN { printf "%.3f", ms / 1000 }')" \ + --write-out '%{time_connect}' "http://${host}:${ETCD_PEER_PORT}/" 2> /dev/null || true)" + awk -v s="$seconds" -v timeout="$ELECTION_TIMEOUT_MS" 'BEGIN { if (s + 0 == 0) printf "%d\n", timeout; else printf "%.3f\n", s * 1000 }' +} + +# The tail of fdatasync latency in this directory's filesystem, in milliseconds, from etcd's own fio check (--rw=write --ioengine=sync --fdatasync=1 --bs=2300) at the fewest writes that give the tail a sample of its own. +fsync_ms() { + fio --rw=write --ioengine=sync --fdatasync=1 --directory="$FSYNC_DIR" --size=$((etcd_fio_write_bytes * window_samples)) --bs="$etcd_fio_write_bytes" \ + --name=fleet-role-probe --unlink=1 --output-format=json \ + | jq -r --arg percentile "${tail_percentile}.000000" '.jobs[0].sync.lat_ns.percentile[$percentile] / 1000000' +} + +# Takes one sample and publishes it. $1 is "true" on the first pass of this process, which for a node that is not a voter is the only sign this probe has of k3s having restarted (a voter sees it in etcd's counters going backwards). +report() { + local first=$1 now entry voters is_voter eligible tcp='{}' peer_name peer_ip ms etcd='null' restarted=false cumulative='null' etcd_id='null' + local iface fsync='null' fsync_at patch metrics reading change psi='' + now="$(date +%s)" + # This node's entry is read from the ConfigMap once, when the process starts, and kept from then on: the probe is the only writer of its key, and two API calls a sample (the nodes, then the patch) are all a probe on a poor path can be asked to make in a period. + if [ "$entry_loaded" != true ]; then + entry="$(kubectl get configmap "$PROBE_CONFIGMAP" -n "$NAMESPACE" -o json | jq -c --arg node "$NODE_NAME" '(.data[$node] // "null") | fromjson')" || return 1 + entry_loaded=true + else + entry="$kept_entry" + fi + nodes_json="$(kubectl get nodes -o json)" || return 1 + voters="$(jq -c --arg label "${VOTER_LABEL%%=*}" --arg value "${VOTER_LABEL#*=}" '[.items[] | select((.metadata.labels // {})[$label] == $value) | {name: .metadata.name, ip: ([.status.addresses[]? | select(.type == "InternalIP") | .address] | .[0])}]' <<< "$nodes_json")" + is_voter="$(jq --arg node "$NODE_NAME" 'any(.[]; .name == $node)' <<< "$voters")" + eligible="$(jq --arg node "$NODE_NAME" --arg label "$ELIGIBLE_LABEL" 'any(.items[]; .metadata.name == $node and (.metadata.labels // {})[$label] == "true")' <<< "$nodes_json")" + if [ "$is_voter" != true ] && [ "$eligible" != true ]; then + # A key set to null in a merge patch is removed. + if [ "$entry" != null ]; then + patch="$(mktemp)" + jq -n --arg node "$NODE_NAME" '{data: {($node): null}}' > "$patch" + kubectl patch configmap "$PROBE_CONFIGMAP" -n "$NAMESPACE" --type merge --patch-file "$patch" > /dev/null + rm -f "$patch" + kept_entry=null + fi + return 0 + fi + + while IFS=$'\t' read -r peer_name peer_ip; do + ms="$(connect_ms "$peer_ip")" + tcp="$(jq -c --arg peer "$peer_name" --argjson ms "$ms" '. + {($peer): $ms}' <<< "$tcp")" + done < <(jq -r --arg node "$NODE_NAME" '.[] | select(.name != $node) | [.name, .ip] | @tsv' <<< "$voters") + + if [ "$is_voter" = true ]; then + if metrics="$(curl --silent --show-error --fail --max-time "$REPORT_SECONDS" "$ETCD_METRICS_URL")"; then + reading="$(jq -Rs -L "$MODULES" 'include "fleet-role-stats"; etcd_reading' <<< "$metrics")" + change="$(jq -c -L "$MODULES" --argjson previous "$(jq -c '.last' <<< "$entry")" --argjson now "$reading" 'include "fleet-role-stats"; etcd_sample($previous; $now)' <<< null)" + etcd="$(jq -c '.sample' <<< "$change")" + restarted="$(jq -c '.restarted' <<< "$change")" + cumulative="$(jq -c '.cumulative' <<< "$change")" + etcd_id="$(jq -c '.id' <<< "$reading")" + else + log "could not read ${ETCD_METRICS_URL}; this sample has no etcd reading" + cumulative="$(jq -c '.last' <<< "$entry")" + etcd_id="$(jq -c '.etcd_id' <<< "$entry")" + fi + else + restarted="$first" + fi + + iface="$(default_route_interface)" + fsync_at="$(jq -c '.fsync_at' <<< "$entry")" + if [ "$is_voter" != true ] && { [ "$fsync_at" = null ] || [ $((now - fsync_at)) -ge "$window_seconds" ]; }; then + fsync="$(fsync_ms)" + fsync_at="$now" + fi + + # A kernel without pressure stall information has no such file, and then the sample has no pressure reading. + if [ -r "$PSI_FILE" ]; then + psi="$(psi_avg300)" + fi + + # The entry goes in on standard input, never as an argument: it holds a window of samples, and one argument is limited to a fraction of the argument space (128 KiB on Linux). + new_entry="$(jq -c -L "$MODULES" --argjson now "$now" --argjson tcp "$tcp" --argjson etcd "$etcd" \ + --argjson restarted "$restarted" --argjson cumulative "$cumulative" --argjson etcd_id "$etcd_id" --argjson fsync "$fsync" --argjson fsync_at "$fsync_at" \ + --arg psi "$psi" --arg avail "$(meminfo_kb MemAvailable)" --arg total "$(meminfo_kb MemTotal)" --arg iface "$iface" --arg link "$(link_type "$iface")" ' + include "fleet-role-stats"; + def number: if . == "" then null else tonumber end; + . as $entry + | {t: $now, tcp: $tcp, etcd: $etcd, restarted: $restarted, psi: ($psi | number), avail_kb: ($avail | number), total_kb: ($total | number), + link: $link, iface: (if $iface == "" then null else $iface end), fsync_ms: $fsync} as $sample + | {etcd_id: $etcd_id, last: $cumulative, fsync_at: $fsync_at, samples: ((($entry.samples // []) + [$sample]) | .[-window_samples:])}' <<< "$entry")" || return 1 + patch="$(mktemp)" + jq --arg node "$NODE_NAME" '{data: {($node): tojson}}' <<< "$new_entry" > "$patch" + kubectl patch configmap "$PROBE_CONFIGMAP" -n "$NAMESPACE" --type merge --patch-file "$patch" > /dev/null || { rm -f "$patch"; return 1; } + rm -f "$patch" + # Kept only once published, so a patch that failed is read again from the ConfigMap and not taken as written. + kept_entry="$new_entry" +} + +# One reading alone, for comparing with a reading taken by hand. +if [ "${1:-}" = check ]; then + case "${2:-}" in + link) + iface="$(default_route_interface)" + jq -nc --arg interface "$iface" --arg link "$(link_type "$iface")" '{interface: $interface, link: $link}' + ;; + connect) connect_ms "${3:?usage: check connect
}" ;; + memory) jq -nc --arg psi "$(psi_avg300)" --arg avail "$(meminfo_kb MemAvailable)" --arg total "$(meminfo_kb MemTotal)" '{psi_avg300: ($psi | tonumber), mem_available_kb: ($avail | tonumber), mem_total_kb: ($total | tonumber)}' ;; + etcd) curl --silent --show-error --fail --max-time "$REPORT_SECONDS" "$ETCD_METRICS_URL" | jq -Rsc -L "$MODULES" 'include "fleet-role-stats"; etcd_reading | {id, has_leader, is_learner, known_members, active_peers, quota_bytes, leader_changes, wal_fsync_p99_ms: (.wal | per_bucket | histogram_quantile(tail_percentile) * 1000), backend_commit_p99_ms: (.commit | per_bucket | histogram_quantile(tail_percentile) * 1000)}' ;; + fsync) fsync_ms ;; + *) echo "usage: $0 check link|connect
|memory|etcd|fsync" >&2; exit 2 ;; + esac + exit 0 +fi + +: "${NODE_NAME:?NODE_NAME is required}" +: "${NAMESPACE:?NAMESPACE is required}" +: "${ELECTION_TIMEOUT_MS:?ELECTION_TIMEOUT_MS is required}" +: "${ELIGIBLE_LABEL:?ELIGIBLE_LABEL is required}" + +# This node's entry as the ConfigMap holds it, read once and then kept (see report). +entry_loaded=false +kept_entry=null +first=true +while true; do + # A sample the API would not take is not recorded, and the next period tries again: a probe on a poor path is the case the controller exists for, and a process that exited on it would be started again as a restart. "First" stays until a sample is recorded, since it is the only sign a node that is not a voter has of its k3s having restarted. + if report "$first"; then + first=false + else + [ "$REPORT_ONCE" != true ] || exit 1 + log "could not record a sample; trying again in ${REPORT_SECONDS}s" + fi + [ "$REPORT_ONCE" = true ] && exit 0 + sleep "$REPORT_SECONDS" +done diff --git a/scripts/fleet-role-stats.jq b/scripts/fleet-role-stats.jq new file mode 100644 index 00000000..8ab28111 --- /dev/null +++ b/scripts/fleet-role-stats.jq @@ -0,0 +1,91 @@ +# Statistics and etcd metrics parsing shared by the role controller's probe (scripts/fleet-role-probe.sh) and the controller itself (scripts/fleet-role.jq), included with `include "fleet-role-stats";` and jq -L naming this directory. + +# The tail the controller judges a host by, as a whole percentile. +def tail_percentile: 99; + +# The fewest samples that give the tail percentile a sample of its own: with this many, the nearest-rank percentile is the second largest, so one outlier in the window never decides alone. 100 / (100 - 99) is exact in floating point, which 1 / (1 - 0.99) is not. +def window_samples: 100 / (100 - tail_percentile); + +# The nearest-rank percentile of an array of numbers: the smallest value that at least $p percent of the array is at or below. Null for an empty array. +def nearest_rank($p): if length == 0 then null else sort | .[((length * $p / 100) | ceil) - 1] end; + +# A histogram's bucket bounds as numbers, with the +Inf bucket as infinity. +def bound: if . == "+Inf" then infinite else tonumber end; + +# The $p percentile of a histogram given as an object from a bucket's upper bound (a string, as Prometheus writes it) to the number of observations that fell in that bucket alone, interpolated linearly inside the bucket the rank falls in, as Prometheus's histogram_quantile does. A rank in the +Inf bucket gives the largest finite bound. Null when there are no observations. +def histogram_quantile($p): + to_entries | map({le: (.key | bound), n: .value}) | sort_by(.le) as $buckets + | (map(.n) | add // 0) as $total + | if $total == 0 then null else + ($total * $p / 100) as $rank + | reduce $buckets[] as $b ({cum: 0, lower: 0, out: null}; + if .out != null then . + else (.cum + $b.n) as $cum + | if $cum >= $rank then + {cum: $cum, lower: $b.le, + out: (if $b.le == infinite then .lower else .lower + ($b.le - .lower) * (($rank - .cum) / $b.n) end)} + else {cum: $cum, lower: $b.le, out: null} end + end) + | .out + end; + +# The sum of several such objects, bucket by bucket. +def sum_buckets: reduce .[] as $o ({}; reduce ($o | keys[]) as $k (.; .[$k] += $o[$k])); + +# A cumulative histogram (upper bound to the number of observations at or below it, as Prometheus exposes it) as the number of observations in each bucket alone. +def per_bucket: + to_entries | map({key: .key, le: (.key | bound), n: .value}) | sort_by(.le) as $b + | [range(0; $b | length) | {(($b[.]).key): (($b[.]).n - (if . == 0 then 0 else ($b[. - 1]).n end))}] | add // {}; + +# What a histogram gained between two readings of it, per bucket and without the empty ones, and whether it went backwards, which only a restart of the process that keeps it does. $previous is null on the first reading, which has nothing to subtract from. +def gained($previous; $now): + ($now | per_bucket) as $after + | if $previous == null then {reset: false, gained: {}} + else ($previous | per_bucket) as $before + | ($after | with_entries(.value -= ($before[.key] // 0))) as $delta + | {reset: ($delta | any(.[]; . < 0)), gained: ($delta | with_entries(select(.value > 0)))} + end; + +# The labels of one line of the Prometheus text format as an object. +def labels: [scan("([A-Za-z_][A-Za-z0-9_]*)=\"([^\"]*)\"") | {(.[0]): .[1]}] | add // {}; + +# The etcd series of a metrics page (read raw, `jq -Rs`), as {name, labels, value}. +def etcd_series: + split("\n") + | map(select(startswith("etcd_")) + | capture("^(?[A-Za-z0-9_:]+)(\\{(?[^}]*)\\})?[ \t]+(?[^ \t]+)$") + | {name, labels: ((.labels // "") | labels), value: (.value | tonumber?)} + | select(.value != null)); + +def histogram_of($series; $name): [$series[] | select(.name == ($name + "_bucket")) | {(.labels.le): .value}] | add // {}; + +# What the probe keeps of an etcd metrics page, cumulative as etcd reports it: the member's own id (the `To` label of its peers' round trip histograms is this id), whether it has a leader, whether it is a learner, the number of members it knows (itself included, and a learner or a member that has not started too, which is how a membership change in progress shows) and the number of peers it is connected to (etcd keeps one gauge per peer that counts its active streams to it, up to two, so a peer counts when its gauge is at least 1), its storage quota, the leader changes it has seen, and the three histograms the controller judges (the WAL fsync, the backend commit and, per peer id, the round trip). +def etcd_reading: + etcd_series as $s + | def gauge($name): [$s[] | select(.name == $name) | .value] | .[0]; + {id: ([$s[] | select(.name == "etcd_server_id") | .labels.server_id] | .[0]), + has_leader: (gauge("etcd_server_has_leader") == 1), + is_learner: (gauge("etcd_server_is_learner") == 1), + known_members: ([$s[] | select(.name == "etcd_network_known_peers" and .value == 1)] | length), + active_peers: ([$s[] | select(.name == "etcd_network_active_peers" and .value >= 1) | .labels.Remote] | unique | length), + quota_bytes: gauge("etcd_server_quota_backend_bytes"), + leader_changes: gauge("etcd_server_leader_changes_seen_total"), + wal: histogram_of($s; "etcd_disk_wal_fsync_duration_seconds"), + commit: histogram_of($s; "etcd_disk_backend_commit_duration_seconds"), + rtt: ([$s[] | select(.name == "etcd_network_peer_round_trip_time_seconds_bucket") | {to: .labels.To, le: .labels.le, value}] + | group_by(.to) | map({(.[0].to): (map({(.le): .value}) | add)}) | add // {})}; + +# The change from the cumulative readings kept after the last sample ($previous, null for the first) to the one just read ($now, an etcd_reading): the gauges as they are, each histogram as what it gained, and whether any went backwards, which means etcd restarted. A peer whose round trip histogram has not been seen before has nothing to subtract from, so it gains nothing in this sample. +def etcd_sample($previous; $now): + gained($previous.wal; $now.wal) as $wal + | gained($previous.commit; $now.commit) as $commit + | ($now.rtt | to_entries | map({key, gain: gained($previous.rtt[.key]; .value)})) as $rtt + | {sample: {has_leader: $now.has_leader, is_learner: $now.is_learner, known_members: $now.known_members, active_peers: $now.active_peers, quota_bytes: $now.quota_bytes, + leader_changes: (if $previous == null or $now.leader_changes < $previous.leader_changes then 0 else $now.leader_changes - $previous.leader_changes end), + wal: $wal.gained, commit: $commit.gained, + rtt: ($rtt | map({(.key): .gain.gained}) | add // {})}, + restarted: ($previous != null and ($wal.reset or $commit.reset or any($rtt[]; .gain.reset) or $now.leader_changes < $previous.leader_changes)), + cumulative: {wal: $now.wal, commit: $now.commit, rtt: $now.rtt, leader_changes: $now.leader_changes}}; + +# The part of a probe sample's memory reading the regrade core calls over the limit (fleet-regrade-core.sh, is_over): pressure at or above the threshold and MemAvailable below the floor share of MemTotal, both. tests/unit/test_fleet_role.py holds this to the core's own function. $c is the controller's settings (pressure_threshold, available_floor_percent). +def over_regrade_limit($c): .psi >= $c.pressure_threshold and (.avail_kb * 100) < (.total_kb * $c.available_floor_percent); diff --git a/scripts/fleet-role.jq b/scripts/fleet-role.jq new file mode 100644 index 00000000..d10c1e07 --- /dev/null +++ b/scripts/fleet-role.jq @@ -0,0 +1,242 @@ +# The role controller's decision (see scripts/fleet-role.sh, which gathers the inputs and publishes the result): which hosts are fit to hold an etcd vote, and the swap the controller would make. Pure: nothing here reads the cluster or the clock. +# +# Inputs, as jq named arguments: +# - $config: the controller's settings (see scripts/fleet-role.sh for each), with `now` the time of the pass in epoch seconds. +# - $nodes_file: the output of `kubectl get nodes -o json`, as --slurpfile gives it (a one-element array). +# - $probes_file: each node's probe entry (scripts/fleet-role-probe.sh), by node name, likewise. +# - $previous_file: the status this program published on the last pass, or null on the first, likewise. +# +# Every threshold is derived, and every constant below is a fact about etcd or k3s with its source. The figures are read over a window of tail_window samples per peer pair; a host with fewer is "unknown", never fit or unfit. + +include "fleet-role-stats"; + +# etcd's tuning guide: the election timeout must be at least ten times the round trip between members, so a voter whose tail round trip passes a tenth of it is unfit. +def unfit_rtt_divisor: 10; + +# etcd's tuning guide: the heartbeat interval is best at about 0.5 to 1.5 times the round trip. A candidate is fit only at the 0.5 end, where the heartbeat is twice the round trip, a twentieth of the election timeout since the election timeout is ten heartbeats. The gap between this and the unfit limit is the hysteresis, so a host does not alternate between the two on one noisy window. +def fit_rtt_divisor: 20; + +# etcd's FAQ: a disk is fast enough when the WAL fsync tail is under 10 ms and the backend commit tail under 25 ms. +def wal_fsync_limit_ms: 10; +def backend_commit_limit_ms: 25; + +# k3s's learnerMaxStallTime (pkg/etcd/etcd.go): a learner that makes no progress for this long is removed, so a membership change has settled this long after it. +def learner_stall_seconds: 300; + +# A probe report older than this many samples counts as no report, as the autoscaler treats a node pressure report older than three of its polls. +def stale_after_samples: 3; + +# Linux's byte count of one kibibyte, MemAvailable being reported in kB that are kibibytes. +def kib: 1024; + +def link_rank: {"wired": 0, "wifi": 1, "unknown": 2}; + +def round2: if . == null then null else (. * 100 | round) / 100 end; + +def ready_condition: ((.status.conditions // []) | map(select(.type == "Ready")) | .[0]); + +# The pair figure of one observer towards one peer over a window: the worse of the TCP connect tail and etcd's own round trip tail (only a voter has the latter, keyed by the peer's etcd id), in milliseconds, with how many TCP readings it rests on. +def pair($window; $peer; $peer_id): + ([$window[] | .tcp[$peer] | select(. != null)]) as $tcp + | (if $peer_id == null then null else ([$window[] | .etcd.rtt[$peer_id] | select(. != null)] | sum_buckets | histogram_quantile(tail_percentile)) end) as $etcd + | ($tcp | nearest_rank(tail_percentile)) as $tcp_tail + | {peer: $peer, readings: ($tcp | length), + tcp_ms: $tcp_tail, + etcd_ms: (if $etcd == null then null else $etcd * 1000 end), + ms: ([$tcp_tail, (if $etcd == null then null else $etcd * 1000 end)] | map(select(. != null)) | max)}; + +$config as $c +| $nodes_file[0] as $nodes +| $probes_file[0] as $probes +| $previous_file[0] as $previous +| $c.now as $now +| ($c.sample_seconds * window_samples) as $window_seconds +| ($c.election_timeout_ms / unfit_rtt_divisor) as $unfit_rtt_ms +| ($c.election_timeout_ms / fit_rtt_divisor) as $fit_rtt_ms +| (learner_stall_seconds + $window_seconds) as $cooldown_seconds +| ($c.sample_seconds * stale_after_samples) as $stale_seconds + +# The nodes, with what the decision reads off each. +| [$nodes.items[] + | (.metadata.labels // {}) as $labels + | ready_condition as $ready + | {name: .metadata.name, + ready: ($ready.status == "True"), + ready_since: ($ready.lastTransitionTime | fromdateiso8601? // null), + voter: ($labels[$c.voter_label] == "true"), + eligible: ($labels[$c.eligible_label] == "true"), + pinned: ($labels[$c.pinned_label] == "true"), + regrade_level: $labels[$c.level_label]}] as $all +| ($all | map(select(.voter)) | sort_by(.name)) as $voters +| ($voters | map(.name)) as $voter_names + +# The membership, and when it last changed: a voter set that differs from the last pass's. The first pass has nothing to compare with and treats the membership as settled, so its first window is not held back by a change it did not see. +| (if $previous == null then null + elif $previous.membership.voters != $voter_names then $now + else $previous.membership.changed_at end) as $changed_at +| (if $changed_at == null then null else $changed_at + $cooldown_seconds end) as $cooldown_until + +# The storage quota a voter's etcd reports, the largest of them, or etcd's own default when no voter reports one. +| ([$voters[] | $probes[.name].samples[-1].etcd.quota_bytes | select(. != null)] | max // $c.default_quota_bytes) as $quota_bytes + +| def window($name): ($probes[$name].samples // []) | map(select(.t >= ($changed_at // 0))) | .[-window_samples:]; + + def measure($node): + $node.name as $name + | $probes[$name] as $entry + | window($name) as $w + | ($entry.samples[-1].t // null) as $last_at + | (if $node.voter then ($voter_names | map(select(. != $name))) else $voter_names end) as $peers + | [$peers[] as $peer | pair($w; $peer; $probes[$peer].etcd_id)] as $pairs + | ($pairs | map(.ms) | map(select(. != null)) | max) as $rtt_ms + | ($pairs | max_by(.ms // 0) | .peer) as $rtt_peer + | ($w | map(.etcd) | map(select(. != null))) as $etcd + | ([$etcd[].wal] | sum_buckets | histogram_quantile(tail_percentile)) as $wal + | ([$etcd[].commit] | sum_buckets | histogram_quantile(tail_percentile)) as $commit + | ([$w[] | .fsync_ms | select(. != null)] | max) as $fsync + | ($w | map(select(.psi != null and over_regrade_limit($c))) | length) as $over + | ($w | map(.avail_kb | select(. != null)) | min) as $min_avail_kb + | ($w | map(select(.restarted == true)) | length) as $restarts + | ($node.ready | not) as $not_ready + | ($node.ready_since != null and $node.ready_since >= ($now - $window_seconds)) as $spell + | { + node: $node, + reported_at: $last_at, + fresh: ($last_at != null and $last_at >= ($now - $stale_seconds)), + samples: ($w | length), + link: ($w[-1].link // "unknown"), + interface: ($w[-1].iface // null), + pairs: $pairs, + rtt_ms: $rtt_ms, + rtt_peer: $rtt_peer, + pairs_complete: ($pairs | all(.readings >= window_samples)), + wal_p99_ms: (if $wal == null then null else $wal * 1000 end), + commit_p99_ms: (if $commit == null then null else $commit * 1000 end), + fsync_p99_ms: $fsync, + over_limit_samples: $over, + min_available_bytes: (if $min_avail_kb == null then null else $min_avail_kb * kib end), + restarts: $restarts, + not_ready: $not_ready, + not_ready_spell: $spell, + leader_changes: ($etcd | map(.leader_changes) | add // 0), + etcd: ($entry.samples[-1].etcd // null) + }; + + # What makes a measured host unfit, as sentences carrying their numbers; empty when nothing does. A voter is judged by the unfit round trip limit and the disk it serves etcd from, a candidate by the stricter fit limit and a disk probe. + def faults($m): + $m.node as $node + | [ + (if $m.rtt_ms != null and $m.rtt_ms > (if $node.voter then $unfit_rtt_ms else $fit_rtt_ms end) + then "round trip p99 to \($m.rtt_peer) is \($m.rtt_ms | round2) ms, over \(if $node.voter then $unfit_rtt_ms else $fit_rtt_ms end) ms (a \(if $node.voter then "tenth" else "twentieth" end) of the \($c.election_timeout_ms) ms election timeout)" + else empty end), + (if $node.voter then + (if $m.wal_p99_ms != null and $m.wal_p99_ms >= wal_fsync_limit_ms then "WAL fsync p99 is \($m.wal_p99_ms | round2) ms, not under \(wal_fsync_limit_ms) ms" else empty end), + (if $m.commit_p99_ms != null and $m.commit_p99_ms >= backend_commit_limit_ms then "backend commit p99 is \($m.commit_p99_ms | round2) ms, not under \(backend_commit_limit_ms) ms" else empty end) + else + (if $m.fsync_p99_ms != null and $m.fsync_p99_ms >= wal_fsync_limit_ms then "fdatasync p99 is \($m.fsync_p99_ms | round2) ms, not under \(wal_fsync_limit_ms) ms" else empty end) + end), + (if $m.over_limit_samples > 0 then "memory was over the regrade limit in \($m.over_limit_samples) of \($m.samples) samples" else empty end), + (if $m.min_available_bytes != null and $m.min_available_bytes < $quota_bytes then "MemAvailable fell to \(($m.min_available_bytes / 1048576) | round) MiB, under etcd's \(($quota_bytes / 1048576) | round) MiB storage quota" else empty end), + (if $m.not_ready then "the node is not Ready" elif $m.not_ready_spell then "the node was not Ready within the last \($window_seconds) s" else empty end), + (if $m.restarts > 0 then "k3s restarted \($m.restarts) time(s) in the window" else empty end) + ]; + + # What is missing for a verdict, as sentences; empty when the readings are all there. + def gaps($m): + $m.node as $node + | [ + (if $m.reported_at == null then "no probe report" elif ($m.fresh | not) then "the probe's last report is \($now - $m.reported_at) s old" else empty end), + (if $m.samples < window_samples then "\($m.samples) of \(window_samples) samples since the last membership change" else empty end), + (if ($m.pairs | length) == 0 then "no voter to measure against" + elif ($m.pairs_complete | not) then "fewer than \(window_samples) round trip readings to some voter" else empty end), + (if $node.voter and ($m.wal_p99_ms == null or $m.commit_p99_ms == null) then "no etcd disk readings" else empty end), + (if ($node.voter | not) and $m.fsync_p99_ms == null then "no fdatasync reading yet" else empty end) + ]; + + def disk_ms($m): $m.node.voter as $voter | if $voter then $m.wal_p99_ms else $m.fsync_p99_ms end; + + def rank($m): [(link_rank[$m.link] // link_rank.unknown), ($m.rtt_ms // infinite), (disk_ms($m) // infinite), $m.node.name]; + + # A measurement as if $peer were not a voter: a candidate is judged against the voters that would remain, since a slow link to the very voter it would replace says nothing against it. + def without($peer): + .pairs |= map(select(.peer != $peer)) + | (.pairs | map(.ms) | map(select(. != null)) | max) as $rtt + | .rtt_ms = $rtt + | .rtt_peer = (.pairs | max_by(.ms // 0) | .peer) + | .pairs_complete = (.pairs | all(.readings >= window_samples)); + + def host($node): + measure($node) as $m + | gaps($m) as $gaps + | faults($m) as $faults + | {name: $node.name, + role: (if $node.voter then "voter" else "standby" end), + eligible: $node.eligible, + pinned: $node.pinned, + verdict: (if ($gaps | length) > 0 then "unknown" elif ($faults | length) > 0 then "unfit" else "fit" end), + reasons: (if ($gaps | length) > 0 then $gaps else $faults end), + link: $m.link, + samples: $m.samples, + figures: {rtt_p99_ms: ($m.rtt_ms | round2), rtt_peer: $m.rtt_peer, + wal_fsync_p99_ms: ($m.wal_p99_ms | round2), backend_commit_p99_ms: ($m.commit_p99_ms | round2), fdatasync_p99_ms: ($m.fsync_p99_ms | round2), + min_available_mib: (if $m.min_available_bytes == null then null else ($m.min_available_bytes / 1048576 | round) end), + leader_changes: $m.leader_changes, restarts: $m.restarts}, + rank: rank($m), + m: $m}; + + # Hosts reported: every voter, and every eligible host that is not one. + ([$all[] | select(.voter or .eligible)] | sort_by(.name) | map(host(.))) as $hosts +| ([$hosts[] | select(.eligible)] | length) as $eligible_count +| ($all | map(select(.voter))) as $voter_nodes + +# Why no swap may start now whatever the hosts' fitness: the epic's conditions on the cluster as a whole. +| ([ + (if ($voters | length) != $c.target then "there are \($voters | length) voters, not the target \($c.target)" else empty end), + ($voter_nodes[] | select(.regrade_level != null) | "voter \(.name) is at regrade level \(.regrade_level), below full"), + ($voter_nodes[] | select(.ready | not) | "voter \(.name) is not Ready"), + ($voter_names[] as $v | $probes[$v].samples[-1] as $last | $last.etcd as $e + | if $last == null or $last.t < ($now - $stale_seconds) then "voter \($v) has no probe report in the last \($stale_seconds) s" + elif $e == null then "etcd's state on voter \($v) is unknown" + elif ($e.has_leader | not) then "voter \($v) reports no leader" + elif $e.is_learner then "voter \($v) is a learner" + elif $e.known_members != ($voters | length) then "voter \($v) knows \($e.known_members) members, not the \($voters | length) voters: a learner or a member that has not started" + elif $e.active_peers != (($voters | length) - 1) then "voter \($v) is connected to \($e.active_peers) of its \(($voters | length) - 1) peers" + else empty end), + (if $cooldown_until != null and $now < $cooldown_until then "the membership changed at \($changed_at), and the cooldown lasts until \($cooldown_until)" else empty end) + ]) as $blocks + +# The swap. A slow link makes both of its ends unfit, since a voter is unfit when its round trip to any other voter is, so the voter to replace is the one at fault: the unfit voter on the most slow links (a link is slow when either end measures it above the unfit limit), the worst-ranked among those on as many. It goes out only if it is eligible and not pinned; when it is not, replacing a voter that shares a slow link with it would fix nothing, so nothing is swapped. The best fit eligible standby that ranks above it comes in, judged against the voters that would remain. +| ([$hosts[] | select(.role == "voter") | .name as $a | .m.pairs[] | select(.ms != null and .ms > $unfit_rtt_ms) | [$a, .peer] | sort] | unique) as $slow_links +| ([$hosts[] | select(.role == "voter" and .verdict == "unfit") | . + {slow_links: (.name as $n | [$slow_links[] | select(index($n) != null)] | length)}] | sort_by(.rank) | reverse | sort_by(-.slow_links) | first) as $culprit +| (if $culprit != null and $culprit.eligible and ($culprit.pinned | not) then $culprit else null end) as $out +| ([$hosts[] | select(.role == "standby" and .eligible) + | (if $out == null then .m else (.m | without($out.name)) end) as $m + | select((gaps($m) | length) == 0 and (faults($m) | length) == 0) + | . + {rank: rank($m)}] | sort_by(.rank)) as $fit_candidates +| ($fit_candidates | map(select($out != null and .rank < $out.rank)) | first) as $in +| (if $eligible_count == 0 then ["no host is marked eligible"] + elif $eligible_count < $c.target then ["\($eligible_count) hosts are eligible, fewer than the target \($c.target)"] + else [] end) as $shortfall +| (if $culprit == null then + {action: "none", swap: null, because: (["no voter is unfit"] + $shortfall)} + elif $out == null then + {action: "none", swap: null, because: (["voter \($culprit.name) is the most at fault and is pinned or not marked eligible, so it is not replaced"] + $shortfall)} + elif $in == null then + {action: "none", swap: null, because: (["voter \($out.name) is unfit and no eligible standby is fit and ranks above it"] + $shortfall)} + elif ($blocks | length) > 0 or ($shortfall | length) > 0 then + {action: "none", swap: null, because: ($blocks + $shortfall), wanted: {out: $out.name, in: $in.name}} + else + {action: "swap", swap: {out: $out.name, in: $in.name}, because: ["voter \($out.name) is unfit (\($out.reasons | join("; "))) and \($in.name) is fit against the voters that would remain and ranks above it"]} + end) as $decision + +| { + config: {target: $c.target, max_servers: $c.max_servers, election_timeout_ms: $c.election_timeout_ms, + unfit_rtt_ms: $unfit_rtt_ms, fit_rtt_ms: $fit_rtt_ms, wal_fsync_limit_ms: wal_fsync_limit_ms, backend_commit_limit_ms: backend_commit_limit_ms, + window_samples: window_samples, sample_seconds: $c.sample_seconds, window_seconds: $window_seconds, cooldown_seconds: $cooldown_seconds, + quota_bytes: $quota_bytes}, + membership: {voters: $voter_names, changed_at: $changed_at, cooldown_until: $cooldown_until}, + hosts: ($hosts | map({key: .name, value: del(.name, .rank, .m)}) | from_entries), + ranking: ($hosts | map(select(.verdict == "fit")) | sort_by(.rank) | map(.name)), + blocks: $blocks, + decision: $decision + } diff --git a/scripts/fleet-role.sh b/scripts/fleet-role.sh new file mode 100755 index 00000000..8a7a78e0 --- /dev/null +++ b/scripts/fleet-role.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# One pass of the role controller (see roles/github_runner_arc/templates/fleet-role.yaml.j2 and fleet-role/loop.sh, which runs this on an interval). It decides which of the hosts the owner marks eligible are fit to hold an etcd vote, from what scripts/fleet-role-probe.sh publishes, and which swap of an unfit voter for a fit standby it would make. It changes nothing in the cluster: its only write is its own status ConfigMap, which the heartbeat republishes. The decision is scripts/fleet-role.jq; this script gathers its inputs and publishes the result. +# +# It runs unchanged with no host marked eligible, and then only reports the voters' fitness. +# +# Environment (provided by the Deployment, see the template): +# - FLEET_ROLE_DRY_RUN: must be "true", the default. The controller has no way to change a role; the setting exists so that a Deployment that names it keeps working when it can. +# - FLEET_ROLE_VOTERS: the number of voters the cluster is held at. Odd, at least 3, and at most FLEET_ROLE_MAX_SERVERS minus one, which leaves a swap one place of headroom (the new voter joins before the old one leaves). Required. +# - FLEET_ROLE_MAX_SERVERS: the most servers the cluster may have (github_runner_cluster_max_servers). Required. +# - FLEET_ROLE_ELECTION_TIMEOUT_MS: the cluster's etcd election timeout in milliseconds, k3s's own default of 1000 where the cluster configures none (the role works that out). Required. +# - FLEET_ROLE_SAMPLE_SECONDS: the probe's sample period. Required; the window the figures are read over is this times the samples a window holds. +# - FLEET_ROLE_PRESSURE_THRESHOLD, FLEET_ROLE_AVAILABLE_FLOOR_PERCENT: the regrade core's limit (github_runner_cluster_regrade_pressure_threshold and github_runner_cluster_regrade_available_floor_percent). Required. +# - FLEET_ROLE_ETCD_QUOTA_BYTES: etcd's storage quota, used for a host's memory floor when no voter reports one; etcd's own default when unset. +# - FLEET_ROLE_ELIGIBLE_LABEL, FLEET_ROLE_PINNED_LABEL: the Node labels, with the value true, that mark a host the controller may promote or demote and one it must never demote. Required. +# - FLEET_ROLE_LEVEL_LABEL: the label regrading puts on a server below full, with the level as its value. Required. +# - FLEET_ROLE_STATE_NAMESPACE, FLEET_ROLE_PROBE_CONFIGMAP, FLEET_ROLE_STATUS_CONFIGMAP: where the probes publish and this controller keeps its status; both ConfigMaps are created by the role. +set -euo pipefail + +# etcd's default storage quota, 2 GiB, in bytes (--quota-backend-bytes). +readonly etcd_default_quota_bytes=2147483648 +# etcd needs a majority, so fewer than three voters tolerate no failure. +readonly min_voters=3 + +dry_run="${FLEET_ROLE_DRY_RUN:-true}" +target="${FLEET_ROLE_VOTERS:?FLEET_ROLE_VOTERS must be set}" +max_servers="${FLEET_ROLE_MAX_SERVERS:?FLEET_ROLE_MAX_SERVERS must be set}" +election_timeout_ms="${FLEET_ROLE_ELECTION_TIMEOUT_MS:?FLEET_ROLE_ELECTION_TIMEOUT_MS must be set}" +sample_seconds="${FLEET_ROLE_SAMPLE_SECONDS:?FLEET_ROLE_SAMPLE_SECONDS must be set}" +pressure_threshold="${FLEET_ROLE_PRESSURE_THRESHOLD:?FLEET_ROLE_PRESSURE_THRESHOLD must be set}" +available_floor_percent="${FLEET_ROLE_AVAILABLE_FLOOR_PERCENT:?FLEET_ROLE_AVAILABLE_FLOOR_PERCENT must be set}" +default_quota_bytes="${FLEET_ROLE_ETCD_QUOTA_BYTES:-$etcd_default_quota_bytes}" +eligible_label="${FLEET_ROLE_ELIGIBLE_LABEL:?FLEET_ROLE_ELIGIBLE_LABEL must be set}" +pinned_label="${FLEET_ROLE_PINNED_LABEL:?FLEET_ROLE_PINNED_LABEL must be set}" +level_label="${FLEET_ROLE_LEVEL_LABEL:?FLEET_ROLE_LEVEL_LABEL must be set}" +namespace="${FLEET_ROLE_STATE_NAMESPACE:-github-runner-platform}" +probe_configmap="${FLEET_ROLE_PROBE_CONFIGMAP:-fleet-role-probe}" +status_configmap="${FLEET_ROLE_STATUS_CONFIGMAP:-fleet-role-status}" +modules="$(dirname "$0")" + +log() { echo "[$(date -u +%Y-%m-%dT%H:%M:%SZ)] $*"; } +fail() { echo "$*" >&2; exit 1; } + +[ "$dry_run" = true ] || fail "FLEET_ROLE_DRY_RUN must be true: this controller reports the swap it would make and has no way to make one (got '${dry_run}')" +for setting in target max_servers election_timeout_ms sample_seconds default_quota_bytes; do + [[ "${!setting}" =~ ^[1-9][0-9]*$ ]] || fail "${setting} must be a whole number, at least 1, not '${!setting}'" +done +for setting in pressure_threshold available_floor_percent; do + [[ "${!setting}" =~ ^[0-9]+([.][0-9]+)?$ ]] || fail "${setting} must be a number, not '${!setting}'" +done +[ $((target % 2)) -eq 1 ] || fail "FLEET_ROLE_VOTERS must be odd, not ${target}: an even count tolerates no more failures than the odd count below it" +[ "$target" -ge "$min_voters" ] || fail "FLEET_ROLE_VOTERS must be at least ${min_voters}, not ${target}" +[ "$target" -le $((max_servers - 1)) ] || fail "FLEET_ROLE_VOTERS (${target}) must leave a swap one place of headroom below FLEET_ROLE_MAX_SERVERS (${max_servers})" + +now="$(date +%s)" +# The inputs go to jq in files, never as arguments: a probe entry holds a window of samples, the entries together can fill a ConfigMap, and one argument is limited to a fraction of the argument space (128 KiB on Linux), which a fleet's nodes and probes pass. +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +kubectl get nodes -o json > "$work/nodes.json" +# Each node's entry is a JSON document in its own key. +kubectl get configmap "$probe_configmap" -n "$namespace" -o json | jq -c '(.data // {}) | map_values(fromjson)' > "$work/probes.json" +kubectl get configmap "$status_configmap" -n "$namespace" -o json | jq -c '(.data["status.json"] // "") | if . == "" then null else fromjson end' > "$work/previous.json" + +config="$(jq -nc \ + --argjson now "$now" --argjson target "$target" --argjson max_servers "$max_servers" \ + --argjson election_timeout_ms "$election_timeout_ms" --argjson sample_seconds "$sample_seconds" \ + --argjson pressure_threshold "$pressure_threshold" --argjson available_floor_percent "$available_floor_percent" \ + --argjson default_quota_bytes "$default_quota_bytes" \ + --arg eligible_label "$eligible_label" --arg pinned_label "$pinned_label" --arg level_label "$level_label" \ + '{now: $now, target: $target, max_servers: $max_servers, election_timeout_ms: $election_timeout_ms, sample_seconds: $sample_seconds, + pressure_threshold: $pressure_threshold, available_floor_percent: $available_floor_percent, default_quota_bytes: $default_quota_bytes, + voter_label: "node-role.kubernetes.io/etcd", eligible_label: $eligible_label, pinned_label: $pinned_label, level_label: $level_label}')" + +result="$(jq -n -L "$modules" --argjson config "$config" --slurpfile nodes_file "$work/nodes.json" --slurpfile probes_file "$work/probes.json" --slurpfile previous_file "$work/previous.json" -f "$modules/fleet-role.jq")" +status="$(jq -c --arg last_run "$(date -u +%Y-%m-%dT%H:%M:%SZ)" --argjson dry_run true '. + {last_run: $last_run, dry_run: $dry_run}' <<< "$result")" + +# The patch goes in a file: it is read back as a whole, and an argument has a size limit. +jq -n --arg status "$status" '{data: {"status.json": $status}}' > "$work/patch.json" +kubectl patch configmap "$status_configmap" -n "$namespace" --type merge --patch-file "$work/patch.json" > /dev/null + +# What each host's verdict rests on, then the decision, in the log. +while IFS= read -r line; do log "$line"; done < <(jq -r ' + .hosts | to_entries[] + | "\(.key) (\(.value.role)\(if .value.pinned then ", pinned" else "" end)\(if .value.eligible then "" else ", not eligible" end)): \(.value.verdict), \(.value.reasons | if length == 0 then "all readings within their limits" else join("; ") end)"' <<< "$result") +while IFS= read -r line; do log "$line"; done < <(jq -r ' + if .decision.action == "swap" then "DRY RUN: would swap out \(.decision.swap.out) for \(.decision.swap.in): \(.decision.because | join("; "))" + else "DRY RUN: would not act: \(.decision.because | join("; "))\(if .decision.wanted == null and (.blocks | length) > 0 then " (also: \(.blocks | join("; ")))" else "" end)" end' <<< "$result") diff --git a/scripts/heartbeat.sh b/scripts/heartbeat.sh index 65e0420b..b8dee073 100755 --- a/scripts/heartbeat.sh +++ b/scripts/heartbeat.sh @@ -10,6 +10,8 @@ # - HEALTH_GIST_FILE: the gist file the health reason (healthy, why not, nodes under pressure) is written to every tick (default fleet-health.json) # - HEARTBEAT_STATE_NAMESPACE: namespace holding the autoscaler's own status ConfigMap (see scripts/autoscaler.sh) # - AUTOSCALER_STATUS_CONFIGMAP: name of that ConfigMap +# - FLEET_ROLE_STATUS_CONFIGMAP: name of the ConfigMap, in the state namespace, the role controller (scripts/fleet-role.sh) publishes its status to; empty (the default) when there is no role controller, and nothing is republished +# - FLEET_ROLE_STATUS_GIST_FILE: the gist file that status is republished to (default fleet-role-status.json) # - HEARTBEAT_REGRADE_LEVEL_LABEL: the label memory-pressure regrading puts on a server it has stepped down, with the level as its value (default github-runner.exadev.com/regrade-level, the cluster role's github_runner_cluster_regrade_level_label) # - HEARTBEAT_DEPLOYED_STATE_CONFIGMAP: name of the ConfigMap, in the state namespace, holding the collection version the ARC role last applied (default deployed-state) # - HEARTBEAT_DEPLOYED_VERSION_ANNOTATION: the node annotation holding the collection version the cluster role last applied on that node's host @@ -26,6 +28,8 @@ HEARTBEAT_EXPECTED_SCALE_SETS="${HEARTBEAT_EXPECTED_SCALE_SETS:-}" HEARTBEAT_STATE_NAMESPACE="${HEARTBEAT_STATE_NAMESPACE:-github-runner-platform}" AUTOSCALER_STATUS_CONFIGMAP="${AUTOSCALER_STATUS_CONFIGMAP:-autoscaler-status}" AUTOSCALER_STATUS_GIST_FILE="${AUTOSCALER_STATUS_GIST_FILE:-autoscaler-status.json}" +FLEET_ROLE_STATUS_CONFIGMAP="${FLEET_ROLE_STATUS_CONFIGMAP:-}" +FLEET_ROLE_STATUS_GIST_FILE="${FLEET_ROLE_STATUS_GIST_FILE:-fleet-role-status.json}" DEPLOYED_STATE_CONFIGMAP="${HEARTBEAT_DEPLOYED_STATE_CONFIGMAP:-deployed-state}" DEPLOYED_VERSION_ANNOTATION="${HEARTBEAT_DEPLOYED_VERSION_ANNOTATION:-github-runner.exadev.com/deployed-version}" REGRADE_LEVEL_LABEL="${HEARTBEAT_REGRADE_LEVEL_LABEL:-github-runner.exadev.com/regrade-level}" @@ -124,6 +128,15 @@ if [ "$healthy" = "true" ]; then files_json="$(jq --arg file "$AUTOSCALER_STATUS_GIST_FILE" --arg status "$status_json" \ '. + {($file): {content: $status}}' <<< "$files_json")" fi + # The role controller's status beside the autoscaler's, on the same terms: while the fleet is vouched for, when there is a status to publish. + if [ -n "$FLEET_ROLE_STATUS_CONFIGMAP" ]; then + role_status_json="$(kubectl get configmap "$FLEET_ROLE_STATUS_CONFIGMAP" -n "$HEARTBEAT_STATE_NAMESPACE" \ + -o jsonpath='{.data.status\.json}' 2>/dev/null || true)" + if [ -n "$role_status_json" ]; then + files_json="$(jq --arg file "$FLEET_ROLE_STATUS_GIST_FILE" --arg status "$role_status_json" \ + '. + {($file): {content: $status}}' <<< "$files_json")" + fi + fi fi # The distribution of the peak memory jobs have measured, so whether the runner pods' memory request can come down is a lookup here and not a query of the cluster. It carries only figures, never a repository, workflow or job name: this gist is readable by anyone holding its link, and the names behind the figures stay in the cluster. Written whatever the fleet's health, since it describes past jobs. diff --git a/scripts/release/stamp_version.py b/scripts/release/stamp_version.py index 3597757e..a637bfcb 100755 --- a/scripts/release/stamp_version.py +++ b/scripts/release/stamp_version.py @@ -29,6 +29,7 @@ "github_runner_arc_autoscaler_image", "github_runner_arc_image_pull_secret_renewer_image", "github_runner_arc_node_recovery_image", + "github_runner_arc_role_controller_image", ) diff --git a/tests/fleet_role/enabled.j2 b/tests/fleet_role/enabled.j2 new file mode 100644 index 00000000..7bbec61f --- /dev/null +++ b/tests/fleet_role/enabled.j2 @@ -0,0 +1 @@ +{{ github_runner_arc_role_controller_enabled | bool }} diff --git a/tests/fleet_role/etcd-metrics/after.txt b/tests/fleet_role/etcd-metrics/after.txt new file mode 100644 index 00000000..f1c53a5c --- /dev/null +++ b/tests/fleet_role/etcd-metrics/after.txt @@ -0,0 +1,128 @@ +# HELP etcd_disk_backend_commit_duration_seconds The latency distributions of commit called by backend. +# TYPE etcd_disk_backend_commit_duration_seconds histogram +etcd_disk_backend_commit_duration_seconds_bucket{le="0.001"} 0 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.002"} 0 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.004"} 0 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.008"} 0 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.016"} 10 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.032"} 25 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.064"} 25 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.128"} 25 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.256"} 26 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.512"} 26 +etcd_disk_backend_commit_duration_seconds_bucket{le="1.024"} 26 +etcd_disk_backend_commit_duration_seconds_bucket{le="2.048"} 26 +etcd_disk_backend_commit_duration_seconds_bucket{le="4.096"} 26 +etcd_disk_backend_commit_duration_seconds_bucket{le="8.192"} 26 +etcd_disk_backend_commit_duration_seconds_bucket{le="+Inf"} 26 +etcd_disk_backend_commit_duration_seconds_sum 0.6412199589999998 +etcd_disk_backend_commit_duration_seconds_count 26 +# HELP etcd_disk_wal_fsync_duration_seconds The latency distributions of fsync called by WAL. +# TYPE etcd_disk_wal_fsync_duration_seconds histogram +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.001"} 0 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.002"} 0 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.004"} 0 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.008"} 5 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.016"} 73 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.032"} 79 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.064"} 79 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.128"} 80 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.256"} 80 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.512"} 80 +etcd_disk_wal_fsync_duration_seconds_bucket{le="1.024"} 80 +etcd_disk_wal_fsync_duration_seconds_bucket{le="2.048"} 80 +etcd_disk_wal_fsync_duration_seconds_bucket{le="4.096"} 80 +etcd_disk_wal_fsync_duration_seconds_bucket{le="8.192"} 80 +etcd_disk_wal_fsync_duration_seconds_bucket{le="+Inf"} 80 +etcd_disk_wal_fsync_duration_seconds_sum 1.0339453739999993 +etcd_disk_wal_fsync_duration_seconds_count 80 +# HELP etcd_mvcc_db_total_size_in_bytes Total size of the underlying database physically allocated in bytes. +# TYPE etcd_mvcc_db_total_size_in_bytes gauge +etcd_mvcc_db_total_size_in_bytes 98304 +# HELP etcd_network_active_peers The current number of active peer connections. +# TYPE etcd_network_active_peers gauge +etcd_network_active_peers{Local="d5811b26ae265d01",Remote="2b6d7f19fe480be8"} 1 +etcd_network_active_peers{Local="d5811b26ae265d01",Remote="819450a5536d704f"} 1 +# HELP etcd_network_known_peers The current number of known peers. +# TYPE etcd_network_known_peers gauge +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="2b6d7f19fe480be8"} 1 +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="6c86a033d0032400"} 0 +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="819450a5536d704f"} 1 +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="d5811b26ae265d01"} 1 +# HELP etcd_network_peer_round_trip_time_seconds Round-Trip-Time histogram between peers +# TYPE etcd_network_peer_round_trip_time_seconds histogram +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0001"} 2 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0002"} 2 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0004"} 8 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0008"} 41 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0016"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0032"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0064"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0128"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0256"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0512"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.1024"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.2048"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.4096"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.8192"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="1.6384"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="3.2768"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="+Inf"} 52 +etcd_network_peer_round_trip_time_seconds_sum{To="2b6d7f19fe480be8"} 0.03344976900000001 +etcd_network_peer_round_trip_time_seconds_count{To="2b6d7f19fe480be8"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0001"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0002"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0004"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0008"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0016"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0032"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0064"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0128"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0256"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0512"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.1024"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.2048"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.4096"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.8192"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="1.6384"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="3.2768"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="+Inf"} 44 +etcd_network_peer_round_trip_time_seconds_sum{To="6c86a033d0032400"} 0 +etcd_network_peer_round_trip_time_seconds_count{To="6c86a033d0032400"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0001"} 2 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0002"} 2 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0004"} 15 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0008"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0016"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0032"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0064"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0128"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0256"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0512"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.1024"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.2048"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.4096"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.8192"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="1.6384"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="3.2768"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="+Inf"} 52 +etcd_network_peer_round_trip_time_seconds_sum{To="819450a5536d704f"} 0.031137343999999994 +etcd_network_peer_round_trip_time_seconds_count{To="819450a5536d704f"} 52 +# HELP etcd_server_has_leader Whether or not a leader exists. 1 is existence, 0 is not. +# TYPE etcd_server_has_leader gauge +etcd_server_has_leader 1 +# HELP etcd_server_id Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID. +# TYPE etcd_server_id gauge +etcd_server_id{server_id="d5811b26ae265d01"} 1 +# HELP etcd_server_is_learner Whether or not this member is a learner. 1 if is, 0 otherwise. +# TYPE etcd_server_is_learner gauge +etcd_server_is_learner 0 +# HELP etcd_server_leader_changes_seen_total The number of leader changes seen. +# TYPE etcd_server_leader_changes_seen_total counter +etcd_server_leader_changes_seen_total 1 +# HELP etcd_server_quota_backend_bytes Current backend storage quota size in bytes. +# TYPE etcd_server_quota_backend_bytes gauge +etcd_server_quota_backend_bytes 2.147483648e+09 +# HELP etcd_server_version Which version is running. 1 for 'server_version' label with current version. +# TYPE etcd_server_version gauge +etcd_server_version{server_version="3.6.15"} 1 diff --git a/tests/fleet_role/etcd-metrics/before.txt b/tests/fleet_role/etcd-metrics/before.txt new file mode 100644 index 00000000..068ea751 --- /dev/null +++ b/tests/fleet_role/etcd-metrics/before.txt @@ -0,0 +1,128 @@ +# HELP etcd_disk_backend_commit_duration_seconds The latency distributions of commit called by backend. +# TYPE etcd_disk_backend_commit_duration_seconds histogram +etcd_disk_backend_commit_duration_seconds_bucket{le="0.001"} 0 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.002"} 0 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.004"} 0 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.008"} 0 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.016"} 7 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.032"} 16 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.064"} 16 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.128"} 16 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.256"} 17 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.512"} 17 +etcd_disk_backend_commit_duration_seconds_bucket{le="1.024"} 17 +etcd_disk_backend_commit_duration_seconds_bucket{le="2.048"} 17 +etcd_disk_backend_commit_duration_seconds_bucket{le="4.096"} 17 +etcd_disk_backend_commit_duration_seconds_bucket{le="8.192"} 17 +etcd_disk_backend_commit_duration_seconds_bucket{le="+Inf"} 17 +etcd_disk_backend_commit_duration_seconds_sum 0.48258645899999997 +etcd_disk_backend_commit_duration_seconds_count 17 +# HELP etcd_disk_wal_fsync_duration_seconds The latency distributions of fsync called by WAL. +# TYPE etcd_disk_wal_fsync_duration_seconds histogram +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.001"} 0 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.002"} 0 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.004"} 0 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.008"} 3 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.016"} 34 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.032"} 39 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.064"} 39 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.128"} 40 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.256"} 40 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.512"} 40 +etcd_disk_wal_fsync_duration_seconds_bucket{le="1.024"} 40 +etcd_disk_wal_fsync_duration_seconds_bucket{le="2.048"} 40 +etcd_disk_wal_fsync_duration_seconds_bucket{le="4.096"} 40 +etcd_disk_wal_fsync_duration_seconds_bucket{le="8.192"} 40 +etcd_disk_wal_fsync_duration_seconds_bucket{le="+Inf"} 40 +etcd_disk_wal_fsync_duration_seconds_sum 0.5518709979999998 +etcd_disk_wal_fsync_duration_seconds_count 40 +# HELP etcd_mvcc_db_total_size_in_bytes Total size of the underlying database physically allocated in bytes. +# TYPE etcd_mvcc_db_total_size_in_bytes gauge +etcd_mvcc_db_total_size_in_bytes 98304 +# HELP etcd_network_active_peers The current number of active peer connections. +# TYPE etcd_network_active_peers gauge +etcd_network_active_peers{Local="d5811b26ae265d01",Remote="2b6d7f19fe480be8"} 1 +etcd_network_active_peers{Local="d5811b26ae265d01",Remote="819450a5536d704f"} 1 +# HELP etcd_network_known_peers The current number of known peers. +# TYPE etcd_network_known_peers gauge +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="2b6d7f19fe480be8"} 1 +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="6c86a033d0032400"} 0 +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="819450a5536d704f"} 1 +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="d5811b26ae265d01"} 1 +# HELP etcd_network_peer_round_trip_time_seconds Round-Trip-Time histogram between peers +# TYPE etcd_network_peer_round_trip_time_seconds histogram +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0001"} 2 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0002"} 2 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0004"} 8 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0008"} 41 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0016"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0032"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0064"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0128"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0256"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0512"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.1024"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.2048"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.4096"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.8192"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="1.6384"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="3.2768"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="+Inf"} 52 +etcd_network_peer_round_trip_time_seconds_sum{To="2b6d7f19fe480be8"} 0.03344976900000001 +etcd_network_peer_round_trip_time_seconds_count{To="2b6d7f19fe480be8"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0001"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0002"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0004"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0008"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0016"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0032"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0064"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0128"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0256"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0512"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.1024"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.2048"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.4096"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.8192"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="1.6384"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="3.2768"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="+Inf"} 44 +etcd_network_peer_round_trip_time_seconds_sum{To="6c86a033d0032400"} 0 +etcd_network_peer_round_trip_time_seconds_count{To="6c86a033d0032400"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0001"} 2 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0002"} 2 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0004"} 15 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0008"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0016"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0032"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0064"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0128"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0256"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0512"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.1024"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.2048"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.4096"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.8192"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="1.6384"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="3.2768"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="+Inf"} 52 +etcd_network_peer_round_trip_time_seconds_sum{To="819450a5536d704f"} 0.031137343999999994 +etcd_network_peer_round_trip_time_seconds_count{To="819450a5536d704f"} 52 +# HELP etcd_server_has_leader Whether or not a leader exists. 1 is existence, 0 is not. +# TYPE etcd_server_has_leader gauge +etcd_server_has_leader 1 +# HELP etcd_server_id Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID. +# TYPE etcd_server_id gauge +etcd_server_id{server_id="d5811b26ae265d01"} 1 +# HELP etcd_server_is_learner Whether or not this member is a learner. 1 if is, 0 otherwise. +# TYPE etcd_server_is_learner gauge +etcd_server_is_learner 0 +# HELP etcd_server_leader_changes_seen_total The number of leader changes seen. +# TYPE etcd_server_leader_changes_seen_total counter +etcd_server_leader_changes_seen_total 1 +# HELP etcd_server_quota_backend_bytes Current backend storage quota size in bytes. +# TYPE etcd_server_quota_backend_bytes gauge +etcd_server_quota_backend_bytes 2.147483648e+09 +# HELP etcd_server_version Which version is running. 1 for 'server_version' label with current version. +# TYPE etcd_server_version gauge +etcd_server_version{server_version="3.6.15"} 1 diff --git a/tests/fleet_role/etcd-metrics/k3s-follower.txt b/tests/fleet_role/etcd-metrics/k3s-follower.txt new file mode 100644 index 00000000..bb735603 --- /dev/null +++ b/tests/fleet_role/etcd-metrics/k3s-follower.txt @@ -0,0 +1,111 @@ +# HELP etcd_disk_backend_commit_duration_seconds The latency distributions of commit called by backend. +# TYPE etcd_disk_backend_commit_duration_seconds histogram +etcd_disk_backend_commit_duration_seconds_bucket{le="0.001"} 3502 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.002"} 3535 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.004"} 3547 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.008"} 3550 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.016"} 3559 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.032"} 3561 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.064"} 3566 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.128"} 3567 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.256"} 3567 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.512"} 3567 +etcd_disk_backend_commit_duration_seconds_bucket{le="1.024"} 3567 +etcd_disk_backend_commit_duration_seconds_bucket{le="2.048"} 3567 +etcd_disk_backend_commit_duration_seconds_bucket{le="4.096"} 3567 +etcd_disk_backend_commit_duration_seconds_bucket{le="8.192"} 3567 +etcd_disk_backend_commit_duration_seconds_bucket{le="+Inf"} 3567 +etcd_disk_backend_commit_duration_seconds_sum 1.089852076999999 +etcd_disk_backend_commit_duration_seconds_count 3567 +# HELP etcd_disk_wal_fsync_duration_seconds The latency distributions of fsync called by WAL. +# TYPE etcd_disk_wal_fsync_duration_seconds histogram +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.001"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.002"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.004"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.008"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.016"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.032"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.064"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.128"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.256"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.512"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="1.024"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="2.048"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="4.096"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="8.192"} 6385 +etcd_disk_wal_fsync_duration_seconds_bucket{le="+Inf"} 6385 +etcd_disk_wal_fsync_duration_seconds_sum 0.009015851000000031 +etcd_disk_wal_fsync_duration_seconds_count 6385 +# HELP etcd_mvcc_db_total_size_in_bytes Total size of the underlying database physically allocated in bytes. +# TYPE etcd_mvcc_db_total_size_in_bytes gauge +etcd_mvcc_db_total_size_in_bytes 6.9697536e+07 +# HELP etcd_network_active_peers The current number of active peer connections. +# TYPE etcd_network_active_peers gauge +etcd_network_active_peers{Local="aa6963edad47e8b5",Remote="756e308686962d26"} 1 +etcd_network_active_peers{Local="aa6963edad47e8b5",Remote="9f5fbb0683a7567a"} 2 +# HELP etcd_network_known_peers The current number of known peers. +# TYPE etcd_network_known_peers gauge +etcd_network_known_peers{Local="aa6963edad47e8b5",Remote="756e308686962d26"} 1 +etcd_network_known_peers{Local="aa6963edad47e8b5",Remote="9f5fbb0683a7567a"} 1 +etcd_network_known_peers{Local="aa6963edad47e8b5",Remote="aa6963edad47e8b5"} 1 +# HELP etcd_network_peer_round_trip_time_seconds Round-Trip-Time histogram between peers +# TYPE etcd_network_peer_round_trip_time_seconds histogram +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.0001"} 0 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.0002"} 0 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.0004"} 0 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.0008"} 0 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.0016"} 10 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.0032"} 20 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.0064"} 39 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.0128"} 53 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.0256"} 60 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.0512"} 70 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.1024"} 76 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.2048"} 80 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.4096"} 83 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="0.8192"} 86 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="1.6384"} 86 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="3.2768"} 86 +etcd_network_peer_round_trip_time_seconds_bucket{To="756e308686962d26",le="+Inf"} 86 +etcd_network_peer_round_trip_time_seconds_sum{To="756e308686962d26"} 3.8235525450000005 +etcd_network_peer_round_trip_time_seconds_count{To="756e308686962d26"} 86 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.0001"} 0 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.0002"} 0 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.0004"} 0 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.0008"} 0 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.0016"} 7 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.0032"} 23 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.0064"} 36 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.0128"} 50 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.0256"} 63 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.0512"} 71 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.1024"} 78 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.2048"} 83 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.4096"} 87 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="0.8192"} 88 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="1.6384"} 88 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="3.2768"} 88 +etcd_network_peer_round_trip_time_seconds_bucket{To="9f5fbb0683a7567a",le="+Inf"} 88 +etcd_network_peer_round_trip_time_seconds_sum{To="9f5fbb0683a7567a"} 3.6984278609999994 +etcd_network_peer_round_trip_time_seconds_count{To="9f5fbb0683a7567a"} 88 +# HELP etcd_server_has_leader Whether or not a leader exists. 1 is existence, 0 is not. +# TYPE etcd_server_has_leader gauge +etcd_server_has_leader 1 +# HELP etcd_server_id Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID. +# TYPE etcd_server_id gauge +etcd_server_id{server_id="aa6963edad47e8b5"} 1 +# HELP etcd_server_is_leader Whether or not this member is a leader. 1 if is, 0 otherwise. +# TYPE etcd_server_is_leader gauge +etcd_server_is_leader 0 +# HELP etcd_server_is_learner Whether or not this member is a learner. 1 if is, 0 otherwise. +# TYPE etcd_server_is_learner gauge +etcd_server_is_learner 0 +# HELP etcd_server_leader_changes_seen_total The number of leader changes seen. +# TYPE etcd_server_leader_changes_seen_total counter +etcd_server_leader_changes_seen_total 1 +# HELP etcd_server_quota_backend_bytes Current backend storage quota size in bytes. +# TYPE etcd_server_quota_backend_bytes gauge +etcd_server_quota_backend_bytes 2.147483648e+09 +# HELP etcd_server_version Which version is running. 1 for 'server_version' label with current version. +# TYPE etcd_server_version gauge +etcd_server_version{server_version="3.6.4"} 1 diff --git a/tests/fleet_role/etcd-metrics/learner-unstarted.txt b/tests/fleet_role/etcd-metrics/learner-unstarted.txt new file mode 100644 index 00000000..1c57f0b4 --- /dev/null +++ b/tests/fleet_role/etcd-metrics/learner-unstarted.txt @@ -0,0 +1,129 @@ +# HELP etcd_disk_backend_commit_duration_seconds The latency distributions of commit called by backend. +# TYPE etcd_disk_backend_commit_duration_seconds histogram +etcd_disk_backend_commit_duration_seconds_bucket{le="0.001"} 0 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.002"} 0 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.004"} 0 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.008"} 0 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.016"} 11 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.032"} 26 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.064"} 26 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.128"} 26 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.256"} 27 +etcd_disk_backend_commit_duration_seconds_bucket{le="0.512"} 27 +etcd_disk_backend_commit_duration_seconds_bucket{le="1.024"} 27 +etcd_disk_backend_commit_duration_seconds_bucket{le="2.048"} 27 +etcd_disk_backend_commit_duration_seconds_bucket{le="4.096"} 27 +etcd_disk_backend_commit_duration_seconds_bucket{le="8.192"} 27 +etcd_disk_backend_commit_duration_seconds_bucket{le="+Inf"} 27 +etcd_disk_backend_commit_duration_seconds_sum 0.6537084589999999 +etcd_disk_backend_commit_duration_seconds_count 27 +# HELP etcd_disk_wal_fsync_duration_seconds The latency distributions of fsync called by WAL. +# TYPE etcd_disk_wal_fsync_duration_seconds histogram +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.001"} 0 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.002"} 0 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.004"} 0 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.008"} 5 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.016"} 73 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.032"} 80 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.064"} 80 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.128"} 81 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.256"} 81 +etcd_disk_wal_fsync_duration_seconds_bucket{le="0.512"} 81 +etcd_disk_wal_fsync_duration_seconds_bucket{le="1.024"} 81 +etcd_disk_wal_fsync_duration_seconds_bucket{le="2.048"} 81 +etcd_disk_wal_fsync_duration_seconds_bucket{le="4.096"} 81 +etcd_disk_wal_fsync_duration_seconds_bucket{le="8.192"} 81 +etcd_disk_wal_fsync_duration_seconds_bucket{le="+Inf"} 81 +etcd_disk_wal_fsync_duration_seconds_sum 1.0567120819999993 +etcd_disk_wal_fsync_duration_seconds_count 81 +# HELP etcd_mvcc_db_total_size_in_bytes Total size of the underlying database physically allocated in bytes. +# TYPE etcd_mvcc_db_total_size_in_bytes gauge +etcd_mvcc_db_total_size_in_bytes 98304 +# HELP etcd_network_active_peers The current number of active peer connections. +# TYPE etcd_network_active_peers gauge +etcd_network_active_peers{Local="d5811b26ae265d01",Remote="2b6d7f19fe480be8"} 1 +etcd_network_active_peers{Local="d5811b26ae265d01",Remote="819450a5536d704f"} 1 +# HELP etcd_network_known_peers The current number of known peers. +# TYPE etcd_network_known_peers gauge +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="2b6d7f19fe480be8"} 1 +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="6c86a033d0032400"} 0 +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="819450a5536d704f"} 1 +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="c65e62daf70a677b"} 1 +etcd_network_known_peers{Local="d5811b26ae265d01",Remote="d5811b26ae265d01"} 1 +# HELP etcd_network_peer_round_trip_time_seconds Round-Trip-Time histogram between peers +# TYPE etcd_network_peer_round_trip_time_seconds histogram +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0001"} 2 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0002"} 2 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0004"} 8 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0008"} 41 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0016"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0032"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0064"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0128"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0256"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.0512"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.1024"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.2048"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.4096"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="0.8192"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="1.6384"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="3.2768"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="2b6d7f19fe480be8",le="+Inf"} 52 +etcd_network_peer_round_trip_time_seconds_sum{To="2b6d7f19fe480be8"} 0.03344976900000001 +etcd_network_peer_round_trip_time_seconds_count{To="2b6d7f19fe480be8"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0001"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0002"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0004"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0008"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0016"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0032"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0064"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0128"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0256"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.0512"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.1024"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.2048"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.4096"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="0.8192"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="1.6384"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="3.2768"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="6c86a033d0032400",le="+Inf"} 44 +etcd_network_peer_round_trip_time_seconds_sum{To="6c86a033d0032400"} 0 +etcd_network_peer_round_trip_time_seconds_count{To="6c86a033d0032400"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0001"} 2 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0002"} 2 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0004"} 15 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0008"} 44 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0016"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0032"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0064"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0128"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0256"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.0512"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.1024"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.2048"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.4096"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="0.8192"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="1.6384"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="3.2768"} 52 +etcd_network_peer_round_trip_time_seconds_bucket{To="819450a5536d704f",le="+Inf"} 52 +etcd_network_peer_round_trip_time_seconds_sum{To="819450a5536d704f"} 0.031137343999999994 +etcd_network_peer_round_trip_time_seconds_count{To="819450a5536d704f"} 52 +# HELP etcd_server_has_leader Whether or not a leader exists. 1 is existence, 0 is not. +# TYPE etcd_server_has_leader gauge +etcd_server_has_leader 1 +# HELP etcd_server_id Server or member ID in hexadecimal format. 1 for 'server_id' label with current ID. +# TYPE etcd_server_id gauge +etcd_server_id{server_id="d5811b26ae265d01"} 1 +# HELP etcd_server_is_learner Whether or not this member is a learner. 1 if is, 0 otherwise. +# TYPE etcd_server_is_learner gauge +etcd_server_is_learner 0 +# HELP etcd_server_leader_changes_seen_total The number of leader changes seen. +# TYPE etcd_server_leader_changes_seen_total counter +etcd_server_leader_changes_seen_total 1 +# HELP etcd_server_quota_backend_bytes Current backend storage quota size in bytes. +# TYPE etcd_server_quota_backend_bytes gauge +etcd_server_quota_backend_bytes 2.147483648e+09 +# HELP etcd_server_version Which version is running. 1 for 'server_version' label with current version. +# TYPE etcd_server_version gauge +etcd_server_version{server_version="3.6.15"} 1 diff --git a/tests/fleet_role/image-smoke.sh b/tests/fleet_role/image-smoke.sh new file mode 100755 index 00000000..97901f3c --- /dev/null +++ b/tests/fleet_role/image-smoke.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Runs what the role controller's pods run, from the image as built (fleet-role/Dockerfile), with nothing that reaches a cluster: the tools and files the pods need are present and executable, the jq modules compile under the image's own jq (not the host's), the probe's check mode reads the memory signals through the regrade core the image carries, and one pass of the controller runs end to end against a kubectl that answers an empty cluster and records the status it is asked to publish. Needs only Docker. +# +# Usage: tests/fleet_role/image-smoke.sh (the image must already be built) +set -euo pipefail + +image="${1:?usage: image-smoke.sh }" +work="$(mktemp -d "${TMPDIR:-/tmp}/fleet-role-image.XXXXXX")" +trap 'rm -rf "$work"' EXIT +# A directory every container user can read, since the files are mounted read-only into a container that runs as a non-root user. +chmod 755 "$work" + +docker run --rm --entrypoint bash "$image" -c ' + set -euo pipefail + for tool in jq kubectl curl fio; do command -v "$tool" > /dev/null; done + test -x /app/loop.sh -a -x /app/scripts/fleet-role.sh -a -x /app/scripts/fleet-role-probe.sh + test -f /app/fleet-regrade-core.sh -a -f /app/scripts/fleet-role.jq -a -f /app/scripts/fleet-role-stats.jq' + +docker run --rm --entrypoint jq "$image" -en -L /app/scripts 'include "fleet-role-stats"; {window: window_samples, tail: tail_percentile} | select(.window == 100 and .tail == 99)' > /dev/null + +printf 'MemTotal: 1000 kB\nMemAvailable: 400 kB\n' > "$work/meminfo" +printf 'some avg10=0.00 avg60=0.00 avg300=1.50 total=0\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=0\n' > "$work/pressure" +chmod 644 "$work/meminfo" "$work/pressure" +docker run --rm -v "$work/meminfo:/meminfo:ro" -v "$work/pressure:/pressure:ro" -e MEMINFO_FILE=/meminfo -e PSI_FILE=/pressure \ + --entrypoint /app/scripts/fleet-role-probe.sh "$image" check memory | jq -e '.psi_avg300 == 1.5 and .mem_available_kb == 400' > /dev/null + +# A kubectl that answers an empty cluster and keeps the patch it is given, in place of the image's own on the PATH. +mkdir "$work/out" +chmod 777 "$work/out" +cat > "$work/kubectl" << 'STUB' +#!/bin/sh +case "$1 $2" in + "get nodes") echo '{"items":[]}' ;; + "get configmap") case "$3" in fleet-role-probe) echo '{"data":{}}' ;; *) echo '{"data":{"status.json":""}}' ;; esac ;; + "patch configmap") + previous='' + for argument in "$@"; do + [ "$previous" = --patch-file ] && cp "$argument" /out/patch.json + previous=$argument + done ;; + *) echo "unexpected kubectl call: $*" >&2; exit 2 ;; +esac +STUB +chmod 755 "$work/kubectl" +output="$(docker run --rm -v "$work/kubectl:/usr/local/bin/kubectl:ro" -v "$work/out:/out" \ + -e FLEET_ROLE_DRY_RUN=true -e FLEET_ROLE_VOTERS=3 -e FLEET_ROLE_MAX_SERVERS=5 -e FLEET_ROLE_ELECTION_TIMEOUT_MS=1000 -e FLEET_ROLE_SAMPLE_SECONDS=15 \ + -e FLEET_ROLE_PRESSURE_THRESHOLD=10 -e FLEET_ROLE_AVAILABLE_FLOOR_PERCENT=15 -e FLEET_ROLE_ELIGIBLE_LABEL=example.com/eligible \ + -e FLEET_ROLE_PINNED_LABEL=example.com/pinned -e FLEET_ROLE_LEVEL_LABEL=example.com/level \ + --entrypoint /app/scripts/fleet-role.sh "$image")" +grep -q 'no host is marked eligible' <<< "$output" +jq -e '.data["status.json"] | fromjson | .dry_run == true and .decision.action == "none"' "$work/out/patch.json" > /dev/null +echo "image smoke test passed: $image" diff --git a/tests/fleet_role/render.yml b/tests/fleet_role/render.yml new file mode 100644 index 00000000..66544632 --- /dev/null +++ b/tests/fleet_role/render.yml @@ -0,0 +1,14 @@ +# Renders the github_runner_arc role's role controller manifests (templates/fleet-role.yaml.j2) with the role's defaults and constants, for tests/unit/test_fleet_role_arc.py. Takes fleet_role_output as an extra variable; a role variable such as the image or the target is overridden with an extra variable of its own name, since the defaults file loaded here outranks play variables. +- name: Render the role controller manifests + hosts: localhost + connection: local + gather_facts: false + vars_files: + - ../../roles/github_runner_arc/defaults/main.yml + - ../../roles/github_runner_arc/vars/main.yml + tasks: + - name: Write the manifests + ansible.builtin.copy: + content: "{{ lookup('ansible.builtin.template', playbook_dir ~ '/../../roles/github_runner_arc/templates/fleet-role.yaml.j2') }}" + dest: "{{ fleet_role_output }}" + mode: "0644" diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index b52fe823..b65a7b28 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). native_server three Docker servers and a native host (a privileged systemd container, tests/mesh/native/Dockerfile, managed over Ansible's docker connection) that joins as an agent, is then made a server by the inventory and promoted in place: it must join etcd as a learner, be promoted to a voter with its mesh address as its peer URL, and keep the cluster writable throughout, and a rerun must not restart it. compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. +# Mesh integration test for the github_runner_cluster role: stands up three k3s nodes in Docker on one machine, each in its own copy of the Compose project, and has the role join them through Headscale. Scenarios: hosted headscale_hosted: the role runs Headscale under Compose beside the first node. in_cluster headscale_in_cluster: the role runs Headscale inside the cluster, on the bootstrap server. existing headscale_existing: the policy check fails, with the entry to add, against a server whose policy lacks autoApprovers, and passes once it has one. Every scenario uses automatic server selection (three hosts, no overrides, so three servers). native_server three Docker servers and a native host (a privileged systemd container, tests/mesh/native/Dockerfile, managed over Ansible's docker connection) that joins as an agent, is then made a server by the inventory and promoted in place: it must join etcd as a learner, be promoted to a voter with its mesh address as its peer URL, and keep the cluster writable throughout, and a rerun must not restart it. role_controller four hosts (three servers and a standby agent, with etcd's data on memory so the machine's disk does not decide): the role controller's probe and controller run in the cluster from the image built from this checkout, with the manifests the ARC role renders; every host is reported fit from a full window of real samples, a voter made slow with tc netem on its mesh interface is reported unfit with the swap named, a paused voter makes the controller report that it would not act, and the controller changes nothing (not in the default set; needs jq, and takes a window of 100 samples before it can judge). compose_faulty the role refuses a Docker Compose release that recreates the containers it has just built, before writing anything (run it under such a release; it is not in the default set). host_network each node runs in a Docker daemon of its own (a docker:dind container standing in for a VM with an address on the LAN, on the shared test network) with github_runner_cluster_network_mode host, joined through a Headscale server the test runs (headscale_existing); the nodes' Tailscale clients must then reach each other directly at those daemons' addresses, and the role must not publish any port; then the role must refuse to move two running nodes to the bridge network in one run, and move one there and back through its guard, the node keeping its mesh address and etcd membership. Linux only, since the control node reaches each daemon at its address on the test network. The cluster scenarios assert that etcd has three members, that every node is Ready, and that pods on different nodes reach each other over the mesh. # # With GRTEST_PIN_CONTEXT=1, the cluster scenarios run the role, and the in_cluster recovery playbook, with github_runner_cluster_docker_context pinned while the current Docker context they see points at nothing (tests/lib/docker_context.sh), so any docker call that ignored the pin fails the run; afterwards the current context must be unchanged. context_checks the role refuses a pinned context that does not exist, and a pinned context alongside DOCKER_HOST, before writing anything (not in the default set). # @@ -30,6 +30,10 @@ native_ip="${subnet_prefix}.20" native_role="" # The Docker hosts the inventory lists, by index; all of them unless a scenario leaves one out (native_server's second run, where node 3 keeps running as a voter outside the inventory). inventory_nodes="" +# Where tc comes from for the role_controller scenario: a throwaway container in a node's network namespace, since the node's own image has none. +tc_image=alpine:3.20 +# The probe's period in the role_controller scenario: long enough that a sample's own work (a few kubectl calls) is a small part of it, since a probe that has not reported for three periods counts as silent. +role_controller_sample_seconds=5 # Tries per pod pair, 5 seconds apart plus each try's own timeout: about two minutes for the mesh to converge after a restart. reach_attempts=8 # etcd's timing for every server, in milliseconds: a heartbeat interval and an election timeout of ten times it, above k3s's own defaults and so told apart from them, and above etcd's required five times. @@ -85,7 +89,9 @@ fi node_name() { echo "grtest-node$1"; } node_ip() { echo "${subnet_prefix}.$((10 + $1))"; } -container() { echo "grtest-node$1-k3s"; } +# The container of node $1: its k3s server, or from agent_from on, the k3s agent the topology makes of a host beyond the servers (only the role_controller scenario has one). +agent_from=99 +container() { if [ "$1" -ge "$agent_from" ]; then echo "grtest-node$1-agent"; else echo "grtest-node$1-k3s"; fi; } vm() { echo "grtest-vm$1"; } # The docker CLI for node $1's daemon: the machine's own, or with host_network that node's own daemon through its context. @@ -98,7 +104,7 @@ docker_on() { teardown() { for index in $(seq 1 "$node_count"); do if [ -f "${work}/$(node_name "$index")/docker-compose.yml" ]; then - docker compose --project-directory "${work}/$(node_name "$index")" -f "${work}/$(node_name "$index")/docker-compose.yml" --profile server --profile regrade down -v --rmi local >/dev/null 2>&1 || true + docker compose --project-directory "${work}/$(node_name "$index")" -f "${work}/$(node_name "$index")/docker-compose.yml" --profile server --profile agent --profile regrade down -v --rmi local >/dev/null 2>&1 || true fi done if [ -f "${work}/headscale/docker-compose.yml" ]; then @@ -155,6 +161,20 @@ prepare_nodes() { else echo " ports: !reset []" fi + # etcd's data on memory when asked: the role_controller scenario judges the voters' and the standby's disk against etcd's own limits (a WAL fsync under 10 ms), which a laptop's or a runner's virtual disk does not reliably meet (a laptop's measured 15 to 30 ms), and what it tests is the controller's pipeline, not the disk of the machine it runs on. + if [ "${etcd_on_memory:-0}" = 1 ]; then + echo " tmpfs:" + echo " - /var/lib/rancher/k3s/server/db" + fi + echo " networks:" + echo " ${network}:" + echo " ipv4_address: $(node_ip "$index")" + # A host the topology makes an agent runs the k3s-agent service, which needs the same fixed address. Its fdatasync probe directory is on memory for the reason etcd's data is. + echo " k3s-agent:" + if [ "${etcd_on_memory:-0}" = 1 ]; then + echo " tmpfs:" + echo " - /var/lib/rancher/k3s/fleet-role-probe" + fi echo " networks:" echo " ${network}:" echo " ipv4_address: $(node_ip "$index")" @@ -204,6 +224,7 @@ EOF echo " github_runner_cluster_etcd_heartbeat_interval: ${etcd_heartbeat_ms}" echo " github_runner_cluster_etcd_election_timeout: ${etcd_election_ms}" echo " github_runner_cluster_regrade: ${regrade}" + if [ "${role_controller:-0}" = 1 ]; then echo " github_runner_cluster_role_controller: true"; fi # The test cluster is throwaway: the role must not install a login item that starts the host's Colima profile. echo " github_runner_cluster_runtime_autostart: false" if [ "$pin_context" = 1 ]; then @@ -218,7 +239,14 @@ EOF echo " $(node_name "$index"):" echo " github_runner_cluster_dir: ${work}/$(node_name "$index")" echo " github_runner_cluster_compose_project: $(node_name "$index")" - echo " github_runner_cluster_container_name: $(container "$index")" + echo " github_runner_cluster_container_name: grtest-node${index}-k3s" + # Used only by a host the topology makes an agent (the role_controller scenario has one); the role refuses one name for both services. + echo " github_runner_cluster_agent_container_name: grtest-node${index}-agent" + # The role_controller scenario marks every host a candidate for a vote, through the inventory, as an owner would, and pins the first. + if [ "${role_controller:-0}" = 1 ]; then + echo " github_runner_cluster_voter_eligible: true" + if [ "$index" = 1 ]; then echo " github_runner_cluster_voter_pinned: true"; fi + fi done if [ -n "$native_role" ]; then echo " ${native_host}:" @@ -1069,6 +1097,135 @@ scenario_native_server() { [ "$before" = "$(native_exec systemctl show k3s --property ActiveEnterTimestampMonotonic)" ] || fail "rerunning the role restarted ${native_host}'s server" } +# The slow link the role_controller scenario puts on a voter, in milliseconds each way on its mesh interface: above the unfit round trip limit, a tenth of the election timeout the inventory sets. +role_controller_delay_ms=$((etcd_election_ms / 10 + 100)) +role_controller_image=grtest/fleet-role:test +platform_namespace=github-runner-platform + +role_status() { kubectl_on 1 --namespace "$platform_namespace" get configmap fleet-role-status -o 'jsonpath={.data.status\.json}' 2>/dev/null; } +# Whether the controller's last status satisfies a jq test, as an exit status. +role_status_is() { role_status | jq -e "$1" >/dev/null 2>&1; } +# The node a pod of the app runs on. +role_pod_node() { kubectl_on 1 --namespace "$platform_namespace" get pods -l "app.kubernetes.io/name=$1" -o 'jsonpath={.items[0].spec.nodeName}'; } +# Puts a delay on a node's mesh interface, or takes it off, from a throwaway container in the node's network namespace (the node's own image has no tc). +mesh_delay() { + local index="$1" action="$2" ms="${3:-0}" command + if [ "$action" = add ]; then command="apk add --quiet iproute2-tc && tc qdisc add dev tailscale0 root netem delay ${ms}ms"; else command="apk add --quiet iproute2-tc && tc qdisc del dev tailscale0 root"; fi + docker run --rm --network "container:$(container "$index")" --cap-add NET_ADMIN --entrypoint sh "$tc_image" -c "$command" /dev/null | grep -c ' Ready ')" = "$node_count" ]; do + [ "$SECONDS" -lt "$deadline" ] || { kubectl_on 1 get nodes -o wide || true; fail "not every node became Ready"; } + sleep 5 + done + [ "$(kubectl_on 1 get nodes -l node-role.kubernetes.io/etcd=true --no-headers | wc -l | tr -d ' ')" = 3 ] || fail "expected three etcd nodes" + if kubectl_on 1 get node "$(node_name 4)" -o 'jsonpath={.metadata.labels}' | grep -F 'node-role.kubernetes.io/etcd' >/dev/null; then fail "the fourth node is a server"; fi + ids=$(etcd_member_ids 1) + + log "Building the role controller image from this checkout and loading it into every node" + docker build --quiet --file "${repo_root}/fleet-role/Dockerfile" --tag "$role_controller_image" "$repo_root" >/dev/null + image_file="${work}/fleet-role-image.tar" + docker save --output "$image_file" "$role_controller_image" + for index in $(seq 1 "$node_count"); do + docker exec -i "$(container "$index")" /bin/ctr --address /run/k3s/containerd/containerd.sock --namespace k8s.io images import - < "$image_file" >/dev/null + done + rm -f "$image_file" + + log "Rendering the ARC role's manifests for the controller and the probe, and applying them" + manifests="${work}/fleet-role.yaml" + ANSIBLE_COLLECTIONS_PATH="${repo_root}/playbooks/collections" ANSIBLE_LOCALHOST_WARNING=false ANSIBLE_INVENTORY_UNPARSED_WARNING=false \ + "$ansible_playbook" "${repo_root}/tests/fleet_role/render.yml" -e "fleet_role_output=${manifests}" \ + -e "github_runner_arc_role_controller_image=${role_controller_image}" -e "github_runner_arc_role_controller_sample_seconds=${role_controller_sample_seconds}" \ + -e "github_runner_arc_role_controller_election_timeout_ms=${etcd_election_ms}" >/dev/null + kubectl_on 1 create namespace "$platform_namespace" >/dev/null + kubectl_on 1 apply -f - >/dev/null </dev/null + log "The role published the labels from the inventory: eligible on every host, pinned on the first alone" + for index in $(seq 1 "$node_count"); do + [ "$(kubectl_on 1 get node "$(node_name "$index")" -o 'jsonpath={.metadata.labels.github-runner\.exadev\.com/voter-eligible}')" = true ] || fail "$(node_name "$index") is not labelled eligible" + expected=false + if [ "$index" = 1 ]; then expected=true; fi + [ "$(kubectl_on 1 get node "$(node_name "$index")" -o 'jsonpath={.metadata.labels.github-runner\.exadev\.com/voter-pinned}')" = "$expected" ] || fail "$(node_name "$index") does not carry voter-pinned=${expected}" + done + kubectl_on 1 --namespace "$platform_namespace" rollout status daemonset/fleet-role-probe --timeout=300s + kubectl_on 1 --namespace "$platform_namespace" rollout status deployment/fleet-role --timeout=300s + kubectl_on 1 --namespace "$platform_namespace" get pods -o wide + + log "The controller and the probe can read nodes and nothing more, and the controller cannot write the probes' ConfigMap" + for account in fleet-role fleet-role-probe; do + [ "$(kubectl_on 1 auth can-i list nodes --as "system:serviceaccount:${platform_namespace}:${account}")" = yes ] || fail "${account} cannot list nodes" + for verb in patch update delete create; do + [ "$(kubectl_on 1 auth can-i "$verb" nodes --as "system:serviceaccount:${platform_namespace}:${account}" || true)" = no ] || fail "${account} can ${verb} nodes" + done + [ "$(kubectl_on 1 auth can-i create configmaps --namespace "$platform_namespace" --as "system:serviceaccount:${platform_namespace}:${account}" || true)" = no ] || fail "${account} can create configmaps" + done + [ "$(kubectl_on 1 auth can-i patch configmaps/fleet-role-probe --namespace "$platform_namespace" --as "system:serviceaccount:${platform_namespace}:fleet-role" || true)" = no ] || fail "the controller can patch the probes' ConfigMap" + + log "Every host is fit, from a window of real samples, and the controller would not act" + wait_for "every host to be reported fit and no swap wanted" 1800 role_status_is '(.hosts | length == 4) and ([.hosts[].verdict] | all(. == "fit")) and .decision.action == "none" and (.blocks | length == 0)' + role_status | jq . + role_status | jq -e "(.hosts | to_entries | all(.value.eligible)) and ([.hosts[] | select(.pinned)] | length == 1) and .hosts[\"$(node_name 1)\"].pinned" >/dev/null || fail "the controller did not read the labels the role published: $(role_status | jq -c '[.hosts[] | {eligible, pinned}]')" + role_status | jq -e '[.hosts[].link] | all(. == "unknown")' >/dev/null || fail "a node in a container should have a link it cannot see: $(role_status | jq -c '[.hosts[].link]')" + + log "A voter is made slow with tc netem on its mesh interface (${role_controller_delay_ms} ms each way): the controller reports it unfit and names the swap" + slow=3 + mesh_delay "$slow" add "$role_controller_delay_ms" + wait_for "$(node_name "$slow") to be reported unfit and the swap named" 900 role_status_is \ + ".hosts[\"$(node_name "$slow")\"].verdict == \"unfit\" and .decision.action == \"swap\" and .decision.swap == {out: \"$(node_name "$slow")\", in: \"$(node_name 4)\"}" + role_status | jq . + role_status | jq -r '.decision.because[]' + role_status | jq -e ".hosts[\"$(node_name "$slow")\"].reasons | any(test(\"round trip p99\"))" >/dev/null || fail "the reason does not name the round trip" + kubectl_on 1 --namespace "$platform_namespace" logs deployment/fleet-role --tail 20 | grep -F "DRY RUN: would swap out $(node_name "$slow") for $(node_name 4)" >/dev/null || fail "the controller did not log the swap it would make" + + log "A voter is paused: the controller reports that it would not act" + controller_node=$(role_pod_node fleet-role) + paused=2 + [ "$(node_name "$paused")" != "$controller_node" ] || paused=$slow + paused_name=$(node_name "$paused") + docker pause "$(container "$paused")" >/dev/null + wait_for "the controller to say it would not act with ${paused_name} paused" 600 role_status_is \ + ".decision.action == \"none\" and (.decision.because + .blocks | any(contains(\"${paused_name}\")))" + role_status | jq '{decision, blocks}' + kubectl_on 1 --namespace "$platform_namespace" logs deployment/fleet-role --tail 5 | grep -F 'DRY RUN: would not act' >/dev/null || fail "the controller did not log that it would not act" + docker unpause "$(container "$paused")" >/dev/null + mesh_delay "$slow" del + + log "The controller changed nothing: the same etcd members, the same servers, no node touched" + [ "$(etcd_member_ids 1)" = "$ids" ] || fail "the etcd members changed: $(etcd_member_ids 1) (was ${ids})" + [ "$(kubectl_on 1 get nodes -l node-role.kubernetes.io/etcd=true --no-headers | wc -l | tr -d ' ')" = 3 ] || fail "the set of etcd nodes changed" + kubectl_on 1 get nodes -o 'jsonpath={range .items[*]}{.metadata.name}{" "}{.spec.unschedulable}{" "}{.spec.taints}{"\n"}{end}' + if kubectl_on 1 get nodes -o json | jq -e '[.items[] | select(.metadata.annotations["etcd.k3s.cattle.io/remove"] != null)] | length > 0' >/dev/null; then fail "a node was annotated for removal"; fi +} + scenarios=("$@") [ "${#scenarios[@]}" -gt 0 ] || scenarios=(existing hosted in_cluster) for scenario in "${scenarios[@]}"; do @@ -1078,6 +1235,7 @@ for scenario in "${scenarios[@]}"; do existing) scenario_existing ;; regrade) scenario_regrade ;; native_server) scenario_native_server ;; + role_controller) scenario_role_controller ;; host_network) scenario_host_network ;; compose_faulty) scenario_compose_faulty ;; context_checks) scenario_context_checks ;; diff --git a/tests/unit/fleet_role_support.py b/tests/unit/fleet_role_support.py new file mode 100644 index 00000000..364d9857 --- /dev/null +++ b/tests/unit/fleet_role_support.py @@ -0,0 +1,298 @@ +"""Shared by the tests of the role controller (test_fleet_role.py) and its probe (test_fleet_role_probe.py): a stand-in for kubectl that keeps nodes and ConfigMaps in a JSON file, and builders for the nodes, probe samples and probe entries the controller reads. The scripts run with the real bash, jq, awk and curl; only the cluster's API is a stand-in.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +import textwrap +import time +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" +REGRADE_CORE = ROOT / "roles" / "github_runner_cluster" / "files" / "regrade" / "fleet-regrade-core.sh" + +NAMESPACE = "github-runner-platform" +PROBE_CONFIGMAP = "fleet-role-probe" +STATUS_CONFIGMAP = "fleet-role-status" +ELIGIBLE_LABEL = "github-runner.exadev.com/voter-eligible" +PINNED_LABEL = "github-runner.exadev.com/voter-pinned" +LEVEL_LABEL = "github-runner.exadev.com/regrade-level" +VOTER_LABEL = "node-role.kubernetes.io/etcd" + +# The settings the tests give the controller; the thresholds below derive from these the way the controller derives them, so a test names the rule it holds the program to and not a bare figure. +ELECTION_TIMEOUT_MS = 1000 +SAMPLE_SECONDS = 15 +TARGET = 3 +MAX_SERVERS = 5 +PRESSURE_THRESHOLD = 10 +AVAILABLE_FLOOR_PERCENT = 15 +TAIL_PERCENTILE = 99 +WINDOW_SAMPLES = 100 // (100 - TAIL_PERCENTILE) +WINDOW_SECONDS = SAMPLE_SECONDS * WINDOW_SAMPLES +UNFIT_RTT_MS = ELECTION_TIMEOUT_MS / 10 +FIT_RTT_MS = ELECTION_TIMEOUT_MS / 20 +LEARNER_STALL_SECONDS = 300 +COOLDOWN_SECONDS = LEARNER_STALL_SECONDS + WINDOW_SECONDS +WAL_LIMIT_MS = 10 +COMMIT_LIMIT_MS = 25 +STALE_AFTER_SAMPLES = 3 +ETCD_QUOTA_BYTES = 2 * 1024**3 +# Plenty of memory on every host unless a test says otherwise: well above the quota, and a share of the total far above the regrade floor. +TOTAL_KB = 16 * 1024 * 1024 +AVAILABLE_KB = 8 * 1024 * 1024 + +# Answers the kubectl calls the probe and the controller make from a JSON file (FAKE_STORE) holding {"nodes": [...], "configmaps": {"namespace/name": {"data": {...}}}}, merges a merge patch's data into a ConfigMap, and records every call in FAKE_CALLS. A ConfigMap the store does not hold is NotFound, as the cluster answers. +FAKE_KUBECTL = textwrap.dedent( + """\ + #!/usr/bin/env python3 + import json, os, sys + args = sys.argv[1:] + with open(os.environ["FAKE_CALLS"], "a") as log: + log.write(json.dumps(args) + "\\n") + store = json.load(open(os.environ["FAKE_STORE"])) + def flag(name): + return args[args.index(name) + 1] if name in args else None + if args[:2] == ["get", "nodes"]: + selector = flag("-l") + items = store["nodes"] + if selector is not None: + key, _, value = selector.partition("=") + items = [n for n in items if n["metadata"].get("labels", {}).get(key) == value] + print(json.dumps({"items": items})) + sys.exit(0) + if args[:2] == ["get", "node"]: + match = [n for n in store["nodes"] if n["metadata"]["name"] == args[2]] + if not match: + sys.stderr.write('Error from server (NotFound): nodes "%s" not found\\n' % args[2]) + sys.exit(1) + print(json.dumps(match[0])) + sys.exit(0) + if args[:2] == ["get", "configmap"]: + key = flag("-n") + "/" + args[2] + if key not in store["configmaps"]: + sys.stderr.write('Error from server (NotFound): configmaps "%s" not found\\n' % args[2]) + sys.exit(1) + print(json.dumps({"metadata": {"name": args[2]}, "data": store["configmaps"][key]["data"]})) + sys.exit(0) + if args[:2] == ["patch", "configmap"] and os.environ.get("FAKE_FAIL_PATCHES"): + # Refuses the next patches as an API server on a poor path does: the count is kept in the file the variable names. + remaining = int(open(os.environ["FAKE_FAIL_PATCHES"]).read() or "0") + if remaining > 0: + open(os.environ["FAKE_FAIL_PATCHES"], "w").write(str(remaining - 1)) + sys.stderr.write("Unable to connect to the server: net/http: TLS handshake timeout\\n") + sys.exit(1) + if args[:2] == ["patch", "configmap"]: + key = flag("-n") + "/" + args[2] + patch = json.load(open(flag("--patch-file"))) + data = store["configmaps"][key]["data"] + for name, value in patch["data"].items(): + if value is None: + data.pop(name, None) + else: + data[name] = value + json.dump(store, open(os.environ["FAKE_STORE"], "w")) + sys.exit(0) + sys.stderr.write("unexpected kubectl call: %s\\n" % args) + sys.exit(2) + """ +) + + +# One argument to a process is limited to 128 KiB on Linux (MAX_ARG_STRLEN, one page short of that for the terminating NUL), whatever the total; macOS allows a mebibyte, so a script that passes a large document as an argument passes here and fails on a Linux runner. This stands in for jq on the PATH and refuses such an argument as the kernel's exec does, before running the real jq. +ARGUMENT_LIMIT_BYTES = 128 * 1024 - 1 +JQ_SHIM = textwrap.dedent( + """\ + #!/usr/bin/env bash + export LC_ALL=C + for argument in "$@"; do + if [ "${#argument}" -gt %d ]; then + echo "jq: Argument list too long" >&2 + exit 126 + fi + done + exec %s "$@" + """ +) + + +class Cluster: + """A scratch directory holding the stand-in kubectl, its store and its call log, with the helpers that run a script against it.""" + + def __init__(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.dir = Path(self.tmp.name) + self.bin = self.dir / "bin" + self.bin.mkdir() + kubectl = self.bin / "kubectl" + kubectl.write_text(FAKE_KUBECTL) + kubectl.chmod(0o755) + jq = self.bin / "jq" + jq.write_text(JQ_SHIM % (ARGUMENT_LIMIT_BYTES, shutil.which("jq"))) + jq.chmod(0o755) + self.store_path = self.dir / "store.json" + self.calls = self.dir / "calls.log" + self.calls.touch() + self.nodes: list[dict[str, Any]] = [] + self.configmaps: dict[str, dict[str, Any]] = { + f"{NAMESPACE}/{PROBE_CONFIGMAP}": {"data": {}}, + f"{NAMESPACE}/{STATUS_CONFIGMAP}": {"data": {"status.json": ""}}, + } + + def cleanup(self) -> None: + self.tmp.cleanup() + + def write_store(self) -> None: + self.store_path.write_text(json.dumps({"nodes": self.nodes, "configmaps": self.configmaps})) + + def read_store(self) -> dict[str, Any]: + return json.loads(self.store_path.read_text()) + + def probes(self) -> dict[str, Any]: + data = self.configmaps[f"{NAMESPACE}/{PROBE_CONFIGMAP}"]["data"] + return {name: json.loads(entry) for name, entry in data.items()} + + def set_probes(self, entries: dict[str, Any]) -> None: + self.configmaps[f"{NAMESPACE}/{PROBE_CONFIGMAP}"]["data"] = {name: json.dumps(entry) for name, entry in entries.items()} + + def set_status(self, status: dict[str, Any] | None) -> None: + self.configmaps[f"{NAMESPACE}/{STATUS_CONFIGMAP}"]["data"] = {"status.json": "" if status is None else json.dumps(status)} + + def status(self) -> dict[str, Any]: + return json.loads(self.read_store()["configmaps"][f"{NAMESPACE}/{STATUS_CONFIGMAP}"]["data"]["status.json"]) + + def env(self, **extra: str) -> dict[str, str]: + env = {**os.environ, "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}", "FAKE_STORE": str(self.store_path), "FAKE_CALLS": str(self.calls)} + env.update(extra) + return env + + def run(self, script: str, **env: str) -> subprocess.CompletedProcess[str]: + self.write_store() + completed = subprocess.run([str(SCRIPTS / script)], env=self.env(**env), capture_output=True, text=True, stdin=subprocess.DEVNULL, check=False) + if self.store_path.exists(): + store = self.read_store() + self.nodes = store["nodes"] + self.configmaps = store["configmaps"] + return completed + + +CONTROLLER_ENV = { + "FLEET_ROLE_VOTERS": str(TARGET), + "FLEET_ROLE_MAX_SERVERS": str(MAX_SERVERS), + "FLEET_ROLE_ELECTION_TIMEOUT_MS": str(ELECTION_TIMEOUT_MS), + "FLEET_ROLE_SAMPLE_SECONDS": str(SAMPLE_SECONDS), + "FLEET_ROLE_PRESSURE_THRESHOLD": str(PRESSURE_THRESHOLD), + "FLEET_ROLE_AVAILABLE_FLOOR_PERCENT": str(AVAILABLE_FLOOR_PERCENT), + "FLEET_ROLE_ELIGIBLE_LABEL": ELIGIBLE_LABEL, + "FLEET_ROLE_PINNED_LABEL": PINNED_LABEL, + "FLEET_ROLE_LEVEL_LABEL": LEVEL_LABEL, + "FLEET_ROLE_STATE_NAMESPACE": NAMESPACE, +} + + +def transition_time(seconds_ago: float) -> str: + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - seconds_ago)) + + +def node( + name: str, + *, + voter: bool = False, + eligible: bool = False, + pinned: bool = False, + ready: bool = True, + ready_for: float = 24 * 3600, + level: str | None = None, + address: str | None = None, +) -> dict[str, Any]: + """A Node as `kubectl get nodes -o json` lists it; ready_for is how long ago its Ready condition last changed.""" + labels: dict[str, str] = {} + if voter: + labels[VOTER_LABEL] = "true" + if eligible: + labels[ELIGIBLE_LABEL] = "true" + if pinned: + labels[PINNED_LABEL] = "true" + if level is not None: + labels[LEVEL_LABEL] = level + return { + "metadata": {"name": name, "labels": labels}, + "status": { + "conditions": [{"type": "Ready", "status": "True" if ready else "Unknown", "lastTransitionTime": transition_time(ready_for)}], + "addresses": [{"type": "InternalIP", "address": address or "127.0.0.1"}, {"type": "Hostname", "address": name}], + }, + } + + +def histogram(milliseconds: float, count: int = 100) -> dict[str, int]: + """Observations that all fell in the one bucket of etcd's WAL and commit histograms (bounds double from a millisecond) holding the given latency; the value is the bucket's upper bound as Prometheus writes it.""" + bound = 0.001 + while bound * 1000 < milliseconds: + bound *= 2 + return {repr(round(bound, 6)): count} + + +def etcd_gain( + *, + wal_ms: float = 4, + commit_ms: float = 8, + rtt: dict[str, dict[str, int]] | None = None, + has_leader: bool = True, + is_learner: bool = False, + known_members: int = TARGET, + active_peers: int = TARGET - 1, + quota_bytes: int = ETCD_QUOTA_BYTES, + leader_changes: int = 0, +) -> dict[str, Any]: + """The etcd part of a sample: what the histograms gained since the last sample, and the gauges.""" + return { + "has_leader": has_leader, + "is_learner": is_learner, + "known_members": known_members, + "active_peers": active_peers, + "quota_bytes": quota_bytes, + "leader_changes": leader_changes, + "wal": histogram(wal_ms, 1), + "commit": histogram(commit_ms, 1), + "rtt": rtt or {}, + } + + +def sample( + at: float, + *, + tcp: dict[str, float] | None = None, + etcd: dict[str, Any] | None = None, + restarted: bool = False, + psi: float | None = 0.0, + avail_kb: int | None = AVAILABLE_KB, + total_kb: int | None = TOTAL_KB, + link: str = "wired", + fsync_ms: float | None = None, +) -> dict[str, Any]: + return { + "t": int(at), + "tcp": tcp or {}, + "etcd": etcd, + "restarted": restarted, + "psi": psi, + "avail_kb": avail_kb, + "total_kb": total_kb, + "link": link, + "iface": "eth0", + "fsync_ms": fsync_ms, + } + + +def entry(samples: list[dict[str, Any]], etcd_id: str | None = None) -> dict[str, Any]: + return {"etcd_id": etcd_id, "last": None, "fsync_at": None, "samples": samples} + + +def times(count: int, now: float | None = None) -> list[float]: + """The times of the last count samples, one sample period apart, the newest now.""" + end = time.time() if now is None else now + return [end - (count - 1 - i) * SAMPLE_SECONDS for i in range(count)] diff --git a/tests/unit/test_autoscaler_image.py b/tests/unit/test_autoscaler_image.py index a90c6526..96f5a3b5 100644 --- a/tests/unit/test_autoscaler_image.py +++ b/tests/unit/test_autoscaler_image.py @@ -17,11 +17,13 @@ def scripts_run_from_the_image() -> set[str]: - """The names of the scripts under /app/scripts that the role's tasks and templates run.""" + """The names of the scripts under /app/scripts that the role's tasks and templates run from this image. The role controller's manifests run theirs from the image of their own (fleet-role/Dockerfile, held by tests/unit/test_fleet_role_arc.py).""" found: set[str] = set() for directory in ("tasks", "templates"): for path in (ROLE / directory).iterdir(): - found.update(re.findall(r"/app/scripts/([A-Za-z0-9._-]+\.sh)", path.read_text())) + text = path.read_text() + if "role_controller_image" not in text: + found.update(re.findall(r"/app/scripts/([A-Za-z0-9._-]+\.sh)", text)) return found diff --git a/tests/unit/test_cluster_voters.py b/tests/unit/test_cluster_voters.py new file mode 100644 index 00000000..c4f6319d --- /dev/null +++ b/tests/unit/test_cluster_voters.py @@ -0,0 +1,246 @@ +"""Tests for the cluster role's settings for the role controller: the defaults, the checks in tasks/voters.yml (run through the role with tests/cluster_versions/checks.yml), and the Node labels that publish which hosts are eligible and pinned, in the Docker node's .env and Compose files, the entrypoint's --node-label flags, and a native node's k3s configuration.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ROLE = ROOT / "roles" / "github_runner_cluster" +ENTRYPOINT = ROLE / "files" / "k3s" / "node-entrypoint.sh" +NODE_LABELS_COMPOSE = ROLE / "files" / "docker-compose.node-labels.yml" +ANSIBLE_PLAYBOOK = os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook") +DEFAULTS: dict[str, Any] = yaml.safe_load((ROLE / "defaults" / "main.yml").read_text()) + +CONTROLLER = "github_runner_cluster_role_controller" +VOTERS = "github_runner_cluster_voters" +ELIGIBLE = "github_runner_cluster_voter_eligible" +PINNED = "github_runner_cluster_voter_pinned" +ELIGIBLE_LABEL = "github-runner.exadev.com/voter-eligible" +PINNED_LABEL = "github-runner.exadev.com/voter-pinned" +MAX_SERVERS = "github_runner_cluster_max_servers" + + +def ansible_env() -> dict[str, str]: + return {**os.environ, "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", "ANSIBLE_RETRY_FILES_ENABLED": "false"} + + +def hosts(count: int, **per_host: dict[str, Any]) -> dict[str, dict[str, Any]]: + """count hosts named node1 to nodeN, with the given variables on the named ones.""" + return {f"node{index}": dict(per_host.get(f"node{index}", {})) for index in range(1, count + 1)} + + +class DefaultsTest(unittest.TestCase): + def test_nothing_is_on_eligible_or_pinned_by_default_and_the_target_is_three(self) -> None: + self.assertIs(DEFAULTS[CONTROLLER], False) + self.assertIs(DEFAULTS[ELIGIBLE], False) + self.assertIs(DEFAULTS[PINNED], False) + self.assertEqual(DEFAULTS[VOTERS], 3) + + def test_the_label_names_are_the_ones_the_controller_reads(self) -> None: + self.assertEqual(DEFAULTS["github_runner_cluster_voter_eligible_label"], ELIGIBLE_LABEL) + self.assertEqual(DEFAULTS["github_runner_cluster_voter_pinned_label"], PINNED_LABEL) + + def test_the_default_target_is_a_valid_one_for_the_default_maximum(self) -> None: + self.assertEqual(DEFAULTS[VOTERS] % 2, 1) + self.assertGreaterEqual(DEFAULTS[VOTERS], 3) + self.assertLessEqual(DEFAULTS[VOTERS], DEFAULTS[MAX_SERVERS] - 1) + + +@unittest.skipIf(shutil.which(ANSIBLE_PLAYBOOK) is None, "ansible-playbook is not installed") +class RoleTest(unittest.TestCase): + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.work) + + def check(self, group_hosts: dict[str, dict[str, Any]] | None = None, **variables: Any) -> subprocess.CompletedProcess[str]: + """Run the topology and role controller checks on a group of hosts (five, so that targets up to 5 and a maximum of 6 can be tried), with ``variables`` as group variables.""" + inventory = self.work / "inventory.yml" + group = {"vars": variables, "hosts": group_hosts or hosts(5)} + inventory.write_text(yaml.safe_dump({"all": {"children": {"github_runner_cluster": group}}})) + args = [ANSIBLE_PLAYBOOK, "-i", str(inventory), str(ROOT / "tests" / "cluster_versions" / "checks.yml"), "-e", "check_tasks=topology,voters"] + return subprocess.run(args, capture_output=True, text=True, check=False, env=ansible_env(), cwd=self.work) + + def refused(self, message: str, **variables: Any) -> None: + result = self.check(**variables) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn(message, result.stdout.replace("\n", " ")) + self.assertIn("Nothing was changed", result.stdout.replace("\n", " ")) + + def test_nothing_is_checked_while_the_controller_is_off(self) -> None: + for variables in ({}, {VOTERS: 4}, {VOTERS: 2}, {VOTERS: 5}, {ELIGIBLE: True}): + with self.subTest(variables=variables): + result = self.check(**variables) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_the_defaults_are_valid_with_the_controller_on_and_no_host_eligible(self) -> None: + result = self.check(**{CONTROLLER: True}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_an_even_target_is_refused(self) -> None: + for voters in (4, 2, 6): + with self.subTest(voters=voters): + self.refused("must be odd", **{CONTROLLER: True, VOTERS: voters, MAX_SERVERS: 9}) + + def test_a_target_under_three_is_refused(self) -> None: + self.refused("must be at least 3", **{CONTROLLER: True, VOTERS: 1}) + + def test_a_target_without_a_place_of_headroom_below_the_maximum_is_refused(self) -> None: + self.refused("can be at most one less than the most servers allowed (4)", **{CONTROLLER: True, VOTERS: 5, MAX_SERVERS: 5}) + self.refused("can be at most one less than the most servers allowed (2)", **{CONTROLLER: True, VOTERS: 3, MAX_SERVERS: 3}) + + def test_the_largest_target_that_leaves_headroom_is_accepted(self) -> None: + for voters, maximum in ((3, 4), (5, 6), (3, 5)): + with self.subTest(voters=voters, maximum=maximum): + result = self.check(**{CONTROLLER: True, VOTERS: voters, MAX_SERVERS: maximum}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_a_target_that_is_not_a_whole_number_is_refused(self) -> None: + for voters in ("three", "3.5", 0, -3, "03"): + with self.subTest(voters=voters): + self.refused("must be a whole number", **{CONTROLLER: True, VOTERS: voters}) + + def test_fewer_eligible_hosts_than_the_target_is_refused_once_any_is_marked(self) -> None: + group = hosts(5, node1={ELIGIBLE: True}, node2={ELIGIBLE: True}) + result = self.check(group, **{CONTROLLER: True}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("2 host(s) of github_runner_cluster set github_runner_cluster_voter_eligible (node1, node2), fewer than github_runner_cluster_voters (3)", result.stdout.replace("\n", " ")) + + def test_as_many_eligible_hosts_as_the_target_or_more_pass(self) -> None: + for count in (3, 4, 5): + with self.subTest(eligible=count): + group = hosts(5, **{f"node{index}": {ELIGIBLE: True} for index in range(1, count + 1)}) + result = self.check(group, **{CONTROLLER: True}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_a_pinned_host_does_not_count_as_eligible(self) -> None: + group = hosts(5, **{f"node{index}": {PINNED: True} for index in range(1, 6)}) + self.assertEqual(self.check(group, **{CONTROLLER: True}).returncode, 0) + + def test_a_datastore_with_no_etcd_is_refused(self) -> None: + self.refused("has no etcd votes to move", **{CONTROLLER: True, "github_runner_cluster_datastore": "sqlite"}) + + def render_env(self, *extra: str) -> tuple[list[str], list[str], list[str]]: + env, agent, files = self.work / "env", self.work / "env-agent", self.work / "files.json" + args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"env_agent_output={agent}", "-e", f"compose_files_output={files}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra] + result = subprocess.run(args, capture_output=True, text=True, check=False, env=ansible_env(), cwd=self.work) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return env.read_text().splitlines(), agent.read_text().splitlines(), json.loads(files.read_text()) + + def test_a_host_that_uses_none_of_it_keeps_its_configuration_exactly(self) -> None: + server, agent, files = self.render_env() + self.assertEqual([line for line in server + agent if "LABEL" in line], []) + self.assertEqual(files, ["docker-compose.yml"], "no override file, so an upgrade recreates no container") + + def test_an_eligible_host_publishes_eligible_true_and_pinned_false_in_its_env_and_its_compose_files(self) -> None: + server, agent, files = self.render_env("-e", f"{ELIGIBLE}=true") + expected = f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=true {PINNED_LABEL}=false" + self.assertIn(expected, server) + self.assertIn(expected, agent) + self.assertEqual(files, ["docker-compose.yml", "docker-compose.node-labels.yml"]) + + def test_a_pinned_host_publishes_pinned_true(self) -> None: + server, _, _ = self.render_env("-e", f"{PINNED}=true") + self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=true", server) + + def test_with_the_controller_on_every_host_publishes_both_as_false_so_that_a_setting_turned_off_is_written_over(self) -> None: + server, agent, files = self.render_env("-e", f"{CONTROLLER}=true") + for lines in (server, agent): + self.assertIn(f"K3S_NODE_LABELS={ELIGIBLE_LABEL}=false {PINNED_LABEL}=false", lines) + self.assertIn("docker-compose.node-labels.yml", files) + + def test_the_label_names_can_be_changed(self) -> None: + server, _, _ = self.render_env("-e", f"{ELIGIBLE}=true", "-e", "github_runner_cluster_voter_eligible_label=example.com/candidate") + self.assertIn(f"K3S_NODE_LABELS=example.com/candidate=true {PINNED_LABEL}=false", server) + + +class ComposeOverrideTest(unittest.TestCase): + def test_it_gives_both_services_the_labels_from_the_env_file(self) -> None: + services = yaml.safe_load(NODE_LABELS_COMPOSE.read_text())["services"] + self.assertEqual(sorted(services), ["k3s", "k3s-agent"]) + for name, service in services.items(): + with self.subTest(service=name): + self.assertEqual(service["environment"], ["K3S_NODE_LABELS=${K3S_NODE_LABELS:-}"]) + + def test_the_base_file_does_not_mention_it_so_a_host_without_labels_is_unchanged(self) -> None: + self.assertNotIn("K3S_NODE_LABELS", (ROLE / "files" / "docker-compose.yml").read_text()) + + def test_the_role_copies_it_to_the_node(self) -> None: + self.assertIn("docker-compose.node-labels.yml", [item["src"] for item in next(task for task in yaml.safe_load((ROLE / "tasks" / "main.yml").read_text()) if "loop" in task and any("docker-compose.yml" == entry.get("src") for entry in task["loop"]))["loop"]]) + + +def has_compose() -> bool: + if shutil.which("docker") is None: + return False + return subprocess.run(["docker", "compose", "version"], capture_output=True, check=False).returncode == 0 + + +@unittest.skipUnless(has_compose(), "docker compose is not installed") +class ComposeMergeTest(unittest.TestCase): + def environment(self, *files: Path, **env: str) -> dict[str, dict[str, str]]: + clean = {key: value for key, value in os.environ.items() if not key.startswith(("DOCKER_", "COMPOSE_", "K3S_", "TAILSCALE_"))} + args = ["docker", "compose", "--project-directory", tempfile.gettempdir(), "--project-name", "example"] + for file in files: + args += ["-f", str(file)] + args += ["--profile", "server", "--profile", "agent", "config", "--format", "json"] + base = {"K3S_TOKEN": "t", "K3S_TLS_SAN_LIST": "n", "K3S_VERSION": "v1.34.1-k3s1", "TAILSCALE_VERSION": "v1.102.4"} + result = subprocess.run(args, capture_output=True, text=True, check=False, env=clean | base | env) + self.assertEqual(result.returncode, 0, result.stderr) + services = json.loads(result.stdout)["services"] + return {name: services[name]["environment"] for name in ("k3s", "k3s-agent")} + + def test_merged_the_containers_get_the_labels(self) -> None: + labels = f"{ELIGIBLE_LABEL}=true {PINNED_LABEL}=false" + merged = self.environment(ROLE / "files" / "docker-compose.yml", NODE_LABELS_COMPOSE, K3S_NODE_LABELS=labels) + for service in merged.values(): + self.assertEqual(service["K3S_NODE_LABELS"], labels) + + def test_unmerged_the_containers_have_no_such_variable(self) -> None: + for service in self.environment(ROLE / "files" / "docker-compose.yml", K3S_NODE_LABELS="ignored").values(): + self.assertNotIn("K3S_NODE_LABELS", service) + + +def function_source(name: str) -> str: + import re + + match = re.search(rf"^{name}\(\) \{{\n.*?^\}}\n", ENTRYPOINT.read_text(), re.MULTILINE | re.DOTALL) + assert match is not None, f"{name} is not defined in node-entrypoint.sh" + return match.group(0) + + +class EntrypointTest(unittest.TestCase): + """Runs the entrypoint's own function in a shell, without Tailscale or k3s.""" + + def flags(self, **env: str) -> list[str]: + script = "\n".join(["set -u", function_source("set_node_label_flags"), "set_node_label_flags", 'printf "%s\\n" "$node_label_flags"']) + clean = {key: value for key, value in os.environ.items() if not key.startswith("K3S_")} + result = subprocess.run(["/bin/sh", "-c", script], capture_output=True, text=True, check=False, env=clean | env) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout.split() + + def test_each_label_becomes_a_node_label_flag(self) -> None: + self.assertEqual(self.flags(K3S_NODE_LABELS=f"{ELIGIBLE_LABEL}=true {PINNED_LABEL}=false"), ["--node-label", f"{ELIGIBLE_LABEL}=true", "--node-label", f"{PINNED_LABEL}=false"]) + + def test_none_set_gives_no_flag(self) -> None: + self.assertEqual(self.flags(), []) + self.assertEqual(self.flags(K3S_NODE_LABELS=""), []) + + def test_the_flags_reach_every_k3s_command_line_the_entrypoint_builds(self) -> None: + source = ENTRYPOINT.read_text() + for command in ("/bin/k3s server", "exec /bin/k3s server", "exec /bin/k3s agent"): + lines = [line for line in source.splitlines() if command in line and "$node_taint_flags" in line] + self.assertTrue(lines, command) + for line in lines: + self.assertIn("$node_label_flags", line) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_fleet_role.py b/tests/unit/test_fleet_role.py new file mode 100644 index 00000000..b39c643b --- /dev/null +++ b/tests/unit/test_fleet_role.py @@ -0,0 +1,632 @@ +"""Tests for scripts/fleet-role.sh and scripts/fleet-role.jq, one pass of the role controller, run against a scripted stand-in for kubectl with the real bash and jq. The probes' entries are built the way scripts/fleet-role-probe.sh writes them (tests/unit/test_fleet_role_probe.py holds the probe to a real etcd's metrics); here every threshold and decision rule is held to its boundary.""" + +from __future__ import annotations + +import json +import os +import subprocess +import time +import unittest +from dataclasses import dataclass, field +from typing import Any + +from fleet_role_support import ( + AVAILABLE_FLOOR_PERCENT, + ARGUMENT_LIMIT_BYTES, + AVAILABLE_KB, + COMMIT_LIMIT_MS, + CONTROLLER_ENV, + COOLDOWN_SECONDS, + ELECTION_TIMEOUT_MS, + ETCD_QUOTA_BYTES, + FIT_RTT_MS, + MAX_SERVERS, + PRESSURE_THRESHOLD, + REGRADE_CORE, + SCRIPTS, + SAMPLE_SECONDS, + STALE_AFTER_SAMPLES, + TARGET, + TOTAL_KB, + UNFIT_RTT_MS, + WAL_LIMIT_MS, + WINDOW_SAMPLES, + WINDOW_SECONDS, + Cluster, + entry, + etcd_gain, + histogram, + node, + sample, + times, +) + +Series = float | list[float] + + +@dataclass +class Host: + """What one host's probe has reported over the window, and the Node it is.""" + + name: str + voter: bool = False + eligible: bool = False + pinned: bool = False + ready: bool = True + ready_for: float = 24 * 3600 + level: str | None = None + link: str = "wired" + # Milliseconds to every voter, or per peer; a list is one reading per sample. + tcp: Series | dict[str, Series] = 5.0 + # What etcd's own round trip histogram holds per peer, in milliseconds, for a voter; none leaves the histograms out. + etcd_rtt: float | dict[str, float] | None = None + # Per-sample gains of a voter's histograms, in milliseconds, or the buckets themselves when given as dicts. + wal: Series | list[dict[str, int]] = 4.0 + commit: Series | list[dict[str, int]] = 8.0 + fsync_ms: float | None = 3.0 + psi: float = 0.0 + avail_kb: int = AVAILABLE_KB + total_kb: int = TOTAL_KB + samples: int = WINDOW_SAMPLES + restarted: bool = False + stale_by: float = 0 + etcd: dict[str, Any] = field(default_factory=dict) + + +def series_value(value: Series | dict[str, Series], peer: str, index: int) -> float: + if isinstance(value, dict): + value = value[peer] + return value[index] if isinstance(value, list) else value + + +def gain(value: Series | list[dict[str, int]], index: int) -> dict[str, int]: + item = value[index] if isinstance(value, list) else value + return item if isinstance(item, dict) else histogram(item, 1) + + +def etcd_id(name: str) -> str: + return f"{name}-etcd-id" + + +def populate(cluster: Cluster, hosts: list[Host]) -> None: + """Gives the cluster these Nodes and, for each host, a probe entry of its last `samples` samples ending now (less stale_by).""" + voters = [h.name for h in hosts if h.voter] + cluster.nodes = [ + node(h.name, voter=h.voter, eligible=h.eligible, pinned=h.pinned, ready=h.ready, ready_for=h.ready_for, level=h.level) for h in hosts + ] + entries = {} + for h in hosts: + peers = [v for v in voters if v != h.name] + stamps = times(h.samples, time.time() - h.stale_by) + built = [] + for i, at in enumerate(stamps): + etcd = None + if h.voter: + rtt = {} + if h.etcd_rtt is not None: + rtt = {etcd_id(p): histogram(h.etcd_rtt[p] if isinstance(h.etcd_rtt, dict) else h.etcd_rtt, 1) for p in peers} + etcd = {**etcd_gain(rtt=rtt, known_members=len(voters), active_peers=len(voters) - 1), "wal": gain(h.wal, i), "commit": gain(h.commit, i), **h.etcd} + built.append( + sample( + at, + tcp={p: series_value(h.tcp, p, i) for p in (voters if not h.voter else peers)}, + etcd=etcd, + restarted=h.restarted and i == len(stamps) // 2, + psi=h.psi, + avail_kb=h.avail_kb, + total_kb=h.total_kb, + link=h.link, + fsync_ms=None if h.voter else h.fsync_ms if i == 0 else None, + ) + ) + entries[h.name] = entry(built, etcd_id(h.name) if h.voter else None) + cluster.set_probes(entries) + + +def healthy(**changes: Any) -> list[Host]: + """Three fit voters, a, b and c, and a fit eligible standby, d; every voter and the standby eligible.""" + hosts = [Host("a", voter=True, eligible=True), Host("b", voter=True, eligible=True), Host("c", voter=True, eligible=True), Host("d", eligible=True)] + for host in hosts: + for key, value in changes.get(host.name, {}).items(): + setattr(host, key, value) + return hosts + + +class ControllerCase(unittest.TestCase): + def setUp(self) -> None: + self.cluster = Cluster() + self.addCleanup(self.cluster.cleanup) + + def pass_(self, hosts: list[Host], **env: str) -> dict[str, Any]: + """Runs one pass over these hosts and returns the status it published.""" + populate(self.cluster, hosts) + completed = self.cluster.run("fleet-role.sh", **{**CONTROLLER_ENV, **env}) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.log = completed.stdout + return self.cluster.status() + + def verdicts(self, status: dict[str, Any]) -> dict[str, str]: + return {name: host["verdict"] for name, host in status["hosts"].items()} + + +class ThresholdTest(ControllerCase): + def test_the_limits_derive_from_the_election_timeout_and_the_window_from_the_tail(self) -> None: + status = self.pass_(healthy()) + config = status["config"] + self.assertEqual(config["unfit_rtt_ms"], ELECTION_TIMEOUT_MS / 10) + self.assertEqual(config["fit_rtt_ms"], ELECTION_TIMEOUT_MS / 20) + self.assertEqual(config["window_samples"], 100) + self.assertEqual(config["window_seconds"], WINDOW_SECONDS) + self.assertEqual(config["cooldown_seconds"], COOLDOWN_SECONDS) + self.assertEqual((config["wal_fsync_limit_ms"], config["backend_commit_limit_ms"]), (WAL_LIMIT_MS, COMMIT_LIMIT_MS)) + + def test_healthy_voters_and_a_healthy_standby_are_all_fit_and_nothing_would_change(self) -> None: + status = self.pass_(healthy()) + self.assertEqual(self.verdicts(status), {"a": "fit", "b": "fit", "c": "fit", "d": "fit"}) + self.assertEqual(status["decision"]["action"], "none") + self.assertEqual(status["decision"]["because"], ["no voter is unfit"]) + self.assertEqual(status["blocks"], []) + self.assertTrue(status["dry_run"]) + + def test_a_voter_is_unfit_only_above_a_tenth_of_the_election_timeout(self) -> None: + for rtt, verdict in ((UNFIT_RTT_MS, "fit"), (UNFIT_RTT_MS + 0.001, "unfit")): + with self.subTest(rtt=rtt): + status = self.pass_(healthy(a={"tcp": rtt})) + self.assertEqual(status["hosts"]["a"]["verdict"], verdict) + + def test_a_candidate_is_fit_only_at_or_below_a_twentieth_of_the_election_timeout(self) -> None: + for rtt, verdict in ((FIT_RTT_MS, "fit"), (FIT_RTT_MS + 0.001, "unfit")): + with self.subTest(rtt=rtt): + status = self.pass_(healthy(d={"tcp": rtt})) + self.assertEqual(status["hosts"]["d"]["verdict"], verdict) + + def test_between_the_two_limits_a_voter_is_kept_and_a_candidate_is_not_promoted(self) -> None: + between = (UNFIT_RTT_MS + FIT_RTT_MS) / 2 + status = self.pass_(healthy(a={"tcp": between}, d={"tcp": between})) + self.assertEqual(self.verdicts(status), {"a": "fit", "b": "fit", "c": "fit", "d": "unfit"}) + self.assertEqual(status["decision"]["action"], "none") + + def test_the_round_trip_figure_is_the_tail_of_the_window_so_one_outlier_never_decides(self) -> None: + one_outlier = [5.0] * (WINDOW_SAMPLES - 1) + [ELECTION_TIMEOUT_MS] + two_outliers = [5.0] * (WINDOW_SAMPLES - 2) + [ELECTION_TIMEOUT_MS] * 2 + self.assertEqual(self.pass_(healthy(a={"tcp": one_outlier}))["hosts"]["a"]["verdict"], "fit") + status = self.pass_(healthy(a={"tcp": two_outliers})) + self.assertEqual(status["hosts"]["a"]["verdict"], "unfit") + self.assertEqual(status["hosts"]["a"]["figures"]["rtt_p99_ms"], ELECTION_TIMEOUT_MS) + + def test_a_window_one_sample_short_is_unknown_not_fit(self) -> None: + status = self.pass_(healthy(a={"samples": WINDOW_SAMPLES - 1})) + self.assertEqual(status["hosts"]["a"]["verdict"], "unknown") + self.assertIn(f"{WINDOW_SAMPLES - 1} of {WINDOW_SAMPLES} samples since the last membership change", status["hosts"]["a"]["reasons"]) + + def test_a_probe_report_older_than_three_samples_is_no_report(self) -> None: + for stale_by, verdict in (((STALE_AFTER_SAMPLES * SAMPLE_SECONDS) - 5, "fit"), ((STALE_AFTER_SAMPLES * SAMPLE_SECONDS) + 5, "unknown")): + with self.subTest(stale_by=stale_by): + status = self.pass_(healthy(a={"stale_by": stale_by})) + self.assertEqual(status["hosts"]["a"]["verdict"], verdict) + + def test_a_node_with_no_probe_entry_is_unknown(self) -> None: + hosts = healthy() + populate(self.cluster, hosts) + entries = self.cluster.probes() + del entries["d"] + self.cluster.set_probes(entries) + completed = self.cluster.run("fleet-role.sh", **CONTROLLER_ENV) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(self.cluster.status()["hosts"]["d"]["reasons"][0], "no probe report") + + def test_etcds_own_round_trip_counts_beside_the_tcp_connect(self) -> None: + status = self.pass_(healthy(a={"tcp": 5.0, "etcd_rtt": {"b": 5.0, "c": 150.0}})) + self.assertEqual(status["hosts"]["a"]["verdict"], "unfit") + self.assertEqual(status["hosts"]["a"]["figures"]["rtt_peer"], "c") + self.assertTrue(status["hosts"]["a"]["figures"]["rtt_p99_ms"] > UNFIT_RTT_MS) + + def test_the_worst_peer_decides(self) -> None: + status = self.pass_(healthy(b={"tcp": {"a": 5.0, "c": UNFIT_RTT_MS + 20}})) + self.assertEqual(status["hosts"]["b"]["verdict"], "unfit") + self.assertEqual(status["hosts"]["b"]["figures"]["rtt_peer"], "c") + + +class DiskTest(ControllerCase): + def gains(self, in_limit: int, past: int, limit_seconds: float, over_seconds: float) -> list[dict[str, int]]: + """One gain per sample: in_limit of them in a bucket whose upper bound is the limit, past in a bucket above it.""" + return [{str(limit_seconds): 1} for _ in range(in_limit)] + [{str(over_seconds): 1} for _ in range(past)] + + def test_a_voters_wal_fsync_tail_must_be_under_ten_milliseconds(self) -> None: + for in_limit, past, bound, expected in ( + (99, 1, 0.01, "unfit"), # the tail interpolates to the bucket's bound, ten milliseconds, which is not under ten + (99, 1, 0.009, "fit"), + ): + with self.subTest(bound=bound): + status = self.pass_(healthy(a={"wal": self.gains(in_limit, past, bound, 0.02)})) + self.assertEqual(status["hosts"]["a"]["verdict"], expected) + + def test_a_voters_backend_commit_tail_must_be_under_twenty_five_milliseconds(self) -> None: + for bound, expected in ((0.025, "unfit"), (0.024, "fit")): + with self.subTest(bound=bound): + status = self.pass_(healthy(a={"commit": self.gains(99, 1, bound, 0.05)})) + self.assertEqual(status["hosts"]["a"]["verdict"], expected) + + def test_a_standbys_fdatasync_tail_must_be_under_the_wal_limit(self) -> None: + for fsync, expected in ((WAL_LIMIT_MS - 0.01, "fit"), (WAL_LIMIT_MS, "unfit")): + with self.subTest(fsync=fsync): + status = self.pass_(healthy(d={"fsync_ms": fsync})) + self.assertEqual(status["hosts"]["d"]["verdict"], expected) + + def test_a_standby_without_an_fdatasync_reading_is_unknown(self) -> None: + status = self.pass_(healthy(d={"fsync_ms": None})) + self.assertEqual(status["hosts"]["d"]["verdict"], "unknown") + self.assertIn("no fdatasync reading yet", status["hosts"]["d"]["reasons"]) + + def test_a_voter_with_no_etcd_disk_observations_is_unknown(self) -> None: + status = self.pass_(healthy(a={"wal": [{}] * WINDOW_SAMPLES})) + self.assertEqual(status["hosts"]["a"]["verdict"], "unknown") + + +class MemoryAndStabilityTest(ControllerCase): + def test_memory_over_the_regrade_limit_needs_both_pressure_and_low_available(self) -> None: + total = 1_000_000 + floor = total * AVAILABLE_FLOOR_PERCENT // 100 + self.assertEqual(floor * 100, total * AVAILABLE_FLOOR_PERCENT, "a total whose floor is a whole number of kilobytes") + cases = ( + ("pressure at the threshold and available just under the floor", PRESSURE_THRESHOLD, floor - 1, "unfit"), + ("pressure under the threshold", PRESSURE_THRESHOLD - 0.01, floor - 1, "fit"), + ("available at the floor", PRESSURE_THRESHOLD, floor, "fit"), + ) + for label, psi, avail, expected in cases: + with self.subTest(label): + # A quota of one byte on every voter, so that only the regrade limit can fail the host. + quota = {"etcd": {"quota_bytes": 1}} + status = self.pass_(healthy(a={"psi": psi, "avail_kb": avail, "total_kb": total, **quota}, b=quota, c=quota)) + self.assertEqual(status["hosts"]["a"]["verdict"], expected) + + def test_available_memory_must_be_at_least_etcds_storage_quota(self) -> None: + quota_kb = ETCD_QUOTA_BYTES // 1024 + for avail, expected in ((quota_kb, "fit"), (quota_kb - 1, "unfit")): + with self.subTest(avail=avail): + status = self.pass_(healthy(a={"avail_kb": avail, "total_kb": avail * 1000})) + self.assertEqual(status["hosts"]["a"]["verdict"], expected) + + def test_the_quota_is_the_largest_a_voter_reports(self) -> None: + quota = 4 * 1024**3 + status = self.pass_(healthy(b={"etcd": {"quota_bytes": quota}}, a={"avail_kb": 3 * 1024**2, "total_kb": 3 * 1024**5})) + self.assertEqual(status["config"]["quota_bytes"], quota) + self.assertEqual(status["hosts"]["a"]["verdict"], "unfit") + + def test_one_sample_over_the_limit_anywhere_in_the_window_is_enough(self) -> None: + populate(self.cluster, healthy()) + entries = self.cluster.probes() + # Under the floor share of the total but above etcd's quota, so only the regrade limit can fail it. + entries["a"]["samples"][10].update(psi=PRESSURE_THRESHOLD + 5, avail_kb=TOTAL_KB * AVAILABLE_FLOOR_PERCENT // 100 - 1) + self.cluster.set_probes(entries) + completed = self.cluster.run("fleet-role.sh", **CONTROLLER_ENV) + self.assertEqual(completed.returncode, 0, completed.stderr) + host = self.cluster.status()["hosts"]["a"] + self.assertEqual(host["verdict"], "unfit") + self.assertIn("memory was over the regrade limit in 1 of 100 samples", host["reasons"]) + + def test_a_node_that_was_not_ready_within_the_window_is_unfit_and_one_that_was_not_is_fit(self) -> None: + for ready_for, expected in ((WINDOW_SECONDS - 60, "unfit"), (WINDOW_SECONDS + 60, "fit")): + with self.subTest(ready_for=ready_for): + status = self.pass_(healthy(d={"ready_for": ready_for})) + self.assertEqual(status["hosts"]["d"]["verdict"], expected) + + def test_a_node_that_is_not_ready_is_unfit(self) -> None: + status = self.pass_(healthy(d={"ready": False})) + self.assertEqual(status["hosts"]["d"]["verdict"], "unfit") + self.assertIn("the node is not Ready", status["hosts"]["d"]["reasons"]) + + def test_a_k3s_restart_in_the_window_is_unfit(self) -> None: + status = self.pass_(healthy(a={"restarted": True}, d={"restarted": True})) + self.assertEqual(status["hosts"]["a"]["verdict"], "unfit") + self.assertEqual(status["hosts"]["d"]["verdict"], "unfit") + self.assertEqual(status["hosts"]["a"]["figures"]["restarts"], 1) + + def test_leader_changes_are_reported(self) -> None: + status = self.pass_(healthy(a={"etcd": {"leader_changes": 2}})) + self.assertEqual(status["hosts"]["a"]["figures"]["leader_changes"], 2 * WINDOW_SAMPLES) + + +class RankingTest(ControllerCase): + def test_an_unfit_voter_is_swapped_for_the_fit_standby(self) -> None: + status = self.pass_(healthy(a={"tcp": UNFIT_RTT_MS + 50})) + self.assertEqual(status["decision"]["action"], "swap") + self.assertEqual(status["decision"]["swap"], {"out": "a", "in": "d"}) + self.assertIn("DRY RUN: would swap out a for d", self.log) + + def test_wired_ranks_before_wifi_before_unknown_whatever_the_round_trip(self) -> None: + hosts = healthy(a={"tcp": UNFIT_RTT_MS + 50, "link": "wifi"}) + hosts = [h for h in hosts if h.name != "d"] + hosts += [Host("e", eligible=True, link="wifi", tcp=1.0), Host("f", eligible=True, link="wired", tcp=FIT_RTT_MS), Host("g", eligible=True, link="unknown", tcp=1.0)] + status = self.pass_(hosts) + self.assertEqual(status["ranking"], ["b", "c", "f", "e", "g"]) + self.assertEqual(status["decision"]["swap"], {"out": "a", "in": "f"}) + + def test_within_a_link_type_the_lower_round_trip_ranks_first_then_the_lower_fsync_then_the_name(self) -> None: + cases = ( + ("round trip", {"e": {"tcp": 10.0}, "f": {"tcp": 20.0}}, "e"), + ("fsync", {"e": {"fsync_ms": 4.0}, "f": {"fsync_ms": 2.0}}, "f"), + ("name", {"e": {}, "f": {}}, "e"), + ) + for label, overrides, expected in cases: + with self.subTest(label): + base = {"tcp": 5.0, "fsync_ms": 3.0} + hosts = healthy(a={"tcp": UNFIT_RTT_MS + 50, "link": "wifi"}, d={"tcp": FIT_RTT_MS + 20}) + hosts += [Host("e", eligible=True, **{**base, **overrides["e"]}), Host("f", eligible=True, **{**base, **overrides["f"]})] + self.assertEqual(self.pass_(hosts)["decision"]["swap"], {"out": "a", "in": expected}) + + def test_a_candidate_must_rank_above_the_voter_it_replaces(self) -> None: + # The unfit voter is wired; the only fit standby is on a link the cluster cannot see, which ranks below wired. + hosts = healthy(a={"tcp": UNFIT_RTT_MS + 50}, d={"link": "unknown"}) + status = self.pass_(hosts) + self.assertEqual(status["decision"]["action"], "none") + self.assertEqual(status["decision"]["because"][0], "voter a is unfit and no eligible standby is fit and ranks above it") + + def test_the_worst_ranked_unfit_voter_goes_out_first(self) -> None: + hosts = healthy(a={"tcp": UNFIT_RTT_MS + 50}, b={"tcp": UNFIT_RTT_MS + 400}) + status = self.pass_(hosts) + self.assertEqual(status["decision"]["swap"], {"out": "b", "in": "d"}) + + def test_a_pinned_voter_is_never_swapped_out(self) -> None: + status = self.pass_(healthy(a={"tcp": UNFIT_RTT_MS + 50, "pinned": True})) + self.assertEqual(status["decision"]["action"], "none") + self.assertEqual(status["decision"]["because"], ["voter a is the most at fault and is pinned or not marked eligible, so it is not replaced"]) + + def test_a_voter_not_marked_eligible_is_never_swapped_out(self) -> None: + status = self.pass_(healthy(a={"tcp": UNFIT_RTT_MS + 50, "eligible": False})) + self.assertEqual(status["decision"]["action"], "none") + self.assertEqual(status["decision"]["because"], ["voter a is the most at fault and is pinned or not marked eligible, so it is not replaced"]) + + SLOW = UNFIT_RTT_MS + 200 + + def slow_voter_c(self, **changes: dict[str, Any]) -> list[Host]: + """Voter c has a slow link to every other host, as a voter on a poor path does: both ends of each of its links measure it slow, so a and b are unfit too, by the rule that a voter is unfit when its round trip to any other voter is.""" + hosts = healthy( + a={"tcp": {"b": 5.0, "c": self.SLOW + 300}}, # the worst single reading is a's, but c is on the most slow links + b={"tcp": {"a": 5.0, "c": self.SLOW}}, + c={"tcp": {"a": self.SLOW, "b": self.SLOW}}, + d={"tcp": {"a": 5.0, "b": 5.0, "c": self.SLOW}}, + ) + for host in hosts: + for key, value in changes.get(host.name, {}).items(): + setattr(host, key, value) + return hosts + + def test_a_slow_link_makes_both_its_ends_unfit_and_the_voter_on_the_most_slow_links_is_replaced(self) -> None: + status = self.pass_(self.slow_voter_c()) + self.assertEqual(self.verdicts(status), {"a": "unfit", "b": "unfit", "c": "unfit", "d": "unfit"}) + self.assertEqual(status["decision"]["swap"], {"out": "c", "in": "d"}, "c is on two slow links, a and b on one each") + + def test_a_candidate_is_judged_against_the_voters_that_would_remain(self) -> None: + # d measures c slow, as everyone does, and is unfit against the full set; without c it is fit, which is what the swap needs. + status = self.pass_(self.slow_voter_c()) + self.assertEqual(status["hosts"]["d"]["verdict"], "unfit") + self.assertEqual(status["decision"]["action"], "swap") + status = self.pass_(self.slow_voter_c(d={"tcp": {"a": self.SLOW, "b": 5.0, "c": self.SLOW}})) + self.assertEqual(status["decision"]["action"], "none", "d is slow to a voter that stays") + self.assertEqual(status["decision"]["because"], ["voter c is unfit and no eligible standby is fit and ranks above it"]) + + def test_nothing_is_swapped_when_the_voter_at_fault_is_pinned_even_though_its_peers_are_unfit_and_replaceable(self) -> None: + status = self.pass_(self.slow_voter_c(c={"pinned": True})) + self.assertEqual(self.verdicts(status)["a"], "unfit") + self.assertEqual(status["decision"]["action"], "none") + self.assertEqual(status["decision"]["because"], ["voter c is the most at fault and is pinned or not marked eligible, so it is not replaced"]) + + def test_an_unfit_standby_is_not_a_candidate(self) -> None: + status = self.pass_(healthy(a={"tcp": UNFIT_RTT_MS + 50}, d={"fsync_ms": 50.0})) + self.assertEqual(status["decision"]["action"], "none") + + +class GateTest(ControllerCase): + """The swap is wanted in each of these (voter a is unfit and d is fit); something about the cluster as a whole says not now.""" + + def refused(self, **changes: dict[str, Any]) -> dict[str, Any]: + status = self.pass_(healthy(a={"tcp": UNFIT_RTT_MS + 50, **changes.get("a", {})}, **{k: v for k, v in changes.items() if k != "a"})) + self.assertEqual(status["decision"]["action"], "none", status["decision"]) + self.assertEqual(status["decision"]["wanted"], {"out": "a", "in": "d"}) + return status + + def test_no_swap_while_a_voter_is_below_full(self) -> None: + status = self.refused(b={"level": "etcd-only"}) + self.assertEqual(status["decision"]["because"], ["voter b is at regrade level etcd-only, below full"]) + + def test_no_swap_while_a_voter_is_not_ready(self) -> None: + status = self.refused(c={"ready": False}) + self.assertIn("voter c is not Ready", status["decision"]["because"]) + + def test_no_swap_while_a_voters_probe_has_stopped_reporting_as_when_it_is_paused(self) -> None: + stale = STALE_AFTER_SAMPLES * SAMPLE_SECONDS + status = self.refused(b={"stale_by": stale + 30}) + self.assertIn(f"voter b has no probe report in the last {stale} s", status["decision"]["because"]) + self.assertEqual(status["hosts"]["b"]["verdict"], "unknown") + + def test_no_swap_while_a_voters_etcd_metrics_could_not_be_read(self) -> None: + populate(self.cluster, healthy(a={"tcp": UNFIT_RTT_MS + 50})) + entries = self.cluster.probes() + entries["b"]["samples"][-1]["etcd"] = None + self.cluster.set_probes(entries) + completed = self.cluster.run("fleet-role.sh", **CONTROLLER_ENV) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertIn("etcd's state on voter b is unknown", self.cluster.status()["decision"]["because"]) + + def test_no_swap_while_etcd_reports_no_leader(self) -> None: + status = self.refused(b={"etcd": {"has_leader": False}}) + self.assertIn("voter b reports no leader", status["decision"]["because"]) + + def test_no_swap_while_a_voter_is_a_learner(self) -> None: + status = self.refused(b={"etcd": {"is_learner": True}}) + self.assertIn("voter b is a learner", status["decision"]["because"]) + + def test_no_swap_while_a_voter_is_not_connected_to_every_other(self) -> None: + status = self.refused(b={"etcd": {"active_peers": TARGET - 2}}) + self.assertIn(f"voter b is connected to {TARGET - 2} of its {TARGET - 1} peers", status["decision"]["because"]) + + def test_no_swap_while_a_voter_knows_a_member_that_is_not_a_voter(self) -> None: + # A learner joining, or a member added and not yet started, is a member etcd knows and no Node is labelled for. + status = self.refused(b={"etcd": {"known_members": TARGET + 1}}) + self.assertIn(f"voter b knows {TARGET + 1} members, not the {TARGET} voters: a learner or a member that has not started", status["decision"]["because"]) + + def test_no_swap_while_the_voter_count_is_not_the_target(self) -> None: + hosts = healthy(a={"tcp": UNFIT_RTT_MS + 50}) + hosts.append(Host("e", voter=True, eligible=True)) + status = self.pass_(hosts) + self.assertEqual(status["decision"]["action"], "none") + self.assertIn(f"there are 4 voters, not the target {TARGET}", status["decision"]["because"]) + + def test_no_swap_during_the_cooldown_after_a_membership_change_and_a_swap_after_it(self) -> None: + for since_change, expected in ((COOLDOWN_SECONDS - 60, "none"), (COOLDOWN_SECONDS + 60, "swap")): + with self.subTest(since_change=since_change): + hosts = healthy(a={"tcp": UNFIT_RTT_MS + 50}) + changed_at = int(time.time()) - since_change + self.cluster.set_status({"membership": {"voters": ["a", "b", "c"], "changed_at": changed_at, "cooldown_until": changed_at + COOLDOWN_SECONDS}}) + # Only samples since the change count, and a window's worth have arrived since in either case (the cooldown outlasts the window). + for host in hosts: + host.samples = WINDOW_SAMPLES + status = self.pass_(hosts) + self.assertEqual(status["decision"]["action"], expected, status["decision"]) + if expected == "none": + self.assertTrue(any("the cooldown lasts until" in reason for reason in status["decision"]["because"])) + + def test_the_cooldown_outlasts_the_window_by_the_learner_stall_time(self) -> None: + self.assertEqual(COOLDOWN_SECONDS - WINDOW_SECONDS, 300) + self.assertGreater(COOLDOWN_SECONDS, WINDOW_SECONDS) + + def test_readings_from_before_a_membership_change_are_not_counted(self) -> None: + changed_at = int(time.time()) - (WINDOW_SECONDS // 2) + self.cluster.set_status({"membership": {"voters": ["a", "b", "c"], "changed_at": changed_at, "cooldown_until": changed_at + COOLDOWN_SECONDS}}) + status = self.pass_(healthy()) + self.assertEqual(status["hosts"]["a"]["verdict"], "unknown") + self.assertEqual(status["hosts"]["a"]["samples"], WINDOW_SAMPLES // 2 + 1) + + def test_fewer_eligible_hosts_than_the_target_is_reported_and_refuses_a_swap(self) -> None: + status = self.pass_(healthy(a={"tcp": UNFIT_RTT_MS + 50}, b={"eligible": False}, c={"eligible": False})) + self.assertEqual(status["decision"]["action"], "none") + self.assertIn(f"2 hosts are eligible, fewer than the target {TARGET}", status["decision"]["because"]) + + +class SizeTest(ControllerCase): + # The probes' entries together can fill a ConfigMap, a mebibyte, and no one argument to a process may pass 128 KiB on Linux (the stand-in jq of tests/unit/fleet_role_support.py refuses one here). + CONFIGMAP_LIMIT_BYTES = 1024 * 1024 + + def test_inputs_the_size_of_a_full_configmap_are_read_whole(self) -> None: + standbys = [Host(f"standby-with-a-long-node-name-{i:02d}", eligible=True) for i in range(14)] + hosts = healthy() + standbys + populate(self.cluster, hosts) + size = sum(len(json.dumps(entry_)) for entry_ in self.cluster.probes().values()) + self.assertGreater(size, 3 * ARGUMENT_LIMIT_BYTES) + self.assertLessEqual(size, self.CONFIGMAP_LIMIT_BYTES) + completed = self.cluster.run("fleet-role.sh", **CONTROLLER_ENV) + self.assertEqual(completed.returncode, 0, completed.stderr) + status = self.cluster.status() + self.assertEqual(len(status["hosts"]), len(hosts)) + self.assertEqual(status["hosts"]["a"]["verdict"], "fit") + + +class MembershipTest(ControllerCase): + def test_the_first_pass_records_the_voters_and_no_change(self) -> None: + status = self.pass_(healthy()) + self.assertEqual(status["membership"], {"voters": ["a", "b", "c"], "changed_at": None, "cooldown_until": None}) + + def test_a_pass_that_sees_the_same_voters_keeps_the_time_of_the_last_change(self) -> None: + self.cluster.set_status({"membership": {"voters": ["a", "b", "c"], "changed_at": 1700000000, "cooldown_until": 1700000000 + COOLDOWN_SECONDS}}) + status = self.pass_(healthy()) + self.assertEqual(status["membership"]["changed_at"], 1700000000) + + def test_a_pass_that_sees_other_voters_records_the_change_now_and_the_cooldown(self) -> None: + self.cluster.set_status({"membership": {"voters": ["a", "b", "x"], "changed_at": None, "cooldown_until": None}}) + before = int(time.time()) + status = self.pass_(healthy()) + self.assertGreaterEqual(status["membership"]["changed_at"], before) + self.assertEqual(status["membership"]["cooldown_until"], status["membership"]["changed_at"] + COOLDOWN_SECONDS) + + +class ReportingTest(ControllerCase): + def test_with_no_host_eligible_it_only_reports_the_voters(self) -> None: + hosts = [Host("a", voter=True), Host("b", voter=True), Host("c", voter=True, tcp=UNFIT_RTT_MS + 50), Host("d")] + status = self.pass_(hosts) + self.assertEqual(sorted(status["hosts"]), ["a", "b", "c"]) + self.assertEqual(status["hosts"]["c"]["verdict"], "unfit") + self.assertEqual(status["decision"]["action"], "none") + self.assertIn("no host is marked eligible", status["decision"]["because"]) + + def test_each_host_says_why_with_the_numbers(self) -> None: + status = self.pass_(healthy(a={"tcp": UNFIT_RTT_MS + 50, "psi": PRESSURE_THRESHOLD, "avail_kb": TOTAL_KB * AVAILABLE_FLOOR_PERCENT // 100 - 1})) + reasons = "; ".join(status["hosts"]["a"]["reasons"]) + self.assertIn("round trip p99 to", reasons) + self.assertIn(f"over {UNFIT_RTT_MS:g} ms", reasons) + self.assertIn(f"{ELECTION_TIMEOUT_MS} ms election timeout", reasons) + self.assertIn("memory was over the regrade limit in 100 of 100 samples", reasons) + self.assertIn("a (voter): unfit", self.log) + + def test_the_link_type_is_reported_for_every_host(self) -> None: + status = self.pass_(healthy(a={"link": "wifi"}, d={"link": "unknown"})) + self.assertEqual({name: host["link"] for name, host in status["hosts"].items()}, {"a": "wifi", "b": "wired", "c": "wired", "d": "unknown"}) + + def test_a_pass_writes_only_its_own_status_and_changes_nothing_in_the_cluster(self) -> None: + self.pass_(healthy(a={"tcp": UNFIT_RTT_MS + 50})) + calls = [json.loads(line) for line in self.cluster.calls.read_text().splitlines()] + writes = [call for call in calls if call[0] not in ("get",)] + self.assertEqual(len(writes), 1, writes) + self.assertEqual(writes[0][:3], ["patch", "configmap", "fleet-role-status"]) + + +class SettingsTest(ControllerCase): + def refuses(self, message: str, **env: str) -> None: + populate(self.cluster, healthy()) + completed = self.cluster.run("fleet-role.sh", **{**CONTROLLER_ENV, **env}) + self.assertNotEqual(completed.returncode, 0) + self.assertIn(message, completed.stderr) + self.assertEqual([c for c in self.cluster.calls.read_text().splitlines() if "patch" in c], [], "nothing is published from refused settings") + + def test_an_even_target_is_refused(self) -> None: + self.refuses("must be odd", FLEET_ROLE_VOTERS="4") + + def test_a_target_under_three_is_refused(self) -> None: + self.refuses("must be at least 3", FLEET_ROLE_VOTERS="1") + + def test_a_target_without_a_place_of_headroom_below_the_maximum_is_refused(self) -> None: + self.refuses("one place of headroom", FLEET_ROLE_VOTERS=str(MAX_SERVERS)) + self.refuses("one place of headroom", FLEET_ROLE_VOTERS="3", FLEET_ROLE_MAX_SERVERS="3") + + def test_the_largest_target_that_leaves_headroom_is_accepted(self) -> None: + status = self.pass_(healthy(), FLEET_ROLE_VOTERS="5", FLEET_ROLE_MAX_SERVERS="6") + self.assertEqual(status["config"]["target"], 5) + + def test_a_pass_that_is_not_a_dry_run_is_refused(self) -> None: + self.refuses("FLEET_ROLE_DRY_RUN must be true", FLEET_ROLE_DRY_RUN="false") + + def test_the_election_timeout_is_required_so_that_the_controller_and_the_probe_cannot_disagree_on_a_default(self) -> None: + populate(self.cluster, healthy()) + env = {k: v for k, v in CONTROLLER_ENV.items() if k != "FLEET_ROLE_ELECTION_TIMEOUT_MS"} + completed = self.cluster.run("fleet-role.sh", **env) + self.assertNotEqual(completed.returncode, 0) + self.assertIn("FLEET_ROLE_ELECTION_TIMEOUT_MS must be set", completed.stderr) + + +class RegradeCoreAgreementTest(unittest.TestCase): + """The memory limit the controller applies is the regrade core's own is_over, checked over a grid of readings on both sides of each limit.""" + + def test_the_controllers_memory_limit_agrees_with_the_regrade_cores_is_over(self) -> None: + total = 1000 + settings = {"pressure_threshold": PRESSURE_THRESHOLD, "available_floor_percent": AVAILABLE_FLOOR_PERCENT} + for psi in (0, PRESSURE_THRESHOLD - 0.5, PRESSURE_THRESHOLD, PRESSURE_THRESHOLD + 0.5, 100): + for available in (0, 149, 150, 151, 1000): + core = subprocess.run( + ["bash", "-c", f'. "{REGRADE_CORE}"; PRESSURE_THRESHOLD={PRESSURE_THRESHOLD} AVAILABLE_FLOOR_PERCENT={AVAILABLE_FLOOR_PERCENT}; is_over {psi} {available} {total}'], + capture_output=True, + text=True, + check=False, + env={**os.environ, "FLEET_REGRADE_ENV": "/nonexistent"}, + ).returncode == 0 + ours = subprocess.run( + ["jq", "-n", "-e", "-L", str(SCRIPTS), "--argjson", "c", json.dumps(settings), "--argjson", "reading", json.dumps({"psi": psi, "avail_kb": available, "total_kb": total}), + 'include "fleet-role-stats"; $reading | over_regrade_limit($c)'], + capture_output=True, + text=True, + check=False, + ).returncode == 0 + self.assertEqual(ours, core, (psi, available)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_fleet_role_arc.py b/tests/unit/test_fleet_role_arc.py new file mode 100644 index 00000000..41451d6f --- /dev/null +++ b/tests/unit/test_fleet_role_arc.py @@ -0,0 +1,407 @@ +"""Tests for how the github_runner_arc role installs the role controller: the manifests templates/fleet-role.yaml.j2 renders (through tests/fleet_role/render.yml, with ansible-playbook), the checks of tasks/validate.yml, and that the settings the scripts require are the ones the manifests give them.""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import tempfile +import unittest +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ROLE = ROOT / "roles" / "github_runner_arc" +CLUSTER_ROLE = ROOT / "roles" / "github_runner_cluster" +RENDER = ROOT / "tests" / "fleet_role" / "render.yml" +# Renders one template file with the role's defaults and constants, the way the role evaluates an expression taken from a task. +EXPRESSION = ROOT / "tests" / "node_cache_pruner" / "kept.yml" +ARC = "github_runner_arc_role_controller_" + + +def ansible(playbook: Path, **variables: Any) -> subprocess.CompletedProcess[str]: + with tempfile.TemporaryDirectory() as work: + extra = Path(work) / "extra.json" + extra.write_text(json.dumps(variables)) + env = {**os.environ, "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", "ANSIBLE_HOME": str(Path(work) / "home")} + return subprocess.run([os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook"), str(playbook), "-e", f"@{extra}"], env=env, cwd=work, capture_output=True, text=True, stdin=subprocess.DEVNULL, check=False) + + +def render(**role_vars: Any) -> list[dict[str, Any]]: + """Renders the manifests and returns them in order.""" + with tempfile.TemporaryDirectory() as work: + output = Path(work) / "fleet-role.yaml" + completed = ansible(RENDER, fleet_role_output=str(output), **role_vars) + assert completed.returncode == 0, completed.stdout + completed.stderr + return [document for document in yaml.safe_load_all(output.read_text()) if document] + + +def of_kind(manifests: list[dict[str, Any]], kind: str, name: str | None = None) -> list[dict[str, Any]]: + return [m for m in manifests if m["kind"] == kind and (name is None or m["metadata"]["name"] == name)] + + +def container(manifest: dict[str, Any]) -> dict[str, Any]: + [only] = manifest["spec"]["template"]["spec"]["containers"] + return only + + +def env_of(manifest: dict[str, Any]) -> dict[str, Any]: + return {item["name"]: item.get("value", item.get("valueFrom")) for item in container(manifest)["env"]} + + +def script_requirements(script: str, prefix: str = "") -> set[str]: + """The environment variables a script requires with `${NAME:?`, or `: "${NAME:?`.""" + return set(re.findall(rf"\$\{{({prefix}[A-Z_]+):\?", (ROOT / "scripts" / script).read_text())) + + +class ManifestTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.manifests = render() + + def test_each_of_the_controller_and_the_probe_has_its_own_account_and_the_workloads_are_a_deployment_and_a_daemonset(self) -> None: + kinds = sorted((m["kind"], m["metadata"]["name"]) for m in self.manifests) + self.assertEqual( + kinds, + sorted( + [("ServiceAccount", n) for n in ("fleet-role", "fleet-role-probe")] + + [("ClusterRole", n) for n in ("github-runner-fleet-role", "github-runner-fleet-role-probe")] + + [("ClusterRoleBinding", n) for n in ("github-runner-fleet-role", "github-runner-fleet-role-probe")] + + [("Role", n) for n in ("fleet-role", "fleet-role-probe")] + + [("RoleBinding", n) for n in ("fleet-role", "fleet-role-probe")] + + [("Deployment", "fleet-role"), ("DaemonSet", "fleet-role-probe")] + ), + ) + + def test_neither_can_write_a_node_and_both_can_only_read_them(self) -> None: + for role in of_kind(self.manifests, "ClusterRole"): + self.assertEqual(role["rules"], [{"apiGroups": [""], "resources": ["nodes"], "verbs": ["get", "list"]}], role["metadata"]["name"]) + + def test_their_only_namespaced_permissions_are_on_their_two_configmaps_by_name(self) -> None: + controller = of_kind(self.manifests, "Role", "fleet-role")[0]["rules"] + probe = of_kind(self.manifests, "Role", "fleet-role-probe")[0]["rules"] + self.assertEqual( + controller, + [ + {"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-probe"], "verbs": ["get"]}, + {"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-status"], "verbs": ["get", "patch"]}, + ], + ) + self.assertEqual(probe, [{"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-probe"], "verbs": ["get", "patch"]}]) + for rule in controller + probe: + self.assertNotIn("create", rule["verbs"]) + self.assertNotIn("delete", rule["verbs"]) + + def test_each_binding_grants_its_role_to_its_own_account(self) -> None: + for kind, role_kind in (("ClusterRoleBinding", "ClusterRole"), ("RoleBinding", "Role")): + for binding in of_kind(self.manifests, kind): + self.assertEqual(binding["roleRef"]["kind"], role_kind) + [subject] = binding["subjects"] + self.assertEqual(subject["kind"], "ServiceAccount") + self.assertIn(subject["name"], ("fleet-role", "fleet-role-probe")) + self.assertTrue(binding["roleRef"]["name"].endswith(subject["name"]), binding) + self.assertEqual(subject["namespace"], "github-runner-platform") + + def test_the_controller_runs_as_an_unprivileged_user_with_a_read_only_root_and_is_a_dry_run(self) -> None: + deployment = of_kind(self.manifests, "Deployment")[0] + pod = deployment["spec"]["template"]["spec"] + self.assertTrue(pod["securityContext"]["runAsNonRoot"]) + self.assertEqual(container(deployment)["securityContext"], {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}}) + self.assertEqual(env_of(deployment)["FLEET_ROLE_DRY_RUN"], "true") + self.assertNotIn("hostNetwork", pod) + + def test_the_controller_gets_every_setting_its_script_requires(self) -> None: + deployment = of_kind(self.manifests, "Deployment")[0] + required = script_requirements("fleet-role.sh", "FLEET_ROLE_") + self.assertTrue(required, "the script's requirements were not found") + self.assertLessEqual(required, set(env_of(deployment))) + self.assertIn("FLEET_ROLE_SAMPLE_SECONDS", set(re.findall(r"\$\{(FLEET_ROLE_[A-Z_]+):\?", (ROOT / "fleet-role" / "loop.sh").read_text()))) + + def test_the_probe_gets_every_setting_its_script_requires(self) -> None: + daemonset = of_kind(self.manifests, "DaemonSet")[0] + required = script_requirements("fleet-role-probe.sh") + self.assertEqual(required, {"NODE_NAME", "NAMESPACE", "ELECTION_TIMEOUT_MS", "ELIGIBLE_LABEL"}) + self.assertLessEqual(required, set(env_of(daemonset))) + + def test_the_default_settings_reach_the_controller_as_strings(self) -> None: + env = env_of(of_kind(self.manifests, "Deployment")[0]) + self.assertEqual( + {k: v for k, v in env.items() if k.startswith("FLEET_ROLE_") and k not in ("FLEET_ROLE_STATE_NAMESPACE",)}, + { + "FLEET_ROLE_DRY_RUN": "true", + "FLEET_ROLE_VOTERS": "3", + "FLEET_ROLE_MAX_SERVERS": "5", + "FLEET_ROLE_ELECTION_TIMEOUT_MS": "1000", + "FLEET_ROLE_SAMPLE_SECONDS": "15", + "FLEET_ROLE_PRESSURE_THRESHOLD": "10", + "FLEET_ROLE_AVAILABLE_FLOOR_PERCENT": "15", + "FLEET_ROLE_ELIGIBLE_LABEL": "github-runner.exadev.com/voter-eligible", + "FLEET_ROLE_PINNED_LABEL": "github-runner.exadev.com/voter-pinned", + "FLEET_ROLE_LEVEL_LABEL": "github-runner.exadev.com/regrade-level", + "FLEET_ROLE_PROBE_CONFIGMAP": "fleet-role-probe", + "FLEET_ROLE_STATUS_CONFIGMAP": "fleet-role-status", + }, + ) + + def test_the_cluster_roles_own_settings_reach_the_controller_when_they_are_defined(self) -> None: + manifests = render( + github_runner_cluster_max_servers=7, + github_runner_cluster_etcd_election_timeout=3000, + github_runner_cluster_regrade_pressure_threshold=20, + github_runner_cluster_regrade_available_floor_percent=25, + github_runner_cluster_regrade_level_label="example.com/level", + github_runner_cluster_voters=5, + github_runner_cluster_voter_eligible_label="example.com/candidate", + github_runner_cluster_voter_pinned_label="example.com/pinned", + ) + env = env_of(of_kind(manifests, "Deployment")[0]) + self.assertEqual( + (env["FLEET_ROLE_MAX_SERVERS"], env["FLEET_ROLE_ELECTION_TIMEOUT_MS"], env["FLEET_ROLE_PRESSURE_THRESHOLD"], env["FLEET_ROLE_AVAILABLE_FLOOR_PERCENT"], env["FLEET_ROLE_LEVEL_LABEL"]), + ("7", "3000", "20", "25", "example.com/level"), + ) + self.assertEqual(env_of(of_kind(manifests, "DaemonSet")[0])["ELECTION_TIMEOUT_MS"], "3000") + self.assertEqual( + (env["FLEET_ROLE_VOTERS"], env["FLEET_ROLE_ELIGIBLE_LABEL"], env["FLEET_ROLE_PINNED_LABEL"], env_of(of_kind(manifests, "DaemonSet")[0])["ELIGIBLE_LABEL"]), + ("5", "example.com/candidate", "example.com/pinned", "example.com/candidate"), + ) + + def test_the_controller_and_the_probe_agree_on_the_election_timeout_and_the_period(self) -> None: + manifests = render(**{ARC + "sample_seconds": 30}) + controller, probe = env_of(of_kind(manifests, "Deployment")[0]), env_of(of_kind(manifests, "DaemonSet")[0]) + self.assertEqual(controller["FLEET_ROLE_ELECTION_TIMEOUT_MS"], probe["ELECTION_TIMEOUT_MS"]) + self.assertEqual((controller["FLEET_ROLE_SAMPLE_SECONDS"], probe["REPORT_SECONDS"]), ("30", "30")) + + def test_the_probe_measures_from_the_nodes_own_network_on_every_linux_node_with_a_kubelet(self) -> None: + daemonset = of_kind(self.manifests, "DaemonSet")[0] + pod = daemonset["spec"]["template"]["spec"] + self.assertTrue(pod["hostNetwork"]) + self.assertEqual(pod["dnsPolicy"], "ClusterFirstWithHostNet") + self.assertEqual(pod["nodeSelector"], {"kubernetes.io/os": "linux"}) + self.assertEqual(pod["tolerations"], [{"operator": "Exists"}]) + self.assertEqual(container(daemonset)["command"], ["/app/scripts/fleet-role-probe.sh"]) + self.assertEqual(env_of(daemonset)["NODE_NAME"], {"fieldRef": {"fieldPath": "spec.nodeName"}}) + + def test_the_probe_is_root_only_for_its_own_directory_and_holds_no_capability(self) -> None: + daemonset = of_kind(self.manifests, "DaemonSet")[0] + self.assertEqual( + container(daemonset)["securityContext"], + {"runAsUser": 0, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"]}}, + ) + self.assertNotIn("privileged", container(daemonset)["securityContext"]) + + def test_the_probe_sees_the_hosts_pressure_read_only_and_writes_only_in_its_own_directory(self) -> None: + daemonset = of_kind(self.manifests, "DaemonSet")[0] + pod = daemonset["spec"]["template"]["spec"] + mounts = {m["name"]: m for m in container(daemonset)["volumeMounts"]} + volumes = {v["name"]: v for v in pod["volumes"]} + self.assertEqual(volumes["pressure"]["hostPath"], {"path": "/proc/pressure"}) + self.assertTrue(mounts["pressure"]["readOnly"]) + self.assertEqual(volumes["fsync"]["hostPath"], {"path": "/var/lib/rancher/k3s/fleet-role-probe", "type": "DirectoryOrCreate"}) + self.assertEqual(mounts["fsync"]["mountPath"], env_of(daemonset)["FSYNC_DIR"]) + self.assertEqual(sorted(volumes), ["fsync", "pressure", "tmp"]) + self.assertEqual({name for name, mount in mounts.items() if not mount.get("readOnly")}, {"tmp", "fsync"}) + + def test_the_controller_and_the_probe_run_one_image(self) -> None: + images = {container(m)["image"] for m in of_kind(self.manifests, "Deployment") + of_kind(self.manifests, "DaemonSet")} + self.assertEqual(len(images), 1) + self.assertRegex(images.pop(), r"^ghcr\.io/exadev/github-runner-fleet-role:\d+\.\d+\.\d+$") + + def test_every_script_the_manifests_run_is_copied_into_the_image_and_made_executable(self) -> None: + dockerfile = (ROOT / "fleet-role" / "Dockerfile").read_text() + template = (ROLE / "templates" / "fleet-role.yaml.j2").read_text() + scripts = set(re.findall(r"/app/scripts/([A-Za-z0-9._-]+\.sh)", template)) + self.assertEqual(scripts, {"fleet-role-probe.sh"}) + scripts.add("fleet-role.sh") # run by the image's entrypoint + chmod = re.search(r"RUN chmod \+x ([^\n]+)", dockerfile) + assert chmod is not None + for script in scripts: + with self.subTest(script): + self.assertTrue((ROOT / "scripts" / script).is_file()) + self.assertRegex(dockerfile, rf"COPY [^\n]*scripts/{re.escape(script)}[^\n]* /app/scripts/") + self.assertIn(f"/app/scripts/{script}", chmod.group(1).split()) + self.assertIn("/app/loop.sh", chmod.group(1).split()) + entrypoint = (ROOT / "fleet-role" / "loop.sh").read_text() + self.assertIn("/app/scripts/fleet-role.sh", entrypoint) + + def test_every_jq_module_the_scripts_include_is_copied_beside_them(self) -> None: + dockerfile = (ROOT / "fleet-role" / "Dockerfile").read_text() + for program in ("fleet-role.jq", "fleet-role-stats.jq"): + for module in re.findall(r'^include "([A-Za-z0-9._-]+)";', (ROOT / "scripts" / program).read_text(), re.MULTILINE): + with self.subTest(program=program, module=module): + self.assertTrue((ROOT / "scripts" / f"{module}.jq").is_file()) + self.assertRegex(dockerfile, rf"COPY [^\n]*scripts/{re.escape(module)}\.jq[^\n]* /app/scripts/") + for program in ("fleet-role.jq", "fleet-role-stats.jq"): + self.assertRegex(dockerfile, rf"COPY [^\n]*scripts/{re.escape(program)}[^\n]* /app/scripts/") + + def test_the_probe_reads_the_regrade_core_the_image_carries(self) -> None: + dockerfile = (ROOT / "fleet-role" / "Dockerfile").read_text() + script = (ROOT / "scripts" / "fleet-role-probe.sh").read_text() + default = re.search(r'FLEET_REGRADE_CORE="\$\{FLEET_REGRADE_CORE:-([^}]+)\}"', script) + assert default is not None + self.assertIn(f"COPY roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh {default.group(1)}", dockerfile) + for name in ("fleet-role.sh", "fleet-role-probe.sh", "fleet-role.jq", "fleet-role-stats.jq"): + self.assertIn(name, dockerfile) + + def test_the_two_workloads_run_under_the_platforms_priority_class_and_ephemeral_storage_bounds(self) -> None: + for manifest in of_kind(self.manifests, "Deployment") + of_kind(self.manifests, "DaemonSet"): + pod = manifest["spec"]["template"]["spec"] + self.assertEqual(pod["priorityClassName"], "github-runner-platform") + resources = container(manifest)["resources"] + self.assertEqual((resources["requests"]["ephemeral-storage"], resources["limits"]["ephemeral-storage"]), ("64Mi", "128Mi")) + + +class NamesTest(unittest.TestCase): + """Names that issue #184 fixes and the fleet's goal gate reads.""" + + def test_the_opt_in_is_the_cluster_roles_variable_and_off_by_default(self) -> None: + cluster = yaml.safe_load((CLUSTER_ROLE / "defaults" / "main.yml").read_text()) + self.assertIs(cluster["github_runner_cluster_role_controller"], False) + + def test_one_inventory_setting_turns_it_on_and_the_status_goes_to_fleet_role_status_in_the_platform_namespace(self) -> None: + manifests = render() + deployment = of_kind(manifests, "Deployment")[0] + self.assertEqual(env_of(deployment)["FLEET_ROLE_STATUS_CONFIGMAP"], "fleet-role-status") + self.assertEqual(deployment["metadata"]["namespace"], "github-runner-platform") + self.assertEqual(env_of(deployment)["FLEET_ROLE_DRY_RUN"], "true") + + def test_the_arc_roles_switch_follows_the_cluster_roles(self) -> None: + template = ROOT / "tests" / "fleet_role" / "enabled.j2" + with tempfile.TemporaryDirectory() as work: + output = Path(work) / "enabled.txt" + for cluster_value, arc_value, expected in ((None, None, "False"), (True, None, "True"), (False, True, "True"), (True, False, "False")): + variables: dict[str, Any] = {"kept_template": str(template), "kept_output": str(output)} + if cluster_value is not None: + variables["github_runner_cluster_role_controller"] = cluster_value + if arc_value is not None: + variables[ARC + "enabled"] = arc_value + completed = ansible(EXPRESSION, **variables) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + self.assertEqual(output.read_text().strip(), expected, (cluster_value, arc_value)) + + +class DefaultsAgreeWithTheClusterRoleTest(unittest.TestCase): + """The fallbacks in the ARC role's defaults stand in for the cluster role's variables when they are not defined, so they must be their defaults.""" + + @classmethod + def setUpClass(cls) -> None: + cls.arc = (ROLE / "defaults" / "main.yml").read_text() + cls.cluster = yaml.safe_load((CLUSTER_ROLE / "defaults" / "main.yml").read_text()) + + def fallback(self, name: str) -> str: + match = re.search(rf"^{ARC}{name}: \"\{{\{{ [a-z_]+ \| default\(([^,)]+)\)", self.arc, re.M) + assert match is not None, name + return match.group(1).strip("'") + + def test_max_servers(self) -> None: + self.assertEqual(int(self.fallback("max_servers")), self.cluster["github_runner_cluster_max_servers"]) + + def test_the_regrade_limit(self) -> None: + self.assertEqual(int(self.fallback("pressure_threshold")), self.cluster["github_runner_cluster_regrade_pressure_threshold"]) + self.assertEqual(int(self.fallback("available_floor_percent")), self.cluster["github_runner_cluster_regrade_available_floor_percent"]) + + def test_the_voter_target_and_the_two_voter_labels(self) -> None: + self.assertEqual(int(self.fallback("voters")), self.cluster["github_runner_cluster_voters"]) + self.assertEqual(self.fallback("eligible_label"), self.cluster["github_runner_cluster_voter_eligible_label"]) + self.assertEqual(self.fallback("pinned_label"), self.cluster["github_runner_cluster_voter_pinned_label"]) + + def test_the_level_label(self) -> None: + self.assertEqual(self.fallback("level_label"), self.cluster["github_runner_cluster_regrade_level_label"]) + + def test_the_election_timeout_falls_back_to_k3ss_own(self) -> None: + # The controller's scripts hold no default of their own; the cluster role's own comment names the figure. + self.assertIn("or 1000 }}", self.arc) + self.assertEqual(self.cluster["github_runner_cluster_etcd_election_timeout"], "") + self.assertIn("1000 ms election timeout", (CLUSTER_ROLE / "defaults" / "main.yml").read_text()) + + +class ValidationTest(unittest.TestCase): + """The role controller's checks in tasks/validate.yml, evaluated by Ansible itself the way the role evaluates them, every case in one run.""" + + VOTERS = ARC + "voters" + MAX = ARC + "max_servers" + ON = {ARC + "enabled": True} + CASES: dict[str, dict[str, Any]] = { + "defaults": {}, + "on with the defaults": {**ON}, + "target 5 of 6": {**ON, VOTERS: 5, MAX: 6}, + "target 5 of 5": {**ON, VOTERS: 5, MAX: 5}, + "target 3 of 4": {**ON, VOTERS: 3, MAX: 4}, + "target 3 of 3": {**ON, VOTERS: 3, MAX: 3}, + "target 4": {**ON, VOTERS: 4, MAX: 9}, + "target 1": {**ON, VOTERS: 1}, + "target 2": {**ON, VOTERS: 2}, + "target zero": {**ON, VOTERS: 0}, + "target text": {**ON, VOTERS: "three"}, + "target as a string": {**ON, VOTERS: "3"}, + "sample zero": {**ON, ARC + "sample_seconds": 0}, + "sample text": {**ON, ARC + "sample_seconds": "fifteen"}, + "election timeout zero": {**ON, ARC + "election_timeout_ms": 0}, + "max servers text": {**ON, MAX: "many"}, + "same label": {**ON, ARC + "pinned_label": "example.com/x", ARC + "eligible_label": "example.com/x"}, + "bad label": {**ON, ARC + "eligible_label": "has space"}, + "good labels": {**ON, ARC + "eligible_label": "example.com/eligible", ARC + "pinned_label": "pinned"}, + "bad settings while off": {VOTERS: 4, ARC + "sample_seconds": 0, ARC + "eligible_label": "has space"}, + } + + @classmethod + def setUpClass(cls) -> None: + tasks = yaml.safe_load((ROLE / "tasks" / "validate.yml").read_text()) + [task] = [task for task in tasks if task["name"].startswith("Fail on invalid controller, metrics, probe, pull Secret, node recovery")] + cls.checks = {check["msg"].split(" ")[0]: check["failed"] for check in task["loop"] if "role_controller" in check["msg"]} + lines = [] + for name, role_vars in cls.CASES.items(): + sets = "".join(f"{{% set {variable} = {json.dumps(value)} %}}" for variable, value in role_vars.items()) + for setting, expression in cls.checks.items(): + lines.append(f"{{% with %}}{sets}{{{{ [{json.dumps(name)}, {json.dumps(setting)}, ({expression.strip().removeprefix('{{').removesuffix('}}')}) | bool] | to_json }}}}{{% endwith %}}") + with tempfile.TemporaryDirectory() as work: + template, output = Path(work) / "failed.j2", Path(work) / "failed.txt" + template.write_text(";".join(lines)) + completed = ansible(EXPRESSION, kept_template=str(template), kept_output=str(output)) + assert completed.returncode == 0, completed.stdout + completed.stderr + cls.results: dict[str, dict[str, bool]] = {} + for line in output.read_text().split(";"): + name, setting, failed = json.loads(line) + cls.results.setdefault(name, {})[setting] = failed + + def refused(self, case: str) -> bool: + return any(self.results[case].values()) + + def refused_by(self, case: str) -> list[str]: + return [setting for setting, failed in self.results[case].items() if failed] + + def test_there_are_three_checks_and_they_are_named_for_the_settings_they_guard(self) -> None: + self.assertEqual(len(self.checks), 3) + self.assertEqual(sorted(self.checks)[0], ARC + "eligible_label") + + def test_the_defaults_and_a_switched_on_default_are_valid(self) -> None: + self.assertFalse(self.refused("defaults")) + self.assertFalse(self.refused("on with the defaults")) + + def test_the_target_must_be_odd_at_least_three_and_leave_a_place_of_headroom(self) -> None: + for case in ("target 5 of 6", "target 3 of 4", "target as a string"): + with self.subTest(case): + self.assertFalse(self.refused(case)) + for case in ("target 5 of 5", "target 3 of 3", "target 4", "target 1", "target 2", "target zero", "target text"): + with self.subTest(case): + self.assertEqual(self.refused_by(case), [ARC + "voters"]) + + def test_the_period_the_timeout_and_the_maximum_must_be_whole_numbers_of_at_least_one(self) -> None: + for case in ("sample zero", "sample text", "election timeout zero", "max servers text"): + with self.subTest(case): + self.assertTrue(self.refused(case)) + + def test_the_two_labels_must_be_label_keys_and_differ(self) -> None: + for case in ("same label", "bad label"): + with self.subTest(case): + self.assertTrue(self.refused(case)) + self.assertFalse(self.refused("good labels")) + + def test_nothing_is_checked_while_the_controller_is_off(self) -> None: + self.assertFalse(self.refused("bad settings while off")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_fleet_role_probe.py b/tests/unit/test_fleet_role_probe.py new file mode 100644 index 00000000..4fd36bf5 --- /dev/null +++ b/tests/unit/test_fleet_role_probe.py @@ -0,0 +1,821 @@ +"""Tests for scripts/fleet-role-probe.sh, one sample of the role controller's per-node probe. The probe reads real host state, and a stand-in with a flatter structure than the real thing has hidden a bug here before (a `find -L` over sysfs passed flat stand-ins and exited 1 on a real host, from a symlink loop), so each reading is taken from the real thing or from a tree that has its structure: + +- the link type from a sysfs tree whose symlinks loop the way the kernel's do (device, subsystem, phy80211), and from the real /sys of the machine running the tests; +- TCP connect times from real listeners, and a connect that fails; +- etcd's metrics from pages a real etcd 3.6.15 three-member cluster served (tests/fleet_role/etcd-metrics), and from a live local cluster when an etcd binary is on the PATH or named by ETCD_BINARY; +- the memory signals in the kernel's own file formats, read by the regrade core's own functions; +- the fdatasync tail from the real fio. + +Only the cluster's API is a stand-in (tests/unit/fleet_role_support.py).""" + +from __future__ import annotations + +import http.server +import json +import os +import re +import shutil +import socket +import socketserver +import subprocess +import tempfile +import threading +import time +import unittest +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +from fleet_role_support import ( + ELECTION_TIMEOUT_MS, + ELIGIBLE_LABEL, + NAMESPACE, + REGRADE_CORE, + SAMPLE_SECONDS, + TARGET, + WINDOW_SAMPLES, + WINDOW_SECONDS, + Cluster, + entry, + node, + sample, +) + +FIXTURES = Path(__file__).resolve().parents[1] / "fleet_role" / "etcd-metrics" +# The members of the cluster the fixtures were captured from (etcd 3.6.15, three members on one host); the page is the first member's. +FIXTURE_SERVER_ID = "d5811b26ae265d01" +FIXTURE_PEER_IDS = ("2b6d7f19fe480be8", "819450a5536d704f") +FIXTURE_LEARNER_ID = "c65e62daf70a677b" +FIXTURE_QUOTA_BYTES = 2 * 1024**3 +ETCD_BINARY = os.environ.get("ETCD_BINARY") or shutil.which("etcd") +FIO = shutil.which("fio") + + +@contextmanager +def listener(host: str = "127.0.0.1") -> Iterator[int]: + """A TCP server that accepts connections and closes them, on a free port; yields the port.""" + + class Handler(socketserver.BaseRequestHandler): + def handle(self) -> None: + return + + class Server(socketserver.ThreadingTCPServer): + allow_reuse_address = True + address_family = socket.AF_INET6 if ":" in host else socket.AF_INET + + server = Server((host, 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server.server_address[1] + finally: + server.shutdown() + server.server_close() + + +def closed_port(host: str = "127.0.0.1") -> int: + with socket.socket(socket.AF_INET6 if ":" in host else socket.AF_INET) as probe: + probe.bind((host, 0)) + return probe.getsockname()[1] + + +@contextmanager +def metrics_server(pages: list[str]) -> Iterator[str]: + """Serves the pages one per request, the last again for any more; yields the metrics URL.""" + remaining = list(pages) + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 - the name http.server calls + page = remaining.pop(0) if len(remaining) > 1 else remaining[0] + body = page.encode() + self.send_response(200) + self.send_header("Content-Type", "text/plain; version=0.0.4") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_: Any) -> None: + return + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}/metrics" + finally: + server.shutdown() + server.server_close() + + +def page(name: str) -> str: + return (FIXTURES / f"{name}.txt").read_text() + + +def cumulative(text: str, family: str, label: str | None = None) -> dict[str, dict[str, float]]: + """The buckets of a histogram family in a page, by the value of `label` (or one group under "" without it), parsed independently of the probe: upper bound to cumulative count.""" + groups: dict[str, dict[str, float]] = {} + pattern = re.compile(rf"^{family}_bucket\{{(?P[^}}]*)\}} (?P\S+)$", re.M) + for match in pattern.finditer(text): + labels = dict(re.findall(r'(\w+)="([^"]*)"', match.group("labels"))) + groups.setdefault(labels[label] if label else "", {})[labels["le"]] = float(match.group("value")) + return groups + + +def per_bucket(buckets: dict[str, float]) -> dict[str, float]: + previous = 0.0 + result = {} + for bound in sorted(buckets, key=lambda b: float("inf") if b == "+Inf" else float(b)): + result[bound] = buckets[bound] - previous + previous = buckets[bound] + return result + + +def gained(before: dict[str, float], after: dict[str, float]) -> dict[str, int]: + b, a = per_bucket(before), per_bucket(after) + return {bound: int(a[bound] - b.get(bound, 0)) for bound in a if a[bound] - b.get(bound, 0) > 0} + + +class ProbeCase(unittest.TestCase): + def setUp(self) -> None: + self.cluster = Cluster() + self.addCleanup(self.cluster.cleanup) + self.sys = self.cluster.dir / "sys" + self.route = self.cluster.dir / "route" + self.psi = self.cluster.dir / "pressure-memory" + self.meminfo = self.cluster.dir / "meminfo" + self.fsync_dir = self.cluster.dir / "k3s-data" + self.fsync_dir.mkdir() + self.nodes: list[dict[str, Any]] = [] + self.write_default_host_files() + + def write_default_host_files(self) -> None: + self.psi.write_text("some avg10=0.00 avg60=0.00 avg300=0.00 total=0\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=0\n") + self.meminfo.write_text("MemTotal: 16384000 kB\nMemFree: 1024000 kB\nMemAvailable: 8192000 kB\nBuffers: 102400 kB\n") + (self.sys / "class" / "net").mkdir(parents=True, exist_ok=True) + self.route.write_text("Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\n") + + def probe(self, name: str = "me", *, voters: list[dict[str, Any]] | None = None, **env: str) -> subprocess.CompletedProcess[str]: + """Takes one sample as node `name`, which is a voter when its Node is among `voters`.""" + self.cluster.nodes = voters if voters is not None else self.nodes + base = { + "NODE_NAME": name, + "NAMESPACE": NAMESPACE, + "ELECTION_TIMEOUT_MS": str(ELECTION_TIMEOUT_MS), + "ELIGIBLE_LABEL": ELIGIBLE_LABEL, + "REPORT_SECONDS": str(SAMPLE_SECONDS), + "REPORT_ONCE": "true", + "ETCD_PEER_PORT": "1", + "FSYNC_DIR": str(self.fsync_dir), + "SYS_CLASS_NET": str(self.sys / "class" / "net"), + "PROC_NET_ROUTE": str(self.route), + "FLEET_REGRADE_CORE": str(REGRADE_CORE), + "PSI_FILE": str(self.psi), + "MEMINFO_FILE": str(self.meminfo), + } + return self.cluster.run("fleet-role-probe.sh", **{**base, **env}) + + def newest(self, name: str = "me") -> dict[str, Any]: + return self.cluster.probes()[name]["samples"][-1] + + def settled(self, name: str = "me") -> None: + """Gives this node an entry whose fdatasync probe ran just now, so that a pass does not run fio.""" + self.cluster.set_probes({**self.cluster.probes(), name: {**entry([]), "fsync_at": int(time.time())}}) + + +class SysfsTree: + """A /sys/class/net and /sys/devices with the structure the kernel's has: class entries are symlinks into the device tree, and each interface has a device link back up to its bus device and a subsystem link back to the class directory, so that a walk following links loops.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.class_net = root / "class" / "net" + self.class_net.mkdir(parents=True, exist_ok=True) + + def _link(self, path: Path, target: Path) -> None: + path.symlink_to(os.path.relpath(target, path.parent)) + + def interface(self, name: str, parent: str, *, wireless: bool = False, devtype: str | None = None) -> Path: + """An interface under the device `parent` (a path below /sys/devices): `parent/net/name`, as for a PCI, USB or SoC network device.""" + bus_device = self.root / "devices" / parent + directory = bus_device / "net" / name + directory.mkdir(parents=True) + (directory / "address").write_text("02:00:00:00:00:01\n") + (directory / "type").write_text("1\n") + uevent = f"INTERFACE={name}\nIFINDEX=2\n" + (f"DEVTYPE={devtype}\n" if devtype else "") + (directory / "uevent").write_text(uevent) + self._link(directory / "subsystem", self.class_net) + if "virtual" not in parent: + self._link(directory / "device", bus_device) + if wireless: + phy = bus_device / "ieee80211" / "phy0" + phy.mkdir(parents=True) + self._link(phy / "device", bus_device) + self._link(directory / "phy80211", phy) + (directory / "wireless").mkdir() + self._link(self.class_net / name, directory) + return directory + + def pci_ethernet(self, name: str = "enp3s0") -> None: + self.interface(name, "pci0000:00/0000:00:1c.2/0000:03:00.0") + + def usb_ethernet(self, name: str = "enx001122334455") -> None: + self.interface(name, "pci0000:00/0000:00:14.0/usb1/1-2/1-2:1.0") + + def pci_wifi(self, name: str = "wlan0") -> None: + self.interface(name, "pci0000:00/0000:00:14.3", wireless=True, devtype="wlan") + + def soc_wifi(self, name: str = "wlan1") -> None: + # A Raspberry Pi's wireless interface hangs below its SDIO host, and the phy link is all that marks it. + self.interface(name, "platform/soc/fe300000.mmcnr/mmc_host/mmc1/mmc1:0001/mmc1:0001:1", wireless=True) + + +ROUTE_HEADER = "Iface\tDestination\tGateway \tFlags\tRefCnt\tUse\tMetric\tMask\t\tMTU\tWindow\tIRTT\n" + + +def route_line(iface: str, *, destination: str = "00000000", gateway: str = "0101A8C0", flags: str = "0003", metric: int = 100, mask: str = "00000000") -> str: + return f"{iface}\t{destination}\t{gateway}\t{flags}\t0\t0\t{metric}\t{mask}\t0\t0\t0\n" + + +class LinkTest(ProbeCase): + def setUp(self) -> None: + super().setUp() + self.tree = SysfsTree(self.sys) + self.settled_node = node("me", eligible=True) + self.nodes = [self.settled_node] + self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + + def link_via(self, *routes: str) -> str: + self.route.write_text(ROUTE_HEADER + "".join(routes)) + completed = self.probe() + self.assertEqual(completed.returncode, 0, completed.stderr) + return self.newest()["link"] + + def test_a_pci_ethernet_interface_is_wired(self) -> None: + self.tree.pci_ethernet() + self.assertEqual(self.link_via(route_line("enp3s0")), "wired") + self.assertEqual(self.newest()["iface"], "enp3s0") + + def test_a_usb_ethernet_adapter_is_wired(self) -> None: + self.tree.usb_ethernet() + self.assertEqual(self.link_via(route_line("enx001122334455")), "wired") + + def test_a_pci_wireless_interface_is_wifi(self) -> None: + self.tree.pci_wifi() + self.assertEqual(self.link_via(route_line("wlan0")), "wifi") + + def test_a_wireless_interface_on_an_soc_bus_is_wifi(self) -> None: + self.tree.soc_wifi() + self.assertEqual(self.link_via(route_line("wlan1")), "wifi") + + def test_a_wireless_interface_is_wifi_by_its_devtype_alone(self) -> None: + directory = self.tree.interface("wlp2s0", "pci0000:00/0000:00:1c.0/0000:02:00.0", devtype="wlan") + self.assertFalse((directory / "phy80211").exists()) + self.assertEqual(self.link_via(route_line("wlp2s0")), "wifi") + + def test_a_virtual_interface_is_unknown_because_the_link_beneath_it_cannot_be_seen(self) -> None: + # What a node running in a container (a veth), a bridge or a tunnel has as its default route. + for name in ("eth0", "br0", "tun0"): + with self.subTest(name): + self.tree.interface(name, "virtual") + self.assertEqual(self.link_via(route_line(name)), "unknown") + + def test_the_interface_with_the_lowest_metric_among_the_default_routes_decides(self) -> None: + self.tree.pci_wifi() + self.tree.pci_ethernet() + wlan = route_line("wlan0", metric=600) + eth = route_line("enp3s0", metric=100) + self.assertEqual(self.link_via(wlan, eth), "wired") + self.assertEqual(self.link_via(eth, wlan), "wired") + self.assertEqual(self.link_via(route_line("wlan0", metric=50), route_line("enp3s0", metric=100)), "wifi") + + def test_a_route_that_is_not_a_default_route_through_a_gateway_does_not_decide(self) -> None: + self.tree.pci_wifi() + self.tree.pci_ethernet() + routes = ( + route_line("enp3s0", destination="0001A8C0", gateway="00000000", flags="0001", mask="00FFFFFF"), + route_line("enp3s0", flags="0001", metric=1), # a default route that is up but has no gateway + route_line("enp3s0", flags="0002", metric=1), # a default route through a gateway that is down + route_line("wlan0", metric=600), + ) + self.assertEqual(self.link_via(*routes), "wifi") + + def test_no_default_route_is_unknown(self) -> None: + self.tree.pci_ethernet() + self.assertEqual(self.link_via(route_line("enp3s0", destination="0001A8C0", gateway="00000000", flags="0001", mask="00FFFFFF")), "unknown") + self.assertIsNone(self.newest()["iface"]) + + def test_an_interface_the_tree_does_not_have_is_unknown(self) -> None: + self.assertEqual(self.link_via(route_line("enp9s0")), "unknown") + + def test_the_kernels_looping_links_do_not_stop_the_reading(self) -> None: + # The tree really loops: from the class directory, an interface's subsystem is the class directory again, and its device leads back to the interface. A find that follows links fails on a tree like this, which is how a real host failed. + self.tree.pci_ethernet() + looped = self.sys / "class" / "net" / "enp3s0" / "subsystem" / "enp3s0" / "subsystem" / "enp3s0" / "device" / "net" / "enp3s0" + self.assertTrue(looped.exists()) + self.assertEqual((self.sys / "class" / "net" / "enp3s0" / "subsystem").resolve(), (self.sys / "class" / "net").resolve()) + self.assertEqual(self.link_via(route_line("enp3s0")), "wired") + + @unittest.skipUnless(Path("/sys/class/net").is_dir() and Path("/proc/net/route").is_file(), "needs a Linux host's own /sys and /proc") + def test_the_hosts_own_sysfs_and_routing_table_give_a_link_type(self) -> None: + completed = self.probe(SYS_CLASS_NET="/sys/class/net", PROC_NET_ROUTE="/proc/net/route") + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertIn(self.newest()["link"], ("wired", "wifi", "unknown")) + self.assertEqual(completed.stderr, "", "the real tree needs no tolerance for anything the probe reads") + + +class ConnectTest(ProbeCase): + def setUp(self) -> None: + super().setUp() + self.settled() + + def peers(self, address: str = "127.0.0.1") -> list[dict[str, Any]]: + return [node("peer", voter=True, address=address), node("me", eligible=True)] + + def test_a_connect_to_a_listening_peer_port_is_its_real_time_in_milliseconds(self) -> None: + with listener() as port: + completed = self.probe(voters=self.peers(), ETCD_PEER_PORT=str(port)) + self.assertEqual(completed.returncode, 0, completed.stderr) + reading = self.newest()["tcp"]["peer"] + self.assertGreater(reading, 0) + self.assertLess(reading, ELECTION_TIMEOUT_MS) + + def test_a_connect_that_does_not_complete_is_recorded_as_the_election_timeout(self) -> None: + port = closed_port() + completed = self.probe(voters=self.peers(), ETCD_PEER_PORT=str(port)) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(self.newest()["tcp"]["peer"], ELECTION_TIMEOUT_MS) + + def test_a_peer_that_never_answers_costs_one_election_timeout_not_more(self) -> None: + started = time.monotonic() + # An address reserved for documentation, which nothing answers: the connect times out or the network is unreachable, and either way it never completes. + completed = self.probe(voters=self.peers("192.0.2.1"), ELECTION_TIMEOUT_MS="400") + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(self.newest()["tcp"]["peer"], 400) + self.assertLess(time.monotonic() - started, 5) + + def test_an_ipv6_peer_address_is_bracketed(self) -> None: + try: + with listener("::1") as port: + completed = self.probe(voters=self.peers("::1"), ETCD_PEER_PORT=str(port)) + except OSError: + self.skipTest("no IPv6 loopback on this host") + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertLess(self.newest()["tcp"]["peer"], ELECTION_TIMEOUT_MS) + + def test_a_node_does_not_measure_itself(self) -> None: + with listener() as port: + completed = self.probe("peer", voters=self.peers(), ETCD_PEER_PORT=str(port)) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(self.newest("peer")["tcp"], {}) + + +class EtcdReadingTest(ProbeCase): + def setUp(self) -> None: + super().setUp() + self.nodes = [node("me", voter=True), node("p1", voter=True), node("p2", voter=True)] + + def pass_with(self, url: str) -> subprocess.CompletedProcess[str]: + return self.probe(ETCD_METRICS_URL=url, ETCD_PEER_PORT=str(closed_port())) + + def test_the_first_sample_records_the_gauges_and_nothing_gained(self) -> None: + with metrics_server([page("before")]) as url: + self.assertEqual(self.pass_with(url).returncode, 0) + etcd = self.newest()["etcd"] + self.assertEqual( + {k: etcd[k] for k in ("has_leader", "is_learner", "known_members", "active_peers", "quota_bytes", "leader_changes")}, + {"has_leader": True, "is_learner": False, "known_members": TARGET, "active_peers": TARGET - 1, "quota_bytes": FIXTURE_QUOTA_BYTES, "leader_changes": 0}, + ) + self.assertEqual((etcd["wal"], etcd["commit"]), ({}, {})) + self.assertFalse(self.newest()["restarted"]) + self.assertEqual(self.cluster.probes()["me"]["etcd_id"], FIXTURE_SERVER_ID) + + def test_the_next_sample_holds_what_each_histogram_gained_as_a_real_etcd_counted_it(self) -> None: + before, after = page("before"), page("after") + with metrics_server([before, after]) as url: + self.assertEqual(self.pass_with(url).returncode, 0) + self.assertEqual(self.pass_with(url).returncode, 0) + etcd = self.newest()["etcd"] + wal = gained(cumulative(before, "etcd_disk_wal_fsync_duration_seconds")[""], cumulative(after, "etcd_disk_wal_fsync_duration_seconds")[""]) + commit = gained(cumulative(before, "etcd_disk_backend_commit_duration_seconds")[""], cumulative(after, "etcd_disk_backend_commit_duration_seconds")[""]) + self.assertGreater(sum(wal.values()), 0, "the captured pages differ, or this test proves nothing") + self.assertEqual(etcd["wal"], wal) + self.assertEqual(etcd["commit"], commit) + rtt_before = cumulative(before, "etcd_network_peer_round_trip_time_seconds", "To") + rtt_after = cumulative(after, "etcd_network_peer_round_trip_time_seconds", "To") + for peer in FIXTURE_PEER_IDS: + self.assertEqual(etcd["rtt"][peer], gained(rtt_before[peer], rtt_after[peer]), peer) + self.assertFalse(self.newest()["restarted"]) + # Cumulative as etcd reports it, so the sample after this one has something to subtract from. + self.assertEqual(self.cluster.probes()["me"]["last"]["wal"], cumulative(after, "etcd_disk_wal_fsync_duration_seconds")[""]) + + def test_a_peer_with_two_active_streams_counts_once_as_k3ss_etcd_reports_it(self) -> None: + # A page from a follower of a three-server k3s cluster (etcd 3.6.4): its gauge for the leader is 2, one for each kind of stream, and for the other follower 1. + page_text = page("k3s-follower") + self.assertRegex(page_text, r'etcd_network_active_peers\{[^}]*\} 2') + with metrics_server([page_text]) as url: + self.pass_with(url) + etcd = self.newest()["etcd"] + self.assertEqual((etcd["active_peers"], etcd["known_members"]), (TARGET - 1, TARGET)) + + def test_counters_that_go_backwards_mean_etcd_restarted(self) -> None: + with metrics_server([page("after"), page("before")]) as url: + self.pass_with(url) + self.pass_with(url) + sample_ = self.newest() + self.assertTrue(sample_["restarted"]) + self.assertEqual(sample_["etcd"]["wal"], {}, "a reset has no gain to count") + + def test_a_member_that_is_added_and_not_started_is_a_known_member_that_is_not_connected(self) -> None: + with metrics_server([page("learner-unstarted")]) as url: + self.pass_with(url) + etcd = self.newest()["etcd"] + self.assertEqual(etcd["known_members"], TARGET + 1) + self.assertEqual(etcd["active_peers"], TARGET - 1) + self.assertIn(FIXTURE_LEARNER_ID, page("learner-unstarted")) + + def test_a_removed_member_that_etcd_still_lists_with_a_zero_is_not_a_known_member(self) -> None: + self.assertRegex(page("before"), r'etcd_network_known_peers\{[^}]*\} 0') + with metrics_server([page("before")]) as url: + self.pass_with(url) + self.assertEqual(self.newest()["etcd"]["known_members"], TARGET) + + def test_a_voter_whose_metrics_cannot_be_read_still_records_the_rest_and_keeps_its_baseline(self) -> None: + with metrics_server([page("before")]) as url: + self.pass_with(url) + baseline = self.cluster.probes()["me"]["last"] + completed = self.pass_with(f"http://127.0.0.1:{closed_port()}/metrics") + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertIn("could not read", completed.stderr) + self.assertIsNone(self.newest()["etcd"]) + self.assertEqual(self.cluster.probes()["me"]["last"], baseline) + self.assertEqual(self.cluster.probes()["me"]["etcd_id"], FIXTURE_SERVER_ID) + + def test_a_node_that_is_not_a_voter_reads_no_etcd_metrics(self) -> None: + self.nodes = [node("me", eligible=True), node("p1", voter=True)] + self.settled() + with metrics_server([page("before")]) as url: + self.assertEqual(self.pass_with(url).returncode, 0) + self.assertIsNone(self.newest()["etcd"]) + self.assertIsNone(self.cluster.probes()["me"]["etcd_id"]) + + @unittest.skipUnless(ETCD_BINARY, "needs an etcd binary (on the PATH, or named by ETCD_BINARY)") + def test_a_live_three_member_cluster_gives_the_readings_the_fixtures_were_captured_from(self) -> None: + assert ETCD_BINARY is not None + with live_etcd(ETCD_BINARY) as cluster_: + url = cluster_.metrics_url(0) + self.assertEqual(self.pass_with(url).returncode, 0) + cluster_.put(20) + time.sleep(1) + self.assertEqual(self.pass_with(url).returncode, 0) + etcd = self.newest()["etcd"] + self.assertEqual((etcd["known_members"], etcd["active_peers"], etcd["has_leader"], etcd["is_learner"]), (TARGET, TARGET - 1, True, False)) + self.assertGreater(sum(etcd["wal"].values()), 0) + self.assertEqual(self.cluster.probes()["me"]["etcd_id"], cluster_.member_id(0)) + cluster_.add_learner() + time.sleep(2) + self.assertEqual(self.pass_with(url).returncode, 0) + self.assertEqual(self.newest()["etcd"]["known_members"], TARGET + 1) + + +class LiveEtcd: + def __init__(self, binary: str, directory: Path) -> None: + self.binary = binary + self.etcdctl = str(Path(binary).with_name("etcdctl")) + self.dir = directory + self.ports = [(closed_port(), closed_port(), closed_port()) for _ in range(3)] + self.processes: list[subprocess.Popen[bytes]] = [] + + def start(self) -> None: + initial = ",".join(f"n{i}=http://127.0.0.1:{peer}" for i, (_, peer, _) in enumerate(self.ports)) + for i, (client, peer, metrics) in enumerate(self.ports): + self.processes.append( + subprocess.Popen( + [ + self.binary, "--name", f"n{i}", "--data-dir", str(self.dir / f"n{i}"), + "--listen-peer-urls", f"http://127.0.0.1:{peer}", "--initial-advertise-peer-urls", f"http://127.0.0.1:{peer}", + "--listen-client-urls", f"http://127.0.0.1:{client}", "--advertise-client-urls", f"http://127.0.0.1:{client}", + "--listen-metrics-urls", f"http://127.0.0.1:{metrics}", "--initial-cluster", initial, "--initial-cluster-token", "fleet-role-probe-test", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + ) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + done = subprocess.run([self.etcdctl, "--endpoints", self.endpoints(), "endpoint", "health"], capture_output=True, check=False) + if done.returncode == 0: + return + time.sleep(0.5) + raise AssertionError("the local etcd cluster did not become healthy") + + def stop(self) -> None: + for process in self.processes: + process.terminate() + for process in self.processes: + process.wait(timeout=20) + + def endpoints(self) -> str: + return ",".join(f"127.0.0.1:{client}" for client, _, _ in self.ports) + + def metrics_url(self, index: int) -> str: + return f"http://127.0.0.1:{self.ports[index][2]}/metrics" + + def put(self, count: int) -> None: + for i in range(count): + subprocess.run([self.etcdctl, "--endpoints", self.endpoints(), "put", f"key{i}", "value"], capture_output=True, check=True) + + def member_id(self, index: int) -> str: + out = subprocess.run([self.etcdctl, "--endpoints", self.endpoints(), "member", "list", "-w", "json"], capture_output=True, text=True, check=True).stdout + for member in json.loads(out)["members"]: + if member["name"] == f"n{index}": + return format(member["ID"], "x") + raise AssertionError("no such member") + + def add_learner(self) -> None: + """Adds a learner that is never started. etcd refuses a membership change until its members have all been connected for a while (strict reconfiguration checks), so it is retried for that long.""" + deadline = time.monotonic() + 30 + while True: + added = subprocess.run( + [self.etcdctl, "--endpoints", self.endpoints(), "member", "add", "n9", "--learner", f"--peer-urls=http://127.0.0.1:{closed_port()}"], capture_output=True, text=True, check=False + ) + if added.returncode == 0: + return + assert "unhealthy cluster" in added.stderr and time.monotonic() < deadline, added.stderr + time.sleep(1) + + +@contextmanager +def live_etcd(binary: str) -> Iterator[LiveEtcd]: + with tempfile.TemporaryDirectory() as directory: + cluster = LiveEtcd(binary, Path(directory)) + cluster.start() + try: + yield cluster + finally: + cluster.stop() + + +class MeasuredNodesTest(ProbeCase): + """Only a voter or a host marked eligible is measured; any other node has nothing recorded, so the ConfigMap grows with the hosts that decide the control plane and not with the fleet.""" + + def test_a_node_that_is_neither_records_nothing(self) -> None: + self.cluster.set_probes({"other": entry([sample(5)], "other-id")}) + completed = self.probe(voters=[node("me"), node("peer", voter=True)]) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(sorted(self.cluster.probes()), ["other"]) + self.assertEqual([json.loads(line)[:2] for line in self.cluster.calls.read_text().splitlines() if '"patch"' in line], []) + self.assertEqual(list(self.fsync_dir.iterdir()), [], "and runs no disk probe") + + def test_a_node_that_stopped_being_either_has_its_entry_removed_and_no_other(self) -> None: + self.cluster.set_probes({"other": entry([sample(5)], "other-id"), "me": entry([sample(6)])}) + completed = self.probe(voters=[node("me"), node("peer", voter=True)]) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(sorted(self.cluster.probes()), ["other"]) + + def test_an_eligible_node_is_measured(self) -> None: + self.settled() + completed = self.probe(voters=[node("me", eligible=True), node("peer", voter=True)]) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(len(self.cluster.probes()["me"]["samples"]), 1) + + def test_a_voter_is_measured_whether_or_not_it_is_eligible(self) -> None: + with metrics_server([page("before")]) as url: + completed = self.probe(voters=[node("me", voter=True), node("peer", voter=True)], ETCD_METRICS_URL=url) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(len(self.cluster.probes()["me"]["samples"]), 1) + + +class MemoryTest(ProbeCase): + def setUp(self) -> None: + super().setUp() + self.nodes = [node("me", eligible=True)] + self.settled() + + def test_pressure_and_available_memory_are_read_from_the_kernels_files_by_the_regrade_cores_functions(self) -> None: + self.psi.write_text("some avg10=0.00 avg60=1.50 avg300=12.34 total=987654\nfull avg10=0.00 avg60=0.10 avg300=3.21 total=12345\n") + self.meminfo.write_text("MemTotal: 32768000 kB\nMemFree: 524288 kB\nMemAvailable: 4096000 kB\nBuffers: 102400 kB\n") + self.assertEqual(self.probe().returncode, 0) + latest = self.newest() + self.assertEqual((latest["psi"], latest["avail_kb"], latest["total_kb"]), (12.34, 4096000, 32768000)) + + def test_a_kernel_without_pressure_stall_information_gives_a_sample_without_a_pressure_reading(self) -> None: + self.psi.unlink() + completed = self.probe() + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(completed.stderr, "") + latest = self.newest() + self.assertIsNone(latest["psi"]) + self.assertEqual(latest["avail_kb"], 8192000) + + +class RingTest(ProbeCase): + def setUp(self) -> None: + super().setUp() + self.nodes = [node("me", eligible=True)] + + def test_a_sample_is_appended_and_the_entry_keeps_the_last_window_of_them(self) -> None: + old = [sample(1000 + i, fsync_ms=None) for i in range(WINDOW_SAMPLES)] + self.cluster.set_probes({"me": {**entry(old), "fsync_at": int(time.time())}}) + self.assertEqual(self.probe().returncode, 0) + kept = self.cluster.probes()["me"]["samples"] + self.assertEqual(len(kept), WINDOW_SAMPLES) + self.assertEqual(kept[0]["t"], 1001, "the oldest was dropped") + self.assertGreater(kept[-1]["t"], 1000 + WINDOW_SAMPLES) + + def test_an_entry_too_large_for_one_argument_is_read_and_extended_whole(self) -> None: + # One argument to a process is limited to 128 KiB on Linux; an entry of a voter in a large cluster, with a connect time to every other voter in each sample, passes that. + peers = {f"peer-with-a-long-node-name-{i:03d}": 1.234 for i in range(60)} + big = [sample(1000 + i, tcp=peers) for i in range(WINDOW_SAMPLES)] + self.cluster.set_probes({"me": {**entry(big), "fsync_at": int(time.time())}}) + self.assertGreater(len(json.dumps(self.cluster.probes()["me"])), 128 * 1024) + completed = self.probe() + self.assertEqual(completed.returncode, 0, completed.stderr) + kept = self.cluster.probes()["me"]["samples"] + self.assertEqual(len(kept), WINDOW_SAMPLES) + self.assertEqual(kept[0]["t"], 1001) + self.assertEqual(kept[0]["tcp"], peers) + self.assertGreater(kept[-1]["t"], 1000 + WINDOW_SAMPLES) + + def test_each_node_patches_only_its_own_key(self) -> None: + other = entry([sample(5)], "other-id") + self.cluster.set_probes({"other": other, "me": {**entry([]), "fsync_at": int(time.time())}}) + self.assertEqual(self.probe().returncode, 0) + self.assertEqual(self.cluster.probes()["other"], other) + patches = [json.loads(line) for line in self.cluster.calls.read_text().splitlines() if '"patch"' in line] + self.assertEqual([call[:3] for call in patches], [["patch", "configmap", "fleet-role-probe"]]) + + def recorded(self) -> list[dict[str, Any]]: + for _ in range(50): + try: + return json.loads(self.cluster.read_store()["configmaps"][f"{NAMESPACE}/fleet-role-probe"]["data"].get("me", "{}")).get("samples", []) + except json.JSONDecodeError: + time.sleep(0.1) # the stand-in kubectl was rewriting the store + raise AssertionError("the store could not be read") + + def loop(self, wanted: int, **env: str) -> list[dict[str, Any]]: + """Runs the probe as the DaemonSet does, one process taking a sample a second, until it has recorded this many samples (or a minute has gone), and returns them.""" + self.cluster.nodes = self.nodes + self.cluster.write_store() + process = subprocess.Popen( + [str(Path(__file__).resolve().parents[2] / "scripts" / "fleet-role-probe.sh")], + env=self.cluster.env( + NODE_NAME="me", NAMESPACE=NAMESPACE, ELECTION_TIMEOUT_MS=str(ELECTION_TIMEOUT_MS), ELIGIBLE_LABEL=ELIGIBLE_LABEL, REPORT_SECONDS="1", ETCD_PEER_PORT="1", FSYNC_DIR=str(self.fsync_dir), + SYS_CLASS_NET=str(self.sys / "class" / "net"), PROC_NET_ROUTE=str(self.route), FLEET_REGRADE_CORE=str(REGRADE_CORE), PSI_FILE=str(self.psi), MEMINFO_FILE=str(self.meminfo), **env, + ), + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 60 + while time.monotonic() < deadline and len(self.recorded()) < wanted: + time.sleep(0.3) + finally: + process.terminate() + process.wait(timeout=20) + return self.recorded() + + def test_a_node_that_is_not_a_voter_marks_its_first_sample_as_a_restart_and_a_later_one_as_none(self) -> None: + self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + samples = self.loop(2) + self.assertGreaterEqual(len(samples), 2) + self.assertEqual([s["restarted"] for s in samples[:2]], [True, False]) + + def test_a_sample_after_the_first_costs_two_api_calls_the_nodes_and_the_patch(self) -> None: + self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + samples = self.loop(3) + calls = [json.loads(line) for line in self.cluster.calls.read_text().splitlines()] + self.assertGreaterEqual(len(samples), 3) + reads = [c[:2] for c in calls if c[0] == "get"] + patches = [c for c in calls if c[0] == "patch"] + self.assertEqual(reads.count(["get", "configmap"]), 1, "this node's entry is read once, when the process starts") + # The process may have been stopped between a sample's read of the nodes and its patch, so the nodes were read once for each patch and at most once more. + self.assertIn(reads.count(["get", "nodes"]) - len(patches), (0, 1), reads) + self.assertGreaterEqual(len(patches), len(samples)) + + def test_a_patch_the_api_refuses_is_retried_next_period_without_the_process_exiting_and_the_first_sample_still_marks_the_restart(self) -> None: + refusals = self.cluster.dir / "refusals" + refusals.write_text("2") + self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + samples = self.loop(2, FAKE_FAIL_PATCHES=str(refusals)) + self.assertEqual(refusals.read_text(), "0") + self.assertGreaterEqual(len(samples), 2, "the probe went on after the refusals") + self.assertEqual([s["restarted"] for s in samples[:2]], [True, False]) + patches = [c for c in (json.loads(line) for line in self.cluster.calls.read_text().splitlines()) if c[0] == "patch"] + self.assertEqual(len(patches), len(samples) + 2) + + def test_a_patch_the_api_refuses_fails_a_single_sample_run(self) -> None: + refusals = self.cluster.dir / "refusals" + refusals.write_text("1") + self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + completed = self.probe(FAKE_FAIL_PATCHES=str(refusals)) + self.assertNotEqual(completed.returncode, 0) + self.assertEqual(self.cluster.probes()["me"]["samples"], []) + + +@unittest.skipUnless(FIO, "needs fio, which the probe's fdatasync reading is taken with") +class FsyncTest(ProbeCase): + def setUp(self) -> None: + super().setUp() + self.nodes = [node("me", eligible=True), node("peer", voter=True)] + + def test_a_node_that_is_not_a_voter_takes_the_reading_from_the_real_fio_and_leaves_no_file_behind(self) -> None: + completed = self.probe() + self.assertEqual(completed.returncode, 0, completed.stderr) + reading = self.newest()["fsync_ms"] + self.assertIsInstance(reading, float) + self.assertGreater(reading, 0) + self.assertEqual(list(self.fsync_dir.iterdir()), []) + self.assertIsNotNone(self.cluster.probes()["me"]["fsync_at"]) + + def test_the_reading_is_taken_once_per_window_not_every_sample(self) -> None: + self.assertEqual(self.probe().returncode, 0) + first_at = self.cluster.probes()["me"]["fsync_at"] + self.assertEqual(self.probe().returncode, 0) + self.assertIsNone(self.newest()["fsync_ms"]) + self.assertEqual(self.cluster.probes()["me"]["fsync_at"], first_at) + entries = self.cluster.probes() + entries["me"]["fsync_at"] = int(time.time()) - WINDOW_SECONDS + self.cluster.set_probes(entries) + self.assertEqual(self.probe().returncode, 0) + self.assertIsNotNone(self.newest()["fsync_ms"]) + + def test_a_voter_never_runs_the_probe_because_its_etcd_is_measured_directly(self) -> None: + marker = self.cluster.dir / "fio-ran" + shim = self.cluster.bin / "fio" + shim.write_text(f"#!/bin/sh\ntouch '{marker}'\nexit 1\n") + shim.chmod(0o755) + with metrics_server([page("before")]) as url: + completed = self.probe(voters=[node("me", voter=True), node("peer", voter=True)], ETCD_METRICS_URL=url) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertFalse(marker.exists()) + self.assertIsNone(self.newest()["fsync_ms"]) + + +class CheckTest(ProbeCase): + """`check` takes one reading alone, with no cluster access, for comparing with a reading taken by hand on a real host.""" + + def check(self, *args: str, **env: str) -> subprocess.CompletedProcess[str]: + base = { + "ELECTION_TIMEOUT_MS": str(ELECTION_TIMEOUT_MS), + "REPORT_SECONDS": str(SAMPLE_SECONDS), + "FSYNC_DIR": str(self.fsync_dir), + "SYS_CLASS_NET": str(self.sys / "class" / "net"), + "PROC_NET_ROUTE": str(self.route), + "FLEET_REGRADE_CORE": str(REGRADE_CORE), + "PSI_FILE": str(self.psi), + "MEMINFO_FILE": str(self.meminfo), + } + completed = subprocess.run( + [str(Path(__file__).resolve().parents[2] / "scripts" / "fleet-role-probe.sh"), "check", *args], + env={"PATH": self.cluster.env()["PATH"], **base, **env}, + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + check=False, + ) + self.assertEqual(self.cluster.calls.read_text(), "", "a check must not reach the cluster") + return completed + + def test_link_prints_the_interface_and_its_link_type(self) -> None: + SysfsTree(self.sys).pci_wifi() + self.route.write_text(ROUTE_HEADER + route_line("wlan0")) + completed = self.check("link") + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(json.loads(completed.stdout), {"interface": "wlan0", "link": "wifi"}) + + def test_connect_prints_the_milliseconds_to_a_listener(self) -> None: + with listener() as port: + completed = self.check("connect", "127.0.0.1", ETCD_PEER_PORT=str(port)) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertLess(float(completed.stdout), ELECTION_TIMEOUT_MS) + self.assertGreater(float(completed.stdout), 0) + + def test_memory_prints_what_the_kernels_files_say(self) -> None: + self.psi.write_text("some avg10=0.00 avg60=0.00 avg300=7.50 total=1\nfull avg10=0.00 avg60=0.00 avg300=0.00 total=0\n") + completed = self.check("memory") + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(json.loads(completed.stdout), {"psi_avg300": 7.5, "mem_available_kb": 8192000, "mem_total_kb": 16384000}) + + def test_etcd_prints_the_gauges_and_the_tails_of_the_cumulative_histograms(self) -> None: + with metrics_server([page("before")]) as url: + completed = self.check("etcd", ETCD_METRICS_URL=url) + self.assertEqual(completed.returncode, 0, completed.stderr) + reading = json.loads(completed.stdout) + self.assertEqual((reading["id"], reading["known_members"], reading["has_leader"]), (FIXTURE_SERVER_ID, TARGET, True)) + self.assertGreater(reading["wal_fsync_p99_ms"], 0) + self.assertGreater(reading["backend_commit_p99_ms"], 0) + + @unittest.skipUnless(FIO, "needs fio") + def test_fsync_prints_the_milliseconds_fio_measured(self) -> None: + completed = self.check("fsync") + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertGreater(float(completed.stdout), 0) + + def test_an_unknown_reading_is_a_usage_error(self) -> None: + completed = self.check("nonsense") + self.assertEqual(completed.returncode, 2) + self.assertIn("usage:", completed.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_heartbeat.py b/tests/unit/test_heartbeat.py index 9f7a3b46..36258b2e 100644 --- a/tests/unit/test_heartbeat.py +++ b/tests/unit/test_heartbeat.py @@ -41,7 +41,7 @@ sys.exit(0) if args[:2] == ["get", "configmap"]: key = args[-1].split("data.")[-1].rstrip("}").replace("\\.", ".") - value = state["configmap"].get(key) + value = state.get("configmaps", {}).get(args[2], state["configmap"]).get(key) if value is None: sys.exit(1) sys.stdout.write(value) @@ -95,6 +95,7 @@ def tick( nodes: list[dict[str, object]] | None = None, previous: dict[str, object] | None = None, env: dict[str, str] | None = None, + state_extra: dict[str, object] | None = None, ) -> tuple[subprocess.CompletedProcess[str], dict[str, str]]: state = self.dir / "state.json" state.write_text( @@ -106,6 +107,7 @@ def tick( "listeners": {"runners-a": 1, "runners-b": 1} if listeners is None else listeners, "configmap": configmap or {}, "nodes": [self.node("node1", None)] if nodes is None else nodes, + **(state_extra or {}), } ) ) @@ -186,6 +188,42 @@ def test_a_controller_without_a_ready_replica_withholds_the_refresh(self) -> Non self.assertEqual(result.returncode, 1) self.assertWithheld(files) + ROLE_STATUS = {"decision": {"action": "none"}, "hosts": {"a": {"verdict": "fit"}}} + + def role_tick(self, *, role_status: str | None, env: dict[str, str] | None = None, **kwargs: object) -> tuple[subprocess.CompletedProcess[str], dict[str, str]]: + configmaps: dict[str, dict[str, str]] = {"autoscaler-status": {"status.json": json.dumps({"mode": "autoscaler"})}} + if role_status is not None: + configmaps["fleet-role-status"] = {"status.json": role_status} + return self.tick(env={"FLEET_ROLE_STATUS_CONFIGMAP": "fleet-role-status", **(env or {})}, state_extra={"configmaps": configmaps}, **kwargs) # type: ignore[arg-type] + + def test_the_role_controllers_status_is_republished_beside_the_autoscalers(self) -> None: + result, files = self.role_tick(role_status=json.dumps(self.ROLE_STATUS)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(files["fleet-role-status.json"]), self.ROLE_STATUS) + self.assertEqual(json.loads(files["autoscaler-status.json"]), {"mode": "autoscaler"}) + + def test_the_role_controllers_status_goes_to_the_gist_file_it_is_given(self) -> None: + result, files = self.role_tick(role_status=json.dumps(self.ROLE_STATUS), env={"FLEET_ROLE_STATUS_GIST_FILE": "roles.json"}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(files["roles.json"]), self.ROLE_STATUS) + self.assertNotIn("fleet-role-status.json", files) + + def test_without_a_role_controller_nothing_is_republished_for_it(self) -> None: + result, files = self.tick(configmap={"status.json": json.dumps({"mode": "autoscaler"})}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("fleet-role-status.json", files) + + def test_an_empty_role_status_is_not_republished(self) -> None: + result, files = self.role_tick(role_status="") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("fleet-role-status.json", files) + + def test_the_role_controllers_status_is_withheld_with_the_autoscalers_while_the_fleet_is_not_vouched_for(self) -> None: + result, files = self.role_tick(role_status=json.dumps(self.ROLE_STATUS), controller_replicas=0) + self.assertEqual(result.returncode, 1) + self.assertNotIn("fleet-role-status.json", files) + self.assertNotIn("autoscaler-status.json", files) + @staticmethod def job_peaks(*peaks: int) -> str: return json.dumps( diff --git a/tests/unit/test_native_node.py b/tests/unit/test_native_node.py index 755aa2d3..c8823f6e 100644 --- a/tests/unit/test_native_node.py +++ b/tests/unit/test_native_node.py @@ -460,6 +460,21 @@ def test_node_taints_are_written_as_k3s_node_taints(self) -> None: config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "agent.yaml").read_text()) self.assertEqual(config["node-taint"], taints) + def test_a_host_that_uses_no_voter_setting_gets_no_node_labels(self) -> None: + self.assertEqual(self.run_role().returncode, 0) + config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "agent.yaml").read_text()) + self.assertNotIn("node-label", config) + + def test_an_eligible_host_gets_its_voter_labels_as_k3s_node_labels(self) -> None: + self.assertEqual(self.run_role(host_vars={"github_runner_cluster_voter_eligible": True}).returncode, 0) + config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "agent.yaml").read_text()) + self.assertEqual(config["node-label"], ["github-runner.exadev.com/voter-eligible=true", "github-runner.exadev.com/voter-pinned=false"]) + + def test_with_the_role_controller_on_every_host_gets_both_labels_false(self) -> None: + self.assertEqual(self.run_role(host_vars={"github_runner_cluster_role_controller": True}).returncode, 0) + config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "agent.yaml").read_text()) + self.assertEqual(config["node-label"], ["github-runner.exadev.com/voter-eligible=false", "github-runner.exadev.com/voter-pinned=false"]) + def test_a_host_that_sets_its_node_name_joins_under_it(self) -> None: self.assertEqual(self.run_role(host_vars={"github_runner_cluster_node_name": "laptop-renamed"}).returncode, 0) config = yaml.safe_load(self.file("github_runner_cluster_native_config_dir", "agent.yaml").read_text())