From 20303c85d0959d6984ecc5bcf0b1f5680f3c2e0d Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Fri, 20 Nov 2026 09:24:59 +0000 Subject: [PATCH 1/6] fix(fleet-role): publish the probe window's summary, not the window Each probe merge-patched its node's whole window of samples into the shared fleet-role-probe ConfigMap every sample period. The datastore stores the whole object on every patch, so each write was the size of every node's window together. The probe now keeps its window in its data directory on the node (window.json, replaced by a rename once a sample's summary is published) and publishes only window_summary, the tails, counts and last readings the controller judges by, from the module both share. An entry is held to probe_entry_limit_bytes (4 KiB, above a summary for etcd's largest recommended cluster of seven voters with the longest Node names); a summary over it is logged and not published. The controller judges the summary as it judged the window. Because it can no longer drop the samples from before a membership change, a window that began before the change is unknown until it has rolled past it, which a full window does once 100 samples have been taken since, as before. The probe now needs the regrade limit to count samples over it, so the DaemonSet passes PRESSURE_THRESHOLD and AVAILABLE_FLOOR_PERCENT from the same role variables as the controller. Refs #266 --- docs/role-controller.md | 4 +- roles/github_runner_arc/defaults/main.yml | 2 +- .../github_runner_arc/meta/argument_specs.yml | 2 +- .../templates/fleet-role.yaml.j2 | 7 +- scripts/fleet-role-probe.sh | 39 ++-- scripts/fleet-role-stats.jq | 32 ++++ scripts/fleet-role.jq | 85 ++++----- scripts/fleet-role.sh | 2 +- tests/unit/fleet_role_support.py | 15 ++ tests/unit/test_fleet_role.py | 27 ++- tests/unit/test_fleet_role_arc.py | 4 +- tests/unit/test_fleet_role_probe.py | 170 +++++++++++++----- 12 files changed, 267 insertions(+), 122 deletions(-) diff --git a/docs/role-controller.md b/docs/role-controller.md index b8745ad3..d876f3a3 100644 --- a/docs/role-controller.md +++ b/docs/role-controller.md @@ -4,7 +4,7 @@ `fleet-role-probe` and `fleet-role` (an optional DaemonSet and Deployment in the platform namespace, see `roles/github_runner_arc/templates/fleet-role.yaml.j2`, `scripts/fleet-role-probe.sh` and `scripts/fleet-role.sh`; `github_runner_cluster_role_controller`, off by default, which `github_runner_arc_role_controller_enabled` takes as its default) measure which hosts are fit to hold an etcd vote, report the swap of an unfit voter for a fit standby that the controller would make and, when asked or when `github_runner_arc_role_controller_dry_run` is false, carry the swap out ([Voter swap](#voter-swap)). The probe only reports: its ClusterRole can `get` and `list` nodes and nothing else, and the only object it writes is its own ConfigMap. The controller runs unchanged when no host is marked eligible, when it reports the voters' fitness alone. -The probe runs on every Linux node that is an etcd voter (k3s's `node-role.kubernetes.io/etcd` label) or is marked eligible and has a kubelet (a server regraded to etcd-only has none, and its side is read from its peers), on the node's own network, through a node affinity on those two labels, so a node that is neither (a Unraid host, an image-flashed agent) has no pod and nothing is asked of its kernel. A pod stopped on a node that has lost both labels, or is gone, removes the node's entry as it ends, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the control plane and not with the fleet. Every `github_runner_arc_role_controller_sample_seconds` (15) it appends one sample to its node's key of the `fleet-role-probe` ConfigMap, which keeps the last window of them so that a replaced pod loses nothing. A sample holds the TCP connect time to every voter's etcd peer port, at the address k3s records for the voter's etcd member (the Node's `etcd.k3s.cattle.io/node-address` annotation, its mesh address or, once moved, its LAN address) (a connect that does not finish within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what its etcd metrics (the `127.0.0.1:2381` listener) gained since the last sample: the WAL fsync and backend commit histograms, the round trip to each peer, leader changes, and whether it has a leader, is a learner and knows as many members as there are voters (a learner or a member that has not started shows as an extra known member); on a node that is not a voter, an fdatasync reading from fio with the write size etcd's own disk check uses, once per window; the memory pressure (`/proc/pressure/memory` of the pod's own procfs, which serves the whole node's, so nothing is mounted) and `MemAvailable` the regrade core reads, with the core's own functions, where a node on a kernel without pressure stall information (built or booted without it, which has no `/proc/pressure` or, on older kernels, has the files and refuses to read them) gives `MemAvailable` and `MemTotal` and no pressure figure; and the medium of the node's link, `wired`, `wifi` or `unknown`: the value of the node's `github-runner.exadev.com/link` label (`github_runner_arc_role_controller_link_label`), which the cluster role reads on the host itself (see the cluster role's README) since a node in a container or a VM sees only its own virtual interface, and otherwise the medium of the interface the pod's default route leaves by (an interface that is virtual, such as the k3s container's veth, hides the link beneath it), each sample saying which (`link_source`, `label` or `interface`). The pod's own reading is taken from the interface's `/sys/class/net` entry and never by walking `/sys`, whose symlinks loop. `scripts/fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of these readings alone (the link as the pod's own interface gives it), with no cluster access, to compare with a reading taken by hand on the host. +The probe runs on every Linux node that is an etcd voter (k3s's `node-role.kubernetes.io/etcd` label) or is marked eligible and has a kubelet (a server regraded to etcd-only has none, and its side is read from its peers), on the node's own network, through a node affinity on those two labels, so a node that is neither (a Unraid host, an image-flashed agent) has no pod and nothing is asked of its kernel. A pod stopped on a node that has lost both labels, or is gone, removes the node's entry as it ends, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the control plane and not with the fleet. Every `github_runner_arc_role_controller_sample_seconds` (15) it takes one sample and appends it to the window of the last 100 it keeps in `github_runner_arc_role_controller_data_dir` on the node (`window.json`), so that a replaced pod loses nothing, and publishes the window's summary to its node's key of the `fleet-role-probe` ConfigMap: the tails, counts and last readings the controller judges by (`window_summary` in `scripts/fleet-role-stats.jq`), never the samples. Every patch of the ConfigMap rewrites the whole object in the datastore, so an entry is held to `probe_entry_limit_bytes` (4 KiB), which a summary for seven voters with the longest Node names stays under; a summary over it is logged and not published, and the host's verdict goes `unknown` once its report is stale. A sample holds the TCP connect time to every voter's etcd peer port, at the address k3s records for the voter's etcd member (the Node's `etcd.k3s.cattle.io/node-address` annotation, its mesh address or, once moved, its LAN address) (a connect that does not finish within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what its etcd metrics (the `127.0.0.1:2381` listener) gained since the last sample: the WAL fsync and backend commit histograms, the round trip to each peer, leader changes, and whether it has a leader, is a learner and knows as many members as there are voters (a learner or a member that has not started shows as an extra known member); on a node that is not a voter, an fdatasync reading from fio with the write size etcd's own disk check uses, once per window; the memory pressure (`/proc/pressure/memory` of the pod's own procfs, which serves the whole node's, so nothing is mounted) and `MemAvailable` the regrade core reads, with the core's own functions, where a node on a kernel without pressure stall information (built or booted without it, which has no `/proc/pressure` or, on older kernels, has the files and refuses to read them) gives `MemAvailable` and `MemTotal` and no pressure figure; and the medium of the node's link, `wired`, `wifi` or `unknown`: the value of the node's `github-runner.exadev.com/link` label (`github_runner_arc_role_controller_link_label`), which the cluster role reads on the host itself (see the cluster role's README) since a node in a container or a VM sees only its own virtual interface, and otherwise the medium of the interface the pod's default route leaves by (an interface that is virtual, such as the k3s container's veth, hides the link beneath it), each sample saying which (`link_source`, `label` or `interface`). The pod's own reading is taken from the interface's `/sys/class/net` entry and never by walking `/sys`, whose symlinks loop. `scripts/fleet-role-probe.sh check link|connect
|memory|etcd|fsync` takes one of these readings alone (the link as the pod's own interface gives it), with no cluster access, to compare with a reading taken by hand on the host. The controller reads the Nodes and the probes' samples every sample period and publishes `fleet-role-status`, which the heartbeat republishes in its gist beside the autoscaler's status. For every voter and every eligible host it gives a verdict, `fit`, `unfit` or `unknown`, with each reason and the numbers behind it. Every limit derives from etcd's guidance and the cluster's own timing, not chosen numbers: @@ -12,7 +12,7 @@ The controller reads the Nodes and the probes' samples every sample period and p - A voter's WAL fsync tail must be under 10 ms and its backend commit tail under 25 ms (etcd's FAQ); a standby's fdatasync tail under 10 ms. - Memory is never over the regrade limit (`github_runner_cluster_regrade_pressure_threshold` and `github_runner_cluster_regrade_available_floor_percent`) in any sample of the window, and `MemAvailable` is at least etcd's storage quota (what the voters report, 2 GiB by default). A host none of whose samples has a pressure reading cannot be over the regrade limit, which needs both, so it is judged by `MemAvailable` against the quota alone, and its status says so: `unknown_signals` lists `memory pressure` (and the log adds `memory pressure unknown` to its verdict), beside `figures.psi_avg300_max`, the worst pressure of the window, which is then `null`. - Stability: no `NotReady` spell in the window and no k3s restart, seen on a voter as etcd's counters going backwards. On a node that is not a voter the only sign of a restart is its probe starting again, so a new probe pod counts as one. -- A window is 100 samples, the fewest that give the 99th percentile a sample of its own, so one outlier never decides; a host with fewer is `unknown`, as is one whose probe has not reported for three sample periods. Readings from before the last change of the voter set are not counted. +- A window is 100 samples, the fewest that give the 99th percentile a sample of its own, so one outlier never decides; a host with fewer is `unknown`, as is one whose probe has not reported for three sample periods. A window that began before the last change of the voter set is not judged (`unknown`) until it has rolled past the change, so every reading a verdict rests on was taken since. - Hosts that pass are ranked wired before Wi-Fi before unknown, then by round trip, then by disk. The link ranks hosts and is never a gate; the status gives each host's link and where it was read (`link` and `link_source`), and the log names it beside the verdict. A slow link makes both of its ends unfit, since a voter is unfit when its round trip to any other voter is, so every unfit verdict is attributed to the node at fault for it (`attributed_to` on each host, and `attribution`, the hosts each node accounts for). A host's own disk, memory, readiness or restarts are its own fault. Its round trip is the link to its worst peer, and the end at fault for a slow link is the one whose links to the other voters are slower too, compared by the best of them, since a node on a poor path is slow to every peer and a node on a good one is fast to its others; when the two ends' other links are as fast as each other's, or one end has none, the link is attributed to both. The voter to replace is the unfit voter the most hosts' unfit verdicts are attributed to, the worst-ranked among those with as many, and the log names the attribution beside each unfit verdict. A swap is reported only when that voter is eligible and not pinned (when it is not, replacing a voter that shares a slow link with it would fix nothing, so nothing is swapped) and a fit eligible standby ranks above it. A candidate is judged against the voters that would remain, since a slow link to the very voter it would replace says nothing against it. A voter is a node that carries k3s's etcd label and whose `EtcdIsVoter` condition is True: k3s labels a server the moment it joins etcd as a learner and never removes the label, so a labelled node without the condition is a member that does not vote: it is not counted as a voter and holds a swap back (`member but not a voter` in the status). It is held back, with the reason in the status and the log, while the voter count is not `github_runner_arc_role_controller_voters`, any voter is below `full`, is not Ready, has no leader, is a learner or knows a member that is not a voter, is not connected to every other voter, fewer hosts are eligible than the target, or the cooldown after a change of the voter set has not passed (k3s's five minutes to remove a stalled learner plus one window). The target must be odd, at least 3, and at most `github_runner_cluster_max_servers` minus one, which leaves a swap one place of headroom; the role refuses anything else. A host is eligible, and a voter pinned, by a Node label with the value `true` (`github_runner_arc_role_controller_eligible_label` and `_pinned_label`), which the cluster role publishes from `github_runner_cluster_voter_eligible` and `github_runner_cluster_voter_pinned` per host (and the target from `github_runner_cluster_voters`, refused unless odd, at least 3 and below `github_runner_cluster_max_servers`; see the cluster role's README); a voter not marked eligible is never swapped out, since nothing has prepared it to come back. diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index de73f7bf..251a69ca 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -234,7 +234,7 @@ github_runner_arc_role_controller_lease_name: "{{ github_runner_cluster_regrade_ # The ConfigMaps in the platform namespace that the probes publish to (a key per node) and that the controller publishes its status to, which the heartbeat republishes beside the autoscaler's. github_runner_arc_role_controller_probe_configmap: fleet-role-probe github_runner_arc_role_controller_status_configmap: fleet-role-status -# Where the probe writes its fdatasync reading on a node that is not a voter: a directory on the filesystem k3s keeps its data on, which the pod mounts from the node, created by the kubelet and owned by root. +# Where the probe keeps its window of samples (window.json) and, on a node that is not a voter, writes its fdatasync reading: a directory on the filesystem k3s keeps its data on, which the pod mounts from the node, created by the kubelet and owned by root. github_runner_arc_role_controller_data_dir: /var/lib/rancher/k3s/fleet-role-probe # Runner reaper (see the role README's Runner reaper section, tasks/install_runner_reaper.yml and files/reap-runners.sh): a CronJob in each runner namespace that deletes an EphemeralRunner whose job's workflow run is completed or cancelled, whose own job is completed, or whose job has been running for longer than github_runner_arc_runner_reaper_max_job_seconds; the scale set replaces it. It reads the runs with an installation token from the org's GitHub App restricted to actions: read, so the App needs the Actions repository permission (read) on every repository whose jobs the runners take, approved on its installation. It never deletes a runner that has not been assigned a job, and deletes nothing when GitHub or the token is unavailable. false removes the CronJob, its RBAC and its ConfigMap. diff --git a/roles/github_runner_arc/meta/argument_specs.yml b/roles/github_runner_arc/meta/argument_specs.yml index 3215317e..68c5bb3e 100644 --- a/roles/github_runner_arc/meta/argument_specs.yml +++ b/roles/github_runner_arc/meta/argument_specs.yml @@ -458,7 +458,7 @@ argument_specs: github_runner_arc_role_controller_data_dir: type: str default: /var/lib/rancher/k3s/fleet-role-probe - description: The directory on the node, on the filesystem k3s keeps its data on, where the probe takes its fdatasync reading. + description: The directory on the node, on the filesystem k3s keeps its data on, where the probe keeps its window of samples and takes its fdatasync reading. github_runner_arc_runner_reaper_enabled: type: bool default: false diff --git a/roles/github_runner_arc/templates/fleet-role.yaml.j2 b/roles/github_runner_arc/templates/fleet-role.yaml.j2 index 1b3a2a30..ec86870f 100644 --- a/roles/github_runner_arc/templates/fleet-role.yaml.j2 +++ b/roles/github_runner_arc/templates/fleet-role.yaml.j2 @@ -72,10 +72,11 @@ rules: resourceNames: {{ [actuator_configmap] | to_json }} verbs: ["get"] {% else %} +{# Patch alone: a probe keeps its window on its own node and never reads the ConfigMap, so it cannot read the other nodes' entries. #} - apiGroups: [""] resources: ["configmaps"] resourceNames: {{ [probe_configmap] | to_json }} - verbs: ["get", "patch"] + verbs: ["patch"] {% endif %} --- apiVersion: rbac.authorization.k8s.io/v1 @@ -271,6 +272,10 @@ spec: value: {{ probe_configmap | to_json }} - name: REPORT_SECONDS value: {{ github_runner_arc_role_controller_sample_seconds | string | to_json }} + - name: PRESSURE_THRESHOLD + value: {{ github_runner_arc_role_controller_pressure_threshold | string | to_json }} + - name: AVAILABLE_FLOOR_PERCENT + value: {{ github_runner_arc_role_controller_available_floor_percent | string | to_json }} - name: FSYNC_DIR value: /fsync # kubectl's cache goes to the in-memory volume, since the root filesystem is read-only. diff --git a/scripts/fleet-role-probe.sh b/scripts/fleet-role-probe.sh index d4cb8b8f..442782f6 100755 --- a/scripts/fleet-role-probe.sh +++ b/scripts/fleet-role-probe.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# The role controller's per-node probe (see roles/github_runner_arc/templates/fleet-role.yaml.j2): the `fleet-role-probe` DaemonSet runs this on every node that is an etcd voter or marked eligible and has a kubelet, with host networking, so it measures the path the node's own etcd and kubelet use. Every REPORT_SECONDS it takes one sample and appends it to this node's entry in the shared probe ConfigMap, which scripts/fleet-role.sh reads. An entry keeps the last window of samples, so it survives this pod being replaced. +# The role controller's per-node probe (see roles/github_runner_arc/templates/fleet-role.yaml.j2): the `fleet-role-probe` DaemonSet runs this on every node that is an etcd voter or marked eligible and has a kubelet, with host networking, so it measures the path the node's own etcd and kubelet use. Every REPORT_SECONDS it takes one sample, appends it to the window of the last window_samples samples it keeps in its data directory on the node (FSYNC_DIR, so the window survives this pod being replaced), and publishes the window's summary (window_summary in scripts/fleet-role-stats.jq) as this node's entry in the shared probe ConfigMap, which scripts/fleet-role.sh reads. The window itself is never published: every patch of the ConfigMap rewrites the whole object in the datastore, so an entry is held to probe_entry_limit_bytes, and a summary over it is refused and not published. # -# Only a node the controller judges is measured: a voter, or a host the owner marks eligible (ELIGIBLE_LABEL, with the value true). The DaemonSet schedules a pod on no other node, and a pod that finds its node is neither (a label removed while it runs) records nothing and removes the entry it has from when it was one, as does a pod that is stopped on a node that is neither any more or is gone, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the cluster's control plane and not with the fleet. +# Only a node the controller judges is measured: a voter, or a host the owner marks eligible (ELIGIBLE_LABEL, with the value true). The DaemonSet schedules a pod on no other node, and a pod that finds its node is neither (a label removed while it runs) records nothing and removes the entry and the window it has from when it was one, as does a pod that is stopped on a node that is neither any more or is gone, so the ConfigMap, which holds at most a mebibyte, grows with the hosts that decide the cluster's control plane and not with the fleet. # # A sample holds, per node: the TCP connect time to every voter's etcd peer port (a connect that does not complete within the election timeout is recorded as the election timeout, so loss shows as tail latency); on a voter, what etcd's own metrics gained since the last sample (the WAL fsync and backend commit histograms, the round trip to each peer, leader changes) and whether it has a leader, is a learner and is connected to its peers; on a node that is not a voter, an fdatasync probe in the k3s data directory once per window, with the write size etcd's own fio check uses; the memory pressure and MemAvailable the regrade core reads; and the medium of the node's link: the value of its LINK_LABEL Node label when it has one, which the cluster role reads on the host itself (the medium of the interface a Mac's Colima VM is bridged to, or of the interface a Linux host's default route leaves by), and otherwise the medium of the interface this pod's default route leaves by. The label decides when the node has it because a node in a container or a VM sees only its own virtual interface: a k3s container's veth, which reads unknown, or a VM's virtio NIC, which reads wired whatever the Mac beneath it uses. # @@ -19,7 +19,8 @@ # - VOTER_LABEL: the Node label that marks an etcd member (default node-role.kubernetes.io/etcd=true). # - ETCD_METRICS_URL: where a voter's etcd serves its metrics (default http://127.0.0.1:2381/metrics, the listener k3s keeps). # - ETCD_PEER_PORT: etcd's peer port (default 2380). -# - FSYNC_DIR: a directory on the filesystem k3s keeps its data on, which the fdatasync probe writes in (default /var/lib/rancher/k3s/fleet-role-probe). +# - PRESSURE_THRESHOLD, AVAILABLE_FLOOR_PERCENT: the regrade core's limit (the controller's FLEET_ROLE_PRESSURE_THRESHOLD and FLEET_ROLE_AVAILABLE_FLOOR_PERCENT), which the summary counts the samples over. Required. +# - FSYNC_DIR: a directory on the filesystem k3s keeps its data on, which the fdatasync probe writes in and the window is kept in, as window.json (default /var/lib/rancher/k3s/fleet-role-probe). # - NODE_LINK: the link reader the cluster role runs on the host too (roles/github_runner_cluster/files/node-link.sh), whose default_route_interface and link_type read this pod's own (default /app/node-link.sh). SYS_CLASS_NET and PROC_NET_ROUTE, which it reads, are where the host's /sys/class/net and /proc/net/route are (the defaults). # - FLEET_REGRADE_CORE: the regrade core (roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh), whose own psi_avg300 and meminfo_kb read the memory signals (default /app/fleet-regrade-core.sh). PSI_FILE and MEMINFO_FILE, which it reads, default here to /proc/pressure/memory and /proc/meminfo, which a pod's own procfs serves for the whole node as it does /proc/meminfo, so nothing is mounted for them. A kernel without pressure stall information (built or booted without it) has no /proc/pressure or, on older kernels, has the files and refuses to read them (EOPNOTSUPP); the sample then has no pressure reading and the memory figures from /proc/meminfo alone, which the controller judges by MemAvailable alone and reports with the pressure unknown. set -euo pipefail @@ -35,6 +36,7 @@ NODE_LINK="${NODE_LINK:-/app/node-link.sh}" FLEET_REGRADE_CORE="${FLEET_REGRADE_CORE:-/app/fleet-regrade-core.sh}" PSI_FILE="${PSI_FILE:-/proc/pressure/memory}" MEMINFO_FILE="${MEMINFO_FILE:-/proc/meminfo}" +WINDOW_FILE="${FSYNC_DIR}/window.json" # The core reads its settings from an environment file when there is one; the probe has none, and must not pick up another by the core's default path. # shellcheck disable=SC2034 # read by the sourced core FLEET_REGRADE_ENV=/nonexistent @@ -100,13 +102,13 @@ labelled_link_in() { jq -r --arg node "$NODE_NAME" --arg label "$LINK_LABEL" '[.items[] | select(.metadata.name == $node) | (.metadata.labels // {})[$label] | strings] | first // ""' <<< "$1" } -# Removes this node's entry from the ConfigMap. A key set to null in a merge patch is removed. +# Removes this node's entry from the ConfigMap, and then its window, which a node measured again starts afresh. A key set to null in a merge patch is removed. remove_entry() { local patch patch="$(mktemp)" jq -n --arg node "$NODE_NAME" '{data: {($node): null}}' > "$patch" kubectl patch configmap "$PROBE_CONFIGMAP" -n "$NAMESPACE" --type merge --patch-file "$patch" > /dev/null || { rm -f "$patch"; return 1; } - rm -f "$patch" + rm -f "$patch" "$WINDOW_FILE" } # What the pod does when it is stopped: if its node is neither a voter nor eligible any more (the DaemonSet stops the pod of a node that has lost both labels) or is gone, its entry is removed, so the ConfigMap holds only the hosts that decide the control plane; a pod stopped for any other reason (a new image, the node shutting down) leaves the entry for the pod that follows it to extend. The API can fail here, and a pod that is going away has nothing to retry it with. @@ -122,11 +124,11 @@ retire() { # Takes one sample and publishes it. $1 is "true" on the first pass of this process, which for a node that is not a voter is the only sign this probe has of k3s having restarted (a voter sees it in etcd's counters going backwards). report() { local first=$1 now entry voters is_voter eligible tcp='{}' peer_name peer_ip ms etcd='null' restarted=false cumulative='null' etcd_id='null' - local iface link link_source fsync='null' fsync_at patch metrics reading change psi='' + local iface link link_source fsync='null' fsync_at patch metrics reading change psi='' new_entry summary size limit_bytes now="$(date +%s)" - # This node's entry is read from the ConfigMap once, when the process starts, and kept from then on: the probe is the only writer of its key, and two API calls a sample (the nodes, then the patch) are all a probe on a poor path can be asked to make in a period. + # This node's window is read from its data directory once, when the process starts, and kept from then on, so a sample costs two API calls (the nodes, then the patch), which are all a probe on a poor path can be asked to make in a period. if [ "$entry_loaded" != true ]; then - entry="$(kubectl get configmap "$PROBE_CONFIGMAP" -n "$NAMESPACE" -o json | jq -c --arg node "$NODE_NAME" '(.data[$node] // "null") | fromjson')" || return 1 + if [ -e "$WINDOW_FILE" ]; then entry="$(jq -c . "$WINDOW_FILE")" || return 1; else entry=null; fi entry_loaded=true else entry="$kept_entry" @@ -136,7 +138,7 @@ report() { is_voter="$(is_voter_in "$voters")" eligible="$(is_eligible_in "$nodes_json")" if [ "$is_voter" != true ] && [ "$eligible" != true ]; then - if [ "$entry" != null ]; then + if [ "$entry" != null ] || [ -e "$WINDOW_FILE" ]; then remove_entry kept_entry=null fi @@ -180,7 +182,7 @@ report() { psi="$(pressure_reading)" - # The entry goes in on standard input, never as an argument: it holds a window of samples, and one argument is limited to a fraction of the argument space (128 KiB on Linux). + # The window goes in on standard input, never as an argument: it holds a window of samples, and one argument is limited to a fraction of the argument space (128 KiB on Linux). new_entry="$(jq -c -L "$MODULES" --argjson now "$now" --argjson tcp "$tcp" --argjson etcd "$etcd" \ --argjson restarted "$restarted" --argjson cumulative "$cumulative" --argjson etcd_id "$etcd_id" --argjson fsync "$fsync" --argjson fsync_at "$fsync_at" \ --arg psi "$psi" --arg avail "$(meminfo_kb MemAvailable)" --arg total "$(meminfo_kb MemTotal)" --arg iface "$iface" --arg link "$link" --arg link_source "$link_source" ' @@ -190,11 +192,19 @@ report() { | {t: $now, tcp: $tcp, etcd: $etcd, restarted: $restarted, psi: ($psi | number), avail_kb: ($avail | number), total_kb: ($total | number), link: $link, link_source: $link_source, iface: (if $iface == "" then null else $iface end), fsync_ms: $fsync} as $sample | {etcd_id: $etcd_id, last: $cumulative, fsync_at: $fsync_at, samples: ((($entry.samples // []) + [$sample]) | .[-window_samples:])}' <<< "$entry")" || return 1 + summary="$(jq -c -L "$MODULES" --argjson limit "$regrade_limit" 'include "fleet-role-stats"; window_summary($limit)' <<< "$new_entry")" || return 1 + size="$(printf '%s' "$summary" | wc -c | tr -d ' ')" + limit_bytes="$(jq -n -L "$MODULES" 'include "fleet-role-stats"; probe_entry_limit_bytes')" + if [ "$size" -gt "$limit_bytes" ]; then + log "this node's summary is ${size} bytes, over the limit of ${limit_bytes} bytes for an entry; it is not published" + return 1 + fi patch="$(mktemp)" - jq --arg node "$NODE_NAME" '{data: {($node): tojson}}' <<< "$new_entry" > "$patch" + jq --arg node "$NODE_NAME" '{data: {($node): tojson}}' <<< "$summary" > "$patch" kubectl patch configmap "$PROBE_CONFIGMAP" -n "$NAMESPACE" --type merge --patch-file "$patch" > /dev/null || { rm -f "$patch"; return 1; } rm -f "$patch" - # Kept only once published, so a patch that failed is read again from the ConfigMap and not taken as written. + # Kept only once its summary is published, so a sample whose patch failed is not in the window; the file is replaced whole by a rename, so a reader never sees half of it. + printf '%s\n' "$new_entry" > "${WINDOW_FILE}.new" && mv "${WINDOW_FILE}.new" "$WINDOW_FILE" || return 1 kept_entry="$new_entry" } @@ -223,10 +233,13 @@ fi : "${ELECTION_TIMEOUT_MS:?ELECTION_TIMEOUT_MS is required}" : "${ELIGIBLE_LABEL:?ELIGIBLE_LABEL is required}" : "${LINK_LABEL:?LINK_LABEL is required}" +: "${PRESSURE_THRESHOLD:?PRESSURE_THRESHOLD is required}" +: "${AVAILABLE_FLOOR_PERCENT:?AVAILABLE_FLOOR_PERCENT is required}" +regrade_limit="$(jq -nc --argjson pressure_threshold "$PRESSURE_THRESHOLD" --argjson available_floor_percent "$AVAILABLE_FLOOR_PERCENT" '{pressure_threshold: $pressure_threshold, available_floor_percent: $available_floor_percent}')" trap retire TERM INT -# This node's entry as the ConfigMap holds it, read once and then kept (see report). +# This node's window as its data directory holds it, read once and then kept (see report). entry_loaded=false kept_entry=null first=true diff --git a/scripts/fleet-role-stats.jq b/scripts/fleet-role-stats.jq index 5a376981..364360f6 100644 --- a/scripts/fleet-role-stats.jq +++ b/scripts/fleet-role-stats.jq @@ -95,3 +95,35 @@ def etcd_sample($previous; $now): # The part of a probe sample's memory reading the regrade core calls over the limit (fleet-regrade-core.sh, is_over): pressure at or above the threshold and MemAvailable below the floor share of MemTotal, both. tests/unit/test_fleet_role.py holds this to the core's own function. $c is the controller's settings (pressure_threshold, available_floor_percent). def over_regrade_limit($c): .psi >= $c.pressure_threshold and (.avail_kb * 100) < (.total_kb * $c.available_floor_percent); + +# The most bytes one probe's published entry may hold. Every patch of the shared probe ConfigMap rewrites the whole object in the datastore, so the size of each entry is part of every node's write. A summary holds a fixed set of figures and two per voter (the TCP tail and its count by Node name, and etcd's round trip tail by member id); at etcd's largest recommended cluster of seven members, each with a Node name of the longest Kubernetes allows (253 characters), it is under 3 KiB, so a summary over this is a fault, never a larger cluster. +def probe_entry_limit_bytes: 4096; + +# What a probe publishes of its window of samples (an object with etcd_id and samples, as the probe keeps it): every figure the controller (scripts/fleet-role.jq) judges a host by, over the whole window, and the last sample's readings that are taken as they are. The window itself stays with the probe. $c is the regrade core's limit (pressure_threshold, available_floor_percent), which the samples over it are counted against. +# - t, since: the times of the newest and the oldest sample, null for an empty window; samples: how many the window holds. +# - tcp: by peer Node name, the nearest-rank tail of the TCP connect times in milliseconds (p99_ms) and how many readings it rests on (readings). +# - etcd_rtt_p99_ms: by peer etcd member id, the tail of etcd's own round trip in milliseconds, from the histograms the window gained. +# - wal_p99_ms, commit_p99_ms: the tails of the WAL fsync and backend commit histograms the window gained, in milliseconds, null without readings; fsync_p99_ms: the worst fdatasync reading. +# - over_limit_samples, pressure_max, pressure_samples, min_avail_kb: the memory readings; restarts, leader_changes: how many samples saw k3s restart and how many leader changes etcd saw. +# - etcd: the last sample's etcd gauges (has_leader, is_learner, known_members, active_peers, quota_bytes), null when it has none. +def window_summary($c): + def ms: if . == null then null else . * 1000 end; + (.samples // []) as $w + | ($w | map(.etcd) | map(select(. != null))) as $etcd + | ($w | map(.psi | select(. != null))) as $pressures + | {etcd_id: .etcd_id, + t: ($w[-1].t // null), since: ($w[0].t // null), samples: ($w | length), + link: ($w[-1].link // null), link_source: ($w[-1].link_source // null), iface: ($w[-1].iface // null), + tcp: ([$w[] | .tcp // {} | keys[]] | unique + | map(. as $peer | [$w[] | .tcp[$peer] | select(. != null)] | {key: $peer, value: {p99_ms: nearest_rank(tail_percentile), readings: length}}) | from_entries), + etcd_rtt_p99_ms: ([$etcd[] | .rtt // {} | keys[]] | unique + | map(. as $id | {key: $id, value: ([$etcd[] | .rtt[$id] | select(. != null)] | sum_buckets | histogram_quantile(tail_percentile) | ms)}) | from_entries), + wal_p99_ms: ([$etcd[].wal] | sum_buckets | histogram_quantile(tail_percentile) | ms), + commit_p99_ms: ([$etcd[].commit] | sum_buckets | histogram_quantile(tail_percentile) | ms), + fsync_p99_ms: ([$w[] | .fsync_ms | select(. != null)] | max), + over_limit_samples: ($w | map(select(.psi != null and over_regrade_limit($c))) | length), + pressure_max: ($pressures | max), pressure_samples: ($pressures | length), + min_avail_kb: ($w | map(.avail_kb | select(. != null)) | min), + restarts: ($w | map(select(.restarted == true)) | length), + leader_changes: ($etcd | map(.leader_changes) | add // 0), + etcd: ($w[-1].etcd // null | if . == null then null else {has_leader, is_learner, known_members, active_peers, quota_bytes} end)}; diff --git a/scripts/fleet-role.jq b/scripts/fleet-role.jq index 8e6ce583..f51ff8ae 100644 --- a/scripts/fleet-role.jq +++ b/scripts/fleet-role.jq @@ -3,10 +3,10 @@ # Inputs, as jq named arguments: # - $config: the controller's settings (see scripts/fleet-role.sh for each), with `now` the time of the pass in epoch seconds, `expected_voters` the number of voters the cluster should hold at this moment (the target, or one more while a swap holds a new voter beside the old) and `ignored_voters`, optional, the voters a swap in progress is taking out, which the gates do not hold to their health (a voter stepped down to etcd-only, or one that is down), and whose connections the others may lack. # - $nodes_file: the output of `kubectl get nodes -o json`, as --slurpfile gives it (a one-element array). -# - $probes_file: each node's probe entry (scripts/fleet-role-probe.sh), by node name, likewise. +# - $probes_file: each node's probe entry (scripts/fleet-role-probe.sh), by node name, likewise: the summary of the probe's window of samples (window_summary in scripts/fleet-role-stats.jq), which the probe keeps to itself. # - $previous_file: the status this program published on the last pass, or null on the first, likewise. # -# Every threshold is derived, and every constant below is a fact about etcd or k3s with its source. The figures are read over a window of tail_window samples per peer pair; a host with fewer is "unknown", never fit or unfit. +# Every threshold is derived, and every constant below is a fact about etcd or k3s with its source. The figures are read over a window of window_samples samples per peer pair, every one taken since the last membership change; a host with fewer, or whose window began before the change, is "unknown", never fit or unfit. include "fleet-role-stats"; @@ -37,15 +37,14 @@ def ready_condition: ((.status.conditions // []) | map(select(.type == "Ready")) def condition_true($type): ((.status.conditions // []) | map(select(.type == $type)) | .[0].status) == "True"; -# The pair figure of one observer towards one peer over a window: the worse of the TCP connect tail and etcd's own round trip tail (only a voter has the latter, keyed by the peer's etcd id), in milliseconds, with how many TCP readings it rests on. -def pair($window; $peer; $peer_id): - ([$window[] | .tcp[$peer] | select(. != null)]) as $tcp - | (if $peer_id == null then null else ([$window[] | .etcd.rtt[$peer_id] | select(. != null)] | sum_buckets | histogram_quantile(tail_percentile)) end) as $etcd - | ($tcp | nearest_rank(tail_percentile)) as $tcp_tail - | {peer: $peer, readings: ($tcp | length), - tcp_ms: $tcp_tail, - etcd_ms: (if $etcd == null then null else $etcd * 1000 end), - ms: ([$tcp_tail, (if $etcd == null then null else $etcd * 1000 end)] | map(select(. != null)) | max)}; +# The pair figure of one observer towards one peer over the observer's window ($summary, its probe's entry): the worse of the TCP connect tail and etcd's own round trip tail (only a voter has the latter, keyed by the peer's etcd id), in milliseconds, with how many TCP readings it rests on. +def pair($summary; $peer; $peer_id): + ($summary.tcp[$peer] // {p99_ms: null, readings: 0}) as $tcp + | (if $peer_id == null then null else $summary.etcd_rtt_p99_ms[$peer_id] end) as $etcd_ms + | {peer: $peer, readings: $tcp.readings, + tcp_ms: $tcp.p99_ms, + etcd_ms: $etcd_ms, + ms: ([$tcp.p99_ms, $etcd_ms] | map(select(. != null)) | max)}; $config as $c | $nodes_file[0] as $nodes @@ -83,53 +82,44 @@ $config as $c | (if $changed_at == null then null else $changed_at + $cooldown_seconds end) as $cooldown_until # The storage quota a voter's etcd reports, the largest of them, or etcd's own default when no voter reports one. -| ([$voters[] | $probes[.name].samples[-1].etcd.quota_bytes | select(. != null)] | max // $c.default_quota_bytes) as $quota_bytes +| ([$voters[] | $probes[.name].etcd.quota_bytes | select(. != null)] | max // $c.default_quota_bytes) as $quota_bytes -| def window($name): ($probes[$name].samples // []) | map(select(.t >= ($changed_at // 0))) | .[-window_samples:]; - - def measure($node): +| def measure($node): $node.name as $name - | $probes[$name] as $entry - | window($name) as $w - | ($entry.samples[-1].t // null) as $last_at + | ($probes[$name] // ({} | window_summary($c))) as $s + # A window that began before the last membership change holds readings of the old membership; it is not judged until it has rolled past the change, which a full window does once window_samples samples have been taken since. + | ($changed_at != null and $s.since != null and $s.since < $changed_at) as $predates | (if $node.voter then ($voter_names | map(select(. != $name))) else $voter_names end) as $peers - | [$peers[] as $peer | pair($w; $peer; $probes[$peer].etcd_id)] as $pairs + | [$peers[] as $peer | pair($s; $peer; $probes[$peer].etcd_id)] as $pairs | ($pairs | map(.ms) | map(select(. != null)) | max) as $rtt_ms | ($pairs | max_by(.ms // 0) | .peer) as $rtt_peer - | ($w | map(.etcd) | map(select(. != null))) as $etcd - | ([$etcd[].wal] | sum_buckets | histogram_quantile(tail_percentile)) as $wal - | ([$etcd[].commit] | sum_buckets | histogram_quantile(tail_percentile)) as $commit - | ([$w[] | .fsync_ms | select(. != null)] | max) as $fsync - | ($w | map(select(.psi != null and over_regrade_limit($c))) | length) as $over - | ($w | map(.psi | select(. != null))) as $pressures - | ($w | map(.avail_kb | select(. != null)) | min) as $min_avail_kb - | ($w | map(select(.restarted == true)) | length) as $restarts | ($node.ready | not) as $not_ready | ($node.ready_since != null and $node.ready_since >= ($now - $window_seconds)) as $spell | { node: $node, - reported_at: $last_at, - fresh: ($last_at != null and $last_at >= ($now - $stale_seconds)), - samples: ($w | length), - link: ($w[-1].link // "unknown"), - link_source: ($w[-1].link_source // null), - interface: ($w[-1].iface // null), + reported_at: $s.t, + fresh: ($s.t != null and $s.t >= ($now - $stale_seconds)), + samples: $s.samples, + predates_change: $predates, + link: ($s.link // "unknown"), + link_source: $s.link_source, + interface: $s.iface, pairs: $pairs, rtt_ms: $rtt_ms, rtt_peer: $rtt_peer, pairs_complete: ($pairs | all(.readings >= window_samples)), - wal_p99_ms: (if $wal == null then null else $wal * 1000 end), - commit_p99_ms: (if $commit == null then null else $commit * 1000 end), - fsync_p99_ms: $fsync, - over_limit_samples: $over, - pressure_max: ($pressures | max), - pressure_samples: ($pressures | length), - min_available_bytes: (if $min_avail_kb == null then null else $min_avail_kb * kib end), - restarts: $restarts, + wal_p99_ms: $s.wal_p99_ms, + commit_p99_ms: $s.commit_p99_ms, + fsync_p99_ms: $s.fsync_p99_ms, + over_limit_samples: $s.over_limit_samples, + pressure_max: $s.pressure_max, + pressure_samples: $s.pressure_samples, + min_available_bytes: (if $s.min_avail_kb == null then null else $s.min_avail_kb * kib end), + restarts: $s.restarts, not_ready: $not_ready, not_ready_spell: $spell, - leader_changes: ($etcd | map(.leader_changes) | add // 0), - etcd: ($entry.samples[-1].etcd // null) + leader_changes: $s.leader_changes, + etcd: $s.etcd }; # What makes a measured host unfit, as sentences carrying their numbers; empty when nothing does. A voter is judged by the unfit round trip limit and the disk it serves etcd from, a candidate by the stricter fit limit and a disk probe. The round trip is a link's, and another node may be at fault for it; every other fault is the host's own. @@ -166,7 +156,8 @@ $config as $c $m.node as $node | [ (if $m.reported_at == null then "no probe report" elif ($m.fresh | not) then "the probe's last report is \($now - $m.reported_at) s old" else empty end), - (if $m.samples < window_samples then "\($m.samples) of \(window_samples) samples since the last membership change" else empty end), + (if $m.predates_change then "the probe's window began before the last membership change at \($changed_at)" + elif $m.samples < window_samples then "\($m.samples) of \(window_samples) samples since the last membership change" else empty end), (if ($m.pairs | length) == 0 then "no voter to measure against" elif ($m.pairs_complete | not) then "fewer than \(window_samples) round trip readings to some voter" else empty end), (if $node.voter and ($m.wal_p99_ms == null or $m.commit_p99_ms == null) then "no etcd disk readings" else empty end), @@ -239,7 +230,7 @@ $config as $c # Why no swap may start now whatever the hosts' fitness: the epic's conditions on the cluster as a whole, as `gates`, which hold for a swap by command as well as by decision, and the cooldown after a membership change, which only a decision waits out. A voter a swap in progress is taking out (`ignored_voters`) is held to none of the gates on its own health, and its peers may lack their connection to it. It is still an etcd member until k3s removes it, whether or not its Node still says EtcdIsVoter (a server stepped down to etcd-only runs no kubelet), so the members etcd lists include it. # How many members each voter's fresh probe report says etcd lists, by voter, null for a voter with no fresh report or no reading: a rollback is settled once every voter lists the target and no more. -| ($voter_names | map({key: ., value: (if ($probes[.].samples[-1].t // 0) >= ($now - $stale_seconds) then $probes[.].samples[-1].etcd.known_members else null end)}) | from_entries) as $known_members +| ($voter_names | map({key: ., value: (if ($probes[.].t // 0) >= ($now - $stale_seconds) then $probes[.].etcd.known_members else null end)}) | from_entries) as $known_members | ($c.ignored_voters // []) as $ignored | ($voter_names | map(select(. as $v | $ignored | index($v) | not))) as $judged # A voter a swap is taking out that no longer says EtcdIsVoter has left etcd once every voter it does not ignore reports, fresh, that etcd lists as many members as there are voters: each voter is a member, so those members are the voters and nothing else. Its Node's etcd label and the absence of k3s's removed-node-name annotation say nothing then, since a server that is still running sets the label again and k3s clears the annotation when a server rejoins under a new name (pkg/etcd/metadata_controller.go and member_controller.go); counting it there would expect a member etcd does not have, for ever. @@ -256,8 +247,8 @@ $config as $c ($all[] | select(.name as $n | $ignored | index($n) | not) | select(.labelled and (.voter | not)) | if .removed then "node \(.name) is a member k3s has removed: it still carries the etcd label" else "node \(.name) is a member but not a voter: it carries the etcd label without \(voter_condition)=True" end), ($voter_nodes[] | select(.name as $n | $ignored | index($n) | not) | select(.regrade_level != null) | "voter \(.name) is at regrade level \(.regrade_level), below full"), ($voter_nodes[] | select(.name as $n | $ignored | index($n) | not) | select(.ready | not) | "voter \(.name) is not Ready"), - ($judged[] as $v | $probes[$v].samples[-1] as $last | $last.etcd as $e - | if $last == null or $last.t < ($now - $stale_seconds) then "voter \($v) has no probe report in the last \($stale_seconds) s" + ($judged[] as $v | $probes[$v] as $last | $last.etcd as $e + | if $last == null or $last.t == null or $last.t < ($now - $stale_seconds) then "voter \($v) has no probe report in the last \($stale_seconds) s" elif $e == null then "etcd's state on voter \($v) is unknown" elif ($e.has_leader | not) then "voter \($v) reports no leader" elif $e.is_learner then "voter \($v) is a learner" diff --git a/scripts/fleet-role.sh b/scripts/fleet-role.sh index e5e72d6f..3f76cb23 100755 --- a/scripts/fleet-role.sh +++ b/scripts/fleet-role.sh @@ -83,7 +83,7 @@ trap 'rm -rf "$work"' EXIT # One kubectl call whose output is not wanted. The actions chain these with `|| return 1`, since errexit is off inside a function that is called as the condition of an `if`. kube() { kubectl_ "$@" > /dev/null; } -# --- Reading the cluster. The inputs go to jq in files, never as arguments: a probe entry holds a window of samples, the entries together can fill a ConfigMap, and one argument is limited to a fraction of the argument space (128 KiB on Linux), which a fleet's nodes and probes pass. --- +# --- Reading the cluster. The inputs go to jq in files, never as arguments: the Nodes of a fleet, and one argument is limited to a fraction of the argument space (128 KiB on Linux), which they pass. --- # The ConfigMap's data key $2 as JSON, null when the ConfigMap or the key has none. read_status_key() { diff --git a/tests/unit/fleet_role_support.py b/tests/unit/fleet_role_support.py index 68068497..f52fdd7c 100644 --- a/tests/unit/fleet_role_support.py +++ b/tests/unit/fleet_role_support.py @@ -38,6 +38,8 @@ SAMPLE_SECONDS = 15 TARGET = 3 MAX_SERVERS = 5 +# etcd's FAQ: the largest cluster it recommends has seven members, the most voters a probe measures. +MAX_VOTERS = 7 PRESSURE_THRESHOLD = 10 AVAILABLE_FLOOR_PERCENT = 15 TAIL_PERCENTILE = 99 @@ -268,6 +270,8 @@ def __init__(self) -> None: self.calls = self.dir / "calls.log" self.calls.touch() self.nodes: list[dict[str, Any]] = [] + # The windows the probes would keep in their data directories, by node, from which the published summaries are made. + self.windows: dict[str, dict[str, Any]] = {} self.configmaps: dict[str, dict[str, Any]] = { f"{NAMESPACE}/{PROBE_CONFIGMAP}": {"data": {}}, f"{NAMESPACE}/{STATUS_CONFIGMAP}": {"data": {"status.json": ""}}, @@ -464,6 +468,17 @@ def entry(samples: list[dict[str, Any]], etcd_id: str | None = None) -> dict[str return {"etcd_id": etcd_id, "last": None, "fsync_at": None, "samples": samples} +def summarise(windows: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]]: + """What the probe publishes for each of these windows (an entry as `entry` builds it, by node name): the window's summary, by the shared module's own function and at the regrade limit the tests' controller is given.""" + completed = subprocess.run( + ["jq", "-c", "-L", str(SCRIPTS), "--argjson", "limit", json.dumps({"pressure_threshold": PRESSURE_THRESHOLD, "available_floor_percent": AVAILABLE_FLOOR_PERCENT}), + 'include "fleet-role-stats"; map_values(window_summary($limit))'], + input=json.dumps(windows), capture_output=True, text=True, check=True, + ) + summaries: dict[str, dict[str, Any]] = json.loads(completed.stdout) + return summaries + + def times(count: int, now: float | None = None) -> list[float]: """The times of the last count samples, one sample period apart, the newest now.""" end = time.time() if now is None else now diff --git a/tests/unit/test_fleet_role.py b/tests/unit/test_fleet_role.py index 0249124f..c4628523 100644 --- a/tests/unit/test_fleet_role.py +++ b/tests/unit/test_fleet_role.py @@ -39,6 +39,7 @@ histogram, node, sample, + summarise, times, ) @@ -127,7 +128,8 @@ def populate(cluster: Cluster, hosts: list[Host]) -> None: ) ) entries[h.name] = entry(built, etcd_id(h.name) if h.voter else None) - cluster.set_probes(entries) + cluster.windows = entries + cluster.set_probes(summarise(entries)) def healthy(**changes: Any) -> list[Host]: @@ -331,10 +333,10 @@ def test_the_quota_is_the_largest_a_voter_reports(self) -> None: def test_one_sample_over_the_limit_anywhere_in_the_window_is_enough(self) -> None: populate(self.cluster, healthy()) - entries = self.cluster.probes() + windows = self.cluster.windows # Under the floor share of the total but above etcd's quota, so only the regrade limit can fail it. - entries["a"]["samples"][10].update(psi=PRESSURE_THRESHOLD + 5, avail_kb=TOTAL_KB * AVAILABLE_FLOOR_PERCENT // 100 - 1) - self.cluster.set_probes(entries) + windows["a"]["samples"][10].update(psi=PRESSURE_THRESHOLD + 5, avail_kb=TOTAL_KB * AVAILABLE_FLOOR_PERCENT // 100 - 1) + self.cluster.set_probes(summarise(windows)) completed = self.cluster.run("fleet-role.sh", **CONTROLLER_ENV) self.assertEqual(completed.returncode, 0, completed.stderr) host = self.cluster.status()["hosts"]["a"] @@ -537,9 +539,9 @@ def test_no_swap_while_a_voters_probe_has_stopped_reporting_as_when_it_is_paused def test_no_swap_while_a_voters_etcd_metrics_could_not_be_read(self) -> None: populate(self.cluster, healthy(a={"tcp": UNFIT_RTT_MS + 50})) - entries = self.cluster.probes() - entries["b"]["samples"][-1]["etcd"] = None - self.cluster.set_probes(entries) + windows = self.cluster.windows + windows["b"]["samples"][-1]["etcd"] = None + self.cluster.set_probes(summarise(windows)) completed = self.cluster.run("fleet-role.sh", **CONTROLLER_ENV) self.assertEqual(completed.returncode, 0, completed.stderr) self.assertIn("etcd's state on voter b is unknown", self.cluster.status()["decision"]["because"]) @@ -630,12 +632,19 @@ def test_the_cooldown_outlasts_the_window_by_the_learner_stall_time(self) -> Non self.assertEqual(COOLDOWN_SECONDS - WINDOW_SECONDS, 300) self.assertGreater(COOLDOWN_SECONDS, WINDOW_SECONDS) - def test_readings_from_before_a_membership_change_are_not_counted(self) -> None: + def test_a_window_that_began_before_a_membership_change_is_not_judged(self) -> None: + # The probe publishes its window's summary, so a window holding readings from before the change cannot be judged until it has rolled past the change. changed_at = int(time.time()) - (WINDOW_SECONDS // 2) self.cluster.set_status({"membership": {"voters": ["a", "b", "c"], "changed_at": changed_at, "cooldown_until": changed_at + COOLDOWN_SECONDS}}) status = self.pass_(healthy()) self.assertEqual(status["hosts"]["a"]["verdict"], "unknown") - self.assertEqual(status["hosts"]["a"]["samples"], WINDOW_SAMPLES // 2 + 1) + self.assertIn(f"the probe's window began before the last membership change at {changed_at}", status["hosts"]["a"]["reasons"]) + + def test_a_window_that_began_after_a_membership_change_is_judged(self) -> None: + changed_at = int(time.time()) - WINDOW_SECONDS - SAMPLE_SECONDS + self.cluster.set_status({"membership": {"voters": ["a", "b", "c"], "changed_at": changed_at, "cooldown_until": changed_at + COOLDOWN_SECONDS}}) + status = self.pass_(healthy()) + self.assertEqual(status["hosts"]["a"]["verdict"], "fit") def test_fewer_eligible_hosts_than_the_target_is_reported_and_refuses_a_swap(self) -> None: status = self.pass_(healthy(a={"tcp": UNFIT_RTT_MS + 50}, b={"eligible": False}, c={"eligible": False})) diff --git a/tests/unit/test_fleet_role_arc.py b/tests/unit/test_fleet_role_arc.py index 0974defe..7dfc9143 100644 --- a/tests/unit/test_fleet_role_arc.py +++ b/tests/unit/test_fleet_role_arc.py @@ -131,7 +131,7 @@ def test_their_namespaced_permissions_in_the_platform_namespace_are_on_configmap {"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-actuators"], "verbs": ["get"]}, ], ) - self.assertEqual(probe, [{"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-probe"], "verbs": ["get", "patch"]}]) + self.assertEqual(probe, [{"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-probe"], "verbs": ["patch"]}]) for rule in controller + probe: self.assertNotIn("create", rule["verbs"]) self.assertNotIn("delete", rule["verbs"]) @@ -164,7 +164,7 @@ def test_the_controller_gets_every_setting_its_script_requires(self) -> None: def test_the_probe_gets_every_setting_its_script_requires(self) -> None: daemonset = of_kind(self.manifests, "DaemonSet")[0] required = script_requirements("fleet-role-probe.sh") - self.assertEqual(required, {"NODE_NAME", "NAMESPACE", "ELECTION_TIMEOUT_MS", "ELIGIBLE_LABEL", "LINK_LABEL"}) + self.assertEqual(required, {"NODE_NAME", "NAMESPACE", "ELECTION_TIMEOUT_MS", "ELIGIBLE_LABEL", "LINK_LABEL", "PRESSURE_THRESHOLD", "AVAILABLE_FLOOR_PERCENT"}) self.assertLessEqual(required, set(env_of(daemonset))) def test_the_default_settings_reach_the_controller_as_strings(self) -> None: diff --git a/tests/unit/test_fleet_role_probe.py b/tests/unit/test_fleet_role_probe.py index 9478847e..5029abe1 100644 --- a/tests/unit/test_fleet_role_probe.py +++ b/tests/unit/test_fleet_role_probe.py @@ -28,11 +28,14 @@ from typing import Any from fleet_role_support import ( + AVAILABLE_FLOOR_PERCENT, ELECTION_TIMEOUT_MS, ELIGIBLE_LABEL, + MAX_VOTERS, NAMESPACE, LINK_LABEL, NODE_LINK, + PRESSURE_THRESHOLD, REGRADE_CORE, SAMPLE_SECONDS, TARGET, @@ -42,6 +45,7 @@ entry, node, sample, + summarise, ) SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "fleet-role-probe.sh" @@ -177,15 +181,28 @@ def probe(self, name: str = "me", *, voters: list[dict[str, Any]] | None = None, "LINK_LABEL": LINK_LABEL, "PSI_FILE": str(self.psi), "MEMINFO_FILE": str(self.meminfo), + "PRESSURE_THRESHOLD": str(PRESSURE_THRESHOLD), + "AVAILABLE_FLOOR_PERCENT": str(AVAILABLE_FLOOR_PERCENT), } return self.cluster.run("fleet-role-probe.sh", **{**base, **env}) - def newest(self, name: str = "me") -> dict[str, Any]: - return self.cluster.probes()[name]["samples"][-1] + @property + def state_file(self) -> Path: + """Where the probe keeps its window, in the data directory the DaemonSet mounts from the node.""" + return self.fsync_dir / "window.json" - def settled(self, name: str = "me") -> None: - """Gives this node an entry whose fdatasync probe ran just now, so that a pass does not run fio.""" - self.cluster.set_probes({**self.cluster.probes(), name: {**entry([]), "fsync_at": int(time.time())}}) + def state(self) -> dict[str, Any]: + return json.loads(self.state_file.read_text()) + + def set_state(self, state: dict[str, Any]) -> None: + self.state_file.write_text(json.dumps(state)) + + def newest(self) -> dict[str, Any]: + return self.state()["samples"][-1] + + def settled(self) -> None: + """Gives this node a window whose fdatasync probe ran just now, so that a pass does not run fio.""" + self.set_state({**entry([]), "fsync_at": int(time.time())}) class SysfsTree: @@ -247,7 +264,7 @@ def setUp(self) -> None: self.tree = SysfsTree(self.sys) self.settled_node = node("me", eligible=True) self.nodes = [self.settled_node] - self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + self.settled() def link_via(self, *routes: str) -> str: self.route.write_text(ROUTE_HEADER + "".join(routes)) @@ -419,7 +436,7 @@ def test_the_first_sample_records_the_gauges_and_nothing_gained(self) -> None: ) self.assertEqual((etcd["wal"], etcd["commit"]), ({}, {})) self.assertFalse(self.newest()["restarted"]) - self.assertEqual(self.cluster.probes()["me"]["etcd_id"], FIXTURE_SERVER_ID) + self.assertEqual(self.state()["etcd_id"], FIXTURE_SERVER_ID) def test_the_next_sample_holds_what_each_histogram_gained_as_a_real_etcd_counted_it(self) -> None: before, after = page("before"), page("after") @@ -438,7 +455,7 @@ def test_the_next_sample_holds_what_each_histogram_gained_as_a_real_etcd_counted self.assertEqual(etcd["rtt"][peer], gained(rtt_before[peer], rtt_after[peer]), peer) self.assertFalse(self.newest()["restarted"]) # Cumulative as etcd reports it, so the sample after this one has something to subtract from. - self.assertEqual(self.cluster.probes()["me"]["last"]["wal"], cumulative(after, "etcd_disk_wal_fsync_duration_seconds")[""]) + self.assertEqual(self.state()["last"]["wal"], cumulative(after, "etcd_disk_wal_fsync_duration_seconds")[""]) def test_a_peer_with_two_active_streams_counts_once_as_k3ss_etcd_reports_it(self) -> None: # A page from a follower of a three-server k3s cluster (etcd 3.6.4): its gauge for the leader is 2, one for each kind of stream, and for the other follower 1. @@ -474,13 +491,13 @@ def test_a_removed_member_that_etcd_still_lists_with_a_zero_is_not_a_known_membe def test_a_voter_whose_metrics_cannot_be_read_still_records_the_rest_and_keeps_its_baseline(self) -> None: with metrics_server([page("before")]) as url: self.pass_with(url) - baseline = self.cluster.probes()["me"]["last"] + baseline = self.state()["last"] completed = self.pass_with(f"http://127.0.0.1:{closed_port()}/metrics") self.assertEqual(completed.returncode, 0, completed.stderr) self.assertIn("could not read", completed.stderr) self.assertIsNone(self.newest()["etcd"]) - self.assertEqual(self.cluster.probes()["me"]["last"], baseline) - self.assertEqual(self.cluster.probes()["me"]["etcd_id"], FIXTURE_SERVER_ID) + self.assertEqual(self.state()["last"], baseline) + self.assertEqual(self.state()["etcd_id"], FIXTURE_SERVER_ID) def test_a_node_that_is_not_a_voter_reads_no_etcd_metrics(self) -> None: self.nodes = [node("me", eligible=True), node("p1", voter=True)] @@ -488,7 +505,7 @@ def test_a_node_that_is_not_a_voter_reads_no_etcd_metrics(self) -> None: with metrics_server([page("before")]) as url: self.assertEqual(self.pass_with(url).returncode, 0) self.assertIsNone(self.newest()["etcd"]) - self.assertIsNone(self.cluster.probes()["me"]["etcd_id"]) + self.assertIsNone(self.state()["etcd_id"]) @unittest.skipUnless(ETCD_BINARY, "needs an etcd binary (on the PATH, or named by ETCD_BINARY)") def test_a_live_three_member_cluster_gives_the_readings_the_fixtures_were_captured_from(self) -> None: @@ -502,7 +519,7 @@ def test_a_live_three_member_cluster_gives_the_readings_the_fixtures_were_captur etcd = self.newest()["etcd"] self.assertEqual((etcd["known_members"], etcd["active_peers"], etcd["has_leader"], etcd["is_learner"]), (TARGET, TARGET - 1, True, False)) self.assertGreater(sum(etcd["wal"].values()), 0) - self.assertEqual(self.cluster.probes()["me"]["etcd_id"], cluster_.member_id(0)) + self.assertEqual(self.state()["etcd_id"], cluster_.member_id(0)) cluster_.add_learner() time.sleep(2) self.assertEqual(self.pass_with(url).returncode, 0) @@ -599,22 +616,24 @@ def test_a_node_that_is_neither_records_nothing(self) -> None: self.assertEqual(list(self.fsync_dir.iterdir()), [], "and runs no disk probe") def test_a_node_that_stopped_being_either_has_its_entry_removed_and_no_other(self) -> None: - self.cluster.set_probes({"other": entry([sample(5)], "other-id"), "me": entry([sample(6)])}) + self.cluster.set_probes({"other": entry([sample(5)], "other-id"), "me": summarise({"me": entry([sample(6)])})["me"]}) + self.set_state(entry([sample(6)])) completed = self.probe(voters=[node("me"), node("peer", voter=True)]) self.assertEqual(completed.returncode, 0, completed.stderr) self.assertEqual(sorted(self.cluster.probes()), ["other"]) + self.assertFalse(self.state_file.exists(), "and its window goes with it") def test_an_eligible_node_is_measured(self) -> None: self.settled() completed = self.probe(voters=[node("me", eligible=True), node("peer", voter=True)]) self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(len(self.cluster.probes()["me"]["samples"]), 1) + self.assertEqual(self.cluster.probes()["me"]["samples"], 1) def test_a_voter_is_measured_whether_or_not_it_is_eligible(self) -> None: with metrics_server([page("before")]) as url: completed = self.probe(voters=[node("me", voter=True), node("peer", voter=True)], ETCD_METRICS_URL=url) self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(len(self.cluster.probes()["me"]["samples"]), 1) + self.assertEqual(self.cluster.probes()["me"]["samples"], 1) class MemoryTest(ProbeCase): @@ -676,9 +695,9 @@ def setUp(self) -> None: def test_a_sample_is_appended_and_the_entry_keeps_the_last_window_of_them(self) -> None: old = [sample(1000 + i, fsync_ms=None) for i in range(WINDOW_SAMPLES)] - self.cluster.set_probes({"me": {**entry(old), "fsync_at": int(time.time())}}) + self.set_state({**entry(old), "fsync_at": int(time.time())}) self.assertEqual(self.probe().returncode, 0) - kept = self.cluster.probes()["me"]["samples"] + kept = self.state()["samples"] self.assertEqual(len(kept), WINDOW_SAMPLES) self.assertEqual(kept[0]["t"], 1001, "the oldest was dropped") self.assertGreater(kept[-1]["t"], 1000 + WINDOW_SAMPLES) @@ -687,11 +706,11 @@ def test_an_entry_too_large_for_one_argument_is_read_and_extended_whole(self) -> # One argument to a process is limited to 128 KiB on Linux; an entry of a voter in a large cluster, with a connect time to every other voter in each sample, passes that. peers = {f"peer-with-a-long-node-name-{i:03d}": 1.234 for i in range(60)} big = [sample(1000 + i, tcp=peers) for i in range(WINDOW_SAMPLES)] - self.cluster.set_probes({"me": {**entry(big), "fsync_at": int(time.time())}}) - self.assertGreater(len(json.dumps(self.cluster.probes()["me"])), 128 * 1024) + self.set_state({**entry(big), "fsync_at": int(time.time())}) + self.assertGreater(len(json.dumps(self.state())), 128 * 1024) completed = self.probe() self.assertEqual(completed.returncode, 0, completed.stderr) - kept = self.cluster.probes()["me"]["samples"] + kept = self.state()["samples"] self.assertEqual(len(kept), WINDOW_SAMPLES) self.assertEqual(kept[0]["t"], 1001) self.assertEqual(kept[0]["tcp"], peers) @@ -699,7 +718,8 @@ def test_an_entry_too_large_for_one_argument_is_read_and_extended_whole(self) -> def test_each_node_patches_only_its_own_key(self) -> None: other = entry([sample(5)], "other-id") - self.cluster.set_probes({"other": other, "me": {**entry([]), "fsync_at": int(time.time())}}) + self.cluster.set_probes({"other": other}) + self.settled() self.assertEqual(self.probe().returncode, 0) self.assertEqual(self.cluster.probes()["other"], other) patches = [json.loads(line) for line in self.cluster.calls.read_text().splitlines() if '"patch"' in line] @@ -708,9 +728,10 @@ def test_each_node_patches_only_its_own_key(self) -> None: def recorded(self) -> list[dict[str, Any]]: for _ in range(50): try: - return json.loads(self.cluster.read_store()["configmaps"][f"{NAMESPACE}/fleet-role-probe"]["data"].get("me", "{}")).get("samples", []) + # A sample is kept in the window only once its summary is published. + return self.state()["samples"] if self.state_file.exists() else [] except json.JSONDecodeError: - time.sleep(0.1) # the stand-in kubectl was rewriting the store + time.sleep(0.1) # the probe was replacing the file raise AssertionError("the store could not be read") def loop(self, wanted: int, **env: str) -> list[dict[str, Any]]: @@ -721,7 +742,8 @@ def loop(self, wanted: int, **env: str) -> list[dict[str, Any]]: [str(Path(__file__).resolve().parents[2] / "scripts" / "fleet-role-probe.sh")], env=self.cluster.env( NODE_NAME="me", NAMESPACE=NAMESPACE, ELECTION_TIMEOUT_MS=str(ELECTION_TIMEOUT_MS), ELIGIBLE_LABEL=ELIGIBLE_LABEL, REPORT_SECONDS="1", ETCD_PEER_PORT="1", FSYNC_DIR=str(self.fsync_dir), - SYS_CLASS_NET=str(self.sys / "class" / "net"), PROC_NET_ROUTE=str(self.route), FLEET_REGRADE_CORE=str(REGRADE_CORE), NODE_LINK=str(NODE_LINK), LINK_LABEL=LINK_LABEL, PSI_FILE=str(self.psi), MEMINFO_FILE=str(self.meminfo), **env, + SYS_CLASS_NET=str(self.sys / "class" / "net"), PROC_NET_ROUTE=str(self.route), FLEET_REGRADE_CORE=str(REGRADE_CORE), NODE_LINK=str(NODE_LINK), LINK_LABEL=LINK_LABEL, PSI_FILE=str(self.psi), MEMINFO_FILE=str(self.meminfo), + PRESSURE_THRESHOLD=str(PRESSURE_THRESHOLD), AVAILABLE_FLOOR_PERCENT=str(AVAILABLE_FLOOR_PERCENT), **env, ), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, @@ -736,19 +758,19 @@ def loop(self, wanted: int, **env: str) -> list[dict[str, Any]]: return self.recorded() def test_a_node_that_is_not_a_voter_marks_its_first_sample_as_a_restart_and_a_later_one_as_none(self) -> None: - self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + self.settled() samples = self.loop(2) self.assertGreaterEqual(len(samples), 2) self.assertEqual([s["restarted"] for s in samples[:2]], [True, False]) def test_a_sample_after_the_first_costs_two_api_calls_the_nodes_and_the_patch(self) -> None: - self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + self.settled() samples = self.loop(3) calls = [json.loads(line) for line in self.cluster.calls.read_text().splitlines()] self.assertGreaterEqual(len(samples), 3) reads = [c[:2] for c in calls if c[0] == "get"] patches = [c for c in calls if c[0] == "patch"] - self.assertEqual(reads.count(["get", "configmap"]), 1, "this node's entry is read once, when the process starts") + self.assertEqual(reads.count(["get", "configmap"]), 0, "the window is read from the data directory, never from the ConfigMap") # The process may have been stopped between a sample's read of the nodes and its patch, and it reads the nodes once more as it stops (to see whether its entry is to go), so the nodes were read once for each patch and at most twice more. self.assertIn(reads.count(["get", "nodes"]) - len(patches), (0, 1, 2), reads) self.assertGreaterEqual(len(patches), len(samples)) @@ -761,7 +783,8 @@ def stop_loop(self, **env: str) -> subprocess.Popen[bytes]: [str(SCRIPT)], env=self.cluster.env( NODE_NAME="me", NAMESPACE=NAMESPACE, ELECTION_TIMEOUT_MS=str(ELECTION_TIMEOUT_MS), ELIGIBLE_LABEL=ELIGIBLE_LABEL, REPORT_SECONDS="600", ETCD_PEER_PORT="1", FSYNC_DIR=str(self.fsync_dir), - SYS_CLASS_NET=str(self.sys / "class" / "net"), PROC_NET_ROUTE=str(self.route), FLEET_REGRADE_CORE=str(REGRADE_CORE), NODE_LINK=str(NODE_LINK), LINK_LABEL=LINK_LABEL, PSI_FILE=str(self.psi), MEMINFO_FILE=str(self.meminfo), **env, + SYS_CLASS_NET=str(self.sys / "class" / "net"), PROC_NET_ROUTE=str(self.route), FLEET_REGRADE_CORE=str(REGRADE_CORE), NODE_LINK=str(NODE_LINK), LINK_LABEL=LINK_LABEL, PSI_FILE=str(self.psi), MEMINFO_FILE=str(self.meminfo), + PRESSURE_THRESHOLD=str(PRESSURE_THRESHOLD), AVAILABLE_FLOOR_PERCENT=str(AVAILABLE_FLOOR_PERCENT), **env, ), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, @@ -791,27 +814,30 @@ def stop(self, process: subprocess.Popen[bytes]) -> None: self.assertLess(time.monotonic() - started, 20, "the signal waited for the sleep between samples") def test_a_pod_stopped_on_a_node_that_lost_both_labels_removes_its_entry_and_no_other(self) -> None: - self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}, "other": entry([sample(5)], "other-id")}) + self.cluster.set_probes({"other": entry([sample(5)], "other-id")}) + self.settled() process = self.stop_loop() self.change_nodes([node("me")]) self.stop(process) self.assertEqual(sorted(self.stored_probes()), ["other"]) + self.assertFalse(self.state_file.exists(), "and its window goes with it") def test_a_pod_stopped_on_a_node_that_is_gone_removes_its_entry(self) -> None: - self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + self.settled() process = self.stop_loop() self.change_nodes([]) self.stop(process) self.assertEqual(self.stored_probes(), {}) def test_a_pod_stopped_on_a_node_that_is_still_eligible_keeps_its_entry_for_the_pod_that_follows(self) -> None: - self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + self.settled() process = self.stop_loop() self.stop(process) - self.assertEqual(len(self.stored_probes()["me"]["samples"]), 1) + self.assertEqual(self.stored_probes()["me"]["samples"], 1) + self.assertEqual(len(self.state()["samples"]), 1, "and its window stays for the pod that follows") def test_a_pod_stopped_on_a_node_that_is_a_voter_keeps_its_entry(self) -> None: - self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + self.settled() self.nodes = [node("me", voter=True)] process = self.stop_loop(ETCD_METRICS_URL="http://127.0.0.1:1/metrics") self.stop(process) @@ -820,7 +846,7 @@ def test_a_pod_stopped_on_a_node_that_is_a_voter_keeps_its_entry(self) -> None: def test_a_pod_stopped_when_the_api_refuses_the_removal_exits_and_leaves_the_entry(self) -> None: refusals = self.cluster.dir / "refusals" refusals.write_text("0") - self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + self.settled() process = self.stop_loop(FAKE_FAIL_PATCHES=str(refusals)) self.change_nodes([node("me")]) refusals.write_text("1") @@ -830,7 +856,7 @@ def test_a_pod_stopped_when_the_api_refuses_the_removal_exits_and_leaves_the_ent def test_a_patch_the_api_refuses_is_retried_next_period_without_the_process_exiting_and_the_first_sample_still_marks_the_restart(self) -> None: refusals = self.cluster.dir / "refusals" refusals.write_text("2") - self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + self.settled() samples = self.loop(2, FAKE_FAIL_PATCHES=str(refusals)) self.assertEqual(refusals.read_text(), "0") self.assertGreaterEqual(len(samples), 2, "the probe went on after the refusals") @@ -841,10 +867,64 @@ def test_a_patch_the_api_refuses_is_retried_next_period_without_the_process_exit def test_a_patch_the_api_refuses_fails_a_single_sample_run(self) -> None: refusals = self.cluster.dir / "refusals" refusals.write_text("1") - self.cluster.set_probes({"me": {**entry([]), "fsync_at": int(time.time())}}) + self.settled() completed = self.probe(FAKE_FAIL_PATCHES=str(refusals)) self.assertNotEqual(completed.returncode, 0) - self.assertEqual(self.cluster.probes()["me"]["samples"], []) + self.assertEqual(self.state()["samples"], [], "a sample that was not published is not kept") + self.assertNotIn("me", self.cluster.probes()) + + +def entry_limit_bytes() -> int: + """The bound on one published entry, from the module the probe and the controller share.""" + return int(subprocess.run(["jq", "-n", "-L", str(SCRIPT.parent), 'include "fleet-role-stats"; probe_entry_limit_bytes'], capture_output=True, text=True, check=True).stdout) + + +def long_name(i: int) -> str: + """A Node name of the longest length Kubernetes allows (a DNS subdomain, 253 characters).""" + return f"{i:03d}-" + "n" * 249 + + +class PublishedEntryTest(ProbeCase): + """The probe publishes its window's summary, never the window: every patch of the shared ConfigMap is a rewrite of the whole object in the datastore, so the size of what one node publishes is the size of every node's write.""" + + def setUp(self) -> None: + super().setUp() + self.voters = [node(long_name(i), voter=True) for i in range(MAX_VOTERS)] + + def full_window(self) -> dict[str, Any]: + peers = {n["metadata"]["name"]: 1.234 for n in self.voters} + return {**entry([sample(int(time.time()) - (WINDOW_SAMPLES - i) * SAMPLE_SECONDS, tcp=peers, fsync_ms=2.5 if i == 0 else None) for i in range(WINDOW_SAMPLES - 1)]), "fsync_at": int(time.time())} + + def test_a_full_window_to_the_most_voters_publishes_its_summary_inside_the_entry_limit(self) -> None: + self.set_state(self.full_window()) + completed = self.probe(voters=[node("me", eligible=True), *self.voters]) + self.assertEqual(completed.returncode, 0, completed.stderr) + published = self.cluster.configmaps[f"{NAMESPACE}/fleet-role-probe"]["data"]["me"] + self.assertLessEqual(len(published.encode()), entry_limit_bytes()) + self.assertEqual(json.loads(published), summarise({"me": self.state()})["me"], "the entry is the summary of the window the probe keeps") + self.assertEqual(len(self.state()["samples"]), WINDOW_SAMPLES) + self.assertEqual(json.loads(published)["samples"], WINDOW_SAMPLES) + + def test_the_published_entry_does_not_grow_with_the_window(self) -> None: + nodes = [node("me", eligible=True), *self.voters] + self.settled() + self.assertEqual(self.probe(voters=nodes).returncode, 0) + one = len(self.cluster.configmaps[f"{NAMESPACE}/fleet-role-probe"]["data"]["me"]) + self.set_state(self.full_window()) + self.assertEqual(self.probe(voters=nodes).returncode, 0) + full = len(self.cluster.configmaps[f"{NAMESPACE}/fleet-role-probe"]["data"]["me"]) + # The figures' digits may differ, and the fdatasync reading is in the window only once, never a sample's worth per sample. + self.assertLess(abs(full - one), 64) + + def test_an_entry_over_the_limit_is_refused_loudly_and_not_published(self) -> None: + # More voters than etcd supports, each with the longest name, so the summary cannot fit. + crowd = [node(long_name(i), voter=True) for i in range(entry_limit_bytes() // 200)] + self.settled() + completed = self.probe(voters=[node("me", eligible=True), *crowd]) + self.assertNotEqual(completed.returncode, 0) + self.assertIn("over the limit", completed.stderr) + self.assertNotIn("me", self.cluster.probes()) + self.assertEqual(self.state()["samples"], [], "a sample that was not published is not kept") @unittest.skipUnless(FIO, "needs fio, which the probe's fdatasync reading is taken with") @@ -859,18 +939,16 @@ def test_a_node_that_is_not_a_voter_takes_the_reading_from_the_real_fio_and_leav reading = self.newest()["fsync_ms"] self.assertIsInstance(reading, float) self.assertGreater(reading, 0) - self.assertEqual(list(self.fsync_dir.iterdir()), []) - self.assertIsNotNone(self.cluster.probes()["me"]["fsync_at"]) + self.assertEqual(list(self.fsync_dir.iterdir()), [self.state_file], "only the window is left") + self.assertIsNotNone(self.state()["fsync_at"]) def test_the_reading_is_taken_once_per_window_not_every_sample(self) -> None: self.assertEqual(self.probe().returncode, 0) - first_at = self.cluster.probes()["me"]["fsync_at"] + first_at = self.state()["fsync_at"] self.assertEqual(self.probe().returncode, 0) self.assertIsNone(self.newest()["fsync_ms"]) - self.assertEqual(self.cluster.probes()["me"]["fsync_at"], first_at) - entries = self.cluster.probes() - entries["me"]["fsync_at"] = int(time.time()) - WINDOW_SECONDS - self.cluster.set_probes(entries) + self.assertEqual(self.state()["fsync_at"], first_at) + self.set_state({**self.state(), "fsync_at": int(time.time()) - WINDOW_SECONDS}) self.assertEqual(self.probe().returncode, 0) self.assertIsNotNone(self.newest()["fsync_ms"]) @@ -900,6 +978,8 @@ def check(self, *args: str, **env: str) -> subprocess.CompletedProcess[str]: "NODE_LINK": str(NODE_LINK), "PSI_FILE": str(self.psi), "MEMINFO_FILE": str(self.meminfo), + "PRESSURE_THRESHOLD": str(PRESSURE_THRESHOLD), + "AVAILABLE_FLOOR_PERCENT": str(AVAILABLE_FLOOR_PERCENT), } completed = subprocess.run( [str(Path(__file__).resolve().parents[2] / "scripts" / "fleet-role-probe.sh"), "check", *args], From edfcd26a889e0b363073ca57720597457f47f210 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Fri, 20 Nov 2026 09:25:01 +0000 Subject: [PATCH 2/6] fix(autoscaler): bound each per-job record by bytes, newest first The job-memory and job-peak records were each capped at 2048 entries, which the comment sized at half the ConfigMap limit, so the two together could reach the whole limit, and an entry's size depends on the names in it. Each record is now held to JOB_RECORDS_MAX_BYTES, a thirty-second of CONFIGMAP_LIMIT_BYTES (Kubernetes' 1 MiB), counted in UTF-8 bytes and keeping the newest entries; the first entry that does not fit ends the record. Every patch rewrites the whole ConfigMap in the datastore, so this bounds each write of the object as well as its size. Refs #266 --- docs/autoscaler.md | 2 +- scripts/autoscaler.sh | 19 ++++++++++++++----- tests/unit/test_autoscaler.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/docs/autoscaler.md b/docs/autoscaler.md index 9a7876f7..2a30872a 100644 --- a/docs/autoscaler.md +++ b/docs/autoscaler.md @@ -15,7 +15,7 @@ Each poll (`AUTOSCALER_POLL_SECONDS`, default 45s): - Lowers the combined `maxRunners` immediately, with no delay or averaging, the moment headroom drops below a pod's hard limit or real pressure is detected — lowering never disrupts in-flight jobs, since ARC only gates new claims. Never patches any pooled profile below its own currently-running count, nor below one runner: a scale set at `maxRunners` 0 asks for no runner, so a job queued for it would wait with nothing to start it, and the cluster keeps no record of a queued job the autoscaler could see. A pool therefore never goes below one runner per pooled profile, whatever the floor, pressure or ceiling say, and a profile left at 0 by something else counts as saturated, so rebalancing gives it a slot back. - Treats any unschedulable runner pod (`PodScheduled=False`, reason `Unschedulable`) like pressure: the memory budget sums every node's capacity, so it cannot see a node that is tainted for disk pressure or a request that fits no single node. It lowers the combined `maxRunners` to the running count and never raises while one exists. - Also collects the peak memory each job's runner measured for itself: the runner image's job-completed hook (`runner-hooks/job-completed.sh`, set through `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`) reads the container cgroup's `memory.peak`, which a spike shorter than the poll cannot escape, and posts it as a `JobMemoryPeak` Event in the pod's namespace. The autoscaler reads those Events each poll and keeps the maximum per repository, workflow ref and job id under the status ConfigMap's `job-peak` key (`kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.job-peak}'`), in MiB. Keeping the maximum means a forged low figure lowers nothing and re-reading an Event changes nothing. The hook never fails the job it ends. To post an Event the runner pods' service account, which otherwise holds no permission, gets `create` on `events` in its own namespace and nothing else (`github_runner_arc_record_job_peaks`, default on; `false` removes the grant). A job's code runs in the same container and can read the same token, so the worst it can do with that permission is post events in its own runner namespace. -- Records, per job, the highest memory a runner pod running it has been seen at, in the status ConfigMap's `job-memory` key (`kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.job-memory}'`), keyed by repository, workflow ref and job name with the peak in MiB, the number of observations and when it was last seen. It is sampled once per poll from the metrics server, so it is a lower bound that a spike shorter than the poll escapes; it shows how far below a pod's request jobs sit, which is what decides whether an accurate in-pod measurement (see the per-job sizing issue) is worth building. Entries unseen for 30 days are dropped, and a failure to keep the record never affects a capacity decision. +- Records, per job, the highest memory a runner pod running it has been seen at, in the status ConfigMap's `job-memory` key (`kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.job-memory}'`), keyed by repository, workflow ref and job name with the peak in MiB, the number of observations and when it was last seen. It is sampled once per poll from the metrics server, so it is a lower bound that a spike shorter than the poll escapes; it shows how far below a pod's request jobs sit, which is what decides whether an accurate in-pod measurement (see the per-job sizing issue) is worth building. Entries unseen for 30 days are dropped, and the record, like the measured `job-peak` one, is held to a thirty-second of the ConfigMap's 1 MiB limit, newest entries first, since every patch rewrites the whole object in the datastore; a failure to keep the record never affects a capacity decision. - Optionally reads per-node pressure from the nodes' own `/proc/pressure` (`github_runner_arc_autoscaler_node_pressure_enabled`). A `psi-reporter` DaemonSet (`scripts/psi-reporter.sh`, in the autoscaler's image) runs on each node runner pods may use, tolerating every taint, and patches that node's own key of one ConfigMap with the `avg10` of its io, cpu and memory pressure. The reading is not taken from the kubelet's summary API, because on a cgroup v1 node that reports every pressure field as 0 whatever the node is doing. The autoscaler lowers the combined `maxRunners` by exactly one each poll, never below the running count, while any node's I/O "full" stall is at or over `AUTOSCALER_IO_PRESSURE_PCT` (default 10) or, when set above 0, its cpu "some" stall is at or over `AUTOSCALER_CPU_PRESSURE_PCT`, and it does not raise the pool while any node has no report newer than three polls, since whether that node can take more is then unknown. This is the signal for a pool whose nodes differ: memory can have room on a node whose disk is already the limit. - Fails safe on any memory measurement error (`kubectl top nodes` or the metrics API's pod list failing, or the node list failing, or a `Ready` node's figure that cannot be parsed), but not on a pod that merely has no metrics yet, nor on a node that is not `Ready`: lowers the combined total toward `AUTOSCALER_FLOOR` (`github_runner_arc_autoscaler_floor`) immediately rather than skipping the cycle silently. A failure reading the scale sets' own state or their pods exits non-zero instead, since fail-safe needs that state to lower anything and the API server is then unreachable anyway. This combined floor need not equal any one profile's own values-file `maxRunners`, since each profile's Helm upgrade reverts to its own static floor independently of how the pool's combined floor is set. diff --git a/scripts/autoscaler.sh b/scripts/autoscaler.sh index 1f2b49ae..c65519c5 100755 --- a/scripts/autoscaler.sh +++ b/scripts/autoscaler.sh @@ -35,7 +35,7 @@ NODE_PRESSURE_MAX_AGE_POLLS=3 POLL_SECONDS="${AUTOSCALER_POLL_SECONDS:-45}" STATE_NAMESPACE="${AUTOSCALER_STATE_NAMESPACE:-github-runner-platform}" CONFIGMAP="${AUTOSCALER_STATUS_CONFIGMAP:-autoscaler-status}" -# Where each job's observed peak memory is kept, in the same ConfigMap. A ConfigMap holds at most 1 MiB and an entry is under 512 bytes, so capping the entries at 2048 stays at half the limit; entries unseen for 30 days are dropped, since a job that has not run for that long no longer informs sizing. +# Where each job's observed peak memory is kept, in the same ConfigMap. Entries unseen for 30 days are dropped, since a job that has not run for that long no longer informs sizing, and each record is held to JOB_RECORDS_MAX_BYTES (see there). JOB_MEMORY_KEY="job-memory" # The peaks the runners' own job-completed hook measured (runner-hooks/job-completed.sh), kept apart from the sampled ones above because they are keyed by the job's id, not its display name, and are exact, not a lower bound. JOB_PEAK_KEY="job-peak" @@ -54,7 +54,10 @@ WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT # No pooled target is ever lowered or rebalanced below this many runners. A scale set at 0 asks for no runner, so a job queued for it waits with nothing to start it, and the cluster keeps no record of a queued job the autoscaler could see to bring it back; one slot each is the least that lets every set start a job. MIN_MAX_PER_TARGET=1 -JOB_MEMORY_MAX_ENTRIES=2048 +# Kubernetes' limit on a ConfigMap's data, in bytes (MaxSecretSize in k8s.io/apimachinery, which ConfigMap validation applies). +CONFIGMAP_LIMIT_BYTES=1048576 +# The most bytes each per-job record (JOB_MEMORY_KEY, JOB_PEAK_KEY) may hold, newest entries first. Every patch rewrites the whole ConfigMap in the datastore and a poll patches it several times, so the records, which inform sizing and nothing a poll decides, are each held to a thirty-second of the limit: the two together and the status stay inside an eighth of it. +JOB_RECORDS_MAX_BYTES=$(( CONFIGMAP_LIMIT_BYTES / 32 )) JOB_MEMORY_RETENTION_SECONDS=$(( 30 * 24 * 60 * 60 )) # Splits on whitespace: one token or several, however the Deployment env's own spacing renders. @@ -205,18 +208,24 @@ rebalance_one() { REBALANCE_TO=$saturated } -# Merges {repo, workflow, job, mib} entries into the JSON object kept under a ConfigMap key, one entry per repository, workflow ref and job, holding the highest figure seen and when it was last seen. $3 is true when each merge counts as an observation (the sampled record, where one job is seen many times); the measured record is re-read from Events that stay around, so counting there would count the same job again. Entries unseen for the retention are dropped, and the oldest go first past the cap. +# Merges {repo, workflow, job, mib} entries into the JSON object kept under a ConfigMap key, one entry per repository, workflow ref and job, holding the highest figure seen and when it was last seen. $3 is true when each merge counts as an observation (the sampled record, where one job is seen many times); the measured record is re-read from Events that stay around, so counting there would count the same job again. Entries unseen for the retention are dropped, and past JOB_RECORDS_MAX_BYTES the oldest go first: the record keeps the newest entries whose serialised object fits, counted in UTF-8 bytes as the ConfigMap stores them. store_job_entries() { local key="$1" jobs="$2" count="$3" existing updated existing="$(configmap_get "$key")" [ -n "$existing" ] || existing="{}" - updated="$(jq -c --slurpfile jobs <(printf '%s' "$jobs") --argjson count "$count" --argjson now "$(date +%s)" --argjson retention "$JOB_MEMORY_RETENTION_SECONDS" --argjson cap "$JOB_MEMORY_MAX_ENTRIES" ' + updated="$(jq -c --slurpfile jobs <(printf '%s' "$jobs") --argjson count "$count" --argjson now "$(date +%s)" --argjson retention "$JOB_MEMORY_RETENTION_SECONDS" --argjson max_bytes "$JOB_RECORDS_MAX_BYTES" ' reduce $jobs[0][] as $o (.; ($o.repo + "|" + $o.workflow + "|" + $o.job) as $k | .[$k] = ({repo: $o.repo, workflow: $o.workflow, job: $o.job, peak_mib: ([$o.mib, (.[$k].peak_mib // 0)] | max), last_seen: $now} + (if $count then {samples: ((.[$k].samples // 0) + 1)} else {} end))) | with_entries(select(.value.last_seen >= ($now - $retention))) - | to_entries | sort_by(-.value.last_seen) | .[0:$cap] | from_entries' <<< "$existing")" + # The braces (2 bytes) and, per entry, its key and value as {"k":v} serialises them less those braces, with one comma between entries. + | to_entries | sort_by(-.value.last_seen) + # The first entry that does not fit ends the record, so no older entry outlasts a newer one. + | reduce .[] as $e ({bytes: 2, kept: [], full: false}; + (({($e.key): $e.value} | tojson | utf8bytelength) - 2 + (if .kept == [] then 0 else 1 end)) as $size + | if .full then . elif .bytes + $size <= $max_bytes then .bytes += $size | .kept += [$e] else .full = true end) + | .kept | from_entries' <<< "$existing")" configmap_set "$key" "$updated" } diff --git a/tests/unit/test_autoscaler.py b/tests/unit/test_autoscaler.py index dda7302e..6f0358b9 100644 --- a/tests/unit/test_autoscaler.py +++ b/tests/unit/test_autoscaler.py @@ -477,6 +477,38 @@ def test_a_job_not_seen_within_the_retention_is_dropped(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) self.assertNotIn("old|wf|job", self.recorded()) + def test_each_job_record_is_held_to_its_share_of_the_configmap_newest_first(self) -> None: + # Every patch rewrites the whole ConfigMap in the datastore, so the records are bounded in bytes, not entries: an entry's size depends on the names in it. The bound is read from the script, where it derives from the ConfigMap limit. + script = SCRIPT.read_text() + limit_match = re.search(r"^CONFIGMAP_LIMIT_BYTES=(\d+)$", script, re.M) + share_match = re.search(r"^JOB_RECORDS_MAX_BYTES=\$\(\( CONFIGMAP_LIMIT_BYTES / (\d+) \)\)$", script, re.M) + assert limit_match is not None and share_match is not None, "the script's bounds were not found" + limit = int(limit_match.group(1)) + self.assertEqual(limit, 1024 * 1024, "Kubernetes' limit on a ConfigMap's data") + budget = limit // int(share_match.group(1)) + now = int(time.time()) + name = "n" * 200 + # Each entry is over 300 bytes, so this many hold several times the budget. + many = { + f"org/{name}-{i}|wf|job": {"repo": f"org/{name}-{i}", "workflow": "wf", "job": "job", "peak_mib": 100, "samples": 1, "last_seen": now - 60 - i} + for i in range(budget * 4 // 300) + } + self.assertGreater(len(json.dumps(many).encode()), budget * 3) + peak_event = json.dumps({"repo": "org/repo", "workflow": "wf", "job": "measured", "peak_bytes": 1024 * 1024 * 1024}) + result = self.poll({}, top={"ns-a": {"runner-1": 600}}, runners={"runner-1": JOB_A}, job_memory=json.dumps(many), job_peak=json.dumps(many), events={"ns-a": [peak_event]}) + self.assertEqual(result.returncode, 0, result.stderr) + for out, newest in ((self.job_memory_out, "org/repo|.github/workflows/ci.yml@refs/heads/main|Build"), (self.job_peak_out, "org/repo|wf|measured")): + with self.subTest(key=out.name): + text = out.read_text() + self.assertLessEqual(len(text.encode()), budget) + kept = json.loads(text) + self.assertIn(newest, kept, "the job seen this poll is kept") + self.assertIn(f"org/{name}-0|wf|job", kept, "the newest of the older entries is kept") + self.assertNotIn(f"org/{name}-{len(many) - 1}|wf|job", kept, "the oldest goes first") + # The whole object, which each patch rewrites: the two records' shares and the status keys stay inside an eighth of the limit. + stored = json.loads((self.dir / "state.json.configmap").read_text()) + self.assertLessEqual(sum(len(value.encode()) for value in stored.values()), limit // 8) + def test_failing_to_record_does_not_stop_the_capacity_decision(self) -> None: result = self.poll({"ns-a": [UNSCHEDULABLE_POD]}, top={"ns-a": {"runner-1": 600}}, runners={"runner-1": JOB_A}, job_memory_patch_fails=True) self.assertEqual(result.returncode, 0, result.stderr) From f1b80060bea791da4dc71e05d930d1ea19d09dc6 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Fri, 20 Nov 2026 09:47:15 +0000 Subject: [PATCH 3/6] test: hold the size tests to the bounded probe and job records The controller size test filled the probe ConfigMap with windows and the autoscaler test expected a record over ARG_MAX to be kept whole. Both records are now bounded, so the tests assert the bounds: every probe entry within probe_entry_limit_bytes, and an oversized stored job record read whole through a file and written back within JOB_RECORDS_MAX_BYTES. Refs #266 --- tests/unit/test_autoscaler.py | 7 ++++--- tests/unit/test_fleet_role.py | 13 ++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/unit/test_autoscaler.py b/tests/unit/test_autoscaler.py index 6f0358b9..34afa745 100644 --- a/tests/unit/test_autoscaler.py +++ b/tests/unit/test_autoscaler.py @@ -709,8 +709,8 @@ def test_a_scale_set_left_at_zero_is_given_a_slot_back_by_rebalancing(self) -> N self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(self.patched_maximums(result), {"ns-a": 1, "ns-b": 5}) - def test_a_record_larger_than_one_argument_may_be_is_still_stored(self) -> None: - # More bytes than the kernel accepts for all arguments together (and far more than one argument may hold), so it can reach kubectl only through a file. + def test_a_stored_record_larger_than_one_argument_may_be_is_read_whole_and_trimmed(self) -> None: + # More bytes than the kernel accepts for all arguments together (and far more than one argument may hold), as a record written before the byte bound can be, so it can be read only through a file; it is written back inside the bound. entry_bytes = 4096 count = min(2000, os.sysconf("SC_ARG_MAX") * 2 // entry_bytes + 1) stored = { @@ -721,8 +721,9 @@ def test_a_record_larger_than_one_argument_may_be_is_still_stored(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) self.assertNotIn("could not record", result.stderr) kept = self.recorded() - self.assertEqual(len(kept), count + 1) self.assertIn("org/repo|.github/workflows/ci.yml@refs/heads/main|Build", kept) + self.assertLess(len(kept), count + 1) + self.assertLessEqual(len(self.job_memory_out.read_text().encode()), 1024 * 1024 // 32) # Two nodes runner pods may use, as a fleet's might be: a large one where cpu is the limit at the template's request (four by cpu, five by memory), and a small one where a system pod's requests leave memory for one. Five runners in all. diff --git a/tests/unit/test_fleet_role.py b/tests/unit/test_fleet_role.py index c4628523..6cb05ca2 100644 --- a/tests/unit/test_fleet_role.py +++ b/tests/unit/test_fleet_role.py @@ -12,7 +12,6 @@ from fleet_role_support import ( AVAILABLE_FLOOR_PERCENT, - ARGUMENT_LIMIT_BYTES, AVAILABLE_KB, COMMIT_LIMIT_MS, CONTROLLER_ENV, @@ -653,16 +652,16 @@ def test_fewer_eligible_hosts_than_the_target_is_reported_and_refuses_a_swap(sel class SizeTest(ControllerCase): - # The probes' entries together can fill a ConfigMap, a mebibyte, and no one argument to a process may pass 128 KiB on Linux (the stand-in jq of tests/unit/fleet_role_support.py refuses one here). - CONFIGMAP_LIMIT_BYTES = 1024 * 1024 + # The probes publish summaries, each within the shared module's probe_entry_limit_bytes, so the probes' ConfigMap is bounded by that times the measured hosts whatever their windows hold; no one argument to a process may pass 128 KiB on Linux (the stand-in jq of tests/unit/fleet_role_support.py refuses one here), and the inputs reach jq in files. + ENTRY_LIMIT_BYTES = 4096 - def test_inputs_the_size_of_a_full_configmap_are_read_whole(self) -> None: + def test_the_probes_inputs_stay_inside_the_entry_limit_and_are_read_whole(self) -> None: standbys = [Host(f"standby-with-a-long-node-name-{i:02d}", eligible=True) for i in range(14)] hosts = healthy() + standbys populate(self.cluster, hosts) - size = sum(len(json.dumps(entry_)) for entry_ in self.cluster.probes().values()) - self.assertGreater(size, 3 * ARGUMENT_LIMIT_BYTES) - self.assertLessEqual(size, self.CONFIGMAP_LIMIT_BYTES) + self.assertEqual(int(subprocess.run(["jq", "-n", "-L", str(SCRIPTS), 'include "fleet-role-stats"; probe_entry_limit_bytes'], capture_output=True, text=True, check=True).stdout), self.ENTRY_LIMIT_BYTES) + for name, entry_ in self.cluster.probes().items(): + self.assertLessEqual(len(json.dumps(entry_).encode()), self.ENTRY_LIMIT_BYTES, name) completed = self.cluster.run("fleet-role.sh", **CONTROLLER_ENV) self.assertEqual(completed.returncode, 0, completed.stderr) status = self.cluster.status() From dbaf29cfc1f3f8d6365ba0acb78ef0aa6c522ea2 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Fri, 20 Nov 2026 10:28:14 +0000 Subject: [PATCH 4/6] fix(fleet-role): keep get on the probe ConfigMap for kubectl patch `kubectl patch` reads the object before it patches it, so the probe Role without get had every patch refused and no probe ever published, which the mesh role controller scenarios showed as voters with no probe report. Refs #266 --- roles/github_runner_arc/templates/fleet-role.yaml.j2 | 4 ++-- tests/unit/test_fleet_role_arc.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/github_runner_arc/templates/fleet-role.yaml.j2 b/roles/github_runner_arc/templates/fleet-role.yaml.j2 index ec86870f..1372ff0f 100644 --- a/roles/github_runner_arc/templates/fleet-role.yaml.j2 +++ b/roles/github_runner_arc/templates/fleet-role.yaml.j2 @@ -72,11 +72,11 @@ rules: resourceNames: {{ [actuator_configmap] | to_json }} verbs: ["get"] {% else %} -{# Patch alone: a probe keeps its window on its own node and never reads the ConfigMap, so it cannot read the other nodes' entries. #} +{# Get as well as patch, though the probe keeps its window on its own node and reads no entry: `kubectl patch` reads the object before it patches it, and is refused without get. #} - apiGroups: [""] resources: ["configmaps"] resourceNames: {{ [probe_configmap] | to_json }} - verbs: ["patch"] + verbs: ["get", "patch"] {% endif %} --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/tests/unit/test_fleet_role_arc.py b/tests/unit/test_fleet_role_arc.py index 7dfc9143..4d8351da 100644 --- a/tests/unit/test_fleet_role_arc.py +++ b/tests/unit/test_fleet_role_arc.py @@ -131,7 +131,7 @@ def test_their_namespaced_permissions_in_the_platform_namespace_are_on_configmap {"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-actuators"], "verbs": ["get"]}, ], ) - self.assertEqual(probe, [{"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-probe"], "verbs": ["patch"]}]) + self.assertEqual(probe, [{"apiGroups": [""], "resources": ["configmaps"], "resourceNames": ["fleet-role-probe"], "verbs": ["get", "patch"]}]) for rule in controller + probe: self.assertNotIn("create", rule["verbs"]) self.assertNotIn("delete", rule["verbs"]) From ec8e9b7ae766af6211a0bd9e2b8ba5d2786b9b44 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Fri, 20 Nov 2026 10:28:16 +0000 Subject: [PATCH 5/6] fix(fleet-role): read an entry that is not a summary as no report A probe from before summaries holds raw samples in its key until its new pod publishes. The controller now drops any entry whose samples field is not a count, so such a host is unknown and a voter's gates do not hold, never unfit on figures it cannot read. Refs #266 --- scripts/fleet-role.jq | 3 ++- tests/unit/test_fleet_role.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/fleet-role.jq b/scripts/fleet-role.jq index f51ff8ae..af09e5b0 100644 --- a/scripts/fleet-role.jq +++ b/scripts/fleet-role.jq @@ -48,7 +48,8 @@ def pair($summary; $peer; $peer_id): $config as $c | $nodes_file[0] as $nodes -| $probes_file[0] as $probes +# An entry that is not a summary (one a probe from before summaries published, holding its raw samples) is no report: its host is unknown and a voter's gates do not hold until its probe publishes a summary. +| ($probes_file[0] | map_values(select(type == "object" and (.samples | type) == "number"))) as $probes | $previous_file[0] as $previous | $c.now as $now | ($c.sample_seconds * window_samples) as $window_seconds diff --git a/tests/unit/test_fleet_role.py b/tests/unit/test_fleet_role.py index 6cb05ca2..bf44b8d0 100644 --- a/tests/unit/test_fleet_role.py +++ b/tests/unit/test_fleet_role.py @@ -639,6 +639,19 @@ def test_a_window_that_began_before_a_membership_change_is_not_judged(self) -> N self.assertEqual(status["hosts"]["a"]["verdict"], "unknown") self.assertIn(f"the probe's window began before the last membership change at {changed_at}", status["hosts"]["a"]["reasons"]) + def test_an_entry_of_raw_samples_from_an_older_probe_is_no_report_never_unfit(self) -> None: + hosts = healthy(a={"tcp": UNFIT_RTT_MS + 50}) + populate(self.cluster, hosts) + summaries = self.cluster.probes() + self.cluster.set_probes({**summaries, "a": self.cluster.windows["a"]}) + completed = self.cluster.run("fleet-role.sh", **CONTROLLER_ENV) + self.assertEqual(completed.returncode, 0, completed.stderr) + status = self.cluster.status() + self.assertEqual(status["hosts"]["a"]["verdict"], "unknown") + self.assertIn("no probe report", status["hosts"]["a"]["reasons"]) + self.assertIn(f"voter a has no probe report in the last {STALE_AFTER_SAMPLES * SAMPLE_SECONDS} s", status["gates"]) + self.assertEqual(status["decision"]["action"], "none") + def test_a_window_that_began_after_a_membership_change_is_judged(self) -> None: changed_at = int(time.time()) - WINDOW_SECONDS - SAMPLE_SECONDS self.cluster.set_status({"membership": {"voters": ["a", "b", "c"], "changed_at": changed_at, "cooldown_until": changed_at + COOLDOWN_SECONDS}}) From 1b16548c1fe9187412aceecfd0e1dd0110c8a019 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Fri, 20 Nov 2026 10:28:17 +0000 Subject: [PATCH 6/6] test: fix three probe and autoscaler tests for the bounded records The stored-record test dated its entries in the future, so the newest-first bound rightly kept them over the job seen now; it dates them a minute ago. The probe test measuring itself passed a node name the window reader no longer takes, and the oversized-window test used more voters than etcd recommends, whose summary is rightly refused; it uses the most voters with the longest names. Refs #266 --- tests/unit/test_autoscaler.py | 2 +- tests/unit/test_fleet_role_probe.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_autoscaler.py b/tests/unit/test_autoscaler.py index 34afa745..20847e76 100644 --- a/tests/unit/test_autoscaler.py +++ b/tests/unit/test_autoscaler.py @@ -714,7 +714,7 @@ def test_a_stored_record_larger_than_one_argument_may_be_is_read_whole_and_trimm entry_bytes = 4096 count = min(2000, os.sysconf("SC_ARG_MAX") * 2 // entry_bytes + 1) stored = { - f"org/repo|wf|job-{n}": {"repo": "org/repo", "workflow": "wf", "job": f"job-{n}-" + "x" * entry_bytes, "peak_mib": 1, "samples": 1, "last_seen": 4102444800} + f"org/repo|wf|job-{n}": {"repo": "org/repo", "workflow": "wf", "job": f"job-{n}-" + "x" * entry_bytes, "peak_mib": 1, "samples": 1, "last_seen": int(time.time()) - 60} for n in range(count) } result = self.poll({}, top={"ns-a": {"runner-1": 600}}, runners={"runner-1": JOB_A}, job_memory=json.dumps(stored)) diff --git a/tests/unit/test_fleet_role_probe.py b/tests/unit/test_fleet_role_probe.py index 5029abe1..33bfac7a 100644 --- a/tests/unit/test_fleet_role_probe.py +++ b/tests/unit/test_fleet_role_probe.py @@ -415,7 +415,7 @@ def test_a_node_does_not_measure_itself(self) -> None: with listener() as port: completed = self.probe("peer", voters=self.peers(), ETCD_PEER_PORT=str(port)) self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertEqual(self.newest("peer")["tcp"], {}) + self.assertEqual(self.newest()["tcp"], {}) class EtcdReadingTest(ProbeCase): @@ -703,8 +703,8 @@ def test_a_sample_is_appended_and_the_entry_keeps_the_last_window_of_them(self) self.assertGreater(kept[-1]["t"], 1000 + WINDOW_SAMPLES) def test_an_entry_too_large_for_one_argument_is_read_and_extended_whole(self) -> None: - # One argument to a process is limited to 128 KiB on Linux; an entry of a voter in a large cluster, with a connect time to every other voter in each sample, passes that. - peers = {f"peer-with-a-long-node-name-{i:03d}": 1.234 for i in range(60)} + # One argument to a process is limited to 128 KiB on Linux; the window of a node measuring the most voters, each with the longest Node name, with a connect time to each in every sample, passes that. + peers = {long_name(i): 1.234 for i in range(MAX_VOTERS)} big = [sample(1000 + i, tcp=peers) for i in range(WINDOW_SAMPLES)] self.set_state({**entry(big), "fsync_at": int(time.time())}) self.assertGreater(len(json.dumps(self.state())), 128 * 1024)