From ae1738f2fa993244aea8b14d1438e332c44b6857 Mon Sep 17 00:00:00 2001 From: David Charlot Date: Fri, 9 Oct 2026 04:04:45 -0400 Subject: [PATCH 1/7] sandbox: a broken draw reading condemned the counter it was meant to judge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `plausible_gpu_counter` returned a bool, and it folded two unlike things into the one `false`: if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) The left is the counter running faster than the board can draw — a finding about the counter. The right is the *board's own telemetry* reading past the board's own limit, which says the yardstick is broken, not the counter. An RTX 4050 Laptop GPU reports about 590 W for a second after it wakes, so one such sample among sane ones declared the counter implausible. jcp-energy had the same function and the same bug; #185 fixed it there after the hardware test on jetson-hub refused the first shape of the fix. This is that fix ported, and three crates still carry the original: this one, jouleclaw-energy and eoc-meter. So the reading is dropped, and only that reading — one broken sample is not a broken bracket. Only when nothing usable is left is there no yardstick, and then the verdict is `BoardTelemetryUnusable`, an indeterminate, mapping to a new `CounterStatus::Unjudgeable` rather than to `Implausible` whose doc asserts "is not counting the board's energy" — a conclusion this case does not support. `counter_admits` compares against `Live`, so the new variant is inadmissible without a change there; the distinction is the diagnosis an operator gets, which is the difference between replacing a counter and waiting for the board to settle. Dropping the bad reading is also what the old short-circuit was really for, and it does the job properly. The mean summed every reading, so a 590 W spike inflated it and made the mean-draw bound *more* permissive — a bad reading could have excused an over-running counter. That only went unnoticed because any such reading failed the check first. A test now pins the bound at twice the usable 10 W reading plus 5 W, which it could not be if the spike were still in the mean. The bool also discarded the magnitude: a counter 1.1x over a bound and one 20x over were the same answer, and a past finding could not be re-graded against a different margin because the rate did not survive the check. `GpuPlausibility` carries the rate, the bound, and which bound it was. The run-level verdict is now the most severe of its GPUs' rather than an `all()` over bools, so one board's broken telemetry no longer condemns the run and one healthy GPU cannot excuse it. Checked for aarch64-unknown-linux-gnu as well, since the NVML consumer is Linux-gated. 272 tests pass in the workspace. Co-Authored-By: Claude Opus 5 --- .../sandbox-meter/src/probe/liveness.rs | 308 +++++++++++++++--- .../crates/sandbox-meter/src/probe/mod.rs | 26 +- .../crates/sandbox-meter/src/probe/nvml.rs | 18 +- 3 files changed, 298 insertions(+), 54 deletions(-) diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs index 6f2bcdee2..054869898 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs @@ -523,46 +523,182 @@ pub(super) const GPU_DRAW_SLACK_MW: u128 = 5_000; /// that while it is polled: at the probes' 100-150 ms ticks it ran at /// 44-305 W while the board drew 5-14 W against a 65 W limit, and polled /// every 1-2 s it still read 11-23 W while the board drew 2-10 W. +/// Which of the two bounds a GPU counter ran past. #[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` -pub(super) fn plausible_gpu_counter(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum GpuBound { + /// [`GPU_CEILING_MARGIN`] of the board's maximum power limit. + PowerLimit, + /// [`GPU_DRAW_FACTOR`] times the board's mean draw plus + /// [`GPU_DRAW_SLACK_MW`]. + MeanDraw, +} + +/// What the plausibility check found, with the figures it found it from. +/// +/// A `bool` was the whole of this, and it threw away everything an operator +/// needs: which bound broke, by how much, and the rate it broke it at. A +/// counter 1.1 times over a bound and one 20 times over were the same answer. +/// The figures also make the judgement reproducible — the margins are +/// constants in this module, and re-grading a past finding against a +/// different margin is only possible if the rate survived the check. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum GpuPlausibility { + /// The counter's rate sits inside every bound whose reading was present. + Plausible { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + }, + /// The counter ran faster than the board can draw. + CounterOverRate { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + /// The bound it broke, in milliwatts. + bound_mw: u64, + /// Which bound that was. + bound: GpuBound, + }, + /// The board's own telemetry read past the board's own power limit, so + /// there is no yardstick left to judge the counter against. An + /// indeterminate, not a finding about the counter — see + /// [`CounterStatus::Unjudgeable`]. + BoardTelemetryUnusable { + /// The offending draw reading, in milliwatts. + draw_mw: u64, + /// The limit it exceeded (the margin-adjusted cap), in milliwatts. + limit_mw: u64, + }, +} + +impl GpuPlausibility { + /// Whether the counter was judged to be counting its board's energy. + /// + /// `false` for an indeterminate as well as for an overrun: unjudgeable is + /// not admissible, and erring toward declining is the safe direction. Use + /// the variant itself to tell the two apart. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` + pub(super) fn plausible(&self) -> bool { + matches!(self, GpuPlausibility::Plausible { .. }) + } + + /// How severe this verdict is, for picking the one that speaks for a run + /// of several GPUs: a proven overrun outranks an indeterminate, which + /// outranks a pass. + fn rank(&self) -> u8 { + match self { + GpuPlausibility::Plausible { .. } => 0, + GpuPlausibility::BoardTelemetryUnusable { .. } => 1, + GpuPlausibility::CounterOverRate { .. } => 2, + } + } + + /// The figure this verdict turns on, to order two of equal rank by. + fn magnitude(&self) -> u64 { + match self { + GpuPlausibility::Plausible { rate_mw } => *rate_mw, + GpuPlausibility::CounterOverRate { rate_mw, .. } => *rate_mw, + GpuPlausibility::BoardTelemetryUnusable { draw_mw, .. } => *draw_mw, + } + } +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) fn plausible_gpu_counter( + energy_mj: u64, + window: Duration, + draws_mw: &[u32], + ceiling_mw: Option, +) -> GpuPlausibility { let ns = window.as_nanos().max(1); // mJ per second is mW. let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; - if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { - let cap = u128::from(limit) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1; - if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { - return false; + let clamp = |v: u128| u64::try_from(v).unwrap_or(u64::MAX); + let cap = ceiling_mw + .filter(|&l| l > 0) + .map(|l| u128::from(l) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1); + + // A draw reading past the board's own limit is not a reading: an RTX 4050 + // Laptop GPU reports about 590 W for a second after it wakes. Such a + // reading is dropped, and only that reading — it is one broken sample, + // not a broken bracket. Voiding the whole judgement on it is what this + // code used to do, and jetson-hub's hardware test refused that shape: one + // spike among otherwise sane 14.8 W samples turned a counter running + // persistently at 4-30x the board's draw into an indeterminate, which is + // a worse diagnosis than the overrun it actually is. + // + // Dropping them also keeps the mean honest, which is what the old + // short-circuit was really for. The mean summed every reading, so a 590 W + // spike inflated it and made the mean-draw bound *more* permissive; that + // only went unnoticed because any such reading failed the check first. + let (usable, unusable): (Vec, Vec) = draws_mw + .iter() + .map(|&d| u128::from(d)) + .partition(|&d| cap.is_none_or(|c| d <= c)); + + if let Some(cap) = cap { + if rate_mw > cap { + return GpuPlausibility::CounterOverRate { + rate_mw: clamp(rate_mw), + bound_mw: clamp(cap), + bound: GpuBound::PowerLimit, + }; + } + // Only when nothing usable is left is there no yardstick at all. + if usable.is_empty() && !unusable.is_empty() { + return GpuPlausibility::BoardTelemetryUnusable { + draw_mw: clamp(unusable[0]), + limit_mw: clamp(cap), + }; } } - if !draws_mw.is_empty() { - let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; - if rate_mw > GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW { - return false; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as u128; + let bound = GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW; + if rate_mw > bound { + return GpuPlausibility::CounterOverRate { + rate_mw: clamp(rate_mw), + bound_mw: clamp(bound), + bound: GpuBound::MeanDraw, + }; } } - true + GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) } } -/// Whether every GPU counter of a probe run plausibly counted its board's -/// energy ([`plausible_gpu_counter`]): each rail's rise from the run's first +/// What the GPU counters of a probe run were judged to be counting +/// ([`plausible_gpu_counter`]): each rail's rise from the run's first /// sample to its last, over the window between them, beside that GPU's draw /// readings (`draws`, one row per sample call, one column per GPU, `None` /// where a reading failed) and its maximum power limit (`ceilings`, one per /// GPU). Rails and columns are in device order. A run with fewer than two /// samples has nothing to judge. #[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` -pub(super) fn gpu_counters_plausible(watch: &RailWatch, draws: &[Vec>], ceilings: &[Option]) -> bool { +pub(super) fn gpu_counters_verdict( + watch: &RailWatch, + draws: &[Vec>], + ceilings: &[Option], +) -> GpuPlausibility { let (Some(first), Some(last), Some(t0), Some(t1)) = (watch.observed.first(), watch.observed.last(), watch.at.first(), watch.at.last()) else { - return true; + return GpuPlausibility::Plausible { rate_mw: 0 }; }; let window = t1.saturating_sub(*t0); - first.iter().zip(last).enumerate().all(|(i, ((_, a), (_, b)))| { - let energy_mj = u64::try_from(b.saturating_sub(*a)).unwrap_or(0); - let gpu_draws: Vec = draws.iter().filter_map(|row| row.get(i).copied().flatten()).collect(); - plausible_gpu_counter(energy_mj, window, &gpu_draws, ceilings.get(i).copied().flatten()) - }) + // The verdict that speaks for the run is the most severe one, and among + // equals the largest figure: a run is not excused by its healthiest GPU. + first + .iter() + .zip(last) + .enumerate() + .map(|(i, ((_, a), (_, b)))| { + let energy_mj = u64::try_from(b.saturating_sub(*a)).unwrap_or(0); + let gpu_draws: Vec = + draws.iter().filter_map(|row| row.get(i).copied().flatten()).collect(); + plausible_gpu_counter(energy_mj, window, &gpu_draws, ceilings.get(i).copied().flatten()) + }) + .max_by_key(|v| (v.rank(), v.magnitude())) + .unwrap_or(GpuPlausibility::Plausible { rate_mw: 0 }) } /// A counter probe's constructor behind its gate: `open` runs only when @@ -1939,36 +2075,100 @@ mod tests { #[test] fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { let w = Duration::from_millis(500); - // 66 W counted against a 7.5 W draw: what jetson-hub's probe saw. - assert!(!plausible_gpu_counter(33_000, w, &[7_500, 7_700], Some(65_000))); + // Over a 500 ms window a counter's rate in mW is twice its mJ. + let over = |rate_mw, bound_mw, bound| GpuPlausibility::CounterOverRate { rate_mw, bound_mw, bound }; + + // 66 W counted against a 7.6 W mean draw: what jetson-hub's probe saw. + // The figures, not just a `false` — 66 W against a 20.2 W bound is a + // 3.3x overrun, and an operator cannot act on a bare bool. + assert_eq!( + plausible_gpu_counter(33_000, w, &[7_500, 7_700], Some(65_000)), + over(66_000, 20_200, GpuBound::MeanDraw) + ); // 44 W at the 150 ms tick against 10.4 W: past twice the draw + 5 W. - assert!(!plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000))); + assert_eq!( + plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000)), + over(44_000, 25_800, GpuBound::MeanDraw) + ); // 120 W, past 1.5 x the 65 W limit, even with no draw reading. - assert!(!plausible_gpu_counter(60_000, w, &[], Some(65_000))); - // A draw reading of 590 W (the board's first second after waking) is no - // reading: it would excuse any counter. - assert!(!plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000))); + assert_eq!( + plausible_gpu_counter(60_000, w, &[], Some(65_000)), + over(120_000, 97_500, GpuBound::PowerLimit) + ); + + // A draw reading of 590 W (the board's first second after waking) is + // no reading. It is dropped, and only it: the remaining 10.3 W reading + // still judges the counter, and 88 W against it is an overrun. This + // used to return a bare `false` meaning "a draw reading was broken", + // which is the one thing the hardware test on jetson-hub refused: it + // turned a persistent 4-30x overrun into "cannot judge". + assert_eq!( + plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000)), + over(88_000, 25_600, GpuBound::MeanDraw) + ); + // Dropping it also keeps the mean honest, and this pins that: the + // bound is twice 10 W plus 5 W, computed from the usable reading + // alone. Had the 590 W reading been summed in, the mean would be + // 300 W and the bound 605 W, which would excuse this counter. + assert_eq!( + plausible_gpu_counter(15_000, w, &[590_000, 10_000], Some(65_000)), + over(30_000, 25_000, GpuBound::MeanDraw) + ); + // Only when no usable reading is left is there no yardstick at all, + // and then the verdict is an indeterminate about the board, not a + // finding about the counter. + assert_eq!( + plausible_gpu_counter(44_000, w, &[590_000], Some(65_000)), + GpuPlausibility::BoardTelemetryUnusable { draw_mw: 590_000, limit_mw: 97_500 } + ); + // A counter past the board's own limit is a finding whatever the + // telemetry says: the rate bound is checked before the yardstick is. + assert_eq!( + plausible_gpu_counter(60_000, w, &[590_000], Some(65_000)), + over(120_000, 97_500, GpuBound::PowerLimit) + ); + // A counter tracking its board: 13 W against 13 W, and 300 W against a // 280 W draw on a 700 W board. - assert!(plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000))); - assert!(plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000))); + assert_eq!( + plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000)), + GpuPlausibility::Plausible { rate_mw: 13_000 } + ); + assert_eq!( + plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000)), + GpuPlausibility::Plausible { rate_mw: 300_000 } + ); // A load step the one-second draw has not caught up with, up to twice // the draw plus 5 W: 205 W against 100 W. - assert!(plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000))); - assert!(!plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000))); + assert!(plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000)).plausible()); + assert_eq!( + plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000)), + over(205_002, 205_000, GpuBound::MeanDraw) + ); // The exact bounds. - assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); - assert!(!plausible_gpu_counter(48_751, w, &[], Some(65_000))); - assert!(plausible_gpu_counter(12_500, w, &[10_000], None), "25 W is 2 x 10 W + 5 W"); - assert!(!plausible_gpu_counter(12_501, w, &[10_000], None)); - // Nothing to judge it by: no bound applies. - assert!(plausible_gpu_counter(1_000_000, w, &[], None)); + assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)).plausible(), "97.5 W is 1.5 x 65 W"); + assert_eq!( + plausible_gpu_counter(48_751, w, &[], Some(65_000)), + over(97_502, 97_500, GpuBound::PowerLimit) + ); + assert!(plausible_gpu_counter(12_500, w, &[10_000], None).plausible(), "25 W is 2 x 10 W + 5 W"); + assert_eq!( + plausible_gpu_counter(12_501, w, &[10_000], None), + over(25_002, 25_000, GpuBound::MeanDraw) + ); + // Nothing to judge it by: no bound applies, and with no limit to + // exceed no reading can be unusable either. + assert_eq!( + plausible_gpu_counter(1_000_000, w, &[], None), + GpuPlausibility::Plausible { rate_mw: 2_000_000 } + ); } /// The rule over a probe run: each GPU's rise from the run's first sample /// to its last, over the window between them, beside that GPU's own draws - /// and limit. One implausible GPU of two makes the run implausible, and a - /// GPU's readings are never judged against another's. + /// and limit. One implausible GPU of two makes the run implausible, a + /// GPU's readings are never judged against another's, and the verdict + /// that speaks for the run is the most severe of its GPUs'. #[test] fn every_gpu_of_a_run_must_count_no_faster_than_its_board_draws() { // Two GPUs republished every 100 ms, sampled at the NVML tick: one at @@ -1984,12 +2184,32 @@ mod tests { let rows = r.watch.observed.len(); let draws = |a: u32, b: u32| vec![vec![Some(a), Some(b)]; rows]; let ceilings = [Some(65_000), Some(700_000)]; - assert!(gpu_counters_plausible(&r.watch, &draws(10_000, 290_000), &ceilings), "each at its own draw"); - assert!(!gpu_counters_plausible(&r.watch, &draws(1_000, 290_000), &ceilings), "the first at 9 x its draw"); - assert!(!gpu_counters_plausible(&r.watch, &draws(10_000, 290_000), &[Some(65_000), Some(65_000)]), "300 W on a 65 W board"); - assert!(!gpu_counters_plausible(&r.watch, &draws(290_000, 10_000), &[None, None]), "draws swapped between the GPUs"); - assert!(gpu_counters_plausible(&r.watch, &[], &[None, None]), "no reading to judge by"); - assert!(gpu_counters_plausible(&RailWatch::default(), &[], &[]), "no samples"); + let verdict = |d: &[Vec>], c: &[Option]| gpu_counters_verdict(&r.watch, d, c); + assert!(verdict(&draws(10_000, 290_000), &ceilings).plausible(), "each at its own draw"); + assert!(!verdict(&draws(1_000, 290_000), &ceilings).plausible(), "the first at 9 x its draw"); + assert!( + !verdict(&draws(10_000, 290_000), &[Some(65_000), Some(65_000)]).plausible(), + "300 W on a 65 W board" + ); + assert!(!verdict(&draws(290_000, 10_000), &[None, None]).plausible(), "draws swapped between the GPUs"); + assert!(verdict(&[], &[None, None]).plausible(), "no reading to judge by"); + assert!(gpu_counters_verdict(&RailWatch::default(), &[], &[]).plausible(), "no samples"); + + // Which verdict speaks for a run of several GPUs. One board's broken + // telemetry withholds a judgement on that board, so the run is + // unjudgeable — not implausible, which would blame a counter nothing + // measured, and not plausible, which would let the healthy GPU excuse + // the run. + assert!(matches!( + verdict(&draws(590_000, 290_000), &ceilings), + GpuPlausibility::BoardTelemetryUnusable { .. } + )); + // But a GPU proven to be over-running outranks another's broken + // telemetry: a finding beats an indeterminate. + assert!(matches!( + verdict(&draws(590_000, 290_000), &[Some(65_000), Some(65_000)]), + GpuPlausibility::CounterOverRate { .. } + )); } /// The tick a counter probe burns is the tick it was asked for. diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs index c21c7f2f5..3e0d56297 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs @@ -190,12 +190,24 @@ pub enum CounterStatus { /// Every rail it sums kept advancing, but a GPU counter (NVML's) ran /// faster than its board can draw power: past 1.5 times the board's /// maximum power limit, or past twice the board's own power draw plus - /// 5 W, or the board's draw reading was itself past that limit. Such a - /// counter is not counting the board's energy, so a figure from it would - /// seal a number the device never spent. Seen on an RTX 4050 Laptop GPU - /// (driver 595.84), whose counter runs at 4 to 30 times the board's draw - /// while it is polled. Never used. + /// 5 W. Such a counter is not counting the board's energy, so a figure + /// from it would seal a number the device never spent. Seen on an RTX + /// 4050 Laptop GPU (driver 595.84), whose counter runs at 4 to 30 times + /// the board's draw while it is polled. Never used. Implausible, + /// Every rail it sums kept advancing and the counter could not be judged, + /// because the board's own power telemetry was unusable over the probe: a + /// draw reading past the board's own maximum power limit (an RTX 4050 + /// Laptop GPU reports about 590 W for a second after it wakes). + /// + /// Split out from [`CounterStatus::Implausible`], which used to cover + /// this — its doc then asserted "is not counting the board's energy", a + /// conclusion this case does not support. The yardstick is broken, not + /// necessarily the counter. Never used either way: unjudgeable is not + /// admissible, and erring toward declining is the safe direction. The + /// distinction is the diagnosis an operator gets, which is the difference + /// between replacing a counter and waiting for the board to settle. + Unjudgeable, /// Every rail it sums kept advancing: in at least five sample intervals, /// at least four in five of those observed, over at least 300 ms of wall /// time, while the probing thread burned CPU. The probe is used. @@ -214,6 +226,10 @@ impl std::fmt::Display for CounterStatus { "the counter kept advancing, but faster than its device can draw power, so it does not \ count the device's energy" } + CounterStatus::Unjudgeable => { + "the counter kept advancing, but the device's own power telemetry read past the \ + device's limit, so there is no yardstick to judge the counter against" + } CounterStatus::Live => "every rail of the counter keeps advancing under load", }) } diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs index 98c27ce13..ba357a594 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs @@ -34,7 +34,8 @@ //! that work ([`Measurement::reads_cpu`](super::Measurement::reads_cpu)). use super::liveness::{ - counter_run, counter_status_of, counter_tick, gated, gpu_counters_plausible, RailWatch, Sample, NVML_PROBE_TICK, + counter_run, counter_status_of, counter_tick, gated, gpu_counters_verdict, GpuPlausibility, RailWatch, Sample, + NVML_PROBE_TICK, }; use std::cell::RefCell; use super::{CounterStatus, Probe, ProbeError, CPU_RAIL_LIVENESS_BUDGET}; @@ -78,12 +79,19 @@ pub(super) fn host_status() -> CounterStatus { status_of_run(&run.watch, &run.draws, &run.ceilings) } -/// The status of an NVML probe run: its verdict's, except that a live run -/// whose counters did not plausibly count their boards' energy -/// ([`gpu_counters_plausible`]) is `Implausible`. +/// The status of an NVML probe run: its verdict's, except that a live run is +/// re-judged on whether its counters counted their boards' energy +/// ([`gpu_counters_verdict`]). A counter proven to run faster than its board +/// can draw is `Implausible`; a board whose own telemetry is unusable leaves +/// the counter `Unjudgeable`, which is a different diagnosis and not a claim +/// about the counter. fn status_of_run(watch: &RailWatch, draws: &[Vec>], ceilings: &[Option]) -> CounterStatus { match counter_status_of(watch.verdict()) { - CounterStatus::Live if !gpu_counters_plausible(watch, draws, ceilings) => CounterStatus::Implausible, + CounterStatus::Live => match gpu_counters_verdict(watch, draws, ceilings) { + GpuPlausibility::Plausible { .. } => CounterStatus::Live, + GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible, + GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable, + }, status => status, } } -- 2.54.0 (Apple Git-157) From 10717600e03620f6eca2c6b926fc0aa4903da173 Mon Sep 17 00:00:00 2001 From: David Charlot Date: Fri, 9 Oct 2026 04:46:59 -0400 Subject: [PATCH 2/7] jouleclaw: a broken draw reading condemned the counter it was meant to judge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same defect as the previous commit, in the second of the three crates that still carried it. `plausible_gpu_counter` returned a bool and folded two unlike things into the one `false`: if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) The left is the counter running faster than the board can draw — a finding about the counter. The right is the *board's own telemetry* reading past the board's own limit, which says the yardstick is broken, not the counter. The test even named the case: `"a 590 W draw reading"` asserted `false`, which is the one reading an RTX 4050 Laptop GPU emits for a second after it wakes. So the reading is dropped, and only that reading. Only when nothing usable is left is there no yardstick, and then the verdict is `BoardTelemetryUnusable` mapping to a new `CounterLiveness::Unjudgeable` — kept apart from `Implausible`, whose doc asserts "it is not counting the board's energy", a conclusion this case does not support. In `nvml::mode_for` an unjudgeable counter falls back to the vendor power integral exactly as an implausible one does: neither can be graded `HwShunt`. `GpuPlausibility` also carries the rate, the bound and which bound it was, where the bool discarded the magnitude: a counter 1.1x over and one 20x over were the same answer, and no past finding could be re-graded against a changed margin. A test pins the mean-draw bound at twice the usable 10 W reading plus 5 W, which it could not be if the 590 W spike were still in the mean — the old code summed every reading, so a bad reading made that bound *more* permissive, and only the short-circuit hid it. Three `admits` functions list the variants exhaustively rather than using a wildcard, which is why adding one forced a decision at each: unjudgeable is inadmissible everywhere. The rapl.rs copy only surfaced under a whole- workspace build, and nvml.rs's only under `--features nvml`, neither of which a single-crate default build compiles. Checked for aarch64-unknown-linux-gnu with and without `--features nvml`, warning-free on both; 57 tests pass in the crate and the workspace is clean. Co-Authored-By: Claude Opus 5 --- .../crates/jouleclaw-energy/src/liveness.rs | 306 +++++++++++++++--- .../crates/jouleclaw-energy/src/nvml.rs | 14 +- .../crates/jouleclaw-energy/src/process.rs | 5 +- .../jouleclaw-energy/src/process_gpu.rs | 5 +- .../crates/jouleclaw-energy/src/rapl.rs | 5 +- 5 files changed, 292 insertions(+), 43 deletions(-) diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs index d1e82f633..863f911ec 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs @@ -61,13 +61,26 @@ pub enum CounterLiveness { Live, /// Every channel kept advancing, but a GPU counter (NVML's) ran faster /// than its board can draw power over the probe: past 1.5 times the - /// board's maximum power limit, past twice the board's own power draw plus - /// 5 W, or beside a draw reading itself past that limit - /// ([`plausible_gpu_counter`]). It is not counting the board's energy, so - /// its figure would not be a reading of the GPU's work. Seen on an RTX 4050 - /// Laptop GPU (driver 595.84), whose counter runs at 4 to 30 times the - /// board's draw while it is polled. + /// board's maximum power limit, or past twice the board's own power draw + /// plus 5 W ([`plausible_gpu_counter`]). It is not counting the board's + /// energy, so its figure would not be a reading of the GPU's work. Seen on + /// an RTX 4050 Laptop GPU (driver 595.84), whose counter runs at 4 to 30 + /// times the board's draw while it is polled. Implausible, + /// Every channel kept advancing and the counter could not be judged, + /// because the board's own power telemetry was unusable over the probe: a + /// draw reading past the board's own maximum power limit (an RTX 4050 + /// Laptop GPU reports about 590 W for a second after it wakes). + /// + /// Split out from [`CounterLiveness::Implausible`], which used to cover + /// this — its doc then asserted "it is not counting the board's energy", a + /// conclusion this case does not support. The yardstick is broken, not + /// necessarily the counter. Neither grades a reading as + /// [`crate::Provenance::HwShunt`]: unjudgeable is not admissible, and + /// erring toward declining is the safe direction. The distinction is the + /// diagnosis an operator gets, which is the difference between replacing a + /// counter and waiting for the board to settle. + Unjudgeable, } /// In at least how many separate sample intervals a channel must advance to @@ -313,24 +326,151 @@ pub(crate) const GPU_DRAW_SLACK_MW: u128 = 5_000; /// 595.84) the total-energy counter runs several times faster than the board /// draws for as long as it is polled: at the NVML probe's 100 ms tick it ran /// at 58-305 W while the board drew 5-14 W against a 65 W limit. +/// Which of the two bounds a GPU counter ran past. #[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` -pub(crate) fn plausible_gpu_counter(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuBound { + /// [`GPU_CEILING_MARGIN`] of the board's maximum power limit. + PowerLimit, + /// [`GPU_DRAW_FACTOR`] times the board's mean draw plus + /// [`GPU_DRAW_SLACK_MW`]. + MeanDraw, +} + +/// What the plausibility check found, with the figures it found it from. +/// +/// A `bool` was the whole of this, and it threw away everything an operator +/// needs: which bound broke, by how much, and the rate it broke it at. A +/// counter 1.1 times over a bound and one 20 times over were the same answer. +/// The figures also make the judgement reproducible — the margins are +/// constants in this module, and re-grading a past finding against a +/// different margin is only possible if the rate survived the check. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuPlausibility { + /// The counter's rate sits inside every bound whose reading was present. + Plausible { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + }, + /// The counter ran faster than the board can draw. + CounterOverRate { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + /// The bound it broke, in milliwatts. + bound_mw: u64, + /// Which bound that was. + bound: GpuBound, + }, + /// The board's own telemetry read past the board's own power limit, so + /// there is no yardstick left to judge the counter against. An + /// indeterminate, not a finding about the counter — see + /// [`CounterLiveness::Unjudgeable`]. + BoardTelemetryUnusable { + /// The offending draw reading, in milliwatts. + draw_mw: u64, + /// The limit it exceeded (the margin-adjusted cap), in milliwatts. + limit_mw: u64, + }, +} + +impl GpuPlausibility { + /// Whether the counter was judged to be counting its board's energy. + /// + /// `false` for an indeterminate as well as for an overrun: unjudgeable is + /// not admissible, and erring toward declining is the safe direction. Use + /// the variant itself to tell the two apart. + // + // The crate's own paths match on the variants rather than call this, so + // outside tests it is unused: the `liveness` module is private, which puts + // even a `pub` item here under dead-code analysis. + #[cfg_attr(not(test), allow(dead_code))] + pub fn plausible(&self) -> bool { + matches!(self, GpuPlausibility::Plausible { .. }) + } + + /// How severe this verdict is, for picking the one that speaks for a probe + /// of several GPUs: a proven overrun outranks an indeterminate, which + /// outranks a pass. + fn rank(&self) -> u8 { + match self { + GpuPlausibility::Plausible { .. } => 0, + GpuPlausibility::BoardTelemetryUnusable { .. } => 1, + GpuPlausibility::CounterOverRate { .. } => 2, + } + } + + /// The figure this verdict turns on, to order two of equal rank by. + fn magnitude(&self) -> u64 { + match self { + GpuPlausibility::Plausible { rate_mw } => *rate_mw, + GpuPlausibility::CounterOverRate { rate_mw, .. } => *rate_mw, + GpuPlausibility::BoardTelemetryUnusable { draw_mw, .. } => *draw_mw, + } + } +} + +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` +pub(crate) fn plausible_gpu_counter( + energy_mj: u64, + window: Duration, + draws_mw: &[u32], + ceiling_mw: Option, +) -> GpuPlausibility { let ns = window.as_nanos().max(1); // mJ per second is mW. let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; - if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { - let cap = u128::from(limit) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1; - if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { - return false; + let clamp = |v: u128| u64::try_from(v).unwrap_or(u64::MAX); + let cap = ceiling_mw + .filter(|&l| l > 0) + .map(|l| u128::from(l) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1); + + // A draw reading past the board's own limit is not a reading: an RTX 4050 + // Laptop GPU reports about 590 W for a second after it wakes. Such a + // reading is dropped, and only that reading — it is one broken sample, not + // a broken bracket. Voiding the whole judgement on it is what this code + // used to do, and the hardware test on jetson-hub refused that shape: one + // spike among otherwise sane 14.8 W samples turned a counter running + // persistently at 4-30x the board's draw into an indeterminate, which is a + // worse diagnosis than the overrun it actually is. + // + // Dropping them also keeps the mean honest, which is what the old + // short-circuit was really for. The mean summed every reading, so a 590 W + // spike inflated it and made the mean-draw bound *more* permissive; that + // only went unnoticed because any such reading failed the check first. + let (usable, unusable): (Vec, Vec) = draws_mw + .iter() + .map(|&d| u128::from(d)) + .partition(|&d| cap.is_none_or(|c| d <= c)); + + if let Some(cap) = cap { + if rate_mw > cap { + return GpuPlausibility::CounterOverRate { + rate_mw: clamp(rate_mw), + bound_mw: clamp(cap), + bound: GpuBound::PowerLimit, + }; + } + // Only when nothing usable is left is there no yardstick at all. + if usable.is_empty() && !unusable.is_empty() { + return GpuPlausibility::BoardTelemetryUnusable { + draw_mw: clamp(unusable[0]), + limit_mw: clamp(cap), + }; } } - if !draws_mw.is_empty() { - let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; - if rate_mw > GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW { - return false; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as u128; + let bound = GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW; + if rate_mw > bound { + return GpuPlausibility::CounterOverRate { + rate_mw: clamp(rate_mw), + bound_mw: clamp(bound), + bound: GpuBound::MeanDraw, + }; } } - true + GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) } } /// What a GPU counter's probe saw beside the rule's verdict: each GPU's @@ -362,21 +502,41 @@ impl GpuWatch { } /// The liveness a GPU counter's probe gives it: the rule's `verdict`, - /// except that a live counter one of whose GPUs did not plausibly count - /// its board's energy over the probe, beside each GPU's maximum power - /// limit `ceilings_mw`, is [`CounterLiveness::Implausible`]. + /// except that a live counter is re-judged on whether its GPUs counted + /// their boards' energy over the probe, beside each GPU's maximum power + /// limit `ceilings_mw`: a counter proven to run faster than its board can + /// draw is [`CounterLiveness::Implausible`], and a board whose own + /// telemetry is unusable leaves it [`CounterLiveness::Unjudgeable`]. pub(crate) fn liveness(&self, verdict: CounterLiveness, ceilings_mw: &[Option]) -> CounterLiveness { let (Some((first, t0)), Some((last, t1))) = (&self.first, &self.last) else { return verdict; }; + if verdict != CounterLiveness::Live { + return verdict; + } let window = t1.saturating_duration_since(*t0); - let plausible = first.iter().zip(last).enumerate().all(|(i, (a, b))| { - let draws = self.draws_mw.get(i).map(Vec::as_slice).unwrap_or(&[]); - plausible_gpu_counter(b.saturating_sub(*a) / 1_000, window, draws, ceilings_mw.get(i).copied().flatten()) - }); - match verdict { - CounterLiveness::Live if !plausible => CounterLiveness::Implausible, - other => other, + // The verdict that speaks for the probe is the most severe of its + // GPUs', and among equals the largest figure: one healthy GPU does not + // excuse the probe, and one board's broken telemetry does not condemn + // a counter nothing measured. + let worst = first + .iter() + .zip(last) + .enumerate() + .map(|(i, (a, b))| { + let draws = self.draws_mw.get(i).map(Vec::as_slice).unwrap_or(&[]); + plausible_gpu_counter( + b.saturating_sub(*a) / 1_000, + window, + draws, + ceilings_mw.get(i).copied().flatten(), + ) + }) + .max_by_key(|v| (v.rank(), v.magnitude())); + match worst { + Some(GpuPlausibility::CounterOverRate { .. }) => CounterLiveness::Implausible, + Some(GpuPlausibility::BoardTelemetryUnusable { .. }) => CounterLiveness::Unjudgeable, + _ => CounterLiveness::Live, } } } @@ -404,17 +564,89 @@ pub(crate) mod tests { #[test] fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { let w = Duration::from_millis(500); - assert!(!plausible_gpu_counter(152_250, w, &[5_550, 7_660], Some(65_000)), "304.5 W against a 6.6 W draw"); - assert!(!plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000)), "44 W against a 10.4 W draw"); - assert!(!plausible_gpu_counter(60_000, w, &[], Some(65_000)), "120 W on a 65 W board"); - assert!(!plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000)), "a 590 W draw reading"); - assert!(plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000)), "13 W against a 13 W draw"); - assert!(plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000)), "300 W against a 280 W draw"); - assert!(plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000)), "205 W is 2 x 100 W + 5 W"); - assert!(!plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000))); - assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); - assert!(!plausible_gpu_counter(48_751, w, &[], Some(65_000))); - assert!(plausible_gpu_counter(1_000_000, w, &[], None), "nothing to judge by"); + // Over a 500 ms window a counter's rate in mW is twice its mJ. + let over = |rate_mw, bound_mw, bound| GpuPlausibility::CounterOverRate { rate_mw, bound_mw, bound }; + + // The figures, not just a `false`: 304.5 W against a 97.5 W ceiling is + // a 3.1x overrun, and an operator cannot act on a bare bool. + assert_eq!( + plausible_gpu_counter(152_250, w, &[5_550, 7_660], Some(65_000)), + over(304_500, 97_500, GpuBound::PowerLimit), + "304.5 W against a 6.6 W draw" + ); + assert_eq!( + plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000)), + over(44_000, 25_800, GpuBound::MeanDraw), + "44 W against a 10.4 W draw" + ); + assert_eq!( + plausible_gpu_counter(60_000, w, &[], Some(65_000)), + over(120_000, 97_500, GpuBound::PowerLimit), + "120 W on a 65 W board" + ); + + // A draw reading of 590 W (the board's first second after waking) is + // no reading. It is dropped, and only it: the remaining 10.3 W reading + // still judges the counter, and 88 W against it is an overrun. This + // used to assert a bare `false` meaning "a 590 W draw reading", which + // is the one thing the hardware test on jetson-hub refused: it turned + // a persistent 4-30x overrun into "cannot judge". + assert_eq!( + plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000)), + over(88_000, 25_600, GpuBound::MeanDraw) + ); + // Dropping it also keeps the mean honest, and this pins that: the + // bound is twice 10 W plus 5 W, from the usable reading alone. Had the + // 590 W reading been summed in, the mean would be 300 W and the bound + // 605 W, which would excuse this counter. + assert_eq!( + plausible_gpu_counter(15_000, w, &[590_000, 10_000], Some(65_000)), + over(30_000, 25_000, GpuBound::MeanDraw) + ); + // Only when no usable reading is left is there no yardstick at all, + // and then the verdict is an indeterminate about the board, not a + // finding about the counter. + assert_eq!( + plausible_gpu_counter(44_000, w, &[590_000], Some(65_000)), + GpuPlausibility::BoardTelemetryUnusable { draw_mw: 590_000, limit_mw: 97_500 } + ); + // A counter past the board's own limit is a finding whatever the + // telemetry says: the rate bound is checked before the yardstick is. + assert_eq!( + plausible_gpu_counter(60_000, w, &[590_000], Some(65_000)), + over(120_000, 97_500, GpuBound::PowerLimit) + ); + + assert_eq!( + plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000)), + GpuPlausibility::Plausible { rate_mw: 13_000 }, + "13 W against a 13 W draw" + ); + assert_eq!( + plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000)), + GpuPlausibility::Plausible { rate_mw: 300_000 }, + "300 W against a 280 W draw" + ); + assert!( + plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000)).plausible(), + "205 W is 2 x 100 W + 5 W" + ); + assert_eq!( + plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000)), + over(205_002, 205_000, GpuBound::MeanDraw) + ); + assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)).plausible(), "97.5 W is 1.5 x 65 W"); + assert_eq!( + plausible_gpu_counter(48_751, w, &[], Some(65_000)), + over(97_502, 97_500, GpuBound::PowerLimit) + ); + // Nothing to judge it by: no bound applies, and with no limit to + // exceed no reading can be unusable either. + assert_eq!( + plausible_gpu_counter(1_000_000, w, &[], None), + GpuPlausibility::Plausible { rate_mw: 2_000_000 }, + "nothing to judge by" + ); } /// The liveness a GPU probe gives: each GPU's rise from the first sample diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs index 778950b1f..7b5c13b57 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs @@ -63,9 +63,17 @@ enum Mode { fn mode_for(supported: bool, liveness: CounterLiveness) -> Mode { match (supported, liveness) { (true, CounterLiveness::Live) => Mode::Counter, - (_, CounterLiveness::Live | CounterLiveness::Frozen | CounterLiveness::Unavailable | CounterLiveness::Implausible) => { - Mode::PowerIntegral - } + // An unjudgeable counter is no more admissible than an implausible + // one: neither can be graded `HwShunt`, so both fall back to the + // vendor's power model. + ( + _, + CounterLiveness::Live + | CounterLiveness::Frozen + | CounterLiveness::Unavailable + | CounterLiveness::Implausible + | CounterLiveness::Unjudgeable, + ) => Mode::PowerIntegral, } } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs index 8502f1f63..e33f01bcb 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs @@ -205,7 +205,10 @@ impl ProcessCpuCounter { fn admits(liveness: CounterLiveness) -> bool { match liveness { CounterLiveness::Live => true, - CounterLiveness::Unavailable | CounterLiveness::Frozen | CounterLiveness::Implausible => false, + CounterLiveness::Unavailable + | CounterLiveness::Frozen + | CounterLiveness::Implausible + | CounterLiveness::Unjudgeable => false, } } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process_gpu.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process_gpu.rs index c56ff511a..eacc51962 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process_gpu.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process_gpu.rs @@ -396,7 +396,10 @@ pub fn probe_gpu_liveness_uncached(load: impl FnMut()) -> CounterLiveness { fn admits(liveness: CounterLiveness) -> bool { match liveness { CounterLiveness::Live => true, - CounterLiveness::Unavailable | CounterLiveness::Frozen | CounterLiveness::Implausible => false, + CounterLiveness::Unavailable + | CounterLiveness::Frozen + | CounterLiveness::Implausible + | CounterLiveness::Unjudgeable => false, } } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs index accf2cea8..9be892dbc 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs @@ -270,7 +270,10 @@ impl RaplCounter { fn admits(liveness: CounterLiveness) -> bool { match liveness { CounterLiveness::Live => true, - CounterLiveness::Unavailable | CounterLiveness::Frozen | CounterLiveness::Implausible => false, + CounterLiveness::Unavailable + | CounterLiveness::Frozen + | CounterLiveness::Implausible + | CounterLiveness::Unjudgeable => false, } } -- 2.54.0 (Apple Git-157) From e4e5b3d528aefa23033af78232b0a1c17751d051 Mon Sep 17 00:00:00 2001 From: David Charlot Date: Fri, 9 Oct 2026 08:32:14 -0400 Subject: [PATCH 3/7] sandbox: cover the new status where the declined ones are enumerated Two tests assert that a counter which is not live is never opened, and both listed the declined statuses by hand rather than matching exhaustively, so adding `Unjudgeable` left it the one status whose gating nothing checked. `nvml.rs`'s doc even claims the gate "declines every status but `Live`", which its own test did not cover. `counter_admits` compares against `Live`, so the behaviour was already right; this is the test catching up with it. The second site is Linux-gated, so it needs the aarch64-unknown-linux-gnu cross-check to compile at all. Co-Authored-By: Claude Opus 5 --- .../sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs | 7 ++++++- sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs index 054869898..057eeb07a 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs @@ -2236,7 +2236,12 @@ mod tests { /// its own error passes through. #[test] fn a_gated_counter_is_opened_only_when_live() { - for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible] { + for status in [ + CounterStatus::Unavailable, + CounterStatus::Frozen, + CounterStatus::Implausible, + CounterStatus::Unjudgeable, + ] { let opened = Cell::new(false); let r = gated("RAPL", status, || { opened.set(true); diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs index ba357a594..391965e6d 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs @@ -305,7 +305,12 @@ mod tests { /// `Live` builds the probe wherever NVML opens with a supported GPU. #[test] fn only_a_live_status_builds_the_nvml_probe() { - for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible] { + for status in [ + CounterStatus::Unavailable, + CounterStatus::Frozen, + CounterStatus::Implausible, + CounterStatus::Unjudgeable, + ] { match NvmlProbe::given(status) { Err(ProbeError::Unavailable(why)) => assert!(why.contains("NVML"), "{why}"), Err(e) => panic!("{status:?}: {e}"), -- 2.54.0 (Apple Git-157) From 12986a5db7c400473987614afe84ada12bdb3fdb Mon Sep 17 00:00:00 2001 From: David Charlot Date: Fri, 9 Oct 2026 08:37:44 -0400 Subject: [PATCH 4/7] eoc: a broken draw reading condemned the counter it was meant to judge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The third and last crate carrying the defect #185 fixed in jcp-energy. `gpu_plausibility::plausible` returned a bool and folded two unlike things into the one `false`: if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) The left is the counter running faster than the board can draw — a finding about the counter. The right is the *board's own telemetry* reading past the board's own limit, which says the yardstick is broken, not the counter. Its test named the case too: `"a 590 W draw reading"` asserted `false`, and 590 W is what an RTX 4050 Laptop GPU reports for a second after it wakes. So the reading is dropped, and only that reading. Only when nothing usable is left is there no yardstick, and then the verdict is `BoardTelemetryUnusable` mapping to a new `CounterStatus::Unjudgeable`, kept apart from `Implausible` whose doc asserts "It is not counting the board's energy" — a conclusion this case does not support. `admits_vendor_counter` compares against `Live`, so the new variant is inadmissible without a change there. `GpuPlausibility` carries the rate, the bound and which bound it was, where the bool discarded the magnitude. A test pins the mean-draw bound at twice the usable 10 W reading plus 5 W, which it could not be if the 590 W spike were still in the mean — the old mean summed every reading, so a bad reading made that bound *more* permissive, and only the short-circuit hid it. Five tests enumerate the declined statuses by hand rather than matching exhaustively, so each now lists `Unjudgeable`; without that it would be the one status whose declining nothing checked. Two of them only compile under `--features cuda`. Checked for aarch64-unknown-linux-gnu with and without `--features cuda`; 75 tests pass in the crate and the workspace is clean. Co-Authored-By: Claude Opus 5 --- eoc/eoc-rs/crates/eoc-meter/src/lib.rs | 301 +++++++++++++++++++++---- 1 file changed, 251 insertions(+), 50 deletions(-) diff --git a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs index 513ac80d3..b5005189e 100644 --- a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs +++ b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs @@ -3689,12 +3689,24 @@ pub enum CounterStatus { Frozen, /// The counter kept advancing, but a GPU counter (NVML's) ran faster than /// its board can draw power over the probe: past 1.5 times the board's - /// maximum power limit, past twice the board's own power draw plus 5 W, - /// or beside a draw reading itself past that limit. It is not counting the - /// board's energy. Seen on an RTX 4050 Laptop GPU (driver 595.84), whose - /// counter runs at 4 to 30 times the board's draw while it is polled. - /// Never constructed. + /// maximum power limit, or past twice the board's own power draw plus + /// 5 W. It is not counting the board's energy. Seen on an RTX 4050 Laptop + /// GPU (driver 595.84), whose counter runs at 4 to 30 times the board's + /// draw while it is polled. Never constructed. Implausible, + /// The counter kept advancing and could not be judged, because the board's + /// own power telemetry was unusable over the probe: a draw reading past + /// the board's own maximum power limit (an RTX 4050 Laptop GPU reports + /// about 590 W for a second after it wakes). + /// + /// Split out from [`CounterStatus::Implausible`], which used to cover this + /// — its doc then asserted "it is not counting the board's energy", a + /// conclusion this case does not support. The yardstick is broken, not + /// necessarily the counter. Never selected either way: unjudgeable is not + /// admissible, and erring toward declining is the safe direction. The + /// distinction is the diagnosis an operator gets, which is the difference + /// between replacing a counter and waiting for the board to settle. + Unjudgeable, /// The counter kept advancing while the probe ran: in at least five /// separate sample intervals, at least 80% of those observed, over at /// least 300 ms. The rung is selected. @@ -4787,38 +4799,138 @@ mod gpu_plausibility { /// NVML's documented accuracy of a board's power draw, ±5 W, in mW. pub(crate) const DRAW_SLACK_MW: u128 = 5_000; - /// Whether a GPU energy counter that counted `energy_mj` over `window` - /// plausibly counted its board's energy, beside the board's power draw - /// `draws_mw` read with the samples and its maximum power limit - /// `ceiling_mw`. Not when the counter ran past [`CEILING_MARGIN`] of the - /// limit, or a draw reading did (the board's own telemetry then is not a - /// reading either: an RTX 4050 Laptop GPU reports 590 W for about a second - /// after it wakes), or the counter ran past [`DRAW_FACTOR`] times the mean - /// draw plus [`DRAW_SLACK_MW`]. A bound whose reading is missing is not - /// applied. + /// Which of the two bounds a GPU counter ran past. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum GpuBound { + /// [`CEILING_MARGIN`] of the board's maximum power limit. + PowerLimit, + /// [`DRAW_FACTOR`] times the board's mean draw plus [`DRAW_SLACK_MW`]. + MeanDraw, + } + + /// What the plausibility check found, with the figures it found it from. + /// + /// A `bool` was the whole of this, and it threw away everything an + /// operator needs: which bound broke, by how much, and the rate it broke + /// it at. A counter 1.1 times over a bound and one 20 times over were the + /// same answer. The figures also make the judgement reproducible — the + /// margins are constants in this module, and re-grading a past finding + /// against a different margin is only possible if the rate survived the + /// check. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum GpuPlausibility { + /// The counter's rate sits inside every bound whose reading was + /// present. + Plausible { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + }, + /// The counter ran faster than the board can draw. + CounterOverRate { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + /// The bound it broke, in milliwatts. + bound_mw: u64, + /// Which bound that was. + bound: GpuBound, + }, + /// The board's own telemetry read past the board's own power limit, so + /// there is no yardstick left to judge the counter against. An + /// indeterminate, not a finding about the counter — see + /// [`CounterStatus::Unjudgeable`]. + BoardTelemetryUnusable { + /// The offending draw reading, in milliwatts. + draw_mw: u64, + /// The limit it exceeded (the margin-adjusted cap), in milliwatts. + limit_mw: u64, + }, + } + + impl GpuPlausibility { + /// Whether the counter was judged to be counting its board's energy. + /// + /// `false` for an indeterminate as well as for an overrun: unjudgeable + /// is not admissible, and erring toward declining is the safe + /// direction. Use the variant itself to tell the two apart. + // + // The module's own path matches on the variants rather than call this. + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) fn plausible(&self) -> bool { + matches!(self, GpuPlausibility::Plausible { .. }) + } + } + + /// What a GPU energy counter that counted `energy_mj` over `window` was + /// judged to be counting, beside the board's power draw `draws_mw` read + /// with the samples and its maximum power limit `ceiling_mw`. It over-ran + /// if it ran past [`CEILING_MARGIN`] of the limit, or past [`DRAW_FACTOR`] + /// times the mean draw plus [`DRAW_SLACK_MW`]. A bound whose reading is + /// missing is not applied. /// /// Liveness says the counter is republished continuously; this says what /// it republishes is the board's energy. On the RTX 4050 Laptop GPU /// (driver 595.84) the total-energy counter runs several times faster than /// the board draws for as long as it is polled: at the probe's 100 ms wait /// it ran at 58-305 W while the board drew 5-14 W against a 65 W limit. - pub(crate) fn plausible(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { + pub(crate) fn plausible( + energy_mj: u64, + window: Duration, + draws_mw: &[u32], + ceiling_mw: Option, + ) -> GpuPlausibility { let ns = window.as_nanos().max(1); // mJ per second is mW. let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; - if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { - let cap = u128::from(limit) * CEILING_MARGIN.0 / CEILING_MARGIN.1; - if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { - return false; + let clamp = |v: u128| u64::try_from(v).unwrap_or(u64::MAX); + let cap = + ceiling_mw.filter(|&l| l > 0).map(|l| u128::from(l) * CEILING_MARGIN.0 / CEILING_MARGIN.1); + + // A draw reading past the board's own limit is not a reading: an RTX + // 4050 Laptop GPU reports about 590 W for a second after it wakes. + // Such a reading is dropped, and only that reading — it is one broken + // sample, not a broken bracket. Voiding the whole judgement on it is + // what this code used to do, and the hardware test on jetson-hub + // refused that shape: one spike among otherwise sane 14.8 W samples + // turned a counter running persistently at 4-30x the board's draw into + // an indeterminate, which is a worse diagnosis than the overrun it + // actually is. + // + // Dropping them also keeps the mean honest, which is what the old + // short-circuit was really for. The mean summed every reading, so a + // 590 W spike inflated it and made the mean-draw bound *more* + // permissive; that only went unnoticed because any such reading failed + // the check first. + let (usable, unusable): (Vec, Vec) = + draws_mw.iter().map(|&d| u128::from(d)).partition(|&d| cap.is_none_or(|c| d <= c)); + + if let Some(cap) = cap { + if rate_mw > cap { + return GpuPlausibility::CounterOverRate { + rate_mw: clamp(rate_mw), + bound_mw: clamp(cap), + bound: GpuBound::PowerLimit, + }; + } + // Only when nothing usable is left is there no yardstick at all. + if usable.is_empty() && !unusable.is_empty() { + return GpuPlausibility::BoardTelemetryUnusable { + draw_mw: clamp(unusable[0]), + limit_mw: clamp(cap), + }; } } - if !draws_mw.is_empty() { - let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; - if rate_mw > DRAW_FACTOR * mean + DRAW_SLACK_MW { - return false; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as u128; + let bound = DRAW_FACTOR * mean + DRAW_SLACK_MW; + if rate_mw > bound { + return GpuPlausibility::CounterOverRate { + rate_mw: clamp(rate_mw), + bound_mw: clamp(bound), + bound: GpuBound::MeanDraw, + }; } } - true + GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) } } /// What the GPU probe saw of the board beside its counter: the counter's @@ -4844,19 +4956,28 @@ mod gpu_plausibility { } } - /// The status a GPU probe gives: the liveness rule's `status`, except - /// that a live counter which did not plausibly count its board's energy - /// over the probe is [`CounterStatus::Implausible`]. + /// The status a GPU probe gives: the liveness rule's `status`, except that + /// a live counter is re-judged on what it was counting over the probe. One + /// proven to run faster than its board can draw is + /// [`CounterStatus::Implausible`]; one whose board's own telemetry was + /// unusable is [`CounterStatus::Unjudgeable`], which is a different + /// diagnosis and not a claim about the counter. pub(crate) fn status_of(status: CounterStatus, gpu: &GpuWatch) -> CounterStatus { - let plausible = match (gpu.first, gpu.last) { + if status != CounterStatus::Live { + return status; + } + match (gpu.first, gpu.last) { (Some((a, t0)), Some((b, t1))) => { - plausible(b.saturating_sub(a), t1.saturating_sub(t0), &gpu.draws_mw, gpu.ceiling_mw) + match plausible(b.saturating_sub(a), t1.saturating_sub(t0), &gpu.draws_mw, gpu.ceiling_mw) { + GpuPlausibility::Plausible { .. } => CounterStatus::Live, + GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible, + GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable, + } } - _ => true, - }; - match status { - CounterStatus::Live if !plausible => CounterStatus::Implausible, - other => other, + // No pair of samples to compute a rate from. Not a judgement that + // the counter is sound; liveness already passed, and there is + // nothing here to contradict it. + _ => CounterStatus::Live, } } @@ -4874,17 +4995,92 @@ mod gpu_plausibility { #[test] fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { let w = Duration::from_millis(500); - assert!(!plausible(59_700, w, &[12_970, 13_520], Some(65_000)), "119.4 W against a 13 W draw"); - assert!(!plausible(22_000, w, &[10_400; 6], Some(65_000)), "44 W against a 10.4 W draw"); - assert!(!plausible(60_000, w, &[], Some(65_000)), "120 W on a 65 W board"); - assert!(!plausible(44_000, w, &[590_000, 10_300], Some(65_000)), "a 590 W draw reading"); - assert!(plausible(6_500, w, &[13_000, 13_500], Some(65_000)), "13 W against a 13 W draw"); - assert!(plausible(150_000, w, &[280_000; 5], Some(700_000)), "300 W against a 280 W draw"); - assert!(plausible(102_500, w, &[100_000; 5], Some(700_000)), "205 W is 2 x 100 W + 5 W"); - assert!(!plausible(102_501, w, &[100_000; 5], Some(700_000))); - assert!(plausible(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); - assert!(!plausible(48_751, w, &[], Some(65_000))); - assert!(plausible(1_000_000, w, &[], None), "nothing to judge by"); + // Over a 500 ms window a counter's rate in mW is twice its mJ. + let over = + |rate_mw, bound_mw, bound| GpuPlausibility::CounterOverRate { rate_mw, bound_mw, bound }; + + // The figures, not just a `false`: an operator cannot act on a + // bare bool, and a finding cannot be re-graded against a changed + // margin unless the rate survived the check. + assert_eq!( + plausible(59_700, w, &[12_970, 13_520], Some(65_000)), + over(119_400, 97_500, GpuBound::PowerLimit), + "119.4 W against a 13 W draw" + ); + assert_eq!( + plausible(22_000, w, &[10_400; 6], Some(65_000)), + over(44_000, 25_800, GpuBound::MeanDraw), + "44 W against a 10.4 W draw" + ); + assert_eq!( + plausible(60_000, w, &[], Some(65_000)), + over(120_000, 97_500, GpuBound::PowerLimit), + "120 W on a 65 W board" + ); + + // A draw reading of 590 W (the board's first second after waking) + // is no reading. It is dropped, and only it: the remaining 10.3 W + // reading still judges the counter, and 88 W against it is an + // overrun. This used to assert a bare `false` meaning "a 590 W + // draw reading", which is the one thing the hardware test on + // jetson-hub refused: it turned a persistent 4-30x overrun into + // "cannot judge". + assert_eq!( + plausible(44_000, w, &[590_000, 10_300], Some(65_000)), + over(88_000, 25_600, GpuBound::MeanDraw) + ); + // Dropping it also keeps the mean honest, and this pins that: the + // bound is twice 10 W plus 5 W, from the usable reading alone. Had + // the 590 W reading been summed in, the mean would be 300 W and + // the bound 605 W, which would excuse this counter. + assert_eq!( + plausible(15_000, w, &[590_000, 10_000], Some(65_000)), + over(30_000, 25_000, GpuBound::MeanDraw) + ); + // Only when no usable reading is left is there no yardstick at + // all, and then the verdict is an indeterminate about the board, + // not a finding about the counter. + assert_eq!( + plausible(44_000, w, &[590_000], Some(65_000)), + GpuPlausibility::BoardTelemetryUnusable { draw_mw: 590_000, limit_mw: 97_500 } + ); + // A counter past the board's own limit is a finding whatever the + // telemetry says: the rate bound is checked before the yardstick. + assert_eq!( + plausible(60_000, w, &[590_000], Some(65_000)), + over(120_000, 97_500, GpuBound::PowerLimit) + ); + + assert_eq!( + plausible(6_500, w, &[13_000, 13_500], Some(65_000)), + GpuPlausibility::Plausible { rate_mw: 13_000 }, + "13 W against a 13 W draw" + ); + assert_eq!( + plausible(150_000, w, &[280_000; 5], Some(700_000)), + GpuPlausibility::Plausible { rate_mw: 300_000 }, + "300 W against a 280 W draw" + ); + assert!( + plausible(102_500, w, &[100_000; 5], Some(700_000)).plausible(), + "205 W is 2 x 100 W + 5 W" + ); + assert_eq!( + plausible(102_501, w, &[100_000; 5], Some(700_000)), + over(205_002, 205_000, GpuBound::MeanDraw) + ); + assert!(plausible(48_750, w, &[], Some(65_000)).plausible(), "97.5 W is 1.5 x 65 W"); + assert_eq!( + plausible(48_751, w, &[], Some(65_000)), + over(97_502, 97_500, GpuBound::PowerLimit) + ); + // Nothing to judge it by: no bound applies, and with no limit to + // exceed no reading can be unusable either. + assert_eq!( + plausible(1_000_000, w, &[], None), + GpuPlausibility::Plausible { rate_mw: 2_000_000 }, + "nothing to judge by" + ); } /// A live counter that ran past its board is `Implausible`; one that @@ -4903,7 +5099,7 @@ mod gpu_plausibility { assert_eq!(status_of(S::Live, &watch(13_000, Some(65_000))), S::Implausible); assert_eq!(status_of(S::Live, &watch(115_000, Some(700_000))), S::Live); assert_eq!(status_of(S::Live, &GpuWatch::default()), S::Live, "nothing to judge by"); - for status in [S::Unavailable, S::Frozen] { + for status in [S::Unavailable, S::Frozen, S::Unjudgeable] { assert_eq!(status_of(status, &watch(13_000, Some(65_000))), status); } } @@ -5045,7 +5241,12 @@ mod nvml { #[test] fn only_a_live_nvml_status_constructs_the_counter() { let attempts = || OPEN_ATTEMPTS.with(|n| n.get()); - for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible] { + for status in [ + CounterStatus::Unavailable, + CounterStatus::Frozen, + CounterStatus::Implausible, + CounterStatus::Unjudgeable, + ] { let before = attempts(); assert!( matches!(NvmlCounter::try_detect_given(status), Err(Error::NoCounter)), @@ -5158,7 +5359,7 @@ mod tests { fn a_vendor_counter_is_selected_only_when_live() { use CounterStatus as S; let declined_ioreport = || IoReportStatus::CpuRailFrozen; - for status in [S::Unavailable, S::Frozen, S::Implausible] { + for status in [S::Unavailable, S::Frozen, S::Implausible, S::Unjudgeable] { let picked = detect_with(|| status, declined_ioreport).name(); assert_ne!(picked, "rapl", "a {status:?} RAPL counter was selected"); } @@ -5182,7 +5383,7 @@ mod tests { IoReportStatus::CpuRailFrozen, IoReportStatus::Live, ]; - for rapl in [S::Unavailable, S::Frozen, S::Implausible, S::Live] { + for rapl in [S::Unavailable, S::Frozen, S::Implausible, S::Unjudgeable, S::Live] { for io in ioreport { let picked = detect_with(|| rapl, || io).name(); assert_ne!(picked, "process-energy", "detect_with({rapl:?}, {io:?})"); @@ -5201,7 +5402,7 @@ mod tests { let fallback = || -> Box { Box::new(StubCounter) }; let picked = detect_in_process_with(|| S::Live, fallback).name(); assert_eq!(picked == "process-energy", cfg!(target_os = "macos"), "Live picked {picked}"); - for status in [S::Unavailable, S::Frozen] { + for status in [S::Unavailable, S::Frozen, S::Unjudgeable] { assert_eq!(detect_in_process_with(|| status, fallback).name(), "stub", "{status:?}"); } } -- 2.54.0 (Apple Git-157) From b739eb26eff40cf11462a336a711e710f3330fd4 Mon Sep 17 00:00:00 2001 From: David Charlot Date: Fri, 9 Oct 2026 08:38:04 -0400 Subject: [PATCH 5/7] energy: the hardware oracle still encoded the rule the crates no longer use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each of the four crates' `nvml_hardware` tests carries its own oracle: the plausibility rule rewritten from the rule itself, in f64, from `nvidia-smi` rather than the crate's own sampling, so that `energy-hardware.yml` on jetson-hub checks the crate against an independent implementation and not against itself. That only holds while the oracle encodes the rule the crate implements, and all four still had the old conflation: if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { return false } jcp-energy's included — #185 fixed that crate's implementation and left its oracle behind, so since then the two have disagreed about what a draw reading past the board's own limit means. On jetson-hub's RTX 4050 the disagreement is masked: that counter runs persistently at 4-30x the board's draw, so the implementation reaches `Implausible` by the rate bound and the oracle reaches it by the broken reading, and they agree on the verdict for different reasons. It would surface on a board whose counter is sane and whose telemetry spikes — the implementation would judge on the surviving reading and say `Live`, the oracle would say `Implausible`, and the test asserts they agree. So the oracles now drop a draw reading past the limit, and only that reading, and answer `Unjudgeable` when nothing usable is left — with a `Verdict` variant for it and an arm in the mapping each test makes against the crate's own status. The run's verdict is the most severe of its GPUs', as the crates' now is, so a definite overrun outranks an indeterminate. These tests are all `#![cfg(target_os = "linux")]`, two of them also behind `--features nvml` and one behind `--features cuda`, so none of them compiles in a default build on this machine; checked for aarch64-unknown-linux-gnu with each crate's own feature. Co-Authored-By: Claude Opus 5 --- .../crates/eoc-meter/tests/nvml_hardware.rs | 50 +++++++++++++++---- .../crates/jcp-energy/tests/nvml_hardware.rs | 50 +++++++++++++++---- .../jouleclaw-energy/tests/nvml_hardware.rs | 50 +++++++++++++++---- .../sandbox-meter/tests/nvml_hardware.rs | 50 +++++++++++++++---- 4 files changed, 156 insertions(+), 44 deletions(-) diff --git a/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs b/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs index 9a8980170..bc1401c0a 100644 --- a/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs +++ b/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs @@ -195,9 +195,13 @@ mod nvml_oracle { Unavailable, /// A counter did not keep advancing. Frozen, - /// Every counter kept advancing, but one is not plausibly its board's - /// energy (see the module doc). + /// Every counter kept advancing, but one ran faster than its board + /// can draw (see the module doc). Implausible, + /// Every counter kept advancing, and one could not be judged because + /// its board's own telemetry read past the board's own limit. The + /// yardstick is broken, not necessarily the counter. + Unjudgeable, /// Every counter kept advancing, plausibly its board's energy. Live, } @@ -206,19 +210,36 @@ mod nvml_oracle { /// average `rate_w`, the board's draws `draws_w` around the window and /// its maximum power limit `ceiling_w`. A bound whose input is missing /// is not applied. - pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { - if let Some(c) = ceiling_w { - if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { - return false; + /// + /// A draw reading past the board's own limit is not a reading — an RTX + /// 4050 Laptop GPU reports about 590 W for a second after it wakes — so it + /// is dropped, and only it. Only when nothing usable is left is there no + /// yardstick at all, and then the counter is `Unjudgeable` rather than + /// `Implausible`: the yardstick is broken, not necessarily the counter. + /// + /// This oracle is written from the rule rather than from the crate, in + /// f64 and from `nvidia-smi` rather than the crate's own sampling, so it + /// is an independent check on the same rule — which is only true while it + /// encodes the rule the crate implements. + pub fn judge(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> Verdict { + let cap = ceiling_w.map(|c| 1.5 * c); + let (usable, unusable): (Vec, Vec) = + draws_w.iter().copied().partition(|d| cap.is_none_or(|c| *d <= c)); + if let Some(c) = cap { + if rate_w > c { + return Verdict::Implausible; + } + if usable.is_empty() && !unusable.is_empty() { + return Verdict::Unjudgeable; } } - if !draws_w.is_empty() { - let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as f64; if rate_w > 2.0 * mean + 5.0 { - return false; + return Verdict::Implausible; } } - true + Verdict::Live } /// What the oracle saw. @@ -301,10 +322,16 @@ mod nvml_oracle { let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + // The verdict that speaks for the run is the most severe of its GPUs', + // as the crate's is: a counter proven to over-run is a finding, while + // one board's broken telemetry only withholds a judgement. + let judged: Vec = (0..n).map(|i| judge(rate_w[i], &draws_w[i], ceiling_w[i])).collect(); let verdict = if !live(&advances, intervals, window) { Verdict::Frozen - } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + } else if judged.contains(&Verdict::Implausible) { Verdict::Implausible + } else if judged.contains(&Verdict::Unjudgeable) { + Verdict::Unjudgeable } else { Verdict::Live }; @@ -359,6 +386,7 @@ fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), Verdict::Implausible => assert_eq!(status, CounterStatus::Implausible), + Verdict::Unjudgeable => assert_eq!(status, CounterStatus::Unjudgeable), Verdict::Live => {} } assert_eq!(constructed, status == CounterStatus::Live, "construction disagrees with the status"); diff --git a/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs b/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs index 76096fb1b..3ccbc92db 100644 --- a/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs +++ b/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs @@ -194,9 +194,13 @@ mod nvml_oracle { Unavailable, /// A counter did not keep advancing. Frozen, - /// Every counter kept advancing, but one is not plausibly its board's - /// energy (see the module doc). + /// Every counter kept advancing, but one ran faster than its board + /// can draw (see the module doc). Implausible, + /// Every counter kept advancing, and one could not be judged because + /// its board's own telemetry read past the board's own limit. The + /// yardstick is broken, not necessarily the counter. + Unjudgeable, /// Every counter kept advancing, plausibly its board's energy. Live, } @@ -205,19 +209,36 @@ mod nvml_oracle { /// average `rate_w`, the board's draws `draws_w` around the window and /// its maximum power limit `ceiling_w`. A bound whose input is missing /// is not applied. - pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { - if let Some(c) = ceiling_w { - if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { - return false; + /// + /// A draw reading past the board's own limit is not a reading — an RTX + /// 4050 Laptop GPU reports about 590 W for a second after it wakes — so it + /// is dropped, and only it. Only when nothing usable is left is there no + /// yardstick at all, and then the counter is `Unjudgeable` rather than + /// `Implausible`: the yardstick is broken, not necessarily the counter. + /// + /// This oracle is written from the rule rather than from the crate, in + /// f64 and from `nvidia-smi` rather than the crate's own sampling, so it + /// is an independent check on the same rule — which is only true while it + /// encodes the rule the crate implements. + pub fn judge(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> Verdict { + let cap = ceiling_w.map(|c| 1.5 * c); + let (usable, unusable): (Vec, Vec) = + draws_w.iter().copied().partition(|d| cap.is_none_or(|c| *d <= c)); + if let Some(c) = cap { + if rate_w > c { + return Verdict::Implausible; + } + if usable.is_empty() && !unusable.is_empty() { + return Verdict::Unjudgeable; } } - if !draws_w.is_empty() { - let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as f64; if rate_w > 2.0 * mean + 5.0 { - return false; + return Verdict::Implausible; } } - true + Verdict::Live } /// What the oracle saw. @@ -300,10 +321,16 @@ mod nvml_oracle { let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + // The verdict that speaks for the run is the most severe of its GPUs', + // as the crate's is: a counter proven to over-run is a finding, while + // one board's broken telemetry only withholds a judgement. + let judged: Vec = (0..n).map(|i| judge(rate_w[i], &draws_w[i], ceiling_w[i])).collect(); let verdict = if !live(&advances, intervals, window) { Verdict::Frozen - } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + } else if judged.contains(&Verdict::Implausible) { Verdict::Implausible + } else if judged.contains(&Verdict::Unjudgeable) { + Verdict::Unjudgeable } else { Verdict::Live }; @@ -357,6 +384,7 @@ fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), Verdict::Implausible => assert_eq!(status, CounterStatus::Implausible), + Verdict::Unjudgeable => assert_eq!(status, CounterStatus::Unjudgeable), Verdict::Live => {} } if meter.rung() == Rung::Nvml { diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs index bca4a6ff2..411fdfbdb 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs @@ -196,9 +196,13 @@ mod nvml_oracle { Unavailable, /// A counter did not keep advancing. Frozen, - /// Every counter kept advancing, but one is not plausibly its board's - /// energy (see the module doc). + /// Every counter kept advancing, but one ran faster than its board + /// can draw (see the module doc). Implausible, + /// Every counter kept advancing, and one could not be judged because + /// its board's own telemetry read past the board's own limit. The + /// yardstick is broken, not necessarily the counter. + Unjudgeable, /// Every counter kept advancing, plausibly its board's energy. Live, } @@ -207,19 +211,36 @@ mod nvml_oracle { /// average `rate_w`, the board's draws `draws_w` around the window and /// its maximum power limit `ceiling_w`. A bound whose input is missing /// is not applied. - pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { - if let Some(c) = ceiling_w { - if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { - return false; + /// + /// A draw reading past the board's own limit is not a reading — an RTX + /// 4050 Laptop GPU reports about 590 W for a second after it wakes — so it + /// is dropped, and only it. Only when nothing usable is left is there no + /// yardstick at all, and then the counter is `Unjudgeable` rather than + /// `Implausible`: the yardstick is broken, not necessarily the counter. + /// + /// This oracle is written from the rule rather than from the crate, in + /// f64 and from `nvidia-smi` rather than the crate's own sampling, so it + /// is an independent check on the same rule — which is only true while it + /// encodes the rule the crate implements. + pub fn judge(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> Verdict { + let cap = ceiling_w.map(|c| 1.5 * c); + let (usable, unusable): (Vec, Vec) = + draws_w.iter().copied().partition(|d| cap.is_none_or(|c| *d <= c)); + if let Some(c) = cap { + if rate_w > c { + return Verdict::Implausible; + } + if usable.is_empty() && !unusable.is_empty() { + return Verdict::Unjudgeable; } } - if !draws_w.is_empty() { - let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as f64; if rate_w > 2.0 * mean + 5.0 { - return false; + return Verdict::Implausible; } } - true + Verdict::Live } /// What the oracle saw. @@ -302,10 +323,16 @@ mod nvml_oracle { let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + // The verdict that speaks for the run is the most severe of its GPUs', + // as the crate's is: a counter proven to over-run is a finding, while + // one board's broken telemetry only withholds a judgement. + let judged: Vec = (0..n).map(|i| judge(rate_w[i], &draws_w[i], ceiling_w[i])).collect(); let verdict = if !live(&advances, intervals, window) { Verdict::Frozen - } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + } else if judged.contains(&Verdict::Implausible) { Verdict::Implausible + } else if judged.contains(&Verdict::Unjudgeable) { + Verdict::Unjudgeable } else { Verdict::Live }; @@ -362,6 +389,7 @@ fn the_nvml_liveness_agrees_with_the_oracle_at_its_own_tick() { Verdict::Unavailable => assert_eq!(liveness, CounterLiveness::Unavailable), Verdict::Frozen => assert_eq!(liveness, CounterLiveness::Frozen), Verdict::Implausible => assert_eq!(liveness, CounterLiveness::Implausible), + Verdict::Unjudgeable => assert_eq!(liveness, CounterLiveness::Unjudgeable), Verdict::Live => {} } assert_eq!(counter.uses_true_counter(), liveness == CounterLiveness::Live, "the counter mode disagrees"); diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs b/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs index 170745c5e..bc5812ed0 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs @@ -196,9 +196,13 @@ mod nvml_oracle { Unavailable, /// A counter did not keep advancing. Frozen, - /// Every counter kept advancing, but one is not plausibly its board's - /// energy (see the module doc). + /// Every counter kept advancing, but one ran faster than its board + /// can draw (see the module doc). Implausible, + /// Every counter kept advancing, and one could not be judged because + /// its board's own telemetry read past the board's own limit. The + /// yardstick is broken, not necessarily the counter. + Unjudgeable, /// Every counter kept advancing, plausibly its board's energy. Live, } @@ -207,19 +211,36 @@ mod nvml_oracle { /// average `rate_w`, the board's draws `draws_w` around the window and /// its maximum power limit `ceiling_w`. A bound whose input is missing /// is not applied. - pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { - if let Some(c) = ceiling_w { - if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { - return false; + /// + /// A draw reading past the board's own limit is not a reading — an RTX + /// 4050 Laptop GPU reports about 590 W for a second after it wakes — so it + /// is dropped, and only it. Only when nothing usable is left is there no + /// yardstick at all, and then the counter is `Unjudgeable` rather than + /// `Implausible`: the yardstick is broken, not necessarily the counter. + /// + /// This oracle is written from the rule rather than from the crate, in + /// f64 and from `nvidia-smi` rather than the crate's own sampling, so it + /// is an independent check on the same rule — which is only true while it + /// encodes the rule the crate implements. + pub fn judge(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> Verdict { + let cap = ceiling_w.map(|c| 1.5 * c); + let (usable, unusable): (Vec, Vec) = + draws_w.iter().copied().partition(|d| cap.is_none_or(|c| *d <= c)); + if let Some(c) = cap { + if rate_w > c { + return Verdict::Implausible; + } + if usable.is_empty() && !unusable.is_empty() { + return Verdict::Unjudgeable; } } - if !draws_w.is_empty() { - let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as f64; if rate_w > 2.0 * mean + 5.0 { - return false; + return Verdict::Implausible; } } - true + Verdict::Live } /// What the oracle saw. @@ -302,10 +323,16 @@ mod nvml_oracle { let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + // The verdict that speaks for the run is the most severe of its GPUs', + // as the crate's is: a counter proven to over-run is a finding, while + // one board's broken telemetry only withholds a judgement. + let judged: Vec = (0..n).map(|i| judge(rate_w[i], &draws_w[i], ceiling_w[i])).collect(); let verdict = if !live(&advances, intervals, window) { Verdict::Frozen - } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + } else if judged.contains(&Verdict::Implausible) { Verdict::Implausible + } else if judged.contains(&Verdict::Unjudgeable) { + Verdict::Unjudgeable } else { Verdict::Live }; @@ -359,6 +386,7 @@ fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), Verdict::Implausible => assert_eq!(status, CounterStatus::Implausible), + Verdict::Unjudgeable => assert_eq!(status, CounterStatus::Unjudgeable), Verdict::Live => {} } assert_eq!(sources.contains(&"nvml-total-energy"), status == CounterStatus::Live, "selection disagrees with the status"); -- 2.54.0 (Apple Git-157) From d94fa324f951d9f4ede2cd3cb1f9300d67a6039f Mon Sep 17 00:00:00 2001 From: David Charlot Date: Fri, 9 Oct 2026 08:38:47 -0400 Subject: [PATCH 6/7] jouleclaw: cover the new liveness where the declined ones are enumerated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two tests are titled "A counter given any verdict but `Live` meters nothing" and then enumerate three of what are now four non-live verdicts, so adding `Unjudgeable` left it the one whose declining nothing checked. The behaviour was already right — both `admits` functions compare against `Live` — so this is the tests catching up with their own claim. Co-Authored-By: Claude Opus 5 --- .../jouleclaw-rs/crates/jouleclaw-energy/src/process.rs | 7 ++++++- .../crates/jouleclaw-energy/src/process_gpu.rs | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs index e33f01bcb..698f82291 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs @@ -280,7 +280,12 @@ mod tests { /// measurement, and its work still runs. #[test] fn a_counter_that_is_not_live_meters_nothing() { - for liveness in [CounterLiveness::Frozen, CounterLiveness::Implausible, CounterLiveness::Unavailable] { + for liveness in [ + CounterLiveness::Frozen, + CounterLiveness::Implausible, + CounterLiveness::Unjudgeable, + CounterLiveness::Unavailable, + ] { let c = ProcessCpuCounter::given(liveness); assert!(!c.is_available() && !EnergyMeter::is_available(&c), "{liveness:?}"); assert!(matches!(c.read(), Err(EnergyError::NoCounter(EnergyDomain::ProcessCpu))), "{liveness:?}"); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process_gpu.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process_gpu.rs index eacc51962..1d4500f6d 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process_gpu.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process_gpu.rs @@ -648,7 +648,12 @@ mod tests { /// still runs. #[test] fn a_counter_that_is_not_live_meters_nothing() { - for liveness in [CounterLiveness::Frozen, CounterLiveness::Implausible, CounterLiveness::Unavailable] { + for liveness in [ + CounterLiveness::Frozen, + CounterLiveness::Implausible, + CounterLiveness::Unjudgeable, + CounterLiveness::Unavailable, + ] { let c = ProcessGpuCounter::given(liveness); assert!(!c.is_available(), "{liveness:?}"); assert!(matches!(c.snapshot(), Err(EnergyError::NoCounter(EnergyDomain::ProcessGpu)))); -- 2.54.0 (Apple Git-157) From 86cd2a442fe53b607f1b6ebf1fd8333e682999b2 Mon Sep 17 00:00:00 2001 From: David Charlot Date: Fri, 9 Oct 2026 08:45:20 -0400 Subject: [PATCH 7/7] jouleclaw: the new plausibility types are crate-internal, not public surface `jouleclaw-public-surface-documented.yml` failed on this branch with exactly one Lost reason, `jouleclaw-energy::src::liveness::plausible`. Two causes, and the second is the real one: - The `///` doc was there, but separated from the item by a `//` note, so the scanner walking back from the signature never reached it. - More to the point, `liveness` is a private module, so a `pub` item in it was never consumer-visible. The gate's own advice is to document it or downgrade it if it is not part of the consumer-visible surface, and the honest answer here is the latter: `CounterLiveness` is this crate's public vocabulary, `GpuPlausibility` is how the module reaches its verdict. So `GpuBound`, `GpuPlausibility` and `plausible()` are `pub(crate)`, which is what they always were in effect, and the note moved into the doc comment. This differs from jcp-energy deliberately: there the types sit in the crate root and are genuinely public. eoc-meter's copy was already `pub(crate)`. Co-Authored-By: Claude Opus 5 --- .../crates/jouleclaw-energy/src/liveness.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs index 863f911ec..cae0c2e61 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs @@ -329,7 +329,7 @@ pub(crate) const GPU_DRAW_SLACK_MW: u128 = 5_000; /// Which of the two bounds a GPU counter ran past. #[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum GpuBound { +pub(crate) enum GpuBound { /// [`GPU_CEILING_MARGIN`] of the board's maximum power limit. PowerLimit, /// [`GPU_DRAW_FACTOR`] times the board's mean draw plus @@ -347,7 +347,7 @@ pub enum GpuBound { /// different margin is only possible if the rate survived the check. #[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum GpuPlausibility { +pub(crate) enum GpuPlausibility { /// The counter's rate sits inside every bound whose reading was present. Plausible { /// The rate the counter implies over the window, in milliwatts. @@ -380,12 +380,11 @@ impl GpuPlausibility { /// `false` for an indeterminate as well as for an overrun: unjudgeable is /// not admissible, and erring toward declining is the safe direction. Use /// the variant itself to tell the two apart. - // - // The crate's own paths match on the variants rather than call this, so - // outside tests it is unused: the `liveness` module is private, which puts - // even a `pub` item here under dead-code analysis. + /// + /// The crate's own paths match on the variants rather than call this, so + /// outside tests it is unused. #[cfg_attr(not(test), allow(dead_code))] - pub fn plausible(&self) -> bool { + pub(crate) fn plausible(&self) -> bool { matches!(self, GpuPlausibility::Plausible { .. }) } -- 2.54.0 (Apple Git-157)