diff --git a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs index 513ac80d3..b5005189e 100644 --- a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs +++ b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs @@ -3689,12 +3689,24 @@ pub enum CounterStatus { Frozen, /// The counter kept advancing, but a GPU counter (NVML's) ran faster than /// its board can draw power over the probe: past 1.5 times the board's - /// maximum power limit, past twice the board's own power draw plus 5 W, - /// or beside a draw reading itself past that limit. It is not counting the - /// board's energy. Seen on an RTX 4050 Laptop GPU (driver 595.84), whose - /// counter runs at 4 to 30 times the board's draw while it is polled. - /// Never constructed. + /// maximum power limit, or past twice the board's own power draw plus + /// 5 W. It is not counting the board's energy. Seen on an RTX 4050 Laptop + /// GPU (driver 595.84), whose counter runs at 4 to 30 times the board's + /// draw while it is polled. Never constructed. Implausible, + /// The counter kept advancing and could not be judged, because the board's + /// own power telemetry was unusable over the probe: a draw reading past + /// the board's own maximum power limit (an RTX 4050 Laptop GPU reports + /// about 590 W for a second after it wakes). + /// + /// Split out from [`CounterStatus::Implausible`], which used to cover this + /// — its doc then asserted "it is not counting the board's energy", a + /// conclusion this case does not support. The yardstick is broken, not + /// necessarily the counter. Never selected either way: unjudgeable is not + /// admissible, and erring toward declining is the safe direction. The + /// distinction is the diagnosis an operator gets, which is the difference + /// between replacing a counter and waiting for the board to settle. + Unjudgeable, /// The counter kept advancing while the probe ran: in at least five /// separate sample intervals, at least 80% of those observed, over at /// least 300 ms. The rung is selected. @@ -4787,38 +4799,138 @@ mod gpu_plausibility { /// NVML's documented accuracy of a board's power draw, ±5 W, in mW. pub(crate) const DRAW_SLACK_MW: u128 = 5_000; - /// Whether a GPU energy counter that counted `energy_mj` over `window` - /// plausibly counted its board's energy, beside the board's power draw - /// `draws_mw` read with the samples and its maximum power limit - /// `ceiling_mw`. Not when the counter ran past [`CEILING_MARGIN`] of the - /// limit, or a draw reading did (the board's own telemetry then is not a - /// reading either: an RTX 4050 Laptop GPU reports 590 W for about a second - /// after it wakes), or the counter ran past [`DRAW_FACTOR`] times the mean - /// draw plus [`DRAW_SLACK_MW`]. A bound whose reading is missing is not - /// applied. + /// Which of the two bounds a GPU counter ran past. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum GpuBound { + /// [`CEILING_MARGIN`] of the board's maximum power limit. + PowerLimit, + /// [`DRAW_FACTOR`] times the board's mean draw plus [`DRAW_SLACK_MW`]. + MeanDraw, + } + + /// What the plausibility check found, with the figures it found it from. + /// + /// A `bool` was the whole of this, and it threw away everything an + /// operator needs: which bound broke, by how much, and the rate it broke + /// it at. A counter 1.1 times over a bound and one 20 times over were the + /// same answer. The figures also make the judgement reproducible — the + /// margins are constants in this module, and re-grading a past finding + /// against a different margin is only possible if the rate survived the + /// check. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum GpuPlausibility { + /// The counter's rate sits inside every bound whose reading was + /// present. + Plausible { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + }, + /// The counter ran faster than the board can draw. + CounterOverRate { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + /// The bound it broke, in milliwatts. + bound_mw: u64, + /// Which bound that was. + bound: GpuBound, + }, + /// The board's own telemetry read past the board's own power limit, so + /// there is no yardstick left to judge the counter against. An + /// indeterminate, not a finding about the counter — see + /// [`CounterStatus::Unjudgeable`]. + BoardTelemetryUnusable { + /// The offending draw reading, in milliwatts. + draw_mw: u64, + /// The limit it exceeded (the margin-adjusted cap), in milliwatts. + limit_mw: u64, + }, + } + + impl GpuPlausibility { + /// Whether the counter was judged to be counting its board's energy. + /// + /// `false` for an indeterminate as well as for an overrun: unjudgeable + /// is not admissible, and erring toward declining is the safe + /// direction. Use the variant itself to tell the two apart. + // + // The module's own path matches on the variants rather than call this. + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) fn plausible(&self) -> bool { + matches!(self, GpuPlausibility::Plausible { .. }) + } + } + + /// What a GPU energy counter that counted `energy_mj` over `window` was + /// judged to be counting, beside the board's power draw `draws_mw` read + /// with the samples and its maximum power limit `ceiling_mw`. It over-ran + /// if it ran past [`CEILING_MARGIN`] of the limit, or past [`DRAW_FACTOR`] + /// times the mean draw plus [`DRAW_SLACK_MW`]. A bound whose reading is + /// missing is not applied. /// /// Liveness says the counter is republished continuously; this says what /// it republishes is the board's energy. On the RTX 4050 Laptop GPU /// (driver 595.84) the total-energy counter runs several times faster than /// the board draws for as long as it is polled: at the probe's 100 ms wait /// it ran at 58-305 W while the board drew 5-14 W against a 65 W limit. - pub(crate) fn plausible(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { + pub(crate) fn plausible( + energy_mj: u64, + window: Duration, + draws_mw: &[u32], + ceiling_mw: Option, + ) -> GpuPlausibility { let ns = window.as_nanos().max(1); // mJ per second is mW. let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; - if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { - let cap = u128::from(limit) * CEILING_MARGIN.0 / CEILING_MARGIN.1; - if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { - return false; + let clamp = |v: u128| u64::try_from(v).unwrap_or(u64::MAX); + let cap = + ceiling_mw.filter(|&l| l > 0).map(|l| u128::from(l) * CEILING_MARGIN.0 / CEILING_MARGIN.1); + + // A draw reading past the board's own limit is not a reading: an RTX + // 4050 Laptop GPU reports about 590 W for a second after it wakes. + // Such a reading is dropped, and only that reading — it is one broken + // sample, not a broken bracket. Voiding the whole judgement on it is + // what this code used to do, and the hardware test on jetson-hub + // refused that shape: one spike among otherwise sane 14.8 W samples + // turned a counter running persistently at 4-30x the board's draw into + // an indeterminate, which is a worse diagnosis than the overrun it + // actually is. + // + // Dropping them also keeps the mean honest, which is what the old + // short-circuit was really for. The mean summed every reading, so a + // 590 W spike inflated it and made the mean-draw bound *more* + // permissive; that only went unnoticed because any such reading failed + // the check first. + let (usable, unusable): (Vec, Vec) = + draws_mw.iter().map(|&d| u128::from(d)).partition(|&d| cap.is_none_or(|c| d <= c)); + + if let Some(cap) = cap { + if rate_mw > cap { + return GpuPlausibility::CounterOverRate { + rate_mw: clamp(rate_mw), + bound_mw: clamp(cap), + bound: GpuBound::PowerLimit, + }; + } + // Only when nothing usable is left is there no yardstick at all. + if usable.is_empty() && !unusable.is_empty() { + return GpuPlausibility::BoardTelemetryUnusable { + draw_mw: clamp(unusable[0]), + limit_mw: clamp(cap), + }; } } - if !draws_mw.is_empty() { - let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; - if rate_mw > DRAW_FACTOR * mean + DRAW_SLACK_MW { - return false; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as u128; + let bound = DRAW_FACTOR * mean + DRAW_SLACK_MW; + if rate_mw > bound { + return GpuPlausibility::CounterOverRate { + rate_mw: clamp(rate_mw), + bound_mw: clamp(bound), + bound: GpuBound::MeanDraw, + }; } } - true + GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) } } /// What the GPU probe saw of the board beside its counter: the counter's @@ -4844,19 +4956,28 @@ mod gpu_plausibility { } } - /// The status a GPU probe gives: the liveness rule's `status`, except - /// that a live counter which did not plausibly count its board's energy - /// over the probe is [`CounterStatus::Implausible`]. + /// The status a GPU probe gives: the liveness rule's `status`, except that + /// a live counter is re-judged on what it was counting over the probe. One + /// proven to run faster than its board can draw is + /// [`CounterStatus::Implausible`]; one whose board's own telemetry was + /// unusable is [`CounterStatus::Unjudgeable`], which is a different + /// diagnosis and not a claim about the counter. pub(crate) fn status_of(status: CounterStatus, gpu: &GpuWatch) -> CounterStatus { - let plausible = match (gpu.first, gpu.last) { + if status != CounterStatus::Live { + return status; + } + match (gpu.first, gpu.last) { (Some((a, t0)), Some((b, t1))) => { - plausible(b.saturating_sub(a), t1.saturating_sub(t0), &gpu.draws_mw, gpu.ceiling_mw) + match plausible(b.saturating_sub(a), t1.saturating_sub(t0), &gpu.draws_mw, gpu.ceiling_mw) { + GpuPlausibility::Plausible { .. } => CounterStatus::Live, + GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible, + GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable, + } } - _ => true, - }; - match status { - CounterStatus::Live if !plausible => CounterStatus::Implausible, - other => other, + // No pair of samples to compute a rate from. Not a judgement that + // the counter is sound; liveness already passed, and there is + // nothing here to contradict it. + _ => CounterStatus::Live, } } @@ -4874,17 +4995,92 @@ mod gpu_plausibility { #[test] fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { let w = Duration::from_millis(500); - assert!(!plausible(59_700, w, &[12_970, 13_520], Some(65_000)), "119.4 W against a 13 W draw"); - assert!(!plausible(22_000, w, &[10_400; 6], Some(65_000)), "44 W against a 10.4 W draw"); - assert!(!plausible(60_000, w, &[], Some(65_000)), "120 W on a 65 W board"); - assert!(!plausible(44_000, w, &[590_000, 10_300], Some(65_000)), "a 590 W draw reading"); - assert!(plausible(6_500, w, &[13_000, 13_500], Some(65_000)), "13 W against a 13 W draw"); - assert!(plausible(150_000, w, &[280_000; 5], Some(700_000)), "300 W against a 280 W draw"); - assert!(plausible(102_500, w, &[100_000; 5], Some(700_000)), "205 W is 2 x 100 W + 5 W"); - assert!(!plausible(102_501, w, &[100_000; 5], Some(700_000))); - assert!(plausible(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); - assert!(!plausible(48_751, w, &[], Some(65_000))); - assert!(plausible(1_000_000, w, &[], None), "nothing to judge by"); + // Over a 500 ms window a counter's rate in mW is twice its mJ. + let over = + |rate_mw, bound_mw, bound| GpuPlausibility::CounterOverRate { rate_mw, bound_mw, bound }; + + // The figures, not just a `false`: an operator cannot act on a + // bare bool, and a finding cannot be re-graded against a changed + // margin unless the rate survived the check. + assert_eq!( + plausible(59_700, w, &[12_970, 13_520], Some(65_000)), + over(119_400, 97_500, GpuBound::PowerLimit), + "119.4 W against a 13 W draw" + ); + assert_eq!( + plausible(22_000, w, &[10_400; 6], Some(65_000)), + over(44_000, 25_800, GpuBound::MeanDraw), + "44 W against a 10.4 W draw" + ); + assert_eq!( + plausible(60_000, w, &[], Some(65_000)), + over(120_000, 97_500, GpuBound::PowerLimit), + "120 W on a 65 W board" + ); + + // A draw reading of 590 W (the board's first second after waking) + // is no reading. It is dropped, and only it: the remaining 10.3 W + // reading still judges the counter, and 88 W against it is an + // overrun. This used to assert a bare `false` meaning "a 590 W + // draw reading", which is the one thing the hardware test on + // jetson-hub refused: it turned a persistent 4-30x overrun into + // "cannot judge". + assert_eq!( + plausible(44_000, w, &[590_000, 10_300], Some(65_000)), + over(88_000, 25_600, GpuBound::MeanDraw) + ); + // Dropping it also keeps the mean honest, and this pins that: the + // bound is twice 10 W plus 5 W, from the usable reading alone. Had + // the 590 W reading been summed in, the mean would be 300 W and + // the bound 605 W, which would excuse this counter. + assert_eq!( + plausible(15_000, w, &[590_000, 10_000], Some(65_000)), + over(30_000, 25_000, GpuBound::MeanDraw) + ); + // Only when no usable reading is left is there no yardstick at + // all, and then the verdict is an indeterminate about the board, + // not a finding about the counter. + assert_eq!( + plausible(44_000, w, &[590_000], Some(65_000)), + GpuPlausibility::BoardTelemetryUnusable { draw_mw: 590_000, limit_mw: 97_500 } + ); + // A counter past the board's own limit is a finding whatever the + // telemetry says: the rate bound is checked before the yardstick. + assert_eq!( + plausible(60_000, w, &[590_000], Some(65_000)), + over(120_000, 97_500, GpuBound::PowerLimit) + ); + + assert_eq!( + plausible(6_500, w, &[13_000, 13_500], Some(65_000)), + GpuPlausibility::Plausible { rate_mw: 13_000 }, + "13 W against a 13 W draw" + ); + assert_eq!( + plausible(150_000, w, &[280_000; 5], Some(700_000)), + GpuPlausibility::Plausible { rate_mw: 300_000 }, + "300 W against a 280 W draw" + ); + assert!( + plausible(102_500, w, &[100_000; 5], Some(700_000)).plausible(), + "205 W is 2 x 100 W + 5 W" + ); + assert_eq!( + plausible(102_501, w, &[100_000; 5], Some(700_000)), + over(205_002, 205_000, GpuBound::MeanDraw) + ); + assert!(plausible(48_750, w, &[], Some(65_000)).plausible(), "97.5 W is 1.5 x 65 W"); + assert_eq!( + plausible(48_751, w, &[], Some(65_000)), + over(97_502, 97_500, GpuBound::PowerLimit) + ); + // Nothing to judge it by: no bound applies, and with no limit to + // exceed no reading can be unusable either. + assert_eq!( + plausible(1_000_000, w, &[], None), + GpuPlausibility::Plausible { rate_mw: 2_000_000 }, + "nothing to judge by" + ); } /// A live counter that ran past its board is `Implausible`; one that @@ -4903,7 +5099,7 @@ mod gpu_plausibility { assert_eq!(status_of(S::Live, &watch(13_000, Some(65_000))), S::Implausible); assert_eq!(status_of(S::Live, &watch(115_000, Some(700_000))), S::Live); assert_eq!(status_of(S::Live, &GpuWatch::default()), S::Live, "nothing to judge by"); - for status in [S::Unavailable, S::Frozen] { + for status in [S::Unavailable, S::Frozen, S::Unjudgeable] { assert_eq!(status_of(status, &watch(13_000, Some(65_000))), status); } } @@ -5045,7 +5241,12 @@ mod nvml { #[test] fn only_a_live_nvml_status_constructs_the_counter() { let attempts = || OPEN_ATTEMPTS.with(|n| n.get()); - for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible] { + for status in [ + CounterStatus::Unavailable, + CounterStatus::Frozen, + CounterStatus::Implausible, + CounterStatus::Unjudgeable, + ] { let before = attempts(); assert!( matches!(NvmlCounter::try_detect_given(status), Err(Error::NoCounter)), @@ -5158,7 +5359,7 @@ mod tests { fn a_vendor_counter_is_selected_only_when_live() { use CounterStatus as S; let declined_ioreport = || IoReportStatus::CpuRailFrozen; - for status in [S::Unavailable, S::Frozen, S::Implausible] { + for status in [S::Unavailable, S::Frozen, S::Implausible, S::Unjudgeable] { let picked = detect_with(|| status, declined_ioreport).name(); assert_ne!(picked, "rapl", "a {status:?} RAPL counter was selected"); } @@ -5182,7 +5383,7 @@ mod tests { IoReportStatus::CpuRailFrozen, IoReportStatus::Live, ]; - for rapl in [S::Unavailable, S::Frozen, S::Implausible, S::Live] { + for rapl in [S::Unavailable, S::Frozen, S::Implausible, S::Unjudgeable, S::Live] { for io in ioreport { let picked = detect_with(|| rapl, || io).name(); assert_ne!(picked, "process-energy", "detect_with({rapl:?}, {io:?})"); @@ -5201,7 +5402,7 @@ mod tests { let fallback = || -> Box { Box::new(StubCounter) }; let picked = detect_in_process_with(|| S::Live, fallback).name(); assert_eq!(picked == "process-energy", cfg!(target_os = "macos"), "Live picked {picked}"); - for status in [S::Unavailable, S::Frozen] { + for status in [S::Unavailable, S::Frozen, S::Unjudgeable] { assert_eq!(detect_in_process_with(|| status, fallback).name(), "stub", "{status:?}"); } } diff --git a/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs b/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs index 9a8980170..bc1401c0a 100644 --- a/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs +++ b/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs @@ -195,9 +195,13 @@ mod nvml_oracle { Unavailable, /// A counter did not keep advancing. Frozen, - /// Every counter kept advancing, but one is not plausibly its board's - /// energy (see the module doc). + /// Every counter kept advancing, but one ran faster than its board + /// can draw (see the module doc). Implausible, + /// Every counter kept advancing, and one could not be judged because + /// its board's own telemetry read past the board's own limit. The + /// yardstick is broken, not necessarily the counter. + Unjudgeable, /// Every counter kept advancing, plausibly its board's energy. Live, } @@ -206,19 +210,36 @@ mod nvml_oracle { /// average `rate_w`, the board's draws `draws_w` around the window and /// its maximum power limit `ceiling_w`. A bound whose input is missing /// is not applied. - pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { - if let Some(c) = ceiling_w { - if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { - return false; + /// + /// A draw reading past the board's own limit is not a reading — an RTX + /// 4050 Laptop GPU reports about 590 W for a second after it wakes — so it + /// is dropped, and only it. Only when nothing usable is left is there no + /// yardstick at all, and then the counter is `Unjudgeable` rather than + /// `Implausible`: the yardstick is broken, not necessarily the counter. + /// + /// This oracle is written from the rule rather than from the crate, in + /// f64 and from `nvidia-smi` rather than the crate's own sampling, so it + /// is an independent check on the same rule — which is only true while it + /// encodes the rule the crate implements. + pub fn judge(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> Verdict { + let cap = ceiling_w.map(|c| 1.5 * c); + let (usable, unusable): (Vec, Vec) = + draws_w.iter().copied().partition(|d| cap.is_none_or(|c| *d <= c)); + if let Some(c) = cap { + if rate_w > c { + return Verdict::Implausible; + } + if usable.is_empty() && !unusable.is_empty() { + return Verdict::Unjudgeable; } } - if !draws_w.is_empty() { - let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as f64; if rate_w > 2.0 * mean + 5.0 { - return false; + return Verdict::Implausible; } } - true + Verdict::Live } /// What the oracle saw. @@ -301,10 +322,16 @@ mod nvml_oracle { let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + // The verdict that speaks for the run is the most severe of its GPUs', + // as the crate's is: a counter proven to over-run is a finding, while + // one board's broken telemetry only withholds a judgement. + let judged: Vec = (0..n).map(|i| judge(rate_w[i], &draws_w[i], ceiling_w[i])).collect(); let verdict = if !live(&advances, intervals, window) { Verdict::Frozen - } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + } else if judged.contains(&Verdict::Implausible) { Verdict::Implausible + } else if judged.contains(&Verdict::Unjudgeable) { + Verdict::Unjudgeable } else { Verdict::Live }; @@ -359,6 +386,7 @@ fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), Verdict::Implausible => assert_eq!(status, CounterStatus::Implausible), + Verdict::Unjudgeable => assert_eq!(status, CounterStatus::Unjudgeable), Verdict::Live => {} } assert_eq!(constructed, status == CounterStatus::Live, "construction disagrees with the status"); diff --git a/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs b/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs index 76096fb1b..3ccbc92db 100644 --- a/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs +++ b/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs @@ -194,9 +194,13 @@ mod nvml_oracle { Unavailable, /// A counter did not keep advancing. Frozen, - /// Every counter kept advancing, but one is not plausibly its board's - /// energy (see the module doc). + /// Every counter kept advancing, but one ran faster than its board + /// can draw (see the module doc). Implausible, + /// Every counter kept advancing, and one could not be judged because + /// its board's own telemetry read past the board's own limit. The + /// yardstick is broken, not necessarily the counter. + Unjudgeable, /// Every counter kept advancing, plausibly its board's energy. Live, } @@ -205,19 +209,36 @@ mod nvml_oracle { /// average `rate_w`, the board's draws `draws_w` around the window and /// its maximum power limit `ceiling_w`. A bound whose input is missing /// is not applied. - pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { - if let Some(c) = ceiling_w { - if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { - return false; + /// + /// A draw reading past the board's own limit is not a reading — an RTX + /// 4050 Laptop GPU reports about 590 W for a second after it wakes — so it + /// is dropped, and only it. Only when nothing usable is left is there no + /// yardstick at all, and then the counter is `Unjudgeable` rather than + /// `Implausible`: the yardstick is broken, not necessarily the counter. + /// + /// This oracle is written from the rule rather than from the crate, in + /// f64 and from `nvidia-smi` rather than the crate's own sampling, so it + /// is an independent check on the same rule — which is only true while it + /// encodes the rule the crate implements. + pub fn judge(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> Verdict { + let cap = ceiling_w.map(|c| 1.5 * c); + let (usable, unusable): (Vec, Vec) = + draws_w.iter().copied().partition(|d| cap.is_none_or(|c| *d <= c)); + if let Some(c) = cap { + if rate_w > c { + return Verdict::Implausible; + } + if usable.is_empty() && !unusable.is_empty() { + return Verdict::Unjudgeable; } } - if !draws_w.is_empty() { - let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as f64; if rate_w > 2.0 * mean + 5.0 { - return false; + return Verdict::Implausible; } } - true + Verdict::Live } /// What the oracle saw. @@ -300,10 +321,16 @@ mod nvml_oracle { let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + // The verdict that speaks for the run is the most severe of its GPUs', + // as the crate's is: a counter proven to over-run is a finding, while + // one board's broken telemetry only withholds a judgement. + let judged: Vec = (0..n).map(|i| judge(rate_w[i], &draws_w[i], ceiling_w[i])).collect(); let verdict = if !live(&advances, intervals, window) { Verdict::Frozen - } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + } else if judged.contains(&Verdict::Implausible) { Verdict::Implausible + } else if judged.contains(&Verdict::Unjudgeable) { + Verdict::Unjudgeable } else { Verdict::Live }; @@ -357,6 +384,7 @@ fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), Verdict::Implausible => assert_eq!(status, CounterStatus::Implausible), + Verdict::Unjudgeable => assert_eq!(status, CounterStatus::Unjudgeable), Verdict::Live => {} } if meter.rung() == Rung::Nvml { diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs index d1e82f633..cae0c2e61 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs @@ -61,13 +61,26 @@ pub enum CounterLiveness { Live, /// Every channel kept advancing, but a GPU counter (NVML's) ran faster /// than its board can draw power over the probe: past 1.5 times the - /// board's maximum power limit, past twice the board's own power draw plus - /// 5 W, or beside a draw reading itself past that limit - /// ([`plausible_gpu_counter`]). It is not counting the board's energy, so - /// its figure would not be a reading of the GPU's work. Seen on an RTX 4050 - /// Laptop GPU (driver 595.84), whose counter runs at 4 to 30 times the - /// board's draw while it is polled. + /// board's maximum power limit, or past twice the board's own power draw + /// plus 5 W ([`plausible_gpu_counter`]). It is not counting the board's + /// energy, so its figure would not be a reading of the GPU's work. Seen on + /// an RTX 4050 Laptop GPU (driver 595.84), whose counter runs at 4 to 30 + /// times the board's draw while it is polled. Implausible, + /// Every channel kept advancing and the counter could not be judged, + /// because the board's own power telemetry was unusable over the probe: a + /// draw reading past the board's own maximum power limit (an RTX 4050 + /// Laptop GPU reports about 590 W for a second after it wakes). + /// + /// Split out from [`CounterLiveness::Implausible`], which used to cover + /// this — its doc then asserted "it is not counting the board's energy", a + /// conclusion this case does not support. The yardstick is broken, not + /// necessarily the counter. Neither grades a reading as + /// [`crate::Provenance::HwShunt`]: unjudgeable is not admissible, and + /// erring toward declining is the safe direction. The distinction is the + /// diagnosis an operator gets, which is the difference between replacing a + /// counter and waiting for the board to settle. + Unjudgeable, } /// In at least how many separate sample intervals a channel must advance to @@ -313,24 +326,150 @@ pub(crate) const GPU_DRAW_SLACK_MW: u128 = 5_000; /// 595.84) the total-energy counter runs several times faster than the board /// draws for as long as it is polled: at the NVML probe's 100 ms tick it ran /// at 58-305 W while the board drew 5-14 W against a 65 W limit. +/// Which of the two bounds a GPU counter ran past. #[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` -pub(crate) fn plausible_gpu_counter(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum GpuBound { + /// [`GPU_CEILING_MARGIN`] of the board's maximum power limit. + PowerLimit, + /// [`GPU_DRAW_FACTOR`] times the board's mean draw plus + /// [`GPU_DRAW_SLACK_MW`]. + MeanDraw, +} + +/// What the plausibility check found, with the figures it found it from. +/// +/// A `bool` was the whole of this, and it threw away everything an operator +/// needs: which bound broke, by how much, and the rate it broke it at. A +/// counter 1.1 times over a bound and one 20 times over were the same answer. +/// The figures also make the judgement reproducible — the margins are +/// constants in this module, and re-grading a past finding against a +/// different margin is only possible if the rate survived the check. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum GpuPlausibility { + /// The counter's rate sits inside every bound whose reading was present. + Plausible { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + }, + /// The counter ran faster than the board can draw. + CounterOverRate { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + /// The bound it broke, in milliwatts. + bound_mw: u64, + /// Which bound that was. + bound: GpuBound, + }, + /// The board's own telemetry read past the board's own power limit, so + /// there is no yardstick left to judge the counter against. An + /// indeterminate, not a finding about the counter — see + /// [`CounterLiveness::Unjudgeable`]. + BoardTelemetryUnusable { + /// The offending draw reading, in milliwatts. + draw_mw: u64, + /// The limit it exceeded (the margin-adjusted cap), in milliwatts. + limit_mw: u64, + }, +} + +impl GpuPlausibility { + /// Whether the counter was judged to be counting its board's energy. + /// + /// `false` for an indeterminate as well as for an overrun: unjudgeable is + /// not admissible, and erring toward declining is the safe direction. Use + /// the variant itself to tell the two apart. + /// + /// The crate's own paths match on the variants rather than call this, so + /// outside tests it is unused. + #[cfg_attr(not(test), allow(dead_code))] + pub(crate) fn plausible(&self) -> bool { + matches!(self, GpuPlausibility::Plausible { .. }) + } + + /// How severe this verdict is, for picking the one that speaks for a probe + /// of several GPUs: a proven overrun outranks an indeterminate, which + /// outranks a pass. + fn rank(&self) -> u8 { + match self { + GpuPlausibility::Plausible { .. } => 0, + GpuPlausibility::BoardTelemetryUnusable { .. } => 1, + GpuPlausibility::CounterOverRate { .. } => 2, + } + } + + /// The figure this verdict turns on, to order two of equal rank by. + fn magnitude(&self) -> u64 { + match self { + GpuPlausibility::Plausible { rate_mw } => *rate_mw, + GpuPlausibility::CounterOverRate { rate_mw, .. } => *rate_mw, + GpuPlausibility::BoardTelemetryUnusable { draw_mw, .. } => *draw_mw, + } + } +} + +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` +pub(crate) fn plausible_gpu_counter( + energy_mj: u64, + window: Duration, + draws_mw: &[u32], + ceiling_mw: Option, +) -> GpuPlausibility { let ns = window.as_nanos().max(1); // mJ per second is mW. let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; - if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { - let cap = u128::from(limit) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1; - if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { - return false; + let clamp = |v: u128| u64::try_from(v).unwrap_or(u64::MAX); + let cap = ceiling_mw + .filter(|&l| l > 0) + .map(|l| u128::from(l) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1); + + // A draw reading past the board's own limit is not a reading: an RTX 4050 + // Laptop GPU reports about 590 W for a second after it wakes. Such a + // reading is dropped, and only that reading — it is one broken sample, not + // a broken bracket. Voiding the whole judgement on it is what this code + // used to do, and the hardware test on jetson-hub refused that shape: one + // spike among otherwise sane 14.8 W samples turned a counter running + // persistently at 4-30x the board's draw into an indeterminate, which is a + // worse diagnosis than the overrun it actually is. + // + // Dropping them also keeps the mean honest, which is what the old + // short-circuit was really for. The mean summed every reading, so a 590 W + // spike inflated it and made the mean-draw bound *more* permissive; that + // only went unnoticed because any such reading failed the check first. + let (usable, unusable): (Vec, Vec) = draws_mw + .iter() + .map(|&d| u128::from(d)) + .partition(|&d| cap.is_none_or(|c| d <= c)); + + if let Some(cap) = cap { + if rate_mw > cap { + return GpuPlausibility::CounterOverRate { + rate_mw: clamp(rate_mw), + bound_mw: clamp(cap), + bound: GpuBound::PowerLimit, + }; + } + // Only when nothing usable is left is there no yardstick at all. + if usable.is_empty() && !unusable.is_empty() { + return GpuPlausibility::BoardTelemetryUnusable { + draw_mw: clamp(unusable[0]), + limit_mw: clamp(cap), + }; } } - if !draws_mw.is_empty() { - let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; - if rate_mw > GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW { - return false; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as u128; + let bound = GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW; + if rate_mw > bound { + return GpuPlausibility::CounterOverRate { + rate_mw: clamp(rate_mw), + bound_mw: clamp(bound), + bound: GpuBound::MeanDraw, + }; } } - true + GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) } } /// What a GPU counter's probe saw beside the rule's verdict: each GPU's @@ -362,21 +501,41 @@ impl GpuWatch { } /// The liveness a GPU counter's probe gives it: the rule's `verdict`, - /// except that a live counter one of whose GPUs did not plausibly count - /// its board's energy over the probe, beside each GPU's maximum power - /// limit `ceilings_mw`, is [`CounterLiveness::Implausible`]. + /// except that a live counter is re-judged on whether its GPUs counted + /// their boards' energy over the probe, beside each GPU's maximum power + /// limit `ceilings_mw`: a counter proven to run faster than its board can + /// draw is [`CounterLiveness::Implausible`], and a board whose own + /// telemetry is unusable leaves it [`CounterLiveness::Unjudgeable`]. pub(crate) fn liveness(&self, verdict: CounterLiveness, ceilings_mw: &[Option]) -> CounterLiveness { let (Some((first, t0)), Some((last, t1))) = (&self.first, &self.last) else { return verdict; }; + if verdict != CounterLiveness::Live { + return verdict; + } let window = t1.saturating_duration_since(*t0); - let plausible = first.iter().zip(last).enumerate().all(|(i, (a, b))| { - let draws = self.draws_mw.get(i).map(Vec::as_slice).unwrap_or(&[]); - plausible_gpu_counter(b.saturating_sub(*a) / 1_000, window, draws, ceilings_mw.get(i).copied().flatten()) - }); - match verdict { - CounterLiveness::Live if !plausible => CounterLiveness::Implausible, - other => other, + // The verdict that speaks for the probe is the most severe of its + // GPUs', and among equals the largest figure: one healthy GPU does not + // excuse the probe, and one board's broken telemetry does not condemn + // a counter nothing measured. + let worst = first + .iter() + .zip(last) + .enumerate() + .map(|(i, (a, b))| { + let draws = self.draws_mw.get(i).map(Vec::as_slice).unwrap_or(&[]); + plausible_gpu_counter( + b.saturating_sub(*a) / 1_000, + window, + draws, + ceilings_mw.get(i).copied().flatten(), + ) + }) + .max_by_key(|v| (v.rank(), v.magnitude())); + match worst { + Some(GpuPlausibility::CounterOverRate { .. }) => CounterLiveness::Implausible, + Some(GpuPlausibility::BoardTelemetryUnusable { .. }) => CounterLiveness::Unjudgeable, + _ => CounterLiveness::Live, } } } @@ -404,17 +563,89 @@ pub(crate) mod tests { #[test] fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { let w = Duration::from_millis(500); - assert!(!plausible_gpu_counter(152_250, w, &[5_550, 7_660], Some(65_000)), "304.5 W against a 6.6 W draw"); - assert!(!plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000)), "44 W against a 10.4 W draw"); - assert!(!plausible_gpu_counter(60_000, w, &[], Some(65_000)), "120 W on a 65 W board"); - assert!(!plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000)), "a 590 W draw reading"); - assert!(plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000)), "13 W against a 13 W draw"); - assert!(plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000)), "300 W against a 280 W draw"); - assert!(plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000)), "205 W is 2 x 100 W + 5 W"); - assert!(!plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000))); - assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); - assert!(!plausible_gpu_counter(48_751, w, &[], Some(65_000))); - assert!(plausible_gpu_counter(1_000_000, w, &[], None), "nothing to judge by"); + // Over a 500 ms window a counter's rate in mW is twice its mJ. + let over = |rate_mw, bound_mw, bound| GpuPlausibility::CounterOverRate { rate_mw, bound_mw, bound }; + + // The figures, not just a `false`: 304.5 W against a 97.5 W ceiling is + // a 3.1x overrun, and an operator cannot act on a bare bool. + assert_eq!( + plausible_gpu_counter(152_250, w, &[5_550, 7_660], Some(65_000)), + over(304_500, 97_500, GpuBound::PowerLimit), + "304.5 W against a 6.6 W draw" + ); + assert_eq!( + plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000)), + over(44_000, 25_800, GpuBound::MeanDraw), + "44 W against a 10.4 W draw" + ); + assert_eq!( + plausible_gpu_counter(60_000, w, &[], Some(65_000)), + over(120_000, 97_500, GpuBound::PowerLimit), + "120 W on a 65 W board" + ); + + // A draw reading of 590 W (the board's first second after waking) is + // no reading. It is dropped, and only it: the remaining 10.3 W reading + // still judges the counter, and 88 W against it is an overrun. This + // used to assert a bare `false` meaning "a 590 W draw reading", which + // is the one thing the hardware test on jetson-hub refused: it turned + // a persistent 4-30x overrun into "cannot judge". + assert_eq!( + plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000)), + over(88_000, 25_600, GpuBound::MeanDraw) + ); + // Dropping it also keeps the mean honest, and this pins that: the + // bound is twice 10 W plus 5 W, from the usable reading alone. Had the + // 590 W reading been summed in, the mean would be 300 W and the bound + // 605 W, which would excuse this counter. + assert_eq!( + plausible_gpu_counter(15_000, w, &[590_000, 10_000], Some(65_000)), + over(30_000, 25_000, GpuBound::MeanDraw) + ); + // Only when no usable reading is left is there no yardstick at all, + // and then the verdict is an indeterminate about the board, not a + // finding about the counter. + assert_eq!( + plausible_gpu_counter(44_000, w, &[590_000], Some(65_000)), + GpuPlausibility::BoardTelemetryUnusable { draw_mw: 590_000, limit_mw: 97_500 } + ); + // A counter past the board's own limit is a finding whatever the + // telemetry says: the rate bound is checked before the yardstick is. + assert_eq!( + plausible_gpu_counter(60_000, w, &[590_000], Some(65_000)), + over(120_000, 97_500, GpuBound::PowerLimit) + ); + + assert_eq!( + plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000)), + GpuPlausibility::Plausible { rate_mw: 13_000 }, + "13 W against a 13 W draw" + ); + assert_eq!( + plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000)), + GpuPlausibility::Plausible { rate_mw: 300_000 }, + "300 W against a 280 W draw" + ); + assert!( + plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000)).plausible(), + "205 W is 2 x 100 W + 5 W" + ); + assert_eq!( + plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000)), + over(205_002, 205_000, GpuBound::MeanDraw) + ); + assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)).plausible(), "97.5 W is 1.5 x 65 W"); + assert_eq!( + plausible_gpu_counter(48_751, w, &[], Some(65_000)), + over(97_502, 97_500, GpuBound::PowerLimit) + ); + // Nothing to judge it by: no bound applies, and with no limit to + // exceed no reading can be unusable either. + assert_eq!( + plausible_gpu_counter(1_000_000, w, &[], None), + GpuPlausibility::Plausible { rate_mw: 2_000_000 }, + "nothing to judge by" + ); } /// The liveness a GPU probe gives: each GPU's rise from the first sample diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs index 778950b1f..7b5c13b57 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs @@ -63,9 +63,17 @@ enum Mode { fn mode_for(supported: bool, liveness: CounterLiveness) -> Mode { match (supported, liveness) { (true, CounterLiveness::Live) => Mode::Counter, - (_, CounterLiveness::Live | CounterLiveness::Frozen | CounterLiveness::Unavailable | CounterLiveness::Implausible) => { - Mode::PowerIntegral - } + // An unjudgeable counter is no more admissible than an implausible + // one: neither can be graded `HwShunt`, so both fall back to the + // vendor's power model. + ( + _, + CounterLiveness::Live + | CounterLiveness::Frozen + | CounterLiveness::Unavailable + | CounterLiveness::Implausible + | CounterLiveness::Unjudgeable, + ) => Mode::PowerIntegral, } } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs index 8502f1f63..698f82291 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs @@ -205,7 +205,10 @@ impl ProcessCpuCounter { fn admits(liveness: CounterLiveness) -> bool { match liveness { CounterLiveness::Live => true, - CounterLiveness::Unavailable | CounterLiveness::Frozen | CounterLiveness::Implausible => false, + CounterLiveness::Unavailable + | CounterLiveness::Frozen + | CounterLiveness::Implausible + | CounterLiveness::Unjudgeable => false, } } @@ -277,7 +280,12 @@ mod tests { /// measurement, and its work still runs. #[test] fn a_counter_that_is_not_live_meters_nothing() { - for liveness in [CounterLiveness::Frozen, CounterLiveness::Implausible, CounterLiveness::Unavailable] { + for liveness in [ + CounterLiveness::Frozen, + CounterLiveness::Implausible, + CounterLiveness::Unjudgeable, + CounterLiveness::Unavailable, + ] { let c = ProcessCpuCounter::given(liveness); assert!(!c.is_available() && !EnergyMeter::is_available(&c), "{liveness:?}"); assert!(matches!(c.read(), Err(EnergyError::NoCounter(EnergyDomain::ProcessCpu))), "{liveness:?}"); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process_gpu.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process_gpu.rs index c56ff511a..1d4500f6d 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process_gpu.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process_gpu.rs @@ -396,7 +396,10 @@ pub fn probe_gpu_liveness_uncached(load: impl FnMut()) -> CounterLiveness { fn admits(liveness: CounterLiveness) -> bool { match liveness { CounterLiveness::Live => true, - CounterLiveness::Unavailable | CounterLiveness::Frozen | CounterLiveness::Implausible => false, + CounterLiveness::Unavailable + | CounterLiveness::Frozen + | CounterLiveness::Implausible + | CounterLiveness::Unjudgeable => false, } } @@ -645,7 +648,12 @@ mod tests { /// still runs. #[test] fn a_counter_that_is_not_live_meters_nothing() { - for liveness in [CounterLiveness::Frozen, CounterLiveness::Implausible, CounterLiveness::Unavailable] { + for liveness in [ + CounterLiveness::Frozen, + CounterLiveness::Implausible, + CounterLiveness::Unjudgeable, + CounterLiveness::Unavailable, + ] { let c = ProcessGpuCounter::given(liveness); assert!(!c.is_available(), "{liveness:?}"); assert!(matches!(c.snapshot(), Err(EnergyError::NoCounter(EnergyDomain::ProcessGpu)))); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs index accf2cea8..9be892dbc 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs @@ -270,7 +270,10 @@ impl RaplCounter { fn admits(liveness: CounterLiveness) -> bool { match liveness { CounterLiveness::Live => true, - CounterLiveness::Unavailable | CounterLiveness::Frozen | CounterLiveness::Implausible => false, + CounterLiveness::Unavailable + | CounterLiveness::Frozen + | CounterLiveness::Implausible + | CounterLiveness::Unjudgeable => false, } } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs index bca4a6ff2..411fdfbdb 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs @@ -196,9 +196,13 @@ mod nvml_oracle { Unavailable, /// A counter did not keep advancing. Frozen, - /// Every counter kept advancing, but one is not plausibly its board's - /// energy (see the module doc). + /// Every counter kept advancing, but one ran faster than its board + /// can draw (see the module doc). Implausible, + /// Every counter kept advancing, and one could not be judged because + /// its board's own telemetry read past the board's own limit. The + /// yardstick is broken, not necessarily the counter. + Unjudgeable, /// Every counter kept advancing, plausibly its board's energy. Live, } @@ -207,19 +211,36 @@ mod nvml_oracle { /// average `rate_w`, the board's draws `draws_w` around the window and /// its maximum power limit `ceiling_w`. A bound whose input is missing /// is not applied. - pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { - if let Some(c) = ceiling_w { - if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { - return false; + /// + /// A draw reading past the board's own limit is not a reading — an RTX + /// 4050 Laptop GPU reports about 590 W for a second after it wakes — so it + /// is dropped, and only it. Only when nothing usable is left is there no + /// yardstick at all, and then the counter is `Unjudgeable` rather than + /// `Implausible`: the yardstick is broken, not necessarily the counter. + /// + /// This oracle is written from the rule rather than from the crate, in + /// f64 and from `nvidia-smi` rather than the crate's own sampling, so it + /// is an independent check on the same rule — which is only true while it + /// encodes the rule the crate implements. + pub fn judge(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> Verdict { + let cap = ceiling_w.map(|c| 1.5 * c); + let (usable, unusable): (Vec, Vec) = + draws_w.iter().copied().partition(|d| cap.is_none_or(|c| *d <= c)); + if let Some(c) = cap { + if rate_w > c { + return Verdict::Implausible; + } + if usable.is_empty() && !unusable.is_empty() { + return Verdict::Unjudgeable; } } - if !draws_w.is_empty() { - let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as f64; if rate_w > 2.0 * mean + 5.0 { - return false; + return Verdict::Implausible; } } - true + Verdict::Live } /// What the oracle saw. @@ -302,10 +323,16 @@ mod nvml_oracle { let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + // The verdict that speaks for the run is the most severe of its GPUs', + // as the crate's is: a counter proven to over-run is a finding, while + // one board's broken telemetry only withholds a judgement. + let judged: Vec = (0..n).map(|i| judge(rate_w[i], &draws_w[i], ceiling_w[i])).collect(); let verdict = if !live(&advances, intervals, window) { Verdict::Frozen - } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + } else if judged.contains(&Verdict::Implausible) { Verdict::Implausible + } else if judged.contains(&Verdict::Unjudgeable) { + Verdict::Unjudgeable } else { Verdict::Live }; @@ -362,6 +389,7 @@ fn the_nvml_liveness_agrees_with_the_oracle_at_its_own_tick() { Verdict::Unavailable => assert_eq!(liveness, CounterLiveness::Unavailable), Verdict::Frozen => assert_eq!(liveness, CounterLiveness::Frozen), Verdict::Implausible => assert_eq!(liveness, CounterLiveness::Implausible), + Verdict::Unjudgeable => assert_eq!(liveness, CounterLiveness::Unjudgeable), Verdict::Live => {} } assert_eq!(counter.uses_true_counter(), liveness == CounterLiveness::Live, "the counter mode disagrees"); diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs index 6f2bcdee2..057eeb07a 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs @@ -523,46 +523,182 @@ pub(super) const GPU_DRAW_SLACK_MW: u128 = 5_000; /// that while it is polled: at the probes' 100-150 ms ticks it ran at /// 44-305 W while the board drew 5-14 W against a 65 W limit, and polled /// every 1-2 s it still read 11-23 W while the board drew 2-10 W. +/// Which of the two bounds a GPU counter ran past. #[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` -pub(super) fn plausible_gpu_counter(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum GpuBound { + /// [`GPU_CEILING_MARGIN`] of the board's maximum power limit. + PowerLimit, + /// [`GPU_DRAW_FACTOR`] times the board's mean draw plus + /// [`GPU_DRAW_SLACK_MW`]. + MeanDraw, +} + +/// What the plausibility check found, with the figures it found it from. +/// +/// A `bool` was the whole of this, and it threw away everything an operator +/// needs: which bound broke, by how much, and the rate it broke it at. A +/// counter 1.1 times over a bound and one 20 times over were the same answer. +/// The figures also make the judgement reproducible — the margins are +/// constants in this module, and re-grading a past finding against a +/// different margin is only possible if the rate survived the check. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum GpuPlausibility { + /// The counter's rate sits inside every bound whose reading was present. + Plausible { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + }, + /// The counter ran faster than the board can draw. + CounterOverRate { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + /// The bound it broke, in milliwatts. + bound_mw: u64, + /// Which bound that was. + bound: GpuBound, + }, + /// The board's own telemetry read past the board's own power limit, so + /// there is no yardstick left to judge the counter against. An + /// indeterminate, not a finding about the counter — see + /// [`CounterStatus::Unjudgeable`]. + BoardTelemetryUnusable { + /// The offending draw reading, in milliwatts. + draw_mw: u64, + /// The limit it exceeded (the margin-adjusted cap), in milliwatts. + limit_mw: u64, + }, +} + +impl GpuPlausibility { + /// Whether the counter was judged to be counting its board's energy. + /// + /// `false` for an indeterminate as well as for an overrun: unjudgeable is + /// not admissible, and erring toward declining is the safe direction. Use + /// the variant itself to tell the two apart. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` + pub(super) fn plausible(&self) -> bool { + matches!(self, GpuPlausibility::Plausible { .. }) + } + + /// How severe this verdict is, for picking the one that speaks for a run + /// of several GPUs: a proven overrun outranks an indeterminate, which + /// outranks a pass. + fn rank(&self) -> u8 { + match self { + GpuPlausibility::Plausible { .. } => 0, + GpuPlausibility::BoardTelemetryUnusable { .. } => 1, + GpuPlausibility::CounterOverRate { .. } => 2, + } + } + + /// The figure this verdict turns on, to order two of equal rank by. + fn magnitude(&self) -> u64 { + match self { + GpuPlausibility::Plausible { rate_mw } => *rate_mw, + GpuPlausibility::CounterOverRate { rate_mw, .. } => *rate_mw, + GpuPlausibility::BoardTelemetryUnusable { draw_mw, .. } => *draw_mw, + } + } +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) fn plausible_gpu_counter( + energy_mj: u64, + window: Duration, + draws_mw: &[u32], + ceiling_mw: Option, +) -> GpuPlausibility { let ns = window.as_nanos().max(1); // mJ per second is mW. let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; - if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { - let cap = u128::from(limit) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1; - if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { - return false; + let clamp = |v: u128| u64::try_from(v).unwrap_or(u64::MAX); + let cap = ceiling_mw + .filter(|&l| l > 0) + .map(|l| u128::from(l) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1); + + // A draw reading past the board's own limit is not a reading: an RTX 4050 + // Laptop GPU reports about 590 W for a second after it wakes. Such a + // reading is dropped, and only that reading — it is one broken sample, + // not a broken bracket. Voiding the whole judgement on it is what this + // code used to do, and jetson-hub's hardware test refused that shape: one + // spike among otherwise sane 14.8 W samples turned a counter running + // persistently at 4-30x the board's draw into an indeterminate, which is + // a worse diagnosis than the overrun it actually is. + // + // Dropping them also keeps the mean honest, which is what the old + // short-circuit was really for. The mean summed every reading, so a 590 W + // spike inflated it and made the mean-draw bound *more* permissive; that + // only went unnoticed because any such reading failed the check first. + let (usable, unusable): (Vec, Vec) = draws_mw + .iter() + .map(|&d| u128::from(d)) + .partition(|&d| cap.is_none_or(|c| d <= c)); + + if let Some(cap) = cap { + if rate_mw > cap { + return GpuPlausibility::CounterOverRate { + rate_mw: clamp(rate_mw), + bound_mw: clamp(cap), + bound: GpuBound::PowerLimit, + }; + } + // Only when nothing usable is left is there no yardstick at all. + if usable.is_empty() && !unusable.is_empty() { + return GpuPlausibility::BoardTelemetryUnusable { + draw_mw: clamp(unusable[0]), + limit_mw: clamp(cap), + }; } } - if !draws_mw.is_empty() { - let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; - if rate_mw > GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW { - return false; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as u128; + let bound = GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW; + if rate_mw > bound { + return GpuPlausibility::CounterOverRate { + rate_mw: clamp(rate_mw), + bound_mw: clamp(bound), + bound: GpuBound::MeanDraw, + }; } } - true + GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) } } -/// Whether every GPU counter of a probe run plausibly counted its board's -/// energy ([`plausible_gpu_counter`]): each rail's rise from the run's first +/// What the GPU counters of a probe run were judged to be counting +/// ([`plausible_gpu_counter`]): each rail's rise from the run's first /// sample to its last, over the window between them, beside that GPU's draw /// readings (`draws`, one row per sample call, one column per GPU, `None` /// where a reading failed) and its maximum power limit (`ceilings`, one per /// GPU). Rails and columns are in device order. A run with fewer than two /// samples has nothing to judge. #[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` -pub(super) fn gpu_counters_plausible(watch: &RailWatch, draws: &[Vec>], ceilings: &[Option]) -> bool { +pub(super) fn gpu_counters_verdict( + watch: &RailWatch, + draws: &[Vec>], + ceilings: &[Option], +) -> GpuPlausibility { let (Some(first), Some(last), Some(t0), Some(t1)) = (watch.observed.first(), watch.observed.last(), watch.at.first(), watch.at.last()) else { - return true; + return GpuPlausibility::Plausible { rate_mw: 0 }; }; let window = t1.saturating_sub(*t0); - first.iter().zip(last).enumerate().all(|(i, ((_, a), (_, b)))| { - let energy_mj = u64::try_from(b.saturating_sub(*a)).unwrap_or(0); - let gpu_draws: Vec = draws.iter().filter_map(|row| row.get(i).copied().flatten()).collect(); - plausible_gpu_counter(energy_mj, window, &gpu_draws, ceilings.get(i).copied().flatten()) - }) + // The verdict that speaks for the run is the most severe one, and among + // equals the largest figure: a run is not excused by its healthiest GPU. + first + .iter() + .zip(last) + .enumerate() + .map(|(i, ((_, a), (_, b)))| { + let energy_mj = u64::try_from(b.saturating_sub(*a)).unwrap_or(0); + let gpu_draws: Vec = + draws.iter().filter_map(|row| row.get(i).copied().flatten()).collect(); + plausible_gpu_counter(energy_mj, window, &gpu_draws, ceilings.get(i).copied().flatten()) + }) + .max_by_key(|v| (v.rank(), v.magnitude())) + .unwrap_or(GpuPlausibility::Plausible { rate_mw: 0 }) } /// A counter probe's constructor behind its gate: `open` runs only when @@ -1939,36 +2075,100 @@ mod tests { #[test] fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { let w = Duration::from_millis(500); - // 66 W counted against a 7.5 W draw: what jetson-hub's probe saw. - assert!(!plausible_gpu_counter(33_000, w, &[7_500, 7_700], Some(65_000))); + // Over a 500 ms window a counter's rate in mW is twice its mJ. + let over = |rate_mw, bound_mw, bound| GpuPlausibility::CounterOverRate { rate_mw, bound_mw, bound }; + + // 66 W counted against a 7.6 W mean draw: what jetson-hub's probe saw. + // The figures, not just a `false` — 66 W against a 20.2 W bound is a + // 3.3x overrun, and an operator cannot act on a bare bool. + assert_eq!( + plausible_gpu_counter(33_000, w, &[7_500, 7_700], Some(65_000)), + over(66_000, 20_200, GpuBound::MeanDraw) + ); // 44 W at the 150 ms tick against 10.4 W: past twice the draw + 5 W. - assert!(!plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000))); + assert_eq!( + plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000)), + over(44_000, 25_800, GpuBound::MeanDraw) + ); // 120 W, past 1.5 x the 65 W limit, even with no draw reading. - assert!(!plausible_gpu_counter(60_000, w, &[], Some(65_000))); - // A draw reading of 590 W (the board's first second after waking) is no - // reading: it would excuse any counter. - assert!(!plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000))); + assert_eq!( + plausible_gpu_counter(60_000, w, &[], Some(65_000)), + over(120_000, 97_500, GpuBound::PowerLimit) + ); + + // A draw reading of 590 W (the board's first second after waking) is + // no reading. It is dropped, and only it: the remaining 10.3 W reading + // still judges the counter, and 88 W against it is an overrun. This + // used to return a bare `false` meaning "a draw reading was broken", + // which is the one thing the hardware test on jetson-hub refused: it + // turned a persistent 4-30x overrun into "cannot judge". + assert_eq!( + plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000)), + over(88_000, 25_600, GpuBound::MeanDraw) + ); + // Dropping it also keeps the mean honest, and this pins that: the + // bound is twice 10 W plus 5 W, computed from the usable reading + // alone. Had the 590 W reading been summed in, the mean would be + // 300 W and the bound 605 W, which would excuse this counter. + assert_eq!( + plausible_gpu_counter(15_000, w, &[590_000, 10_000], Some(65_000)), + over(30_000, 25_000, GpuBound::MeanDraw) + ); + // Only when no usable reading is left is there no yardstick at all, + // and then the verdict is an indeterminate about the board, not a + // finding about the counter. + assert_eq!( + plausible_gpu_counter(44_000, w, &[590_000], Some(65_000)), + GpuPlausibility::BoardTelemetryUnusable { draw_mw: 590_000, limit_mw: 97_500 } + ); + // A counter past the board's own limit is a finding whatever the + // telemetry says: the rate bound is checked before the yardstick is. + assert_eq!( + plausible_gpu_counter(60_000, w, &[590_000], Some(65_000)), + over(120_000, 97_500, GpuBound::PowerLimit) + ); + // A counter tracking its board: 13 W against 13 W, and 300 W against a // 280 W draw on a 700 W board. - assert!(plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000))); - assert!(plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000))); + assert_eq!( + plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000)), + GpuPlausibility::Plausible { rate_mw: 13_000 } + ); + assert_eq!( + plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000)), + GpuPlausibility::Plausible { rate_mw: 300_000 } + ); // A load step the one-second draw has not caught up with, up to twice // the draw plus 5 W: 205 W against 100 W. - assert!(plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000))); - assert!(!plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000))); + assert!(plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000)).plausible()); + assert_eq!( + plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000)), + over(205_002, 205_000, GpuBound::MeanDraw) + ); // The exact bounds. - assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); - assert!(!plausible_gpu_counter(48_751, w, &[], Some(65_000))); - assert!(plausible_gpu_counter(12_500, w, &[10_000], None), "25 W is 2 x 10 W + 5 W"); - assert!(!plausible_gpu_counter(12_501, w, &[10_000], None)); - // Nothing to judge it by: no bound applies. - assert!(plausible_gpu_counter(1_000_000, w, &[], None)); + assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)).plausible(), "97.5 W is 1.5 x 65 W"); + assert_eq!( + plausible_gpu_counter(48_751, w, &[], Some(65_000)), + over(97_502, 97_500, GpuBound::PowerLimit) + ); + assert!(plausible_gpu_counter(12_500, w, &[10_000], None).plausible(), "25 W is 2 x 10 W + 5 W"); + assert_eq!( + plausible_gpu_counter(12_501, w, &[10_000], None), + over(25_002, 25_000, GpuBound::MeanDraw) + ); + // Nothing to judge it by: no bound applies, and with no limit to + // exceed no reading can be unusable either. + assert_eq!( + plausible_gpu_counter(1_000_000, w, &[], None), + GpuPlausibility::Plausible { rate_mw: 2_000_000 } + ); } /// The rule over a probe run: each GPU's rise from the run's first sample /// to its last, over the window between them, beside that GPU's own draws - /// and limit. One implausible GPU of two makes the run implausible, and a - /// GPU's readings are never judged against another's. + /// and limit. One implausible GPU of two makes the run implausible, a + /// GPU's readings are never judged against another's, and the verdict + /// that speaks for the run is the most severe of its GPUs'. #[test] fn every_gpu_of_a_run_must_count_no_faster_than_its_board_draws() { // Two GPUs republished every 100 ms, sampled at the NVML tick: one at @@ -1984,12 +2184,32 @@ mod tests { let rows = r.watch.observed.len(); let draws = |a: u32, b: u32| vec![vec![Some(a), Some(b)]; rows]; let ceilings = [Some(65_000), Some(700_000)]; - assert!(gpu_counters_plausible(&r.watch, &draws(10_000, 290_000), &ceilings), "each at its own draw"); - assert!(!gpu_counters_plausible(&r.watch, &draws(1_000, 290_000), &ceilings), "the first at 9 x its draw"); - assert!(!gpu_counters_plausible(&r.watch, &draws(10_000, 290_000), &[Some(65_000), Some(65_000)]), "300 W on a 65 W board"); - assert!(!gpu_counters_plausible(&r.watch, &draws(290_000, 10_000), &[None, None]), "draws swapped between the GPUs"); - assert!(gpu_counters_plausible(&r.watch, &[], &[None, None]), "no reading to judge by"); - assert!(gpu_counters_plausible(&RailWatch::default(), &[], &[]), "no samples"); + let verdict = |d: &[Vec>], c: &[Option]| gpu_counters_verdict(&r.watch, d, c); + assert!(verdict(&draws(10_000, 290_000), &ceilings).plausible(), "each at its own draw"); + assert!(!verdict(&draws(1_000, 290_000), &ceilings).plausible(), "the first at 9 x its draw"); + assert!( + !verdict(&draws(10_000, 290_000), &[Some(65_000), Some(65_000)]).plausible(), + "300 W on a 65 W board" + ); + assert!(!verdict(&draws(290_000, 10_000), &[None, None]).plausible(), "draws swapped between the GPUs"); + assert!(verdict(&[], &[None, None]).plausible(), "no reading to judge by"); + assert!(gpu_counters_verdict(&RailWatch::default(), &[], &[]).plausible(), "no samples"); + + // Which verdict speaks for a run of several GPUs. One board's broken + // telemetry withholds a judgement on that board, so the run is + // unjudgeable — not implausible, which would blame a counter nothing + // measured, and not plausible, which would let the healthy GPU excuse + // the run. + assert!(matches!( + verdict(&draws(590_000, 290_000), &ceilings), + GpuPlausibility::BoardTelemetryUnusable { .. } + )); + // But a GPU proven to be over-running outranks another's broken + // telemetry: a finding beats an indeterminate. + assert!(matches!( + verdict(&draws(590_000, 290_000), &[Some(65_000), Some(65_000)]), + GpuPlausibility::CounterOverRate { .. } + )); } /// The tick a counter probe burns is the tick it was asked for. @@ -2016,7 +2236,12 @@ mod tests { /// its own error passes through. #[test] fn a_gated_counter_is_opened_only_when_live() { - for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible] { + for status in [ + CounterStatus::Unavailable, + CounterStatus::Frozen, + CounterStatus::Implausible, + CounterStatus::Unjudgeable, + ] { let opened = Cell::new(false); let r = gated("RAPL", status, || { opened.set(true); diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs index c21c7f2f5..3e0d56297 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs @@ -190,12 +190,24 @@ pub enum CounterStatus { /// Every rail it sums kept advancing, but a GPU counter (NVML's) ran /// faster than its board can draw power: past 1.5 times the board's /// maximum power limit, or past twice the board's own power draw plus - /// 5 W, or the board's draw reading was itself past that limit. Such a - /// counter is not counting the board's energy, so a figure from it would - /// seal a number the device never spent. Seen on an RTX 4050 Laptop GPU - /// (driver 595.84), whose counter runs at 4 to 30 times the board's draw - /// while it is polled. Never used. + /// 5 W. Such a counter is not counting the board's energy, so a figure + /// from it would seal a number the device never spent. Seen on an RTX + /// 4050 Laptop GPU (driver 595.84), whose counter runs at 4 to 30 times + /// the board's draw while it is polled. Never used. Implausible, + /// Every rail it sums kept advancing and the counter could not be judged, + /// because the board's own power telemetry was unusable over the probe: a + /// draw reading past the board's own maximum power limit (an RTX 4050 + /// Laptop GPU reports about 590 W for a second after it wakes). + /// + /// Split out from [`CounterStatus::Implausible`], which used to cover + /// this — its doc then asserted "is not counting the board's energy", a + /// conclusion this case does not support. The yardstick is broken, not + /// necessarily the counter. Never used either way: unjudgeable is not + /// admissible, and erring toward declining is the safe direction. The + /// distinction is the diagnosis an operator gets, which is the difference + /// between replacing a counter and waiting for the board to settle. + Unjudgeable, /// Every rail it sums kept advancing: in at least five sample intervals, /// at least four in five of those observed, over at least 300 ms of wall /// time, while the probing thread burned CPU. The probe is used. @@ -214,6 +226,10 @@ impl std::fmt::Display for CounterStatus { "the counter kept advancing, but faster than its device can draw power, so it does not \ count the device's energy" } + CounterStatus::Unjudgeable => { + "the counter kept advancing, but the device's own power telemetry read past the \ + device's limit, so there is no yardstick to judge the counter against" + } CounterStatus::Live => "every rail of the counter keeps advancing under load", }) } diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs index 98c27ce13..391965e6d 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs @@ -34,7 +34,8 @@ //! that work ([`Measurement::reads_cpu`](super::Measurement::reads_cpu)). use super::liveness::{ - counter_run, counter_status_of, counter_tick, gated, gpu_counters_plausible, RailWatch, Sample, NVML_PROBE_TICK, + counter_run, counter_status_of, counter_tick, gated, gpu_counters_verdict, GpuPlausibility, RailWatch, Sample, + NVML_PROBE_TICK, }; use std::cell::RefCell; use super::{CounterStatus, Probe, ProbeError, CPU_RAIL_LIVENESS_BUDGET}; @@ -78,12 +79,19 @@ pub(super) fn host_status() -> CounterStatus { status_of_run(&run.watch, &run.draws, &run.ceilings) } -/// The status of an NVML probe run: its verdict's, except that a live run -/// whose counters did not plausibly count their boards' energy -/// ([`gpu_counters_plausible`]) is `Implausible`. +/// The status of an NVML probe run: its verdict's, except that a live run is +/// re-judged on whether its counters counted their boards' energy +/// ([`gpu_counters_verdict`]). A counter proven to run faster than its board +/// can draw is `Implausible`; a board whose own telemetry is unusable leaves +/// the counter `Unjudgeable`, which is a different diagnosis and not a claim +/// about the counter. fn status_of_run(watch: &RailWatch, draws: &[Vec>], ceilings: &[Option]) -> CounterStatus { match counter_status_of(watch.verdict()) { - CounterStatus::Live if !gpu_counters_plausible(watch, draws, ceilings) => CounterStatus::Implausible, + CounterStatus::Live => match gpu_counters_verdict(watch, draws, ceilings) { + GpuPlausibility::Plausible { .. } => CounterStatus::Live, + GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible, + GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable, + }, status => status, } } @@ -297,7 +305,12 @@ mod tests { /// `Live` builds the probe wherever NVML opens with a supported GPU. #[test] fn only_a_live_status_builds_the_nvml_probe() { - for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible] { + for status in [ + CounterStatus::Unavailable, + CounterStatus::Frozen, + CounterStatus::Implausible, + CounterStatus::Unjudgeable, + ] { match NvmlProbe::given(status) { Err(ProbeError::Unavailable(why)) => assert!(why.contains("NVML"), "{why}"), Err(e) => panic!("{status:?}: {e}"), diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs b/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs index 170745c5e..bc5812ed0 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs @@ -196,9 +196,13 @@ mod nvml_oracle { Unavailable, /// A counter did not keep advancing. Frozen, - /// Every counter kept advancing, but one is not plausibly its board's - /// energy (see the module doc). + /// Every counter kept advancing, but one ran faster than its board + /// can draw (see the module doc). Implausible, + /// Every counter kept advancing, and one could not be judged because + /// its board's own telemetry read past the board's own limit. The + /// yardstick is broken, not necessarily the counter. + Unjudgeable, /// Every counter kept advancing, plausibly its board's energy. Live, } @@ -207,19 +211,36 @@ mod nvml_oracle { /// average `rate_w`, the board's draws `draws_w` around the window and /// its maximum power limit `ceiling_w`. A bound whose input is missing /// is not applied. - pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { - if let Some(c) = ceiling_w { - if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { - return false; + /// + /// A draw reading past the board's own limit is not a reading — an RTX + /// 4050 Laptop GPU reports about 590 W for a second after it wakes — so it + /// is dropped, and only it. Only when nothing usable is left is there no + /// yardstick at all, and then the counter is `Unjudgeable` rather than + /// `Implausible`: the yardstick is broken, not necessarily the counter. + /// + /// This oracle is written from the rule rather than from the crate, in + /// f64 and from `nvidia-smi` rather than the crate's own sampling, so it + /// is an independent check on the same rule — which is only true while it + /// encodes the rule the crate implements. + pub fn judge(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> Verdict { + let cap = ceiling_w.map(|c| 1.5 * c); + let (usable, unusable): (Vec, Vec) = + draws_w.iter().copied().partition(|d| cap.is_none_or(|c| *d <= c)); + if let Some(c) = cap { + if rate_w > c { + return Verdict::Implausible; + } + if usable.is_empty() && !unusable.is_empty() { + return Verdict::Unjudgeable; } } - if !draws_w.is_empty() { - let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if !usable.is_empty() { + let mean = usable.iter().sum::() / usable.len() as f64; if rate_w > 2.0 * mean + 5.0 { - return false; + return Verdict::Implausible; } } - true + Verdict::Live } /// What the oracle saw. @@ -302,10 +323,16 @@ mod nvml_oracle { let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + // The verdict that speaks for the run is the most severe of its GPUs', + // as the crate's is: a counter proven to over-run is a finding, while + // one board's broken telemetry only withholds a judgement. + let judged: Vec = (0..n).map(|i| judge(rate_w[i], &draws_w[i], ceiling_w[i])).collect(); let verdict = if !live(&advances, intervals, window) { Verdict::Frozen - } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + } else if judged.contains(&Verdict::Implausible) { Verdict::Implausible + } else if judged.contains(&Verdict::Unjudgeable) { + Verdict::Unjudgeable } else { Verdict::Live }; @@ -359,6 +386,7 @@ fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), Verdict::Implausible => assert_eq!(status, CounterStatus::Implausible), + Verdict::Unjudgeable => assert_eq!(status, CounterStatus::Unjudgeable), Verdict::Live => {} } assert_eq!(sources.contains(&"nvml-total-energy"), status == CounterStatus::Live, "selection disagrees with the status");