From f18337dc8133a061d4da543ba4d4a69e0ebdc01f Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Tue, 20 Oct 2026 04:15:36 -0400 Subject: [PATCH] energy meters: a GPU counter reading low is refuted in sandbox-meter, eoc-meter and jcp-energy too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #341 bounded an NVML cumulative-energy counter from below in jouleclaw-energy. The same plausibility check exists in three more meter crates, and each still judged the counter in one direction only. A counter that accrued a fraction of what its board drew still advanced, so the liveness rule called it live. It was then graded `HwShunt` and spent an energy ceiling it was not measuring. Under-reporting is the unsafe direction at a ceiling (jcp SPEC §5): it admits work the budget cannot cover. A counter running fast fails safe. sandbox-meter, eoc-meter and jcp-energy now carry `CounterUnderRate`, judged against the same board telemetry and with the same constants as #341: floor = (mean_usable_draw - 5 W) / 4 So a counter one meter declines, another does not admit. The slack comes off the draw before the division, so an idle board makes no finding. The verdict carries the floor and the draw it was taken from. In sandbox-meter, which judges several GPUs per run, an under-report outranks an over-report, and the larger shortfall speaks. sandbox-meter's NVML probe only builds on Linux, so its verdict-to-status mapping was tested only by CI's Linux legs. The mapping now lives in the shared `liveness` module (`gpu_counter_status`), which builds and is tested on every platform. SPEC changes: - jcp §5.1: the plausibility rule is now two-sided, with the floor allowing for the draw's imprecision and never firing near idle. - jouleclaw §1.3: no longer claims the reference implementation already falls back to the power integral on an unvalidated GPU. It falls back only where it can refute the counter, and the bound is wide: a counter 13.3 % low passes it and is graded `HwShunt` until something it shares no error with validates it. Tests: 7 new across the three crates. All 12 mutations of the new logic are killed. Co-Authored-By: Claude Opus 5.5 --- eoc/eoc-rs/crates/eoc-meter/src/lib.rs | 99 ++++++++++- jcp/SPEC.md | 14 ++ jcp/jcp-rs/crates/jcp-energy/src/lib.rs | 99 ++++++++++- jouleclaw/SPEC.md | 16 +- .../sandbox-meter/src/probe/liveness.rs | 160 +++++++++++++++++- .../crates/sandbox-meter/src/probe/nvml.rs | 25 +-- 6 files changed, 386 insertions(+), 27 deletions(-) diff --git a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs index 7c5c9c351..0109f9776 100644 --- a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs +++ b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs @@ -5153,6 +5153,24 @@ mod gpu_plausibility { /// NVML's documented accuracy of a board's power draw, ±5 W, in mW. pub(crate) const DRAW_SLACK_MW: u128 = 5_000; + /// How far **below** its board's mean draw a counter may accrue before it + /// is refuted: a shortfall of more than this factor, after + /// [`DRAW_SLACK_MW`] is taken off the draw, is no count of the board's + /// energy. + /// + /// The bound has to exist in this direction because **under-reporting is + /// the unsafe direction at an energy ceiling**: it admits work the budget + /// cannot cover. The ceiling bounds only catch a counter running *fast*, + /// which fails safe. A counter republished in lumps and sampled across its + /// republish period reads low rather than frozen, so liveness alone does + /// not see it. It is looser than [`DRAW_FACTOR`] because the draw is a + /// one-second average a bursty load integrates below, and the slack comes + /// off before the division so an idle board (at or under the slack) makes + /// no finding. The same factor as `jouleclaw-energy`'s and + /// `sandbox-meter`'s, so a counter one meter declines another does not + /// admit. + pub(crate) const UNDER_FACTOR: u128 = 4; + /// Which of the two bounds a GPU counter ran past. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum GpuBound { @@ -5188,6 +5206,19 @@ mod gpu_plausibility { /// Which bound that was. bound: GpuBound, }, + /// The counter accrued far less energy than the board's own telemetry + /// says it drew: the dangerous finding, because a counter reading low + /// is graded `HwShunt` and then spends a budget it is not measuring, + /// while it still advances, so liveness calls it live. + CounterUnderRate { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + /// The floor it fell below, in milliwatts. + floor_mw: u64, + /// The board's mean usable draw the floor was taken from, in + /// milliwatts: the yardstick, so the judgement can be re-made. + mean_draw_mw: u64, + }, /// The board's own telemetry read past the board's own power limit, so /// there is no yardstick left to judge the counter against. An /// indeterminate, not a finding about the counter — see @@ -5218,8 +5249,9 @@ mod gpu_plausibility { /// judged to be counting, beside the board's power draw `draws_mw` read /// with the samples and its maximum power limit `ceiling_mw`. It over-ran /// if it ran past [`CEILING_MARGIN`] of the limit, or past [`DRAW_FACTOR`] - /// times the mean draw plus [`DRAW_SLACK_MW`]. A bound whose reading is - /// missing is not applied. + /// times the mean draw plus [`DRAW_SLACK_MW`]; it under-ran if it fell + /// short of the mean draw, less the slack, by more than [`UNDER_FACTOR`]. + /// A bound whose reading is missing is not applied. /// /// Liveness says the counter is republished continuously; this says what /// it republishes is the board's energy. On the RTX 4050 Laptop GPU @@ -5283,6 +5315,17 @@ mod gpu_plausibility { bound: GpuBound::MeanDraw, }; } + // And the same yardstick from below. The slack comes off the draw + // before the division, so the floor is zero at or under + // `DRAW_SLACK_MW` of mean draw: an idle board makes no finding. + let floor = mean.saturating_sub(DRAW_SLACK_MW) / UNDER_FACTOR; + if floor > 0 && rate_mw < floor { + return GpuPlausibility::CounterUnderRate { + rate_mw: clamp(rate_mw), + floor_mw: clamp(floor), + mean_draw_mw: clamp(mean), + }; + } } GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) } } @@ -5312,8 +5355,8 @@ mod gpu_plausibility { /// The status a GPU probe gives: the liveness rule's `status`, except that /// a live counter is re-judged on what it was counting over the probe. One - /// proven to run faster than its board can draw is - /// [`CounterStatus::Implausible`]; one whose board's own telemetry was + /// proven to run faster than its board can draw, or far slower than it + /// draws, is [`CounterStatus::Implausible`]; one whose board's own telemetry was /// unusable is [`CounterStatus::Unjudgeable`], which is a different /// diagnosis and not a claim about the counter. pub(crate) fn status_of(status: CounterStatus, gpu: &GpuWatch) -> CounterStatus { @@ -5324,7 +5367,9 @@ mod gpu_plausibility { (Some((a, t0)), Some((b, t1))) => { match plausible(b.saturating_sub(a), t1.saturating_sub(t0), &gpu.draws_mw, gpu.ceiling_mw) { GpuPlausibility::Plausible { .. } => CounterStatus::Live, - GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible, + GpuPlausibility::CounterOverRate { .. } | GpuPlausibility::CounterUnderRate { .. } => { + CounterStatus::Implausible + } GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable, } } @@ -5437,6 +5482,50 @@ mod gpu_plausibility { ); } + /// **The bound that was missing: a counter reading LOW.** It still + /// advances, so liveness calls it live, and graded `HwShunt` it would + /// spend a ceiling it is not measuring. Figures are the jetson-hub RTX + /// 4050's own. + #[test] + fn a_gpu_counter_must_count_no_slower_than_its_board_draws() { + let w = Duration::from_millis(500); + // floor = (14_800 - 5_000) / 4 = 2_450 mW. + assert_eq!( + plausible(250, w, &[14_800, 14_800], Some(65_000)), + GpuPlausibility::CounterUnderRate { rate_mw: 500, floor_mw: 2_450, mean_draw_mw: 14_800 }, + "0.5 W counted against a 14.8 W draw" + ); + assert!(plausible(1_300, w, &[14_800, 14_800], Some(65_000)).plausible(), "lagging, not refuted"); + assert!(plausible(1_225, w, &[14_800, 14_800], Some(65_000)).plausible(), "at the floor"); + // An idle board makes no finding; nor does a board with no reading. + for mean in [0u32, 1_000, 5_000] { + assert!(plausible(0, w, &[mean, mean], Some(65_000)).plausible(), "{mean} mW"); + } + assert!(plausible(0, w, &[], Some(65_000)).plausible()); + assert_eq!( + plausible(0, w, &[9_000, 9_000], Some(65_000)), + GpuPlausibility::CounterUnderRate { rate_mw: 0, floor_mw: 1_000, mean_draw_mw: 9_000 } + ); + } + + /// The wiring, not only the verdict: a live counter that accrued a + /// fraction of what its board drew is `Implausible`, and the same slow + /// counter beside an idle board stays `Live`. + #[test] + fn a_live_gpu_counter_short_of_its_board_is_implausible() { + let watch = |draw_mw| { + let mut g = GpuWatch { ceiling_mw: Some(65_000), ..GpuWatch::default() }; + for k in 0..6u64 { + // 10 mJ every 100 ms: 100 mW. + g.saw(Some(1_000_000 + 10 * k), Some(draw_mw), Duration::from_millis(100 * k)); + } + g + }; + use CounterStatus as S; + assert_eq!(status_of(S::Live, &watch(14_800)), S::Implausible, "0.1 W counted, 14.8 W drawn"); + assert_eq!(status_of(S::Live, &watch(2_000)), S::Live, "0.1 W beside a 2 W board is a counter doing its job"); + } + /// A live counter that ran past its board is `Implausible`; one that /// tracked it stays `Live`; a status that was not `Live` stands. #[test] diff --git a/jcp/SPEC.md b/jcp/SPEC.md index 4c75b2d94..cd55eb1c3 100644 --- a/jcp/SPEC.md +++ b/jcp/SPEC.md @@ -623,6 +623,20 @@ it broke one: a counter a tenth over a bound and one twenty times over are the same refusal and not the same finding, and re-grading a past refusal against a revised bound is only possible if the rate survived the check. +The check is **two-sided**. Where the bound is the board's own draw, a counter +whose rate falls far below that draw MUST NOT produce a hardware-graded figure +either. This is the more dangerous of the two directions. Under-reporting is the +unsafe direction at a ceiling (§5), because it admits work the budget cannot +cover; a counter running fast fails safe, because it only spends a budget +sooner. A counter republished in lumps and sampled across its republish period +reads low rather than frozen, so liveness alone does not see it. The floor MUST +allow for the draw reading's own imprecision. It MUST NOT fire where the board +is near idle, because a correctly reporting counter accrues almost nothing there. +The reference meters refute a rate under a quarter of the board's mean draw, +after the draw's ±5 W accuracy is taken off it. Either bound refutes and never +verifies: a counter inside both is not refuted, and that is not a finding that +it is correct. + A bound reading that is itself impossible is not a reading: the same RTX 4050 reports about 590 W, past its own 65 W limit, for a second after it wakes. Such a reading MUST be discarded — and only that reading. One broken sample is not a diff --git a/jcp/jcp-rs/crates/jcp-energy/src/lib.rs b/jcp/jcp-rs/crates/jcp-energy/src/lib.rs index e84fe99a4..1a67640a9 100644 --- a/jcp/jcp-rs/crates/jcp-energy/src/lib.rs +++ b/jcp/jcp-rs/crates/jcp-energy/src/lib.rs @@ -28,7 +28,8 @@ //! GPU, so the CPU's share of the work is unmetered and the figure never //! claims the whole task. Only while the counter is live ([`nvml_status`]) //! and plausibly counts the board's energy: over the probe it ran no -//! faster than the board can draw ([`CounterStatus::Implausible`]); a span +//! faster than the board can draw, and not far slower than it drew +//! ([`CounterStatus::Implausible`]); a span //! over which it did not advance is the whole-task default-power estimate. //! 3. **Per-process CPU energy** (macOS) — the kernel's count of this //! process's own CPU energy, `ri_energy_nj` from @@ -682,6 +683,11 @@ pub enum CounterStatus { /// carries the rate and the bound it broke: a counter 1.1 times over a /// bound and one 20 times over are the same status but not the same /// finding, and tuning the margins later needs the figures. + /// + /// A counter that accrued far less than its board drew is `Implausible` + /// too ([`GpuPlausibility::CounterUnderRate`]), and that is the dangerous + /// direction: graded a reading, it would spend a budget it is not + /// measuring. Implausible, /// The counter kept advancing and could not be judged, because the board's /// own power telemetry was unusable over the probe: a draw reading past the @@ -976,6 +982,22 @@ const GPU_DRAW_FACTOR: u128 = 2; /// NVML's documented accuracy of a board's power draw, ±5 W, in mW. const GPU_DRAW_SLACK_MW: u128 = 5_000; +/// How far **below** its board's mean draw a counter may accrue before it is +/// refuted: a shortfall of more than this factor, after [`GPU_DRAW_SLACK_MW`] +/// is taken off the draw, is no count of the board's energy. +/// +/// The bound has to exist in this direction because **under-reporting is the +/// unsafe direction at an energy ceiling** (SPEC §5): it admits work the +/// budget cannot cover. The ceiling bounds only catch a counter running +/// *fast*, which fails safe. A counter republished in lumps and sampled across +/// its republish period reads low rather than frozen, so liveness alone does +/// not see it. It is looser than [`GPU_DRAW_FACTOR`] because the draw is a +/// one-second average a bursty load integrates below, and the slack comes off +/// before the division so an idle board (at or under the slack) makes no +/// finding. The same factor as `jouleclaw-energy`'s, `sandbox-meter`'s and +/// `eoc-meter`'s, so a counter one meter declines another does not admit. +const GPU_UNDER_FACTOR: u128 = 4; + /// Whether a GPU energy counter that counted `energy_mj` over `window` /// plausibly counted its board's energy, beside the board's power draw /// `draws_mw` read with the samples and its maximum power limit `ceiling_mw`. @@ -983,7 +1005,8 @@ const GPU_DRAW_SLACK_MW: u128 = 5_000; /// draw reading did (the board's own telemetry then is not a reading either: /// an RTX 4050 Laptop GPU reports 590 W for about a second after it wakes), /// or the counter ran past [`GPU_DRAW_FACTOR`] times the mean draw plus -/// [`GPU_DRAW_SLACK_MW`]. A bound whose reading is missing is not applied. +/// [`GPU_DRAW_SLACK_MW`], or fell short of that mean, less the slack, by more +/// than [`GPU_UNDER_FACTOR`]. A bound whose reading is missing is not applied. /// /// Liveness says the counter is republished continuously; this says what it /// republishes is the board's energy. On the RTX 4050 Laptop GPU (driver @@ -1024,6 +1047,19 @@ pub enum GpuPlausibility { /// Which bound that was. bound: GpuBound, }, + /// The counter accrued far less energy than the board's own telemetry says + /// it drew: the dangerous finding, because a counter reading low is graded + /// `HwShunt` and then spends a budget it is not measuring, while it still + /// advances, so liveness calls it live. + CounterUnderRate { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + /// The floor it fell below, in milliwatts. + floor_mw: u64, + /// The board's mean usable draw the floor was taken from, in + /// milliwatts: the yardstick, so the judgement can be re-made. + mean_draw_mw: u64, + }, /// The board's own telemetry read past the board's own power limit, so /// there is no yardstick to judge the counter against. An indeterminate, /// not a finding about the counter — see [`CounterStatus::Unjudgeable`]. @@ -1104,13 +1140,25 @@ fn plausible_gpu_counter( bound: GpuBound::MeanDraw, }; } + // And the same yardstick from below. The slack comes off the draw + // before the division, so the floor is zero at or under + // `GPU_DRAW_SLACK_MW` of mean draw: an idle board makes no finding. + let floor = mean.saturating_sub(GPU_DRAW_SLACK_MW) / GPU_UNDER_FACTOR; + if floor > 0 && rate_mw < floor { + return GpuPlausibility::CounterUnderRate { + rate_mw: clamp(rate_mw), + floor_mw: clamp(floor), + mean_draw_mw: clamp(mean), + }; + } } GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) } } /// The status an NVML probe reports: [`liveness::status_of`] its run, except /// that a live counter which did not plausibly count its board's energy over -/// the run ([`plausible_gpu_counter`]) is [`CounterStatus::Implausible`]. +/// the run ([`plausible_gpu_counter`]), running faster than its board or far +/// slower, is [`CounterStatus::Implausible`]. #[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] fn nvml_status_of(probed: Option<&liveness::Advances>, gpu: &GpuWatch) -> CounterStatus { match liveness::status_of(probed) { @@ -1123,7 +1171,9 @@ fn nvml_status_of(probed: Option<&liveness::Advances>, gpu: &GpuWatch) -> Counte gpu.ceiling_mw, ) { GpuPlausibility::Plausible { .. } => CounterStatus::Live, - GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible, + GpuPlausibility::CounterOverRate { .. } | GpuPlausibility::CounterUnderRate { .. } => { + CounterStatus::Implausible + } GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable, } } @@ -7240,6 +7290,47 @@ mod tests { } } + /// **The bound that was missing: a counter reading LOW.** It still + /// advances, so liveness calls it live; graded `HwShunt` it would spend a + /// ceiling it is not measuring. Figures are the jetson-hub RTX 4050's own. + #[test] + fn a_gpu_counter_must_count_no_slower_than_its_board_draws() { + let w = Duration::from_millis(500); + // floor = (14_800 - 5_000) / 4 = 2_450 mW. + assert_eq!( + plausible_gpu_counter(250, w, &[14_800, 14_800], Some(65_000)), + GpuPlausibility::CounterUnderRate { rate_mw: 500, floor_mw: 2_450, mean_draw_mw: 14_800 }, + "0.5 W counted against a 14.8 W draw" + ); + assert!(plausible_gpu_counter(1_300, w, &[14_800, 14_800], Some(65_000)).plausible(), "lagging"); + assert!(plausible_gpu_counter(1_225, w, &[14_800, 14_800], Some(65_000)).plausible(), "at the floor"); + // An idle board makes no finding, and nor does a board with no reading. + for mean in [0u32, 1_000, 5_000] { + assert!(plausible_gpu_counter(0, w, &[mean, mean], Some(65_000)).plausible(), "{mean} mW"); + } + assert!(plausible_gpu_counter(0, w, &[], Some(65_000)).plausible()); + + // The wiring, not only the verdict: a live counter at 0.1 W beside a + // 14.8 W board is `Implausible`, and beside a 2 W board stays `Live`. + let rail = |v| liveness::every_rail(&[("nvml device 0 total energy".to_string(), v)]); + let mut live = liveness::Advances::default(); + let t0 = Instant::now(); + let watch = |draw| { + let mut g = GpuWatch { ceiling_mw: Some(65_000), ..GpuWatch::default() }; + for k in 0..6u32 { + // 10 mJ every 100 ms: 100 mW. + g.saw(Some(1_000_000 + 10 * u64::from(k)), Some(draw), t0 + Duration::from_millis(100) * k); + } + g + }; + for k in 0..6u32 { + live.observe(rail(1_000_000 + 10 * i64::from(k)), Duration::from_millis(100) * k); + } + assert_eq!(liveness::status_of(Some(&live)), CounterStatus::Live); + assert_eq!(nvml_status_of(Some(&live), &watch(14_800)), CounterStatus::Implausible); + assert_eq!(nvml_status_of(Some(&live), &watch(2_000)), CounterStatus::Live); + } + /// The NVML probe's status: a live counter that ran past its board is /// `Implausible`; one that tracked it stays `Live`; a counter that was not /// live keeps its status, whatever its board drew. diff --git a/jouleclaw/SPEC.md b/jouleclaw/SPEC.md index 1d8749023..406fa96ee 100644 --- a/jouleclaw/SPEC.md +++ b/jouleclaw/SPEC.md @@ -145,10 +145,18 @@ Realistic floors: data-center part (A100) agreed to ±0.2 % at every interval, while an unvalidated consumer part (RTX 4090) was still 13.3 % low at 100 ms and came within 1.6 % only at 1 s. So on a GPU whose counter has not - been validated against integrated power, prefer the power integral — - which this standard's reference implementation already falls back to, - graded `ModelBased`. A counter read below its own update rate is not a - reading of the span, whatever the interface. + been validated against integrated power, prefer the power integral, + graded `ModelBased`. The reference implementation does not tell a + validated part from an unvalidated one. It falls back to the power + integral only where it can refute the counter: not live, or outside + the two-sided bound below. That bound is deliberately wide (a quarter + of the board's draw once 5 W is taken off), so a counter 13.3 % low + passes it and is graded `HwShunt`. Telling the two apart needs the + device's counter checked against something it does not share an error + with, such as an external meter. Until that is done, an `HwShunt` NVML + figure carries its device's unvalidated error. A counter read below + its own update rate is not a reading of the span, whatever the + interface. - Apple Silicon IOReport: ~1 mJ, ~10 ms. `HwShunt` while its CPU energy rail is live. On macOS 27.0.1 an unprivileged reader's CPU rails are frozen (observed on an M5 Max and an M3 Ultra), so the diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs index 057eeb07a..77182b547 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs @@ -507,6 +507,27 @@ pub(super) const GPU_DRAW_FACTOR: u128 = 2; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` pub(super) const GPU_DRAW_SLACK_MW: u128 = 5_000; +/// How far **below** its board's mean draw a counter may accrue before it is +/// refuted: a shortfall of more than this factor, after [`GPU_DRAW_SLACK_MW`] +/// is taken off the draw, is no count of the board's energy. +/// +/// The bound has to exist in this direction at all because **under-reporting +/// is the unsafe direction at an energy ceiling**: it admits work the budget +/// cannot cover. The ceiling bounds only catch a counter running *fast*, which +/// is the direction that fails safe. A counter republished in lumps and +/// sampled across its republish period reads low rather than frozen, so the +/// liveness rule alone does not see it. +/// +/// It is deliberately looser than [`GPU_DRAW_FACTOR`]. The draw readings are +/// one-second averages, so a bursty load that is idle between samples really +/// does integrate to less than its sampled mean. And the slack comes off the +/// draw before the division, so the floor collapses to zero at or below +/// [`GPU_DRAW_SLACK_MW`]: at idle draw there is no finding to make, only noise +/// to avoid. The same factor as `jouleclaw-energy`'s, so a counter one meter +/// declines the other does not admit. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) const GPU_UNDER_FACTOR: u128 = 4; + /// Whether one GPU's energy counter, which counted `energy_mj` over `window`, /// plausibly counted its board's energy, beside the board's power draw /// `draws_mw` read with the samples and its maximum power limit `ceiling_mw`. @@ -514,7 +535,9 @@ pub(super) const GPU_DRAW_SLACK_MW: u128 = 5_000; /// draw reading did (the board's own telemetry then is not a reading either: /// an RTX 4050 Laptop GPU reports 590 W for about a second after it wakes), /// or the counter ran past [`GPU_DRAW_FACTOR`] times the mean draw plus -/// [`GPU_DRAW_SLACK_MW`]. A bound whose reading is missing is not applied. +/// [`GPU_DRAW_SLACK_MW`], or fell short of that mean draw, less the slack, by +/// more than [`GPU_UNDER_FACTOR`]. A bound whose reading is missing is not +/// applied. /// /// Liveness says the counter is republished continuously; this says what it /// republishes is the board's energy. A counter that runs several times @@ -559,6 +582,24 @@ pub(super) enum GpuPlausibility { /// Which bound that was. bound: GpuBound, }, + /// The counter accrued far less energy than the board's own telemetry says + /// it drew. + /// + /// This is the finding the ceiling bounds cannot make, and the dangerous + /// one: a counter reading low is graded `HwShunt` and then spends a budget + /// it is not measuring. A counter republished in lumps and sampled across + /// the republish period looks exactly like this. It advances, so the + /// liveness rule calls it live, while it accrues a fraction of the energy. + CounterUnderRate { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + /// The floor it fell below, in milliwatts. + floor_mw: u64, + /// The board's mean usable draw the floor was taken from, in + /// milliwatts: the yardstick, so the judgement can be re-made against + /// a different factor. + mean_draw_mw: u64, + }, /// The board's own telemetry read past the board's own power limit, so /// there is no yardstick left to judge the counter against. An /// indeterminate, not a finding about the counter — see @@ -583,13 +624,18 @@ impl GpuPlausibility { } /// How severe this verdict is, for picking the one that speaks for a run - /// of several GPUs: a proven overrun outranks an indeterminate, which - /// outranks a pass. + /// of several GPUs: a proven shortfall outranks a proven overrun, which + /// outranks an indeterminate, which outranks a pass. fn rank(&self) -> u8 { match self { GpuPlausibility::Plausible { .. } => 0, GpuPlausibility::BoardTelemetryUnusable { .. } => 1, GpuPlausibility::CounterOverRate { .. } => 2, + // Above a proven overrun: both are refutations, but only this one + // is the direction that admits work a budget cannot cover, so where + // one GPU of a run over-reports and another under-reports, the + // under-report is the verdict an operator has to act on. + GpuPlausibility::CounterUnderRate { .. } => 3, } } @@ -598,6 +644,10 @@ impl GpuPlausibility { match self { GpuPlausibility::Plausible { rate_mw } => *rate_mw, GpuPlausibility::CounterOverRate { rate_mw, .. } => *rate_mw, + // The shortfall, so that among two under-reporting GPUs the worse + // one speaks: a lower rate is a bigger shortfall, so it is ordered + // by how far the floor was missed rather than by the rate itself. + GpuPlausibility::CounterUnderRate { rate_mw, floor_mw, .. } => floor_mw.saturating_sub(*rate_mw), GpuPlausibility::BoardTelemetryUnusable { draw_mw, .. } => *draw_mw, } } @@ -662,6 +712,17 @@ pub(super) fn plausible_gpu_counter( bound: GpuBound::MeanDraw, }; } + // And the same yardstick from below. The slack comes off the draw + // before the division, so the floor is zero at or under + // `GPU_DRAW_SLACK_MW` of mean draw: an idle board makes no finding. + let floor = mean.saturating_sub(GPU_DRAW_SLACK_MW) / GPU_UNDER_FACTOR; + if floor > 0 && rate_mw < floor { + return GpuPlausibility::CounterUnderRate { + rate_mw: clamp(rate_mw), + floor_mw: clamp(floor), + mean_draw_mw: clamp(mean), + }; + } } GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) } } @@ -701,6 +762,24 @@ pub(super) fn gpu_counters_verdict( .unwrap_or(GpuPlausibility::Plausible { rate_mw: 0 }) } +/// The status a live GPU counter keeps once its run is judged on what it was +/// counting ([`gpu_counters_verdict`]). A counter proven to run faster than its +/// board can draw, or far slower than it draws, is `Implausible`. A board +/// whose own telemetry is unusable leaves the counter `Unjudgeable`, which is +/// a different diagnosis and not a claim about the counter. Kept here, apart +/// from the NVML probe that only builds on Linux, so that the mapping is +/// tested on every platform. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) fn gpu_counter_status(verdict: GpuPlausibility) -> CounterStatus { + match verdict { + GpuPlausibility::Plausible { .. } => CounterStatus::Live, + GpuPlausibility::CounterOverRate { .. } | GpuPlausibility::CounterUnderRate { .. } => { + CounterStatus::Implausible + } + GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable, + } +} + /// A counter probe's constructor behind its gate: `open` runs only when /// [`counter_admits`] the `status`; otherwise `Unavailable`, naming the /// counter (`what`) and why, and the counter is never touched. Every counter @@ -2164,6 +2243,81 @@ mod tests { ); } + /// **The bound that was missing: a counter reading LOW.** A counter + /// running fast fails safe, since an over-reported figure spends a budget + /// sooner. A counter reading low is graded `HwShunt` and then spends a + /// ceiling it is not measuring, and it still advances, so the liveness + /// rule calls it live. Figures are the jetson-hub RTX 4050's own. + #[test] + fn a_gpu_counter_must_count_no_slower_than_its_board_draws() { + let w = Duration::from_millis(500); + // Over a 500 ms window a counter's rate in mW is twice its mJ. + let draws = &[14_800u32, 14_800]; + // floor = (14_800 - 5_000) / 4 = 2_450 mW. + assert_eq!( + plausible_gpu_counter(250, w, draws, Some(65_000)), + GpuPlausibility::CounterUnderRate { rate_mw: 500, floor_mw: 2_450, mean_draw_mw: 14_800 }, + "0.5 W counted against a 14.8 W draw is no count of the board's energy" + ); + // A counter that merely lags its board passes: the draws are + // one-second averages, and a bursty load integrates to less. + assert!(plausible_gpu_counter(1_300, w, draws, Some(65_000)).plausible()); + // Exactly at the floor is not below it. + assert!(plausible_gpu_counter(1_225, w, draws, Some(65_000)).plausible()); + } + + /// An idle board makes no finding: the slack comes off the draw before the + /// division, so the floor is zero at or below it, and a correct counter on + /// an idle GPU, which accrues almost nothing, is not refuted for it. + #[test] + fn an_idle_board_yields_no_under_rate_finding() { + let w = Duration::from_millis(500); + for mean in [0u32, 1_000, 4_000, 5_000] { + assert!( + plausible_gpu_counter(0, w, &[mean, mean], Some(65_000)).plausible(), + "a counter accruing nothing beside a {mean} mW board must not be refuted" + ); + } + assert_eq!( + plausible_gpu_counter(0, w, &[9_000, 9_000], Some(65_000)), + GpuPlausibility::CounterUnderRate { rate_mw: 0, floor_mw: 1_000, mean_draw_mw: 9_000 } + ); + // No draw reading: no yardstick, so no finding in either direction. + assert!(plausible_gpu_counter(0, w, &[], Some(65_000)).plausible()); + } + + /// What a live counter's status becomes from each verdict: refuted in + /// either direction is `Implausible`, never `Live`, and an unusable board + /// is `Unjudgeable`. + #[test] + fn a_gpu_counter_refuted_either_way_is_implausible() { + let w = Duration::from_millis(500); + let under = plausible_gpu_counter(250, w, &[14_800], Some(65_000)); + let over = plausible_gpu_counter(33_000, w, &[7_500], Some(65_000)); + let idle = plausible_gpu_counter(250, w, &[2_000], Some(65_000)); + let unusable = plausible_gpu_counter(1_000, w, &[590_000], Some(65_000)); + assert_eq!(gpu_counter_status(under), CounterStatus::Implausible, "{under:?}"); + assert_eq!(gpu_counter_status(over), CounterStatus::Implausible, "{over:?}"); + assert_eq!(gpu_counter_status(idle), CounterStatus::Live, "the same slow counter beside an idle board"); + assert_eq!(gpu_counter_status(unusable), CounterStatus::Unjudgeable); + } + + /// Under-reporting outranks over-reporting, so across a run's GPUs the + /// verdict that speaks is the one that admits unbudgeted work; and among + /// two under-reports, the larger shortfall. + #[test] + fn an_under_reporting_gpu_speaks_for_a_run_before_an_over_reporting_one() { + let w = Duration::from_millis(500); + let under = plausible_gpu_counter(250, w, &[14_800], Some(65_000)); + let worse = plausible_gpu_counter(0, w, &[14_800], Some(65_000)); + let over = plausible_gpu_counter(152_250, w, &[5_550], Some(65_000)); + assert!(matches!(over, GpuPlausibility::CounterOverRate { .. })); + let speaks = |vs: [GpuPlausibility; 2]| vs.into_iter().max_by_key(|v| (v.rank(), v.magnitude())); + assert_eq!(speaks([over, under]), Some(under), "the unsafe direction is the one that speaks"); + assert_eq!(speaks([under, worse]), Some(worse), "and the bigger shortfall among two"); + assert!(!under.plausible()); + } + /// The rule over a probe run: each GPU's rise from the run's first sample /// to its last, over the window between them, beside that GPU's own draws /// and limit. One implausible GPU of two makes the run implausible, a diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs index 391965e6d..1441b3a84 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs @@ -24,8 +24,11 @@ //! probe each counter's rise is held to its board's maximum power limit and //! to the board's own draw, read with every sample //! ([`plausible_gpu_counter`](super::liveness::plausible_gpu_counter)); one -//! that runs past them is [`Implausible`](super::CounterStatus::Implausible) -//! and the meter has no GPU reading. +//! that runs past them, or falls far short of the board's draw, is +//! [`Implausible`](super::CounterStatus::Implausible) and the meter has no +//! GPU reading. The short side is the one that matters at a ceiling: a +//! counter reading low would be graded a hardware reading and then spend a +//! budget it is not measuring. //! //! A GPU-only figure is the GPU's energy, not the work's: the meter's span //! rule (a CPU counter that held estimates the span) does not apply to a @@ -34,7 +37,7 @@ //! that work ([`Measurement::reads_cpu`](super::Measurement::reads_cpu)). use super::liveness::{ - counter_run, counter_status_of, counter_tick, gated, gpu_counters_verdict, GpuPlausibility, RailWatch, Sample, + counter_run, counter_status_of, counter_tick, gated, gpu_counter_status, gpu_counters_verdict, RailWatch, Sample, NVML_PROBE_TICK, }; use std::cell::RefCell; @@ -82,16 +85,12 @@ pub(super) fn host_status() -> CounterStatus { /// The status of an NVML probe run: its verdict's, except that a live run is /// re-judged on whether its counters counted their boards' energy /// ([`gpu_counters_verdict`]). A counter proven to run faster than its board -/// can draw is `Implausible`; a board whose own telemetry is unusable leaves -/// the counter `Unjudgeable`, which is a different diagnosis and not a claim -/// about the counter. +/// can draw, or far slower than it draws, is `Implausible`; a board whose own +/// telemetry is unusable leaves the counter `Unjudgeable`, which is a +/// different diagnosis and not a claim about the counter. fn status_of_run(watch: &RailWatch, draws: &[Vec>], ceilings: &[Option]) -> CounterStatus { match counter_status_of(watch.verdict()) { - CounterStatus::Live => match gpu_counters_verdict(watch, draws, ceilings) { - GpuPlausibility::Plausible { .. } => CounterStatus::Live, - GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible, - GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable, - }, + CounterStatus::Live => gpu_counter_status(gpu_counters_verdict(watch, draws, ceilings)), status => status, } } @@ -297,6 +296,10 @@ mod tests { assert_eq!(status_of_run(&live, &[vec![Some(95_000)]], &[Some(350_000)]), CounterStatus::Live); assert_eq!(status_of_run(&live, &[vec![Some(10_000)]], &[Some(65_000)]), CounterStatus::Implausible); assert_eq!(status_of_run(&live, &[vec![Some(95_000)]], &[Some(65_000)]), CounterStatus::Implausible); + // 100 W counted while the board drew 500 W, under a 700 W limit: the + // counter advanced in every interval, so the rule called it live, and + // it counted a fifth of the board. Reading low is refuted too. + assert_eq!(status_of_run(&live, &[vec![Some(500_000)]], &[Some(700_000)]), CounterStatus::Implausible); assert_eq!(status_of_run(&frozen, &[vec![Some(10_000)]], &[Some(65_000)]), CounterStatus::Frozen); assert_eq!(status_of_run(&RailWatch::default(), &[], &[]), CounterStatus::Unavailable); } -- 2.54.0 (Apple Git-157)