diff --git a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs index 7c5c9c351..0109f9776 100644 --- a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs +++ b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs @@ -5153,6 +5153,24 @@ mod gpu_plausibility { /// NVML's documented accuracy of a board's power draw, ±5 W, in mW. pub(crate) const DRAW_SLACK_MW: u128 = 5_000; + /// How far **below** its board's mean draw a counter may accrue before it + /// is refuted: a shortfall of more than this factor, after + /// [`DRAW_SLACK_MW`] is taken off the draw, is no count of the board's + /// energy. + /// + /// The bound has to exist in this direction because **under-reporting is + /// the unsafe direction at an energy ceiling**: it admits work the budget + /// cannot cover. The ceiling bounds only catch a counter running *fast*, + /// which fails safe. A counter republished in lumps and sampled across its + /// republish period reads low rather than frozen, so liveness alone does + /// not see it. It is looser than [`DRAW_FACTOR`] because the draw is a + /// one-second average a bursty load integrates below, and the slack comes + /// off before the division so an idle board (at or under the slack) makes + /// no finding. The same factor as `jouleclaw-energy`'s and + /// `sandbox-meter`'s, so a counter one meter declines another does not + /// admit. + pub(crate) const UNDER_FACTOR: u128 = 4; + /// Which of the two bounds a GPU counter ran past. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum GpuBound { @@ -5188,6 +5206,19 @@ mod gpu_plausibility { /// Which bound that was. bound: GpuBound, }, + /// The counter accrued far less energy than the board's own telemetry + /// says it drew: the dangerous finding, because a counter reading low + /// is graded `HwShunt` and then spends a budget it is not measuring, + /// while it still advances, so liveness calls it live. + CounterUnderRate { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + /// The floor it fell below, in milliwatts. + floor_mw: u64, + /// The board's mean usable draw the floor was taken from, in + /// milliwatts: the yardstick, so the judgement can be re-made. + mean_draw_mw: u64, + }, /// The board's own telemetry read past the board's own power limit, so /// there is no yardstick left to judge the counter against. An /// indeterminate, not a finding about the counter — see @@ -5218,8 +5249,9 @@ mod gpu_plausibility { /// judged to be counting, beside the board's power draw `draws_mw` read /// with the samples and its maximum power limit `ceiling_mw`. It over-ran /// if it ran past [`CEILING_MARGIN`] of the limit, or past [`DRAW_FACTOR`] - /// times the mean draw plus [`DRAW_SLACK_MW`]. A bound whose reading is - /// missing is not applied. + /// times the mean draw plus [`DRAW_SLACK_MW`]; it under-ran if it fell + /// short of the mean draw, less the slack, by more than [`UNDER_FACTOR`]. + /// A bound whose reading is missing is not applied. /// /// Liveness says the counter is republished continuously; this says what /// it republishes is the board's energy. On the RTX 4050 Laptop GPU @@ -5283,6 +5315,17 @@ mod gpu_plausibility { bound: GpuBound::MeanDraw, }; } + // And the same yardstick from below. The slack comes off the draw + // before the division, so the floor is zero at or under + // `DRAW_SLACK_MW` of mean draw: an idle board makes no finding. + let floor = mean.saturating_sub(DRAW_SLACK_MW) / UNDER_FACTOR; + if floor > 0 && rate_mw < floor { + return GpuPlausibility::CounterUnderRate { + rate_mw: clamp(rate_mw), + floor_mw: clamp(floor), + mean_draw_mw: clamp(mean), + }; + } } GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) } } @@ -5312,8 +5355,8 @@ mod gpu_plausibility { /// The status a GPU probe gives: the liveness rule's `status`, except that /// a live counter is re-judged on what it was counting over the probe. One - /// proven to run faster than its board can draw is - /// [`CounterStatus::Implausible`]; one whose board's own telemetry was + /// proven to run faster than its board can draw, or far slower than it + /// draws, is [`CounterStatus::Implausible`]; one whose board's own telemetry was /// unusable is [`CounterStatus::Unjudgeable`], which is a different /// diagnosis and not a claim about the counter. pub(crate) fn status_of(status: CounterStatus, gpu: &GpuWatch) -> CounterStatus { @@ -5324,7 +5367,9 @@ mod gpu_plausibility { (Some((a, t0)), Some((b, t1))) => { match plausible(b.saturating_sub(a), t1.saturating_sub(t0), &gpu.draws_mw, gpu.ceiling_mw) { GpuPlausibility::Plausible { .. } => CounterStatus::Live, - GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible, + GpuPlausibility::CounterOverRate { .. } | GpuPlausibility::CounterUnderRate { .. } => { + CounterStatus::Implausible + } GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable, } } @@ -5437,6 +5482,50 @@ mod gpu_plausibility { ); } + /// **The bound that was missing: a counter reading LOW.** It still + /// advances, so liveness calls it live, and graded `HwShunt` it would + /// spend a ceiling it is not measuring. Figures are the jetson-hub RTX + /// 4050's own. + #[test] + fn a_gpu_counter_must_count_no_slower_than_its_board_draws() { + let w = Duration::from_millis(500); + // floor = (14_800 - 5_000) / 4 = 2_450 mW. + assert_eq!( + plausible(250, w, &[14_800, 14_800], Some(65_000)), + GpuPlausibility::CounterUnderRate { rate_mw: 500, floor_mw: 2_450, mean_draw_mw: 14_800 }, + "0.5 W counted against a 14.8 W draw" + ); + assert!(plausible(1_300, w, &[14_800, 14_800], Some(65_000)).plausible(), "lagging, not refuted"); + assert!(plausible(1_225, w, &[14_800, 14_800], Some(65_000)).plausible(), "at the floor"); + // An idle board makes no finding; nor does a board with no reading. + for mean in [0u32, 1_000, 5_000] { + assert!(plausible(0, w, &[mean, mean], Some(65_000)).plausible(), "{mean} mW"); + } + assert!(plausible(0, w, &[], Some(65_000)).plausible()); + assert_eq!( + plausible(0, w, &[9_000, 9_000], Some(65_000)), + GpuPlausibility::CounterUnderRate { rate_mw: 0, floor_mw: 1_000, mean_draw_mw: 9_000 } + ); + } + + /// The wiring, not only the verdict: a live counter that accrued a + /// fraction of what its board drew is `Implausible`, and the same slow + /// counter beside an idle board stays `Live`. + #[test] + fn a_live_gpu_counter_short_of_its_board_is_implausible() { + let watch = |draw_mw| { + let mut g = GpuWatch { ceiling_mw: Some(65_000), ..GpuWatch::default() }; + for k in 0..6u64 { + // 10 mJ every 100 ms: 100 mW. + g.saw(Some(1_000_000 + 10 * k), Some(draw_mw), Duration::from_millis(100 * k)); + } + g + }; + use CounterStatus as S; + assert_eq!(status_of(S::Live, &watch(14_800)), S::Implausible, "0.1 W counted, 14.8 W drawn"); + assert_eq!(status_of(S::Live, &watch(2_000)), S::Live, "0.1 W beside a 2 W board is a counter doing its job"); + } + /// A live counter that ran past its board is `Implausible`; one that /// tracked it stays `Live`; a status that was not `Live` stands. #[test] diff --git a/jcp/SPEC.md b/jcp/SPEC.md index 4c75b2d94..cd55eb1c3 100644 --- a/jcp/SPEC.md +++ b/jcp/SPEC.md @@ -623,6 +623,20 @@ it broke one: a counter a tenth over a bound and one twenty times over are the same refusal and not the same finding, and re-grading a past refusal against a revised bound is only possible if the rate survived the check. +The check is **two-sided**. Where the bound is the board's own draw, a counter +whose rate falls far below that draw MUST NOT produce a hardware-graded figure +either. This is the more dangerous of the two directions. Under-reporting is the +unsafe direction at a ceiling (§5), because it admits work the budget cannot +cover; a counter running fast fails safe, because it only spends a budget +sooner. A counter republished in lumps and sampled across its republish period +reads low rather than frozen, so liveness alone does not see it. The floor MUST +allow for the draw reading's own imprecision. It MUST NOT fire where the board +is near idle, because a correctly reporting counter accrues almost nothing there. +The reference meters refute a rate under a quarter of the board's mean draw, +after the draw's ±5 W accuracy is taken off it. Either bound refutes and never +verifies: a counter inside both is not refuted, and that is not a finding that +it is correct. + A bound reading that is itself impossible is not a reading: the same RTX 4050 reports about 590 W, past its own 65 W limit, for a second after it wakes. Such a reading MUST be discarded — and only that reading. One broken sample is not a diff --git a/jcp/jcp-rs/crates/jcp-energy/src/lib.rs b/jcp/jcp-rs/crates/jcp-energy/src/lib.rs index e84fe99a4..1a67640a9 100644 --- a/jcp/jcp-rs/crates/jcp-energy/src/lib.rs +++ b/jcp/jcp-rs/crates/jcp-energy/src/lib.rs @@ -28,7 +28,8 @@ //! GPU, so the CPU's share of the work is unmetered and the figure never //! claims the whole task. Only while the counter is live ([`nvml_status`]) //! and plausibly counts the board's energy: over the probe it ran no -//! faster than the board can draw ([`CounterStatus::Implausible`]); a span +//! faster than the board can draw, and not far slower than it drew +//! ([`CounterStatus::Implausible`]); a span //! over which it did not advance is the whole-task default-power estimate. //! 3. **Per-process CPU energy** (macOS) — the kernel's count of this //! process's own CPU energy, `ri_energy_nj` from @@ -682,6 +683,11 @@ pub enum CounterStatus { /// carries the rate and the bound it broke: a counter 1.1 times over a /// bound and one 20 times over are the same status but not the same /// finding, and tuning the margins later needs the figures. + /// + /// A counter that accrued far less than its board drew is `Implausible` + /// too ([`GpuPlausibility::CounterUnderRate`]), and that is the dangerous + /// direction: graded a reading, it would spend a budget it is not + /// measuring. Implausible, /// The counter kept advancing and could not be judged, because the board's /// own power telemetry was unusable over the probe: a draw reading past the @@ -976,6 +982,22 @@ const GPU_DRAW_FACTOR: u128 = 2; /// NVML's documented accuracy of a board's power draw, ±5 W, in mW. const GPU_DRAW_SLACK_MW: u128 = 5_000; +/// How far **below** its board's mean draw a counter may accrue before it is +/// refuted: a shortfall of more than this factor, after [`GPU_DRAW_SLACK_MW`] +/// is taken off the draw, is no count of the board's energy. +/// +/// The bound has to exist in this direction because **under-reporting is the +/// unsafe direction at an energy ceiling** (SPEC §5): it admits work the +/// budget cannot cover. The ceiling bounds only catch a counter running +/// *fast*, which fails safe. A counter republished in lumps and sampled across +/// its republish period reads low rather than frozen, so liveness alone does +/// not see it. It is looser than [`GPU_DRAW_FACTOR`] because the draw is a +/// one-second average a bursty load integrates below, and the slack comes off +/// before the division so an idle board (at or under the slack) makes no +/// finding. The same factor as `jouleclaw-energy`'s, `sandbox-meter`'s and +/// `eoc-meter`'s, so a counter one meter declines another does not admit. +const GPU_UNDER_FACTOR: u128 = 4; + /// Whether a GPU energy counter that counted `energy_mj` over `window` /// plausibly counted its board's energy, beside the board's power draw /// `draws_mw` read with the samples and its maximum power limit `ceiling_mw`. @@ -983,7 +1005,8 @@ const GPU_DRAW_SLACK_MW: u128 = 5_000; /// draw reading did (the board's own telemetry then is not a reading either: /// an RTX 4050 Laptop GPU reports 590 W for about a second after it wakes), /// or the counter ran past [`GPU_DRAW_FACTOR`] times the mean draw plus -/// [`GPU_DRAW_SLACK_MW`]. A bound whose reading is missing is not applied. +/// [`GPU_DRAW_SLACK_MW`], or fell short of that mean, less the slack, by more +/// than [`GPU_UNDER_FACTOR`]. A bound whose reading is missing is not applied. /// /// Liveness says the counter is republished continuously; this says what it /// republishes is the board's energy. On the RTX 4050 Laptop GPU (driver @@ -1024,6 +1047,19 @@ pub enum GpuPlausibility { /// Which bound that was. bound: GpuBound, }, + /// The counter accrued far less energy than the board's own telemetry says + /// it drew: the dangerous finding, because a counter reading low is graded + /// `HwShunt` and then spends a budget it is not measuring, while it still + /// advances, so liveness calls it live. + CounterUnderRate { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + /// The floor it fell below, in milliwatts. + floor_mw: u64, + /// The board's mean usable draw the floor was taken from, in + /// milliwatts: the yardstick, so the judgement can be re-made. + mean_draw_mw: u64, + }, /// The board's own telemetry read past the board's own power limit, so /// there is no yardstick to judge the counter against. An indeterminate, /// not a finding about the counter — see [`CounterStatus::Unjudgeable`]. @@ -1104,13 +1140,25 @@ fn plausible_gpu_counter( bound: GpuBound::MeanDraw, }; } + // And the same yardstick from below. The slack comes off the draw + // before the division, so the floor is zero at or under + // `GPU_DRAW_SLACK_MW` of mean draw: an idle board makes no finding. + let floor = mean.saturating_sub(GPU_DRAW_SLACK_MW) / GPU_UNDER_FACTOR; + if floor > 0 && rate_mw < floor { + return GpuPlausibility::CounterUnderRate { + rate_mw: clamp(rate_mw), + floor_mw: clamp(floor), + mean_draw_mw: clamp(mean), + }; + } } GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) } } /// The status an NVML probe reports: [`liveness::status_of`] its run, except /// that a live counter which did not plausibly count its board's energy over -/// the run ([`plausible_gpu_counter`]) is [`CounterStatus::Implausible`]. +/// the run ([`plausible_gpu_counter`]), running faster than its board or far +/// slower, is [`CounterStatus::Implausible`]. #[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] fn nvml_status_of(probed: Option<&liveness::Advances>, gpu: &GpuWatch) -> CounterStatus { match liveness::status_of(probed) { @@ -1123,7 +1171,9 @@ fn nvml_status_of(probed: Option<&liveness::Advances>, gpu: &GpuWatch) -> Counte gpu.ceiling_mw, ) { GpuPlausibility::Plausible { .. } => CounterStatus::Live, - GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible, + GpuPlausibility::CounterOverRate { .. } | GpuPlausibility::CounterUnderRate { .. } => { + CounterStatus::Implausible + } GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable, } } @@ -7240,6 +7290,47 @@ mod tests { } } + /// **The bound that was missing: a counter reading LOW.** It still + /// advances, so liveness calls it live; graded `HwShunt` it would spend a + /// ceiling it is not measuring. Figures are the jetson-hub RTX 4050's own. + #[test] + fn a_gpu_counter_must_count_no_slower_than_its_board_draws() { + let w = Duration::from_millis(500); + // floor = (14_800 - 5_000) / 4 = 2_450 mW. + assert_eq!( + plausible_gpu_counter(250, w, &[14_800, 14_800], Some(65_000)), + GpuPlausibility::CounterUnderRate { rate_mw: 500, floor_mw: 2_450, mean_draw_mw: 14_800 }, + "0.5 W counted against a 14.8 W draw" + ); + assert!(plausible_gpu_counter(1_300, w, &[14_800, 14_800], Some(65_000)).plausible(), "lagging"); + assert!(plausible_gpu_counter(1_225, w, &[14_800, 14_800], Some(65_000)).plausible(), "at the floor"); + // An idle board makes no finding, and nor does a board with no reading. + for mean in [0u32, 1_000, 5_000] { + assert!(plausible_gpu_counter(0, w, &[mean, mean], Some(65_000)).plausible(), "{mean} mW"); + } + assert!(plausible_gpu_counter(0, w, &[], Some(65_000)).plausible()); + + // The wiring, not only the verdict: a live counter at 0.1 W beside a + // 14.8 W board is `Implausible`, and beside a 2 W board stays `Live`. + let rail = |v| liveness::every_rail(&[("nvml device 0 total energy".to_string(), v)]); + let mut live = liveness::Advances::default(); + let t0 = Instant::now(); + let watch = |draw| { + let mut g = GpuWatch { ceiling_mw: Some(65_000), ..GpuWatch::default() }; + for k in 0..6u32 { + // 10 mJ every 100 ms: 100 mW. + g.saw(Some(1_000_000 + 10 * u64::from(k)), Some(draw), t0 + Duration::from_millis(100) * k); + } + g + }; + for k in 0..6u32 { + live.observe(rail(1_000_000 + 10 * i64::from(k)), Duration::from_millis(100) * k); + } + assert_eq!(liveness::status_of(Some(&live)), CounterStatus::Live); + assert_eq!(nvml_status_of(Some(&live), &watch(14_800)), CounterStatus::Implausible); + assert_eq!(nvml_status_of(Some(&live), &watch(2_000)), CounterStatus::Live); + } + /// The NVML probe's status: a live counter that ran past its board is /// `Implausible`; one that tracked it stays `Live`; a counter that was not /// live keeps its status, whatever its board drew. diff --git a/jouleclaw/SPEC.md b/jouleclaw/SPEC.md index 1d8749023..406fa96ee 100644 --- a/jouleclaw/SPEC.md +++ b/jouleclaw/SPEC.md @@ -145,10 +145,18 @@ Realistic floors: data-center part (A100) agreed to ±0.2 % at every interval, while an unvalidated consumer part (RTX 4090) was still 13.3 % low at 100 ms and came within 1.6 % only at 1 s. So on a GPU whose counter has not - been validated against integrated power, prefer the power integral — - which this standard's reference implementation already falls back to, - graded `ModelBased`. A counter read below its own update rate is not a - reading of the span, whatever the interface. + been validated against integrated power, prefer the power integral, + graded `ModelBased`. The reference implementation does not tell a + validated part from an unvalidated one. It falls back to the power + integral only where it can refute the counter: not live, or outside + the two-sided bound below. That bound is deliberately wide (a quarter + of the board's draw once 5 W is taken off), so a counter 13.3 % low + passes it and is graded `HwShunt`. Telling the two apart needs the + device's counter checked against something it does not share an error + with, such as an external meter. Until that is done, an `HwShunt` NVML + figure carries its device's unvalidated error. A counter read below + its own update rate is not a reading of the span, whatever the + interface. - Apple Silicon IOReport: ~1 mJ, ~10 ms. `HwShunt` while its CPU energy rail is live. On macOS 27.0.1 an unprivileged reader's CPU rails are frozen (observed on an M5 Max and an M3 Ultra), so the diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs index 057eeb07a..77182b547 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs @@ -507,6 +507,27 @@ pub(super) const GPU_DRAW_FACTOR: u128 = 2; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` pub(super) const GPU_DRAW_SLACK_MW: u128 = 5_000; +/// How far **below** its board's mean draw a counter may accrue before it is +/// refuted: a shortfall of more than this factor, after [`GPU_DRAW_SLACK_MW`] +/// is taken off the draw, is no count of the board's energy. +/// +/// The bound has to exist in this direction at all because **under-reporting +/// is the unsafe direction at an energy ceiling**: it admits work the budget +/// cannot cover. The ceiling bounds only catch a counter running *fast*, which +/// is the direction that fails safe. A counter republished in lumps and +/// sampled across its republish period reads low rather than frozen, so the +/// liveness rule alone does not see it. +/// +/// It is deliberately looser than [`GPU_DRAW_FACTOR`]. The draw readings are +/// one-second averages, so a bursty load that is idle between samples really +/// does integrate to less than its sampled mean. And the slack comes off the +/// draw before the division, so the floor collapses to zero at or below +/// [`GPU_DRAW_SLACK_MW`]: at idle draw there is no finding to make, only noise +/// to avoid. The same factor as `jouleclaw-energy`'s, so a counter one meter +/// declines the other does not admit. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) const GPU_UNDER_FACTOR: u128 = 4; + /// Whether one GPU's energy counter, which counted `energy_mj` over `window`, /// plausibly counted its board's energy, beside the board's power draw /// `draws_mw` read with the samples and its maximum power limit `ceiling_mw`. @@ -514,7 +535,9 @@ pub(super) const GPU_DRAW_SLACK_MW: u128 = 5_000; /// draw reading did (the board's own telemetry then is not a reading either: /// an RTX 4050 Laptop GPU reports 590 W for about a second after it wakes), /// or the counter ran past [`GPU_DRAW_FACTOR`] times the mean draw plus -/// [`GPU_DRAW_SLACK_MW`]. A bound whose reading is missing is not applied. +/// [`GPU_DRAW_SLACK_MW`], or fell short of that mean draw, less the slack, by +/// more than [`GPU_UNDER_FACTOR`]. A bound whose reading is missing is not +/// applied. /// /// Liveness says the counter is republished continuously; this says what it /// republishes is the board's energy. A counter that runs several times @@ -559,6 +582,24 @@ pub(super) enum GpuPlausibility { /// Which bound that was. bound: GpuBound, }, + /// The counter accrued far less energy than the board's own telemetry says + /// it drew. + /// + /// This is the finding the ceiling bounds cannot make, and the dangerous + /// one: a counter reading low is graded `HwShunt` and then spends a budget + /// it is not measuring. A counter republished in lumps and sampled across + /// the republish period looks exactly like this. It advances, so the + /// liveness rule calls it live, while it accrues a fraction of the energy. + CounterUnderRate { + /// The rate the counter implies over the window, in milliwatts. + rate_mw: u64, + /// The floor it fell below, in milliwatts. + floor_mw: u64, + /// The board's mean usable draw the floor was taken from, in + /// milliwatts: the yardstick, so the judgement can be re-made against + /// a different factor. + mean_draw_mw: u64, + }, /// The board's own telemetry read past the board's own power limit, so /// there is no yardstick left to judge the counter against. An /// indeterminate, not a finding about the counter — see @@ -583,13 +624,18 @@ impl GpuPlausibility { } /// How severe this verdict is, for picking the one that speaks for a run - /// of several GPUs: a proven overrun outranks an indeterminate, which - /// outranks a pass. + /// of several GPUs: a proven shortfall outranks a proven overrun, which + /// outranks an indeterminate, which outranks a pass. fn rank(&self) -> u8 { match self { GpuPlausibility::Plausible { .. } => 0, GpuPlausibility::BoardTelemetryUnusable { .. } => 1, GpuPlausibility::CounterOverRate { .. } => 2, + // Above a proven overrun: both are refutations, but only this one + // is the direction that admits work a budget cannot cover, so where + // one GPU of a run over-reports and another under-reports, the + // under-report is the verdict an operator has to act on. + GpuPlausibility::CounterUnderRate { .. } => 3, } } @@ -598,6 +644,10 @@ impl GpuPlausibility { match self { GpuPlausibility::Plausible { rate_mw } => *rate_mw, GpuPlausibility::CounterOverRate { rate_mw, .. } => *rate_mw, + // The shortfall, so that among two under-reporting GPUs the worse + // one speaks: a lower rate is a bigger shortfall, so it is ordered + // by how far the floor was missed rather than by the rate itself. + GpuPlausibility::CounterUnderRate { rate_mw, floor_mw, .. } => floor_mw.saturating_sub(*rate_mw), GpuPlausibility::BoardTelemetryUnusable { draw_mw, .. } => *draw_mw, } } @@ -662,6 +712,17 @@ pub(super) fn plausible_gpu_counter( bound: GpuBound::MeanDraw, }; } + // And the same yardstick from below. The slack comes off the draw + // before the division, so the floor is zero at or under + // `GPU_DRAW_SLACK_MW` of mean draw: an idle board makes no finding. + let floor = mean.saturating_sub(GPU_DRAW_SLACK_MW) / GPU_UNDER_FACTOR; + if floor > 0 && rate_mw < floor { + return GpuPlausibility::CounterUnderRate { + rate_mw: clamp(rate_mw), + floor_mw: clamp(floor), + mean_draw_mw: clamp(mean), + }; + } } GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) } } @@ -701,6 +762,24 @@ pub(super) fn gpu_counters_verdict( .unwrap_or(GpuPlausibility::Plausible { rate_mw: 0 }) } +/// The status a live GPU counter keeps once its run is judged on what it was +/// counting ([`gpu_counters_verdict`]). A counter proven to run faster than its +/// board can draw, or far slower than it draws, is `Implausible`. A board +/// whose own telemetry is unusable leaves the counter `Unjudgeable`, which is +/// a different diagnosis and not a claim about the counter. Kept here, apart +/// from the NVML probe that only builds on Linux, so that the mapping is +/// tested on every platform. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) fn gpu_counter_status(verdict: GpuPlausibility) -> CounterStatus { + match verdict { + GpuPlausibility::Plausible { .. } => CounterStatus::Live, + GpuPlausibility::CounterOverRate { .. } | GpuPlausibility::CounterUnderRate { .. } => { + CounterStatus::Implausible + } + GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable, + } +} + /// A counter probe's constructor behind its gate: `open` runs only when /// [`counter_admits`] the `status`; otherwise `Unavailable`, naming the /// counter (`what`) and why, and the counter is never touched. Every counter @@ -2164,6 +2243,81 @@ mod tests { ); } + /// **The bound that was missing: a counter reading LOW.** A counter + /// running fast fails safe, since an over-reported figure spends a budget + /// sooner. A counter reading low is graded `HwShunt` and then spends a + /// ceiling it is not measuring, and it still advances, so the liveness + /// rule calls it live. Figures are the jetson-hub RTX 4050's own. + #[test] + fn a_gpu_counter_must_count_no_slower_than_its_board_draws() { + let w = Duration::from_millis(500); + // Over a 500 ms window a counter's rate in mW is twice its mJ. + let draws = &[14_800u32, 14_800]; + // floor = (14_800 - 5_000) / 4 = 2_450 mW. + assert_eq!( + plausible_gpu_counter(250, w, draws, Some(65_000)), + GpuPlausibility::CounterUnderRate { rate_mw: 500, floor_mw: 2_450, mean_draw_mw: 14_800 }, + "0.5 W counted against a 14.8 W draw is no count of the board's energy" + ); + // A counter that merely lags its board passes: the draws are + // one-second averages, and a bursty load integrates to less. + assert!(plausible_gpu_counter(1_300, w, draws, Some(65_000)).plausible()); + // Exactly at the floor is not below it. + assert!(plausible_gpu_counter(1_225, w, draws, Some(65_000)).plausible()); + } + + /// An idle board makes no finding: the slack comes off the draw before the + /// division, so the floor is zero at or below it, and a correct counter on + /// an idle GPU, which accrues almost nothing, is not refuted for it. + #[test] + fn an_idle_board_yields_no_under_rate_finding() { + let w = Duration::from_millis(500); + for mean in [0u32, 1_000, 4_000, 5_000] { + assert!( + plausible_gpu_counter(0, w, &[mean, mean], Some(65_000)).plausible(), + "a counter accruing nothing beside a {mean} mW board must not be refuted" + ); + } + assert_eq!( + plausible_gpu_counter(0, w, &[9_000, 9_000], Some(65_000)), + GpuPlausibility::CounterUnderRate { rate_mw: 0, floor_mw: 1_000, mean_draw_mw: 9_000 } + ); + // No draw reading: no yardstick, so no finding in either direction. + assert!(plausible_gpu_counter(0, w, &[], Some(65_000)).plausible()); + } + + /// What a live counter's status becomes from each verdict: refuted in + /// either direction is `Implausible`, never `Live`, and an unusable board + /// is `Unjudgeable`. + #[test] + fn a_gpu_counter_refuted_either_way_is_implausible() { + let w = Duration::from_millis(500); + let under = plausible_gpu_counter(250, w, &[14_800], Some(65_000)); + let over = plausible_gpu_counter(33_000, w, &[7_500], Some(65_000)); + let idle = plausible_gpu_counter(250, w, &[2_000], Some(65_000)); + let unusable = plausible_gpu_counter(1_000, w, &[590_000], Some(65_000)); + assert_eq!(gpu_counter_status(under), CounterStatus::Implausible, "{under:?}"); + assert_eq!(gpu_counter_status(over), CounterStatus::Implausible, "{over:?}"); + assert_eq!(gpu_counter_status(idle), CounterStatus::Live, "the same slow counter beside an idle board"); + assert_eq!(gpu_counter_status(unusable), CounterStatus::Unjudgeable); + } + + /// Under-reporting outranks over-reporting, so across a run's GPUs the + /// verdict that speaks is the one that admits unbudgeted work; and among + /// two under-reports, the larger shortfall. + #[test] + fn an_under_reporting_gpu_speaks_for_a_run_before_an_over_reporting_one() { + let w = Duration::from_millis(500); + let under = plausible_gpu_counter(250, w, &[14_800], Some(65_000)); + let worse = plausible_gpu_counter(0, w, &[14_800], Some(65_000)); + let over = plausible_gpu_counter(152_250, w, &[5_550], Some(65_000)); + assert!(matches!(over, GpuPlausibility::CounterOverRate { .. })); + let speaks = |vs: [GpuPlausibility; 2]| vs.into_iter().max_by_key(|v| (v.rank(), v.magnitude())); + assert_eq!(speaks([over, under]), Some(under), "the unsafe direction is the one that speaks"); + assert_eq!(speaks([under, worse]), Some(worse), "and the bigger shortfall among two"); + assert!(!under.plausible()); + } + /// The rule over a probe run: each GPU's rise from the run's first sample /// to its last, over the window between them, beside that GPU's own draws /// and limit. One implausible GPU of two makes the run implausible, a diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs index 391965e6d..1441b3a84 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs @@ -24,8 +24,11 @@ //! probe each counter's rise is held to its board's maximum power limit and //! to the board's own draw, read with every sample //! ([`plausible_gpu_counter`](super::liveness::plausible_gpu_counter)); one -//! that runs past them is [`Implausible`](super::CounterStatus::Implausible) -//! and the meter has no GPU reading. +//! that runs past them, or falls far short of the board's draw, is +//! [`Implausible`](super::CounterStatus::Implausible) and the meter has no +//! GPU reading. The short side is the one that matters at a ceiling: a +//! counter reading low would be graded a hardware reading and then spend a +//! budget it is not measuring. //! //! A GPU-only figure is the GPU's energy, not the work's: the meter's span //! rule (a CPU counter that held estimates the span) does not apply to a @@ -34,7 +37,7 @@ //! that work ([`Measurement::reads_cpu`](super::Measurement::reads_cpu)). use super::liveness::{ - counter_run, counter_status_of, counter_tick, gated, gpu_counters_verdict, GpuPlausibility, RailWatch, Sample, + counter_run, counter_status_of, counter_tick, gated, gpu_counter_status, gpu_counters_verdict, RailWatch, Sample, NVML_PROBE_TICK, }; use std::cell::RefCell; @@ -82,16 +85,12 @@ pub(super) fn host_status() -> CounterStatus { /// The status of an NVML probe run: its verdict's, except that a live run is /// re-judged on whether its counters counted their boards' energy /// ([`gpu_counters_verdict`]). A counter proven to run faster than its board -/// can draw is `Implausible`; a board whose own telemetry is unusable leaves -/// the counter `Unjudgeable`, which is a different diagnosis and not a claim -/// about the counter. +/// can draw, or far slower than it draws, is `Implausible`; a board whose own +/// telemetry is unusable leaves the counter `Unjudgeable`, which is a +/// different diagnosis and not a claim about the counter. fn status_of_run(watch: &RailWatch, draws: &[Vec>], ceilings: &[Option]) -> CounterStatus { match counter_status_of(watch.verdict()) { - CounterStatus::Live => match gpu_counters_verdict(watch, draws, ceilings) { - GpuPlausibility::Plausible { .. } => CounterStatus::Live, - GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible, - GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable, - }, + CounterStatus::Live => gpu_counter_status(gpu_counters_verdict(watch, draws, ceilings)), status => status, } } @@ -297,6 +296,10 @@ mod tests { assert_eq!(status_of_run(&live, &[vec![Some(95_000)]], &[Some(350_000)]), CounterStatus::Live); assert_eq!(status_of_run(&live, &[vec![Some(10_000)]], &[Some(65_000)]), CounterStatus::Implausible); assert_eq!(status_of_run(&live, &[vec![Some(95_000)]], &[Some(65_000)]), CounterStatus::Implausible); + // 100 W counted while the board drew 500 W, under a 700 W limit: the + // counter advanced in every interval, so the rule called it live, and + // it counted a fifth of the board. Reading low is refuted too. + assert_eq!(status_of_run(&live, &[vec![Some(500_000)]], &[Some(700_000)]), CounterStatus::Implausible); assert_eq!(status_of_run(&frozen, &[vec![Some(10_000)]], &[Some(65_000)]), CounterStatus::Frozen); assert_eq!(status_of_run(&RailWatch::default(), &[], &[]), CounterStatus::Unavailable); }