energy meters: a GPU counter reading low is refuted in sandbox-meter, eoc-meter and jcp-energy too #354
6 changed files with 386 additions and 27 deletions
|
|
@ -5153,6 +5153,24 @@ mod gpu_plausibility {
|
|||
/// NVML's documented accuracy of a board's power draw, ±5 W, in mW.
|
||||
pub(crate) const DRAW_SLACK_MW: u128 = 5_000;
|
||||
|
||||
/// How far **below** its board's mean draw a counter may accrue before it
|
||||
/// is refuted: a shortfall of more than this factor, after
|
||||
/// [`DRAW_SLACK_MW`] is taken off the draw, is no count of the board's
|
||||
/// energy.
|
||||
///
|
||||
/// The bound has to exist in this direction because **under-reporting is
|
||||
/// the unsafe direction at an energy ceiling**: it admits work the budget
|
||||
/// cannot cover. The ceiling bounds only catch a counter running *fast*,
|
||||
/// which fails safe. A counter republished in lumps and sampled across its
|
||||
/// republish period reads low rather than frozen, so liveness alone does
|
||||
/// not see it. It is looser than [`DRAW_FACTOR`] because the draw is a
|
||||
/// one-second average a bursty load integrates below, and the slack comes
|
||||
/// off before the division so an idle board (at or under the slack) makes
|
||||
/// no finding. The same factor as `jouleclaw-energy`'s and
|
||||
/// `sandbox-meter`'s, so a counter one meter declines another does not
|
||||
/// admit.
|
||||
pub(crate) const UNDER_FACTOR: u128 = 4;
|
||||
|
||||
/// Which of the two bounds a GPU counter ran past.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum GpuBound {
|
||||
|
|
@ -5188,6 +5206,19 @@ mod gpu_plausibility {
|
|||
/// Which bound that was.
|
||||
bound: GpuBound,
|
||||
},
|
||||
/// The counter accrued far less energy than the board's own telemetry
|
||||
/// says it drew: the dangerous finding, because a counter reading low
|
||||
/// is graded `HwShunt` and then spends a budget it is not measuring,
|
||||
/// while it still advances, so liveness calls it live.
|
||||
CounterUnderRate {
|
||||
/// The rate the counter implies over the window, in milliwatts.
|
||||
rate_mw: u64,
|
||||
/// The floor it fell below, in milliwatts.
|
||||
floor_mw: u64,
|
||||
/// The board's mean usable draw the floor was taken from, in
|
||||
/// milliwatts: the yardstick, so the judgement can be re-made.
|
||||
mean_draw_mw: u64,
|
||||
},
|
||||
/// The board's own telemetry read past the board's own power limit, so
|
||||
/// there is no yardstick left to judge the counter against. An
|
||||
/// indeterminate, not a finding about the counter — see
|
||||
|
|
@ -5218,8 +5249,9 @@ mod gpu_plausibility {
|
|||
/// judged to be counting, beside the board's power draw `draws_mw` read
|
||||
/// with the samples and its maximum power limit `ceiling_mw`. It over-ran
|
||||
/// if it ran past [`CEILING_MARGIN`] of the limit, or past [`DRAW_FACTOR`]
|
||||
/// times the mean draw plus [`DRAW_SLACK_MW`]. A bound whose reading is
|
||||
/// missing is not applied.
|
||||
/// times the mean draw plus [`DRAW_SLACK_MW`]; it under-ran if it fell
|
||||
/// short of the mean draw, less the slack, by more than [`UNDER_FACTOR`].
|
||||
/// A bound whose reading is missing is not applied.
|
||||
///
|
||||
/// Liveness says the counter is republished continuously; this says what
|
||||
/// it republishes is the board's energy. On the RTX 4050 Laptop GPU
|
||||
|
|
@ -5283,6 +5315,17 @@ mod gpu_plausibility {
|
|||
bound: GpuBound::MeanDraw,
|
||||
};
|
||||
}
|
||||
// And the same yardstick from below. The slack comes off the draw
|
||||
// before the division, so the floor is zero at or under
|
||||
// `DRAW_SLACK_MW` of mean draw: an idle board makes no finding.
|
||||
let floor = mean.saturating_sub(DRAW_SLACK_MW) / UNDER_FACTOR;
|
||||
if floor > 0 && rate_mw < floor {
|
||||
return GpuPlausibility::CounterUnderRate {
|
||||
rate_mw: clamp(rate_mw),
|
||||
floor_mw: clamp(floor),
|
||||
mean_draw_mw: clamp(mean),
|
||||
};
|
||||
}
|
||||
}
|
||||
GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) }
|
||||
}
|
||||
|
|
@ -5312,8 +5355,8 @@ mod gpu_plausibility {
|
|||
|
||||
/// The status a GPU probe gives: the liveness rule's `status`, except that
|
||||
/// a live counter is re-judged on what it was counting over the probe. One
|
||||
/// proven to run faster than its board can draw is
|
||||
/// [`CounterStatus::Implausible`]; one whose board's own telemetry was
|
||||
/// proven to run faster than its board can draw, or far slower than it
|
||||
/// draws, is [`CounterStatus::Implausible`]; one whose board's own telemetry was
|
||||
/// unusable is [`CounterStatus::Unjudgeable`], which is a different
|
||||
/// diagnosis and not a claim about the counter.
|
||||
pub(crate) fn status_of(status: CounterStatus, gpu: &GpuWatch) -> CounterStatus {
|
||||
|
|
@ -5324,7 +5367,9 @@ mod gpu_plausibility {
|
|||
(Some((a, t0)), Some((b, t1))) => {
|
||||
match plausible(b.saturating_sub(a), t1.saturating_sub(t0), &gpu.draws_mw, gpu.ceiling_mw) {
|
||||
GpuPlausibility::Plausible { .. } => CounterStatus::Live,
|
||||
GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible,
|
||||
GpuPlausibility::CounterOverRate { .. } | GpuPlausibility::CounterUnderRate { .. } => {
|
||||
CounterStatus::Implausible
|
||||
}
|
||||
GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable,
|
||||
}
|
||||
}
|
||||
|
|
@ -5437,6 +5482,50 @@ mod gpu_plausibility {
|
|||
);
|
||||
}
|
||||
|
||||
/// **The bound that was missing: a counter reading LOW.** It still
|
||||
/// advances, so liveness calls it live, and graded `HwShunt` it would
|
||||
/// spend a ceiling it is not measuring. Figures are the jetson-hub RTX
|
||||
/// 4050's own.
|
||||
#[test]
|
||||
fn a_gpu_counter_must_count_no_slower_than_its_board_draws() {
|
||||
let w = Duration::from_millis(500);
|
||||
// floor = (14_800 - 5_000) / 4 = 2_450 mW.
|
||||
assert_eq!(
|
||||
plausible(250, w, &[14_800, 14_800], Some(65_000)),
|
||||
GpuPlausibility::CounterUnderRate { rate_mw: 500, floor_mw: 2_450, mean_draw_mw: 14_800 },
|
||||
"0.5 W counted against a 14.8 W draw"
|
||||
);
|
||||
assert!(plausible(1_300, w, &[14_800, 14_800], Some(65_000)).plausible(), "lagging, not refuted");
|
||||
assert!(plausible(1_225, w, &[14_800, 14_800], Some(65_000)).plausible(), "at the floor");
|
||||
// An idle board makes no finding; nor does a board with no reading.
|
||||
for mean in [0u32, 1_000, 5_000] {
|
||||
assert!(plausible(0, w, &[mean, mean], Some(65_000)).plausible(), "{mean} mW");
|
||||
}
|
||||
assert!(plausible(0, w, &[], Some(65_000)).plausible());
|
||||
assert_eq!(
|
||||
plausible(0, w, &[9_000, 9_000], Some(65_000)),
|
||||
GpuPlausibility::CounterUnderRate { rate_mw: 0, floor_mw: 1_000, mean_draw_mw: 9_000 }
|
||||
);
|
||||
}
|
||||
|
||||
/// The wiring, not only the verdict: a live counter that accrued a
|
||||
/// fraction of what its board drew is `Implausible`, and the same slow
|
||||
/// counter beside an idle board stays `Live`.
|
||||
#[test]
|
||||
fn a_live_gpu_counter_short_of_its_board_is_implausible() {
|
||||
let watch = |draw_mw| {
|
||||
let mut g = GpuWatch { ceiling_mw: Some(65_000), ..GpuWatch::default() };
|
||||
for k in 0..6u64 {
|
||||
// 10 mJ every 100 ms: 100 mW.
|
||||
g.saw(Some(1_000_000 + 10 * k), Some(draw_mw), Duration::from_millis(100 * k));
|
||||
}
|
||||
g
|
||||
};
|
||||
use CounterStatus as S;
|
||||
assert_eq!(status_of(S::Live, &watch(14_800)), S::Implausible, "0.1 W counted, 14.8 W drawn");
|
||||
assert_eq!(status_of(S::Live, &watch(2_000)), S::Live, "0.1 W beside a 2 W board is a counter doing its job");
|
||||
}
|
||||
|
||||
/// A live counter that ran past its board is `Implausible`; one that
|
||||
/// tracked it stays `Live`; a status that was not `Live` stands.
|
||||
#[test]
|
||||
|
|
|
|||
14
jcp/SPEC.md
14
jcp/SPEC.md
|
|
@ -623,6 +623,20 @@ it broke one: a counter a tenth over a bound and one twenty times over are the
|
|||
same refusal and not the same finding, and re-grading a past refusal against a
|
||||
revised bound is only possible if the rate survived the check.
|
||||
|
||||
The check is **two-sided**. Where the bound is the board's own draw, a counter
|
||||
whose rate falls far below that draw MUST NOT produce a hardware-graded figure
|
||||
either. This is the more dangerous of the two directions. Under-reporting is the
|
||||
unsafe direction at a ceiling (§5), because it admits work the budget cannot
|
||||
cover; a counter running fast fails safe, because it only spends a budget
|
||||
sooner. A counter republished in lumps and sampled across its republish period
|
||||
reads low rather than frozen, so liveness alone does not see it. The floor MUST
|
||||
allow for the draw reading's own imprecision. It MUST NOT fire where the board
|
||||
is near idle, because a correctly reporting counter accrues almost nothing there.
|
||||
The reference meters refute a rate under a quarter of the board's mean draw,
|
||||
after the draw's ±5 W accuracy is taken off it. Either bound refutes and never
|
||||
verifies: a counter inside both is not refuted, and that is not a finding that
|
||||
it is correct.
|
||||
|
||||
A bound reading that is itself impossible is not a reading: the same RTX 4050
|
||||
reports about 590 W, past its own 65 W limit, for a second after it wakes. Such a
|
||||
reading MUST be discarded — and only that reading. One broken sample is not a
|
||||
|
|
|
|||
|
|
@ -28,7 +28,8 @@
|
|||
//! GPU, so the CPU's share of the work is unmetered and the figure never
|
||||
//! claims the whole task. Only while the counter is live ([`nvml_status`])
|
||||
//! and plausibly counts the board's energy: over the probe it ran no
|
||||
//! faster than the board can draw ([`CounterStatus::Implausible`]); a span
|
||||
//! faster than the board can draw, and not far slower than it drew
|
||||
//! ([`CounterStatus::Implausible`]); a span
|
||||
//! over which it did not advance is the whole-task default-power estimate.
|
||||
//! 3. **Per-process CPU energy** (macOS) — the kernel's count of this
|
||||
//! process's own CPU energy, `ri_energy_nj` from
|
||||
|
|
@ -682,6 +683,11 @@ pub enum CounterStatus {
|
|||
/// carries the rate and the bound it broke: a counter 1.1 times over a
|
||||
/// bound and one 20 times over are the same status but not the same
|
||||
/// finding, and tuning the margins later needs the figures.
|
||||
///
|
||||
/// A counter that accrued far less than its board drew is `Implausible`
|
||||
/// too ([`GpuPlausibility::CounterUnderRate`]), and that is the dangerous
|
||||
/// direction: graded a reading, it would spend a budget it is not
|
||||
/// measuring.
|
||||
Implausible,
|
||||
/// The counter kept advancing and could not be judged, because the board's
|
||||
/// own power telemetry was unusable over the probe: a draw reading past the
|
||||
|
|
@ -976,6 +982,22 @@ const GPU_DRAW_FACTOR: u128 = 2;
|
|||
/// NVML's documented accuracy of a board's power draw, ±5 W, in mW.
|
||||
const GPU_DRAW_SLACK_MW: u128 = 5_000;
|
||||
|
||||
/// How far **below** its board's mean draw a counter may accrue before it is
|
||||
/// refuted: a shortfall of more than this factor, after [`GPU_DRAW_SLACK_MW`]
|
||||
/// is taken off the draw, is no count of the board's energy.
|
||||
///
|
||||
/// The bound has to exist in this direction because **under-reporting is the
|
||||
/// unsafe direction at an energy ceiling** (SPEC §5): it admits work the
|
||||
/// budget cannot cover. The ceiling bounds only catch a counter running
|
||||
/// *fast*, which fails safe. A counter republished in lumps and sampled across
|
||||
/// its republish period reads low rather than frozen, so liveness alone does
|
||||
/// not see it. It is looser than [`GPU_DRAW_FACTOR`] because the draw is a
|
||||
/// one-second average a bursty load integrates below, and the slack comes off
|
||||
/// before the division so an idle board (at or under the slack) makes no
|
||||
/// finding. The same factor as `jouleclaw-energy`'s, `sandbox-meter`'s and
|
||||
/// `eoc-meter`'s, so a counter one meter declines another does not admit.
|
||||
const GPU_UNDER_FACTOR: u128 = 4;
|
||||
|
||||
/// Whether a GPU energy counter that counted `energy_mj` over `window`
|
||||
/// plausibly counted its board's energy, beside the board's power draw
|
||||
/// `draws_mw` read with the samples and its maximum power limit `ceiling_mw`.
|
||||
|
|
@ -983,7 +1005,8 @@ const GPU_DRAW_SLACK_MW: u128 = 5_000;
|
|||
/// draw reading did (the board's own telemetry then is not a reading either:
|
||||
/// an RTX 4050 Laptop GPU reports 590 W for about a second after it wakes),
|
||||
/// or the counter ran past [`GPU_DRAW_FACTOR`] times the mean draw plus
|
||||
/// [`GPU_DRAW_SLACK_MW`]. A bound whose reading is missing is not applied.
|
||||
/// [`GPU_DRAW_SLACK_MW`], or fell short of that mean, less the slack, by more
|
||||
/// than [`GPU_UNDER_FACTOR`]. A bound whose reading is missing is not applied.
|
||||
///
|
||||
/// Liveness says the counter is republished continuously; this says what it
|
||||
/// republishes is the board's energy. On the RTX 4050 Laptop GPU (driver
|
||||
|
|
@ -1024,6 +1047,19 @@ pub enum GpuPlausibility {
|
|||
/// Which bound that was.
|
||||
bound: GpuBound,
|
||||
},
|
||||
/// The counter accrued far less energy than the board's own telemetry says
|
||||
/// it drew: the dangerous finding, because a counter reading low is graded
|
||||
/// `HwShunt` and then spends a budget it is not measuring, while it still
|
||||
/// advances, so liveness calls it live.
|
||||
CounterUnderRate {
|
||||
/// The rate the counter implies over the window, in milliwatts.
|
||||
rate_mw: u64,
|
||||
/// The floor it fell below, in milliwatts.
|
||||
floor_mw: u64,
|
||||
/// The board's mean usable draw the floor was taken from, in
|
||||
/// milliwatts: the yardstick, so the judgement can be re-made.
|
||||
mean_draw_mw: u64,
|
||||
},
|
||||
/// The board's own telemetry read past the board's own power limit, so
|
||||
/// there is no yardstick to judge the counter against. An indeterminate,
|
||||
/// not a finding about the counter — see [`CounterStatus::Unjudgeable`].
|
||||
|
|
@ -1104,13 +1140,25 @@ fn plausible_gpu_counter(
|
|||
bound: GpuBound::MeanDraw,
|
||||
};
|
||||
}
|
||||
// And the same yardstick from below. The slack comes off the draw
|
||||
// before the division, so the floor is zero at or under
|
||||
// `GPU_DRAW_SLACK_MW` of mean draw: an idle board makes no finding.
|
||||
let floor = mean.saturating_sub(GPU_DRAW_SLACK_MW) / GPU_UNDER_FACTOR;
|
||||
if floor > 0 && rate_mw < floor {
|
||||
return GpuPlausibility::CounterUnderRate {
|
||||
rate_mw: clamp(rate_mw),
|
||||
floor_mw: clamp(floor),
|
||||
mean_draw_mw: clamp(mean),
|
||||
};
|
||||
}
|
||||
}
|
||||
GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) }
|
||||
}
|
||||
|
||||
/// The status an NVML probe reports: [`liveness::status_of`] its run, except
|
||||
/// that a live counter which did not plausibly count its board's energy over
|
||||
/// the run ([`plausible_gpu_counter`]) is [`CounterStatus::Implausible`].
|
||||
/// the run ([`plausible_gpu_counter`]), running faster than its board or far
|
||||
/// slower, is [`CounterStatus::Implausible`].
|
||||
#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))]
|
||||
fn nvml_status_of(probed: Option<&liveness::Advances>, gpu: &GpuWatch) -> CounterStatus {
|
||||
match liveness::status_of(probed) {
|
||||
|
|
@ -1123,7 +1171,9 @@ fn nvml_status_of(probed: Option<&liveness::Advances>, gpu: &GpuWatch) -> Counte
|
|||
gpu.ceiling_mw,
|
||||
) {
|
||||
GpuPlausibility::Plausible { .. } => CounterStatus::Live,
|
||||
GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible,
|
||||
GpuPlausibility::CounterOverRate { .. } | GpuPlausibility::CounterUnderRate { .. } => {
|
||||
CounterStatus::Implausible
|
||||
}
|
||||
GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable,
|
||||
}
|
||||
}
|
||||
|
|
@ -7240,6 +7290,47 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
/// **The bound that was missing: a counter reading LOW.** It still
|
||||
/// advances, so liveness calls it live; graded `HwShunt` it would spend a
|
||||
/// ceiling it is not measuring. Figures are the jetson-hub RTX 4050's own.
|
||||
#[test]
|
||||
fn a_gpu_counter_must_count_no_slower_than_its_board_draws() {
|
||||
let w = Duration::from_millis(500);
|
||||
// floor = (14_800 - 5_000) / 4 = 2_450 mW.
|
||||
assert_eq!(
|
||||
plausible_gpu_counter(250, w, &[14_800, 14_800], Some(65_000)),
|
||||
GpuPlausibility::CounterUnderRate { rate_mw: 500, floor_mw: 2_450, mean_draw_mw: 14_800 },
|
||||
"0.5 W counted against a 14.8 W draw"
|
||||
);
|
||||
assert!(plausible_gpu_counter(1_300, w, &[14_800, 14_800], Some(65_000)).plausible(), "lagging");
|
||||
assert!(plausible_gpu_counter(1_225, w, &[14_800, 14_800], Some(65_000)).plausible(), "at the floor");
|
||||
// An idle board makes no finding, and nor does a board with no reading.
|
||||
for mean in [0u32, 1_000, 5_000] {
|
||||
assert!(plausible_gpu_counter(0, w, &[mean, mean], Some(65_000)).plausible(), "{mean} mW");
|
||||
}
|
||||
assert!(plausible_gpu_counter(0, w, &[], Some(65_000)).plausible());
|
||||
|
||||
// The wiring, not only the verdict: a live counter at 0.1 W beside a
|
||||
// 14.8 W board is `Implausible`, and beside a 2 W board stays `Live`.
|
||||
let rail = |v| liveness::every_rail(&[("nvml device 0 total energy".to_string(), v)]);
|
||||
let mut live = liveness::Advances::default();
|
||||
let t0 = Instant::now();
|
||||
let watch = |draw| {
|
||||
let mut g = GpuWatch { ceiling_mw: Some(65_000), ..GpuWatch::default() };
|
||||
for k in 0..6u32 {
|
||||
// 10 mJ every 100 ms: 100 mW.
|
||||
g.saw(Some(1_000_000 + 10 * u64::from(k)), Some(draw), t0 + Duration::from_millis(100) * k);
|
||||
}
|
||||
g
|
||||
};
|
||||
for k in 0..6u32 {
|
||||
live.observe(rail(1_000_000 + 10 * i64::from(k)), Duration::from_millis(100) * k);
|
||||
}
|
||||
assert_eq!(liveness::status_of(Some(&live)), CounterStatus::Live);
|
||||
assert_eq!(nvml_status_of(Some(&live), &watch(14_800)), CounterStatus::Implausible);
|
||||
assert_eq!(nvml_status_of(Some(&live), &watch(2_000)), CounterStatus::Live);
|
||||
}
|
||||
|
||||
/// The NVML probe's status: a live counter that ran past its board is
|
||||
/// `Implausible`; one that tracked it stays `Live`; a counter that was not
|
||||
/// live keeps its status, whatever its board drew.
|
||||
|
|
|
|||
|
|
@ -145,10 +145,18 @@ Realistic floors:
|
|||
data-center part (A100) agreed to ±0.2 % at every interval, while an
|
||||
unvalidated consumer part (RTX 4090) was still 13.3 % low at 100 ms
|
||||
and came within 1.6 % only at 1 s. So on a GPU whose counter has not
|
||||
been validated against integrated power, prefer the power integral —
|
||||
which this standard's reference implementation already falls back to,
|
||||
graded `ModelBased`. A counter read below its own update rate is not a
|
||||
reading of the span, whatever the interface.
|
||||
been validated against integrated power, prefer the power integral,
|
||||
graded `ModelBased`. The reference implementation does not tell a
|
||||
validated part from an unvalidated one. It falls back to the power
|
||||
integral only where it can refute the counter: not live, or outside
|
||||
the two-sided bound below. That bound is deliberately wide (a quarter
|
||||
of the board's draw once 5 W is taken off), so a counter 13.3 % low
|
||||
passes it and is graded `HwShunt`. Telling the two apart needs the
|
||||
device's counter checked against something it does not share an error
|
||||
with, such as an external meter. Until that is done, an `HwShunt` NVML
|
||||
figure carries its device's unvalidated error. A counter read below
|
||||
its own update rate is not a reading of the span, whatever the
|
||||
interface.
|
||||
- Apple Silicon IOReport: ~1 mJ, ~10 ms. `HwShunt` while its CPU
|
||||
energy rail is live. On macOS 27.0.1 an unprivileged reader's CPU
|
||||
rails are frozen (observed on an M5 Max and an M3 Ultra), so the
|
||||
|
|
|
|||
|
|
@ -507,6 +507,27 @@ pub(super) const GPU_DRAW_FACTOR: u128 = 2;
|
|||
#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml`
|
||||
pub(super) const GPU_DRAW_SLACK_MW: u128 = 5_000;
|
||||
|
||||
/// How far **below** its board's mean draw a counter may accrue before it is
|
||||
/// refuted: a shortfall of more than this factor, after [`GPU_DRAW_SLACK_MW`]
|
||||
/// is taken off the draw, is no count of the board's energy.
|
||||
///
|
||||
/// The bound has to exist in this direction at all because **under-reporting
|
||||
/// is the unsafe direction at an energy ceiling**: it admits work the budget
|
||||
/// cannot cover. The ceiling bounds only catch a counter running *fast*, which
|
||||
/// is the direction that fails safe. A counter republished in lumps and
|
||||
/// sampled across its republish period reads low rather than frozen, so the
|
||||
/// liveness rule alone does not see it.
|
||||
///
|
||||
/// It is deliberately looser than [`GPU_DRAW_FACTOR`]. The draw readings are
|
||||
/// one-second averages, so a bursty load that is idle between samples really
|
||||
/// does integrate to less than its sampled mean. And the slack comes off the
|
||||
/// draw before the division, so the floor collapses to zero at or below
|
||||
/// [`GPU_DRAW_SLACK_MW`]: at idle draw there is no finding to make, only noise
|
||||
/// to avoid. The same factor as `jouleclaw-energy`'s, so a counter one meter
|
||||
/// declines the other does not admit.
|
||||
#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml`
|
||||
pub(super) const GPU_UNDER_FACTOR: u128 = 4;
|
||||
|
||||
/// Whether one GPU's energy counter, which counted `energy_mj` over `window`,
|
||||
/// plausibly counted its board's energy, beside the board's power draw
|
||||
/// `draws_mw` read with the samples and its maximum power limit `ceiling_mw`.
|
||||
|
|
@ -514,7 +535,9 @@ pub(super) const GPU_DRAW_SLACK_MW: u128 = 5_000;
|
|||
/// draw reading did (the board's own telemetry then is not a reading either:
|
||||
/// an RTX 4050 Laptop GPU reports 590 W for about a second after it wakes),
|
||||
/// or the counter ran past [`GPU_DRAW_FACTOR`] times the mean draw plus
|
||||
/// [`GPU_DRAW_SLACK_MW`]. A bound whose reading is missing is not applied.
|
||||
/// [`GPU_DRAW_SLACK_MW`], or fell short of that mean draw, less the slack, by
|
||||
/// more than [`GPU_UNDER_FACTOR`]. A bound whose reading is missing is not
|
||||
/// applied.
|
||||
///
|
||||
/// Liveness says the counter is republished continuously; this says what it
|
||||
/// republishes is the board's energy. A counter that runs several times
|
||||
|
|
@ -559,6 +582,24 @@ pub(super) enum GpuPlausibility {
|
|||
/// Which bound that was.
|
||||
bound: GpuBound,
|
||||
},
|
||||
/// The counter accrued far less energy than the board's own telemetry says
|
||||
/// it drew.
|
||||
///
|
||||
/// This is the finding the ceiling bounds cannot make, and the dangerous
|
||||
/// one: a counter reading low is graded `HwShunt` and then spends a budget
|
||||
/// it is not measuring. A counter republished in lumps and sampled across
|
||||
/// the republish period looks exactly like this. It advances, so the
|
||||
/// liveness rule calls it live, while it accrues a fraction of the energy.
|
||||
CounterUnderRate {
|
||||
/// The rate the counter implies over the window, in milliwatts.
|
||||
rate_mw: u64,
|
||||
/// The floor it fell below, in milliwatts.
|
||||
floor_mw: u64,
|
||||
/// The board's mean usable draw the floor was taken from, in
|
||||
/// milliwatts: the yardstick, so the judgement can be re-made against
|
||||
/// a different factor.
|
||||
mean_draw_mw: u64,
|
||||
},
|
||||
/// The board's own telemetry read past the board's own power limit, so
|
||||
/// there is no yardstick left to judge the counter against. An
|
||||
/// indeterminate, not a finding about the counter — see
|
||||
|
|
@ -583,13 +624,18 @@ impl GpuPlausibility {
|
|||
}
|
||||
|
||||
/// How severe this verdict is, for picking the one that speaks for a run
|
||||
/// of several GPUs: a proven overrun outranks an indeterminate, which
|
||||
/// outranks a pass.
|
||||
/// of several GPUs: a proven shortfall outranks a proven overrun, which
|
||||
/// outranks an indeterminate, which outranks a pass.
|
||||
fn rank(&self) -> u8 {
|
||||
match self {
|
||||
GpuPlausibility::Plausible { .. } => 0,
|
||||
GpuPlausibility::BoardTelemetryUnusable { .. } => 1,
|
||||
GpuPlausibility::CounterOverRate { .. } => 2,
|
||||
// Above a proven overrun: both are refutations, but only this one
|
||||
// is the direction that admits work a budget cannot cover, so where
|
||||
// one GPU of a run over-reports and another under-reports, the
|
||||
// under-report is the verdict an operator has to act on.
|
||||
GpuPlausibility::CounterUnderRate { .. } => 3,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -598,6 +644,10 @@ impl GpuPlausibility {
|
|||
match self {
|
||||
GpuPlausibility::Plausible { rate_mw } => *rate_mw,
|
||||
GpuPlausibility::CounterOverRate { rate_mw, .. } => *rate_mw,
|
||||
// The shortfall, so that among two under-reporting GPUs the worse
|
||||
// one speaks: a lower rate is a bigger shortfall, so it is ordered
|
||||
// by how far the floor was missed rather than by the rate itself.
|
||||
GpuPlausibility::CounterUnderRate { rate_mw, floor_mw, .. } => floor_mw.saturating_sub(*rate_mw),
|
||||
GpuPlausibility::BoardTelemetryUnusable { draw_mw, .. } => *draw_mw,
|
||||
}
|
||||
}
|
||||
|
|
@ -662,6 +712,17 @@ pub(super) fn plausible_gpu_counter(
|
|||
bound: GpuBound::MeanDraw,
|
||||
};
|
||||
}
|
||||
// And the same yardstick from below. The slack comes off the draw
|
||||
// before the division, so the floor is zero at or under
|
||||
// `GPU_DRAW_SLACK_MW` of mean draw: an idle board makes no finding.
|
||||
let floor = mean.saturating_sub(GPU_DRAW_SLACK_MW) / GPU_UNDER_FACTOR;
|
||||
if floor > 0 && rate_mw < floor {
|
||||
return GpuPlausibility::CounterUnderRate {
|
||||
rate_mw: clamp(rate_mw),
|
||||
floor_mw: clamp(floor),
|
||||
mean_draw_mw: clamp(mean),
|
||||
};
|
||||
}
|
||||
}
|
||||
GpuPlausibility::Plausible { rate_mw: clamp(rate_mw) }
|
||||
}
|
||||
|
|
@ -701,6 +762,24 @@ pub(super) fn gpu_counters_verdict(
|
|||
.unwrap_or(GpuPlausibility::Plausible { rate_mw: 0 })
|
||||
}
|
||||
|
||||
/// The status a live GPU counter keeps once its run is judged on what it was
|
||||
/// counting ([`gpu_counters_verdict`]). A counter proven to run faster than its
|
||||
/// board can draw, or far slower than it draws, is `Implausible`. A board
|
||||
/// whose own telemetry is unusable leaves the counter `Unjudgeable`, which is
|
||||
/// a different diagnosis and not a claim about the counter. Kept here, apart
|
||||
/// from the NVML probe that only builds on Linux, so that the mapping is
|
||||
/// tested on every platform.
|
||||
#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml`
|
||||
pub(super) fn gpu_counter_status(verdict: GpuPlausibility) -> CounterStatus {
|
||||
match verdict {
|
||||
GpuPlausibility::Plausible { .. } => CounterStatus::Live,
|
||||
GpuPlausibility::CounterOverRate { .. } | GpuPlausibility::CounterUnderRate { .. } => {
|
||||
CounterStatus::Implausible
|
||||
}
|
||||
GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable,
|
||||
}
|
||||
}
|
||||
|
||||
/// A counter probe's constructor behind its gate: `open` runs only when
|
||||
/// [`counter_admits`] the `status`; otherwise `Unavailable`, naming the
|
||||
/// counter (`what`) and why, and the counter is never touched. Every counter
|
||||
|
|
@ -2164,6 +2243,81 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// **The bound that was missing: a counter reading LOW.** A counter
|
||||
/// running fast fails safe, since an over-reported figure spends a budget
|
||||
/// sooner. A counter reading low is graded `HwShunt` and then spends a
|
||||
/// ceiling it is not measuring, and it still advances, so the liveness
|
||||
/// rule calls it live. Figures are the jetson-hub RTX 4050's own.
|
||||
#[test]
|
||||
fn a_gpu_counter_must_count_no_slower_than_its_board_draws() {
|
||||
let w = Duration::from_millis(500);
|
||||
// Over a 500 ms window a counter's rate in mW is twice its mJ.
|
||||
let draws = &[14_800u32, 14_800];
|
||||
// floor = (14_800 - 5_000) / 4 = 2_450 mW.
|
||||
assert_eq!(
|
||||
plausible_gpu_counter(250, w, draws, Some(65_000)),
|
||||
GpuPlausibility::CounterUnderRate { rate_mw: 500, floor_mw: 2_450, mean_draw_mw: 14_800 },
|
||||
"0.5 W counted against a 14.8 W draw is no count of the board's energy"
|
||||
);
|
||||
// A counter that merely lags its board passes: the draws are
|
||||
// one-second averages, and a bursty load integrates to less.
|
||||
assert!(plausible_gpu_counter(1_300, w, draws, Some(65_000)).plausible());
|
||||
// Exactly at the floor is not below it.
|
||||
assert!(plausible_gpu_counter(1_225, w, draws, Some(65_000)).plausible());
|
||||
}
|
||||
|
||||
/// An idle board makes no finding: the slack comes off the draw before the
|
||||
/// division, so the floor is zero at or below it, and a correct counter on
|
||||
/// an idle GPU, which accrues almost nothing, is not refuted for it.
|
||||
#[test]
|
||||
fn an_idle_board_yields_no_under_rate_finding() {
|
||||
let w = Duration::from_millis(500);
|
||||
for mean in [0u32, 1_000, 4_000, 5_000] {
|
||||
assert!(
|
||||
plausible_gpu_counter(0, w, &[mean, mean], Some(65_000)).plausible(),
|
||||
"a counter accruing nothing beside a {mean} mW board must not be refuted"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
plausible_gpu_counter(0, w, &[9_000, 9_000], Some(65_000)),
|
||||
GpuPlausibility::CounterUnderRate { rate_mw: 0, floor_mw: 1_000, mean_draw_mw: 9_000 }
|
||||
);
|
||||
// No draw reading: no yardstick, so no finding in either direction.
|
||||
assert!(plausible_gpu_counter(0, w, &[], Some(65_000)).plausible());
|
||||
}
|
||||
|
||||
/// What a live counter's status becomes from each verdict: refuted in
|
||||
/// either direction is `Implausible`, never `Live`, and an unusable board
|
||||
/// is `Unjudgeable`.
|
||||
#[test]
|
||||
fn a_gpu_counter_refuted_either_way_is_implausible() {
|
||||
let w = Duration::from_millis(500);
|
||||
let under = plausible_gpu_counter(250, w, &[14_800], Some(65_000));
|
||||
let over = plausible_gpu_counter(33_000, w, &[7_500], Some(65_000));
|
||||
let idle = plausible_gpu_counter(250, w, &[2_000], Some(65_000));
|
||||
let unusable = plausible_gpu_counter(1_000, w, &[590_000], Some(65_000));
|
||||
assert_eq!(gpu_counter_status(under), CounterStatus::Implausible, "{under:?}");
|
||||
assert_eq!(gpu_counter_status(over), CounterStatus::Implausible, "{over:?}");
|
||||
assert_eq!(gpu_counter_status(idle), CounterStatus::Live, "the same slow counter beside an idle board");
|
||||
assert_eq!(gpu_counter_status(unusable), CounterStatus::Unjudgeable);
|
||||
}
|
||||
|
||||
/// Under-reporting outranks over-reporting, so across a run's GPUs the
|
||||
/// verdict that speaks is the one that admits unbudgeted work; and among
|
||||
/// two under-reports, the larger shortfall.
|
||||
#[test]
|
||||
fn an_under_reporting_gpu_speaks_for_a_run_before_an_over_reporting_one() {
|
||||
let w = Duration::from_millis(500);
|
||||
let under = plausible_gpu_counter(250, w, &[14_800], Some(65_000));
|
||||
let worse = plausible_gpu_counter(0, w, &[14_800], Some(65_000));
|
||||
let over = plausible_gpu_counter(152_250, w, &[5_550], Some(65_000));
|
||||
assert!(matches!(over, GpuPlausibility::CounterOverRate { .. }));
|
||||
let speaks = |vs: [GpuPlausibility; 2]| vs.into_iter().max_by_key(|v| (v.rank(), v.magnitude()));
|
||||
assert_eq!(speaks([over, under]), Some(under), "the unsafe direction is the one that speaks");
|
||||
assert_eq!(speaks([under, worse]), Some(worse), "and the bigger shortfall among two");
|
||||
assert!(!under.plausible());
|
||||
}
|
||||
|
||||
/// The rule over a probe run: each GPU's rise from the run's first sample
|
||||
/// to its last, over the window between them, beside that GPU's own draws
|
||||
/// and limit. One implausible GPU of two makes the run implausible, a
|
||||
|
|
|
|||
|
|
@ -24,8 +24,11 @@
|
|||
//! probe each counter's rise is held to its board's maximum power limit and
|
||||
//! to the board's own draw, read with every sample
|
||||
//! ([`plausible_gpu_counter`](super::liveness::plausible_gpu_counter)); one
|
||||
//! that runs past them is [`Implausible`](super::CounterStatus::Implausible)
|
||||
//! and the meter has no GPU reading.
|
||||
//! that runs past them, or falls far short of the board's draw, is
|
||||
//! [`Implausible`](super::CounterStatus::Implausible) and the meter has no
|
||||
//! GPU reading. The short side is the one that matters at a ceiling: a
|
||||
//! counter reading low would be graded a hardware reading and then spend a
|
||||
//! budget it is not measuring.
|
||||
//!
|
||||
//! A GPU-only figure is the GPU's energy, not the work's: the meter's span
|
||||
//! rule (a CPU counter that held estimates the span) does not apply to a
|
||||
|
|
@ -34,7 +37,7 @@
|
|||
//! that work ([`Measurement::reads_cpu`](super::Measurement::reads_cpu)).
|
||||
|
||||
use super::liveness::{
|
||||
counter_run, counter_status_of, counter_tick, gated, gpu_counters_verdict, GpuPlausibility, RailWatch, Sample,
|
||||
counter_run, counter_status_of, counter_tick, gated, gpu_counter_status, gpu_counters_verdict, RailWatch, Sample,
|
||||
NVML_PROBE_TICK,
|
||||
};
|
||||
use std::cell::RefCell;
|
||||
|
|
@ -82,16 +85,12 @@ pub(super) fn host_status() -> CounterStatus {
|
|||
/// The status of an NVML probe run: its verdict's, except that a live run is
|
||||
/// re-judged on whether its counters counted their boards' energy
|
||||
/// ([`gpu_counters_verdict`]). A counter proven to run faster than its board
|
||||
/// can draw is `Implausible`; a board whose own telemetry is unusable leaves
|
||||
/// the counter `Unjudgeable`, which is a different diagnosis and not a claim
|
||||
/// about the counter.
|
||||
/// can draw, or far slower than it draws, is `Implausible`; a board whose own
|
||||
/// telemetry is unusable leaves the counter `Unjudgeable`, which is a
|
||||
/// different diagnosis and not a claim about the counter.
|
||||
fn status_of_run(watch: &RailWatch, draws: &[Vec<Option<u32>>], ceilings: &[Option<u32>]) -> CounterStatus {
|
||||
match counter_status_of(watch.verdict()) {
|
||||
CounterStatus::Live => match gpu_counters_verdict(watch, draws, ceilings) {
|
||||
GpuPlausibility::Plausible { .. } => CounterStatus::Live,
|
||||
GpuPlausibility::CounterOverRate { .. } => CounterStatus::Implausible,
|
||||
GpuPlausibility::BoardTelemetryUnusable { .. } => CounterStatus::Unjudgeable,
|
||||
},
|
||||
CounterStatus::Live => gpu_counter_status(gpu_counters_verdict(watch, draws, ceilings)),
|
||||
status => status,
|
||||
}
|
||||
}
|
||||
|
|
@ -297,6 +296,10 @@ mod tests {
|
|||
assert_eq!(status_of_run(&live, &[vec![Some(95_000)]], &[Some(350_000)]), CounterStatus::Live);
|
||||
assert_eq!(status_of_run(&live, &[vec![Some(10_000)]], &[Some(65_000)]), CounterStatus::Implausible);
|
||||
assert_eq!(status_of_run(&live, &[vec![Some(95_000)]], &[Some(65_000)]), CounterStatus::Implausible);
|
||||
// 100 W counted while the board drew 500 W, under a 700 W limit: the
|
||||
// counter advanced in every interval, so the rule called it live, and
|
||||
// it counted a fifth of the board. Reading low is refuted too.
|
||||
assert_eq!(status_of_run(&live, &[vec![Some(500_000)]], &[Some(700_000)]), CounterStatus::Implausible);
|
||||
assert_eq!(status_of_run(&frozen, &[vec![Some(10_000)]], &[Some(65_000)]), CounterStatus::Frozen);
|
||||
assert_eq!(status_of_run(&RailWatch::default(), &[], &[]), CounterStatus::Unavailable);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue