diff --git a/eoc/eoc-rs/README.md b/eoc/eoc-rs/README.md index 10ac02428..e194e041d 100644 --- a/eoc/eoc-rs/README.md +++ b/eoc/eoc-rs/README.md @@ -9,7 +9,7 @@ EOC is a four-stage memoizing cascade. Every query is tried at each stage in tur 3. **Graph** (`eoc-graph`) — triple-store retrieval. Reference impl is a small in-memory matcher; production deployments swap in a real graph backend driven by DCY (EOC-4). 4. **Neural** (`eoc-neural`) — last-resort inference. Reference impl ships an `EchoBackend`; production deployments wire in llama.cpp / Ollama / Anthropic API / etc. -Joule cost is the unit of accounting. The `eoc-meter` crate reads cumulative micro-joules from whatever hardware counter is available (Linux RAPL, macOS `powermetrics`, NVIDIA NVML behind the `cuda` feature) and falls back to a `StubCounter` everywhere else — including WASM. +Joule cost is the unit of accounting. The `eoc-meter` crate reads cumulative micro-joules from whatever whole-work hardware counter is available and live (Linux RAPL, Apple Silicon IOReport; NVIDIA NVML behind the `cuda` feature is GPU telemetry, never the cascade's counter) and falls back to a `StubCounter` everywhere else — including WASM. ## Workspace layout @@ -46,7 +46,7 @@ receipt: <64-char hex> ## CUDA / NVML -NVIDIA NVML support is feature-gated and **off by default** so the workspace builds without `libnvidia-ml`. To enable: +NVIDIA NVML support is feature-gated and **off by default** so the workspace builds without `libnvidia-ml`. NVML's total-energy counter reads the GPU alone, so `detect()` never hands it to the cascade, whose costs carry no field to say a figure covers part of the work: `NvmlCounter` is GPU telemetry, read beside the whole-work counter, and it is constructed only while its counter is live and counts no faster than the board can draw. To enable: ```sh cargo build -p eoc-meter --features cuda diff --git a/eoc/eoc-rs/crates/eoc-core/src/joule_cost.rs b/eoc/eoc-rs/crates/eoc-core/src/joule_cost.rs index be5be856d..aa458a6e1 100644 --- a/eoc/eoc-rs/crates/eoc-core/src/joule_cost.rs +++ b/eoc/eoc-rs/crates/eoc-core/src/joule_cost.rs @@ -5,8 +5,11 @@ use serde::{Deserialize, Serialize}; /// Where the joule reading came from. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum JouleSource { - /// Read from a live hardware energy counter (RAPL, NVML, or IOReport - /// while its CPU rail is live): the counter advanced over the span. + /// Read from a live hardware energy counter of the work's whole domain + /// (RAPL's packages, or IOReport while its CPU rail is live): the counter + /// advanced over the span. Never a GPU-only counter (NVML's): with no + /// field to say a figure covers part of the work, a GPU-only figure is an + /// estimate of the work, not a reading of it. Measured, /// Anything else: a synthesized estimate, or a figure no live counter /// read (no counter, a failed read, a counter that did not advance). @@ -47,8 +50,9 @@ impl JouleCost { } } - /// A measured cost: the delta a live hardware counter (RAPL, NVML, a live - /// IOReport rail) read over the span of the work, and nothing else. + /// A measured cost: the delta a live hardware counter of the work's whole + /// domain (RAPL's packages, a live IOReport rail) read over the span of the + /// work, and nothing else. pub fn measured(microjoules: u64) -> Self { Self { microjoules, diff --git a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs index 984a7aef5..33d01e17e 100644 --- a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs +++ b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs @@ -53,8 +53,16 @@ //! reference wrapper does not parse it, so every read returns //! `Err(NoCounter)`: no figure, and callers keep their own estimate. //! - NVIDIA NVML is gated behind the `cuda` feature (off by default), and -//! selected only when GPU 0's total-energy counter is live -//! ([`nvml_status`]). +//! **never selected by [`detect`]**: [`NvmlCounter`] reads GPU 0's +//! total-energy counter, which is the GPU's energy and not the work's (the +//! CPU's share of it goes unread), and eoc's consumers grade any advance of +//! the counter they are handed `Measured`, with no coverage field to say a +//! figure covers part of the work. By the family's rule a figure that does +//! not cover the work it is booked to is an estimate in such a format, so +//! the GPU counter is not a [`JouleCounter`]: it is GPU telemetry, read +//! beside one. It is constructed only while its counter is live and +//! plausibly counts the board's energy ([`nvml_status`], +//! [`CounterStatus::Implausible`]). //! - WASM and unknown targets fall through to `StubCounter`. // `deny` not `forbid`: the whole crate is safe except three audited FFI @@ -2899,7 +2907,7 @@ mod macos_ioreport { let before = script::counter_opens(); let given = MacosIoReportCounter::try_detect_given(status); assert!(matches!(given, Err(Error::NoCounter)), "try_detect_given({status:?}) did not decline with NoCounter"); - let picked = crate::detect_with(|| CounterStatus::Unavailable, || CounterStatus::Unavailable, || status).name(); + let picked = crate::detect_with(|| CounterStatus::Unavailable, || status).name(); assert_ne!(picked, "ioreport", "detect_with({status:?}) selected the IOReport counter"); let (built, picked) = script::with_status(status, || (MacosIoReportCounter::try_detect().is_ok(), crate::detect().name())); @@ -2934,7 +2942,7 @@ mod macos_ioreport { if let Err(e) = given { assert!(matches!(e, Error::NoCounter), "try_detect_given({status:?}) declined with {e:?}"); } - let picked = crate::detect_with(|| CounterStatus::Unavailable, || CounterStatus::Unavailable, || status).name(); + let picked = crate::detect_with(|| CounterStatus::Unavailable, || status).name(); assert_eq!(picked == "ioreport", admitted, "detect_with({status:?}) selected {picked}"); let (built, picked) = script::with_status(status, || { assert_eq!(crate::ioreport_status(), status, "the installed status"); @@ -3654,6 +3662,14 @@ pub enum CounterStatus { /// held, or was republished in lumps. Never selected: a lump landing in /// one caller's span would be sealed as that span's reading. Frozen, + /// The counter kept advancing, but a GPU counter (NVML's) ran faster than + /// its board can draw power over the probe: past 1.5 times the board's + /// maximum power limit, past twice the board's own power draw plus 5 W, + /// or beside a draw reading itself past that limit. It is not counting the + /// board's energy. Seen on an RTX 4050 Laptop GPU (driver 595.84), whose + /// counter runs at 4 to 30 times the board's draw while it is polled. + /// Never constructed. + Implausible, /// The counter kept advancing while the probe ran: in at least five /// separate sample intervals, at least 80% of those observed, over at /// least 300 ms. The rung is selected. @@ -3696,14 +3712,17 @@ pub fn rapl_status() -> CounterStatus { /// This host's NVML status, probed once per process and cached. Always /// [`CounterStatus::Unavailable`] without the `cuda` feature. /// -/// [`detect`] selects the NVML rung only when this is [`CounterStatus::Live`]: +/// [`NvmlCounter`] is constructed only when this is [`CounterStatus::Live`]: /// GPU 0's total-energy counter kept advancing, in at least five sample -/// intervals, at least 80% of them, over at least 300 ms. The probe cannot -/// load the GPU, so it waits between samples rather than burning CPU, and a -/// powered GPU's own draw advances a live counter; this shows the counter is -/// published continuously, not that it tracks a load. One that holds or steps -/// in lumps is declined. The first call where NVML opens waits about half a -/// second (a second if the counter is not live). +/// intervals, at least 80% of them, over at least 300 ms, and over that window +/// counted no faster than the board can draw (with the board's power draw read +/// at every sample and its maximum power limit; otherwise +/// [`CounterStatus::Implausible`]). The probe cannot load the GPU, so it waits +/// between samples rather than burning CPU, and a powered GPU's own draw +/// advances a live counter; this shows the counter is published continuously, +/// not that it tracks a load. One that holds or steps in lumps is declined. +/// [`detect`] never selects the GPU counter, live or not. The first call where +/// NVML opens waits about half a second (a second if the counter is not live). pub fn nvml_status() -> CounterStatus { #[cfg(feature = "cuda")] { @@ -4719,17 +4738,169 @@ pub(crate) fn detect_in_process_with( fallback() } +/// Whether a GPU energy counter that kept advancing over its probe plausibly +/// counted its board's energy. Kept apart from NVML so it runs under test in +/// every build; used by the NVML probe (`cuda`). +#[cfg_attr(not(any(test, feature = "cuda")), allow(dead_code))] +mod gpu_plausibility { + use crate::CounterStatus; + use std::time::Duration; + + /// How far past its board's maximum power limit a GPU energy counter may + /// run, on average over a probe's window, as a fraction (3/2): a board can + /// draw past its limit for milliseconds, not on average over a third of a + /// second. + pub(crate) const CEILING_MARGIN: (u128, u128) = (3, 2); + + /// How much faster than its board's own power draw a GPU energy counter + /// may run over a probe's window: twice the draw's mean over the samples, + /// plus [`DRAW_SLACK_MW`]. The draw is a one-second average, so a load that + /// changed during the window can leave it behind the counter, but not by a + /// factor of two at a probe. + pub(crate) const DRAW_FACTOR: u128 = 2; + + /// NVML's documented accuracy of a board's power draw, ±5 W, in mW. + pub(crate) const DRAW_SLACK_MW: u128 = 5_000; + + /// Whether a GPU energy counter that counted `energy_mj` over `window` + /// plausibly counted its board's energy, beside the board's power draw + /// `draws_mw` read with the samples and its maximum power limit + /// `ceiling_mw`. Not when the counter ran past [`CEILING_MARGIN`] of the + /// limit, or a draw reading did (the board's own telemetry then is not a + /// reading either: an RTX 4050 Laptop GPU reports 590 W for about a second + /// after it wakes), or the counter ran past [`DRAW_FACTOR`] times the mean + /// draw plus [`DRAW_SLACK_MW`]. A bound whose reading is missing is not + /// applied. + /// + /// Liveness says the counter is republished continuously; this says what + /// it republishes is the board's energy. On the RTX 4050 Laptop GPU + /// (driver 595.84) the total-energy counter runs several times faster than + /// the board draws for as long as it is polled: at the probe's 100 ms wait + /// it ran at 58-305 W while the board drew 5-14 W against a 65 W limit. + pub(crate) fn plausible(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { + let ns = window.as_nanos().max(1); + // mJ per second is mW. + let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; + if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { + let cap = u128::from(limit) * CEILING_MARGIN.0 / CEILING_MARGIN.1; + if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { + return false; + } + } + if !draws_mw.is_empty() { + let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; + if rate_mw > DRAW_FACTOR * mean + DRAW_SLACK_MW { + return false; + } + } + true + } + + /// What the GPU probe saw of the board beside its counter: the counter's + /// first and last reading (mJ) and when, the board's power draw read with + /// every sample, and its maximum power limit. + #[derive(Debug, Default, Clone)] + pub(crate) struct GpuWatch { + first: Option<(u64, Duration)>, + last: Option<(u64, Duration)>, + draws_mw: Vec, + pub(crate) ceiling_mw: Option, + } + + impl GpuWatch { + /// Fold in one sample: the counter's reading, if it read, and the + /// board's draw, if that read, at `at`. + pub(crate) fn saw(&mut self, mj: Option, draw_mw: Option, at: Duration) { + if let Some(mj) = mj { + self.first.get_or_insert((mj, at)); + self.last = Some((mj, at)); + } + self.draws_mw.extend(draw_mw); + } + } + + /// The status a GPU probe gives: the liveness rule's `status`, except + /// that a live counter which did not plausibly count its board's energy + /// over the probe is [`CounterStatus::Implausible`]. + pub(crate) fn status_of(status: CounterStatus, gpu: &GpuWatch) -> CounterStatus { + let plausible = match (gpu.first, gpu.last) { + (Some((a, t0)), Some((b, t1))) => { + plausible(b.saturating_sub(a), t1.saturating_sub(t0), &gpu.draws_mw, gpu.ceiling_mw) + } + _ => true, + }; + match status { + CounterStatus::Live if !plausible => CounterStatus::Implausible, + other => other, + } + } + + #[cfg(test)] + mod tests { + use super::*; + + /// The rule with the readings of the RTX 4050 Laptop GPU on + /// jetson-hub (driver 595.84, 65 W maximum limit): polled at the + /// probe's 100 ms wait its counter ran at 58-305 W while the board drew + /// 5-14 W, which is no count of the board's energy. A counter that + /// tracks its board passes, a one-second draw lagging a load step by up + /// to twice still passes, and a bound whose reading is missing is not + /// applied. + #[test] + fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { + let w = Duration::from_millis(500); + assert!(!plausible(59_700, w, &[12_970, 13_520], Some(65_000)), "119.4 W against a 13 W draw"); + assert!(!plausible(22_000, w, &[10_400; 6], Some(65_000)), "44 W against a 10.4 W draw"); + assert!(!plausible(60_000, w, &[], Some(65_000)), "120 W on a 65 W board"); + assert!(!plausible(44_000, w, &[590_000, 10_300], Some(65_000)), "a 590 W draw reading"); + assert!(plausible(6_500, w, &[13_000, 13_500], Some(65_000)), "13 W against a 13 W draw"); + assert!(plausible(150_000, w, &[280_000; 5], Some(700_000)), "300 W against a 280 W draw"); + assert!(plausible(102_500, w, &[100_000; 5], Some(700_000)), "205 W is 2 x 100 W + 5 W"); + assert!(!plausible(102_501, w, &[100_000; 5], Some(700_000))); + assert!(plausible(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); + assert!(!plausible(48_751, w, &[], Some(65_000))); + assert!(plausible(1_000_000, w, &[], None), "nothing to judge by"); + } + + /// A live counter that ran past its board is `Implausible`; one that + /// tracked it stays `Live`; a status that was not `Live` stands. + #[test] + fn a_live_gpu_counter_past_its_board_is_implausible() { + let watch = |draw_mw, ceiling_mw| { + let mut g = GpuWatch { ceiling_mw, ..GpuWatch::default() }; + for k in 0..6u64 { + // 12 J every 100 ms: 120 W. + g.saw(Some(1_000_000 + 12_000 * k), Some(draw_mw), Duration::from_millis(100 * k)); + } + g + }; + use CounterStatus as S; + assert_eq!(status_of(S::Live, &watch(13_000, Some(65_000))), S::Implausible); + assert_eq!(status_of(S::Live, &watch(115_000, Some(700_000))), S::Live); + assert_eq!(status_of(S::Live, &GpuWatch::default()), S::Live, "nothing to judge by"); + for status in [S::Unavailable, S::Frozen] { + assert_eq!(status_of(status, &watch(13_000, Some(65_000))), status); + } + } + } +} + #[cfg(feature = "cuda")] mod nvml { use super::*; use crate::rail_liveness; use nvml_wrapper::Nvml; - /// Reads cumulative GPU energy via NVML. + /// Reads GPU 0's cumulative energy via NVML: GPU telemetry, not a + /// [`JouleCounter`]. /// /// Constructed only on a host whose total-energy counter the liveness - /// probe found live ([`crate::nvml_status`]); a live NVML counter is a - /// vendor energy counter, so a delta it reads is a measurement of GPU 0. + /// probe found live and plausibly counting the board's energy + /// ([`crate::nvml_status`]); a delta it reads is a measurement of GPU 0, + /// and of nothing else. It is not a [`JouleCounter`] because eoc's + /// consumers grade any advance of a `JouleCounter` `Measured` with no + /// coverage field: a GPU-only delta graded so would claim the CPU's share + /// of the work, which it does not read. [`crate::detect`] never selects it. pub struct NvmlCounter { _nvml: Nvml, device_index: u32, @@ -4786,21 +4957,35 @@ mod nvml { const NVML_PROBE_WAIT: std::time::Duration = std::time::Duration::from_millis(100); /// The real NVML probe. Cached by [`crate::nvml_status`], its only caller. + /// With each sample it reads the board's power draw, and once its maximum + /// power limit, so that a live counter that ran faster than the board can + /// draw is [`CounterStatus::Implausible`] ([`crate::gpu_plausibility`]). pub(crate) fn host_status() -> CounterStatus { - rail_liveness::counter_status( - rail_liveness::probe_counter( - NvmlCounter::open, - NvmlCounter::sample, - || std::thread::sleep(NVML_PROBE_WAIT), - crate::nvml_wall_now, - crate::VENDOR_COUNTER_LIVENESS_BUDGET, - ) - .as_ref(), - ) + let seen = std::cell::RefCell::new(crate::gpu_plausibility::GpuWatch::default()); + let watch = rail_liveness::probe_counter( + || { + let c = NvmlCounter::open()?; + seen.borrow_mut().ceiling_mw = c.max_limit_mw(); + Some(c) + }, + |c: &NvmlCounter| { + let sample = c.sample(); + let mj = sample.as_ref().and_then(|rails| rails.first()).and_then(|(_, raw)| u64::try_from(*raw).ok()); + seen.borrow_mut().saw(mj, c.draw_mw(), crate::nvml_wall_now()); + sample + }, + || std::thread::sleep(NVML_PROBE_WAIT), + crate::nvml_wall_now, + crate::VENDOR_COUNTER_LIVENESS_BUDGET, + ); + crate::gpu_plausibility::status_of(rail_liveness::counter_status(watch.as_ref()), &seen.into_inner()) } - impl JouleCounter for NvmlCounter { - fn read_microjoules(&self) -> Result { + impl NvmlCounter { + /// GPU 0's cumulative energy, in µJ (NVML counts millijoules). The + /// GPU's energy alone: never grade a delta of it as a figure for work + /// that also ran on the CPU. + pub fn read_gpu_microjoules(&self) -> Result { let device = self ._nvml .device_by_index(self.device_index) @@ -4811,8 +4996,16 @@ mod nvml { // NVML reports millijoules; convert to microjoules. Ok(mj.saturating_mul(1000)) } - fn name(&self) -> &'static str { - "nvml" + + /// The board's power draw, in mW, as NVML reports it. + fn draw_mw(&self) -> Option { + self._nvml.device_by_index(self.device_index).ok()?.power_usage().ok() + } + + /// The board's maximum power-management limit, in mW. + fn max_limit_mw(&self) -> Option { + let limits = self._nvml.device_by_index(self.device_index).ok()?.power_management_limit_constraints().ok()?; + (limits.max_limit > 0).then_some(limits.max_limit) } } @@ -4827,7 +5020,7 @@ mod nvml { #[test] fn only_a_live_nvml_status_constructs_the_counter() { let attempts = || OPEN_ATTEMPTS.with(|n| n.get()); - for status in [CounterStatus::Unavailable, CounterStatus::Frozen] { + for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible] { let before = attempts(); assert!( matches!(NvmlCounter::try_detect_given(status), Err(Error::NoCounter)), @@ -4860,20 +5053,24 @@ pub use nvml::NvmlCounter; /// Runtime-detect the best available counter on this host. /// -/// Preference order: NVML (if compiled with `cuda`, present, and its counter -/// live — see [`nvml_status`]) → RAPL on Linux (only when every package +/// Preference order: RAPL on Linux (only when every package /// counter is live — see [`rapl_status`]) → IOReport on macOS (Energy-Model counters, /// no sudo; only when every CPU energy rail is live — see [`ioreport_status`]) /// → powermetrics on macOS → `StubCounter`. Every hardware rung is held to /// the same sustained-advance rule; a rung whose counter is not live is /// skipped, so callers estimate rather than seal what it read. /// +/// It never selects NVML's GPU counter ([`NvmlCounter`], feature `cuda`), +/// which reads the GPU and not the CPU's share of the work: eoc's consumers +/// grade whatever this returns `Measured`, with no field to say a figure +/// covers part of the work, so a GPU-only figure would claim the whole. +/// /// It never selects this process's own energy counter /// ([`MacosProcessEnergyCounter`]), whose domain stops at this process's /// CPU threads: what eoc meters may run in a subprocess or on the GPU. A /// caller whose work is in-process asks [`detect_in_process`]. pub fn detect() -> Box { - detect_with(nvml_status, rapl_status, ioreport_status) + detect_with(rapl_status, ioreport_status) } /// The selection behind [`detect`], given where each rung's status comes from @@ -4883,14 +5080,9 @@ pub fn detect() -> Box { // and build some of them go unused. #[allow(unused_variables)] pub(crate) fn detect_with( - nvml: impl FnOnce() -> CounterStatus, rapl: impl FnOnce() -> CounterStatus, ioreport: impl FnOnce() -> IoReportStatus, ) -> Box { - #[cfg(feature = "cuda")] - if let Ok(c) = NvmlCounter::try_detect_given(nvml()) { - return Box::new(c); - } #[cfg(target_os = "linux")] if let Ok(c) = LinuxRaplCounter::try_detect_given(rapl()) { return Box::new(c); @@ -4932,27 +5124,23 @@ mod tests { assert_eq!(ioreport_status(), IoReportStatus::Unavailable); } - /// RAPL and NVML are selected only when their counter is live, and only - /// where their rung exists: RAPL on Linux, NVML with `cuda` where NVML - /// opens. A frozen or unavailable counter is never selected, whatever - /// the other rungs say. + /// RAPL is selected only when its counter is live, and only where its + /// rung exists, on Linux. A frozen, implausible or unavailable counter is + /// never selected, whatever the other rungs say. NVML's GPU counter is + /// never selected at all: it is not a `JouleCounter`, and the ladder has + /// no rung for it, whatever its status. #[test] fn a_vendor_counter_is_selected_only_when_live() { use CounterStatus as S; let declined_ioreport = || IoReportStatus::CpuRailFrozen; - for status in [S::Unavailable, S::Frozen] { - let picked = detect_with(|| status, || status, declined_ioreport).name(); - assert!(!matches!(picked, "rapl" | "nvml"), "a {status:?} counter was selected: {picked}"); - let picked = detect_with(|| S::Unavailable, || status, declined_ioreport).name(); + for status in [S::Unavailable, S::Frozen, S::Implausible] { + let picked = detect_with(|| status, declined_ioreport).name(); assert_ne!(picked, "rapl", "a {status:?} RAPL counter was selected"); } - let picked = detect_with(|| S::Unavailable, || S::Live, declined_ioreport).name(); + let picked = detect_with(|| S::Live, declined_ioreport).name(); assert_eq!(picked == "rapl", cfg!(target_os = "linux"), "a live RAPL status selected {picked}"); - #[cfg(feature = "cuda")] - { - let opens = nvml::NvmlCounter::try_detect_given(S::Live).is_ok(); - let picked = detect_with(|| S::Live, || S::Unavailable, declined_ioreport).name(); - assert_eq!(picked == "nvml", opens, "a live NVML status selected {picked}"); + for status in [S::Unavailable, S::Live] { + assert_ne!(detect_with(|| status, declined_ioreport).name(), "nvml"); } } @@ -4969,12 +5157,10 @@ mod tests { IoReportStatus::CpuRailFrozen, IoReportStatus::Live, ]; - for nvml in [S::Unavailable, S::Frozen, S::Live] { - for rapl in [S::Unavailable, S::Frozen, S::Live] { - for io in ioreport { - let picked = detect_with(|| nvml, || rapl, || io).name(); - assert_ne!(picked, "process-energy", "detect_with({nvml:?}, {rapl:?}, {io:?})"); - } + for rapl in [S::Unavailable, S::Frozen, S::Implausible, S::Live] { + for io in ioreport { + let picked = detect_with(|| rapl, || io).name(); + assert_ne!(picked, "process-energy", "detect_with({rapl:?}, {io:?})"); } } let picked = detect().name(); @@ -5008,8 +5194,8 @@ mod tests { /// Against this host's RAPL package counters, where there are any: the /// status the probe cached agrees with what they do now. Live: every one - /// advances across 200 ms of CPU work, and `detect()` selects RAPL unless - /// NVML is live. Not live: `detect()` never selects it. Logs what it saw. + /// advances across 200 ms of CPU work, and `detect()` selects RAPL. Not + /// live: `detect()` never selects it. Logs what it saw. #[cfg(target_os = "linux")] #[test] fn the_rapl_rung_agrees_with_the_counter() { @@ -5030,7 +5216,7 @@ mod tests { for ((p, b), a) in host.packages.iter().zip(&b).zip(&a) { assert!(a != b, "live package zone intel-rapl:{} did not move across 200 ms of CPU work: {b} -> {a}", p.index); } - assert!(picked == "rapl" || nvml_status() == CounterStatus::Live, "detect() picked {picked}"); + assert_eq!(picked, "rapl", "detect() picked {picked}"); } else { assert_ne!(picked, "rapl", "detect() selected a {status:?} RAPL counter"); } diff --git a/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs b/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs new file mode 100644 index 000000000..9a8980170 --- /dev/null +++ b/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs @@ -0,0 +1,392 @@ +//! NVML against the hardware (feature `cuda`): GPU 0's total-energy counter, +//! read by the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the NVML probe behind `nvml_status()` +//! runs in this process, and the tests take one lock. The oracle opens +//! `libnvidia-ml` itself, measures how often the counter is republished, +//! samples it at the probe's own wait (100 ms) by the sustained-advance rule +//! and holds it to the board's power ceiling, which it asks of `nvidia-smi`. +//! None of these tests passes without asserting: where there is no NVML (a +//! CI container has no GPU), the status must say so and agree with the +//! oracle; where the counter is live, both must say so. Either way the GPU +//! counter is never the counter `detect()` hands eoc's consumers, which +//! grade any advance of it `Measured` with no coverage field to say it read +//! only the GPU: by the family's rule a GPU-only figure in such a format is +//! an estimate, never a whole-work hardware reading. + +#![cfg(all(target_os = "linux", feature = "cuda"))] + +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use eoc_meter::{CounterStatus, NvmlCounter}; + +/// One test at a time: each reads machine-wide counters, and the oracle's +/// own sampling must not overlap the meter's. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The NVML probe's wait between samples (`NVML_PROBE_WAIT`). +const TICK: Duration = Duration::from_millis(100); + +/// The tests' own reading of NVIDIA's total-energy counter: `libnvidia-ml` +/// opened here with `dlopen` and called through symbols looked up here, not +/// through the crate under test or its bindings, and each GPU's power limit +/// and draw asked of `nvidia-smi`, a separate process. It measures how often +/// the counter is republished, and judges it by the family's rule: every +/// GPU's counter advanced in at least five intervals, at least 80% of them, +/// over at least 300 ms, sampled at the crate's own tick; and it is plausibly +/// counting its board's energy: over the window it ran no faster than 1.5 +/// times the board's maximum power limit, the board's own power draw (as +/// `nvidia-smi` reports it just before and just after) was itself within that +/// bound, and the counter ran no faster than twice that draw plus 5 W (the +/// draw's documented accuracy). A counter that runs several times faster than +/// its board draws is not counting that board's energy. +#[allow(dead_code)] +mod nvml_oracle { + use std::ffi::{c_char, c_int, c_uint, c_void, CString}; + use std::time::{Duration, Instant}; + + unsafe extern "C" { + fn dlopen(filename: *const c_char, flag: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; + } + + /// `RTLD_NOW` in ``. + const RTLD_NOW: c_int = 2; + + type Device = *mut c_void; + type EnergyFn = unsafe extern "C" fn(Device, *mut u64) -> c_int; + type LimitsFn = unsafe extern "C" fn(Device, *mut c_uint, *mut c_uint) -> c_int; + type MilliwattsFn = unsafe extern "C" fn(Device, *mut c_uint) -> c_int; + + /// NVML, opened by the oracle. + pub struct Nvml { + energy: EnergyFn, + limits: Option, + power: Option, + /// Every GPU whose total-energy counter reads, in index order. + pub devices: Vec, + } + + fn sym(lib: *mut c_void, name: &str) -> Option<*mut c_void> { + let name = CString::new(name).ok()?; + let p = unsafe { dlsym(lib, name.as_ptr()) }; + (!p.is_null()).then_some(p) + } + + /// `libnvidia-ml.so.1`, initialised, with every GPU whose counter reads; + /// `None` where there is no library, no GPU, or no GPU with the counter. + pub fn open() -> Option { + let file = CString::new("libnvidia-ml.so.1").ok()?; + let lib = unsafe { dlopen(file.as_ptr(), RTLD_NOW) }; + if lib.is_null() { + return None; + } + unsafe { + let init: unsafe extern "C" fn() -> c_int = std::mem::transmute(sym(lib, "nvmlInit_v2")?); + let count: unsafe extern "C" fn(*mut c_uint) -> c_int = std::mem::transmute(sym(lib, "nvmlDeviceGetCount_v2")?); + let by_index: unsafe extern "C" fn(c_uint, *mut Device) -> c_int = + std::mem::transmute(sym(lib, "nvmlDeviceGetHandleByIndex_v2")?); + let energy: EnergyFn = std::mem::transmute(sym(lib, "nvmlDeviceGetTotalEnergyConsumption")?); + let limits = sym(lib, "nvmlDeviceGetPowerManagementLimitConstraints").map(|p| std::mem::transmute::<_, LimitsFn>(p)); + let power = sym(lib, "nvmlDeviceGetPowerUsage").map(|p| std::mem::transmute::<_, MilliwattsFn>(p)); + if init() != 0 { + return None; + } + let mut n: c_uint = 0; + if count(&mut n) != 0 { + return None; + } + let mut devices = Vec::new(); + for i in 0..n { + let mut d: Device = std::ptr::null_mut(); + let mut mj = 0u64; + if by_index(i, &mut d) == 0 && !d.is_null() && energy(d, &mut mj) == 0 { + devices.push(d); + } + } + (!devices.is_empty()).then_some(Nvml { energy, limits, power, devices }) + } + } + + impl Nvml { + /// Every GPU's total-energy counter, in mJ; `None` if one fails. + pub fn energy_mj(&self) -> Option> { + self.devices + .iter() + .map(|&d| { + let mut mj = 0u64; + (unsafe { (self.energy)(d, &mut mj) } == 0).then_some(mj) + }) + .collect() + } + + /// GPU `i`'s maximum power-management limit, in mW, as NVML gives it. + pub fn max_limit_mw(&self, i: usize) -> Option { + let (mut lo, mut hi) = (0, 0); + (unsafe { (self.limits?)(self.devices[i], &mut lo, &mut hi) } == 0 && hi > 0).then_some(hi) + } + + /// GPU `i`'s power draw, in mW, as NVML gives it. + pub fn power_mw(&self, i: usize) -> Option { + let mut mw = 0; + (unsafe { (self.power?)(self.devices[i], &mut mw) } == 0).then_some(mw) + } + } + + /// `nvidia-smi`'s view of every GPU: (maximum power limit, enforced limit, + /// draw), in watts; `None` without `nvidia-smi`. + pub fn smi() -> Option, Option, Option)>> { + let out = std::process::Command::new("nvidia-smi") + .args(["--query-gpu=power.max_limit,enforced.power.limit,power.draw", "--format=csv,noheader,nounits"]) + .output() + .ok()?; + out.status.success().then(|| { + String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| { + let f: Vec> = l.split(',').map(|v| v.trim().parse().ok()).collect(); + (f.first().copied().flatten(), f.get(1).copied().flatten(), f.get(2).copied().flatten()) + }) + .collect() + }) + } + + /// GPU `i`'s ceiling in watts: `nvidia-smi`'s maximum power limit, else + /// NVML's. + pub fn ceiling_w(nvml: &Nvml, smi: &Option, Option, Option)>>, i: usize) -> Option { + smi.as_ref() + .and_then(|s| s.get(i)) + .and_then(|(max, _, _)| *max) + .or_else(|| nvml.max_limit_mw(i).map(|mw| f64::from(mw) / 1000.0)) + } + + /// How often GPU 0's counter is republished: the median gap between its + /// changes, polled every 2 ms for up to 1.5 s (`None` if it changed + /// fewer than 6 times). + pub fn update_period(nvml: &Nvml) -> Option { + let started = Instant::now(); + let mut last = nvml.energy_mj()?[0]; + let mut changes = Vec::new(); + while started.elapsed() < Duration::from_millis(1500) && changes.len() < 16 { + std::thread::sleep(Duration::from_millis(2)); + let now = nvml.energy_mj()?[0]; + if now != last { + changes.push(started.elapsed()); + last = now; + } + } + let mut gaps: Vec = changes.windows(2).map(|w| w[1] - w[0]).collect(); + if gaps.len() < 5 { + return None; + } + gaps.sort(); + Some(gaps[gaps.len() / 2]) + } + + /// The oracle's verdict on the counters. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No NVML, no GPU, or no GPU whose total-energy counter reads. + Unavailable, + /// A counter did not keep advancing. + Frozen, + /// Every counter kept advancing, but one is not plausibly its board's + /// energy (see the module doc). + Implausible, + /// Every counter kept advancing, plausibly its board's energy. + Live, + } + + /// The family's plausibility rule for one GPU over a window: its counter's + /// average `rate_w`, the board's draws `draws_w` around the window and + /// its maximum power limit `ceiling_w`. A bound whose input is missing + /// is not applied. + pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { + if let Some(c) = ceiling_w { + if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { + return false; + } + } + if !draws_w.is_empty() { + let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if rate_w > 2.0 * mean + 5.0 { + return false; + } + } + true + } + + /// What the oracle saw. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + /// Each GPU's average rate over the window, in watts. + pub rate_w: Vec, + /// Each GPU's draw just before and just after the window, in watts. + pub draws_w: Vec>, + /// Each GPU's ceiling, in watts. + pub ceiling_w: Vec>, + } + + impl Watch { + /// The verdict, each GPU's advances, its counter's rate, the board's + /// draw and its maximum limit, for the test's log line. + pub fn summary(&self) -> String { + format!( + "{:?} (advanced {:?} of {} intervals over {} ms; counter rate {:?} W, board draw {:?} W, max power limit {:?} W)", + self.verdict, + self.advances, + self.intervals, + self.window.as_millis(), + self.rate_w.iter().map(|w| (w * 10.0).round() / 10.0).collect::>(), + self.draws_w, + self.ceiling_w + ) + } + } + + /// The oracle's run over the first `gpus` GPUs (all, if more), sleeping + /// `tick` between samples, by the sustained-advance rule (stopping when + /// live, or after a second with at least five intervals, or five + /// seconds); then each counter's plausibility over the whole window. + pub fn watch(tick: Duration, gpus: usize) -> Watch { + let none = Watch { + verdict: Verdict::Unavailable, + intervals: 0, + advances: Vec::new(), + window: Duration::ZERO, + rate_w: Vec::new(), + draws_w: Vec::new(), + ceiling_w: Vec::new(), + }; + let Some(nvml) = open() else { return none }; + let n = nvml.devices.len().min(gpus); + let before = smi(); + let read = || nvml.energy_mj().map(|v| v[..n].to_vec()); + let started = Instant::now(); + let Some(first) = read() else { return none }; + let (mut prev, mut last_at) = (first.clone(), Duration::ZERO); + let mut advances = vec![0u32; n]; + let mut intervals = 0u32; + let live = |adv: &[u32], k: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(k) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, last_at) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + std::thread::sleep(tick); + if let Some(now) = read() { + last_at = started.elapsed(); + intervals += 1; + for ((a, x), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(x > p); + } + prev = now; + } + } + let after = smi(); + let window = last_at; + let rate_w: Vec = + first.iter().zip(&prev).map(|(a, b)| b.saturating_sub(*a) as f64 / 1000.0 / window.as_secs_f64().max(1e-9)).collect(); + let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); + let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); + let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + let verdict = if !live(&advances, intervals, window) { + Verdict::Frozen + } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + Verdict::Implausible + } else { + Verdict::Live + }; + Watch { verdict, intervals, advances, window, rate_w, draws_w, ceiling_w } + } + + /// One line on the GPUs for the log: count, counters, ceilings, draw, and + /// the counter's measured republication `period`. + pub fn listing(period: Option) -> String { + let Some(nvml) = open() else { return "no NVML device with a total-energy counter".to_string() }; + format!( + "{} GPU(s) with a total-energy counter, counters {:?} mJ; NVML max power limit {:?} mW, power draw {:?} mW; \ + nvidia-smi (max limit, enforced limit, draw) W {:?}; counter republished every {:?}", + nvml.devices.len(), + nvml.energy_mj(), + (0..nvml.devices.len()).map(|i| nvml.max_limit_mw(i)).collect::>(), + (0..nvml.devices.len()).map(|i| nvml.power_mw(i)).collect::>(), + smi(), + period + ) + } + + /// GPU 0's counter's republication period, where there is one. + pub fn period() -> Option { + open().and_then(|n| update_period(&n)) + } +} + +use nvml_oracle::Verdict; + +/// The status agrees with the oracle, judged at the probe's own tick, and the +/// tick is no faster than the counter is republished; only a live status +/// constructs the counter, and `detect()` never selects it, live or not. +#[test] +fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { + let _one = serial(); + let period = nvml_oracle::period(); + let status = eoc_meter::nvml_status(); + let watch = nvml_oracle::watch(TICK, 1); + let constructed = NvmlCounter::try_detect().is_ok(); + let selected = eoc_meter::detect().name(); + eprintln!( + "nvml hardware: {}; oracle at a {TICK:?} tick {}; status={status:?} constructed={constructed} detect()={selected}", + nvml_oracle::listing(period), + watch.summary() + ); + if let Some(p) = period { + assert!(TICK >= p * 9 / 10, "the {TICK:?} tick is faster than the counter's {p:?} republication"); + } + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Implausible => assert_eq!(status, CounterStatus::Implausible), + Verdict::Live => {} + } + assert_eq!(constructed, status == CounterStatus::Live, "construction disagrees with the status"); + assert_ne!(selected, "nvml", "detect() handed eoc's consumers the GPU-only counter, which they grade Measured"); +} + +/// Over a one-second span a live GPU counter advances, inside the oracle's +/// own reads around the span, and no faster than the board can draw. +#[test] +fn a_live_gpu_counter_reads_inside_the_oracles_bracket() { + let _one = serial(); + let Ok(counter) = NvmlCounter::try_detect() else { + eprintln!("nvml span: status {:?}; no counter", eoc_meter::nvml_status()); + assert_ne!(eoc_meter::nvml_status(), CounterStatus::Live); + return; + }; + let oracle = nvml_oracle::open().expect("the meter reads NVML, yet the oracle cannot open it"); + let o0 = oracle.energy_mj().expect("oracle read")[0]; + let started = std::time::Instant::now(); + let m0 = counter.read_gpu_microjoules().expect("a live counter reads"); + std::thread::sleep(Duration::from_secs(1)); + let m1 = counter.read_gpu_microjoules().expect("a live counter reads"); + let wall = started.elapsed(); + let o1 = oracle.energy_mj().expect("oracle read")[0]; + let ceiling = nvml_oracle::ceiling_w(&oracle, &nvml_oracle::smi(), 0).unwrap_or(f64::INFINITY); + let uj = m1 - m0; + eprintln!("nvml span: {wall:?}; meter {uj} µJ; oracle {o0} -> {o1} mJ; ceiling {ceiling} W"); + assert!(uj > 0 && uj <= (o1 - o0) * 1_000, "{uj} µJ outside the oracle's (0, {}] µJ", (o1 - o0) * 1_000); + let watts = uj as f64 / 1e6 / wall.as_secs_f64(); + assert!(watts <= 1.5 * ceiling, "the GPU counter reads {watts:.1} W, past the board's {ceiling} W ceiling"); +} diff --git a/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs b/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs new file mode 100644 index 000000000..5fb841e47 --- /dev/null +++ b/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs @@ -0,0 +1,425 @@ +//! Linux RAPL against the hardware: this host's powercap counters, read by +//! the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the RAPL probe behind `rapl_status()` +//! runs in this process, and the tests take one lock: each burns CPU and +//! reads machine-wide counters. The oracle reads `/sys/class/powercap` +//! itself — every zone's `name`, `energy_uj` and `max_energy_range_uj` — and +//! applies the sustained-advance rule to the top-level `package-` zones +//! on its own. None of these tests passes without asserting: where the +//! counters cannot be read (no powercap; a CI container, whose +//! `/sys/devices/virtual/powercap` Docker masks), the status must say so and +//! agree with the oracle; where they are live, both must say live, and the +//! meter's figure over a CPU burn must sit inside the oracle's package-only +//! bracket and clearly below package+psys and package+core. + +#![cfg(target_os = "linux")] + +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use eoc_meter::{CounterStatus, JouleCounter, LinuxRaplCounter}; + +/// One test at a time: each burns CPU and reads machine-wide counters. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The tests' own reading of `/sys/class/powercap`: every zone listed, its +/// `name`, `energy_uj` and `max_energy_range_uj`, read here and nowhere else, +/// and the sustained-advance verdict over the package zones by the family's +/// rule. Nothing in it comes from the crate under test. +#[allow(dead_code)] +mod oracle { + use std::os::raw::{c_int, c_long}; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// Where the kernel lists its powercap zones. + pub const POWERCAP: &str = "/sys/class/powercap"; + + #[repr(C)] + struct Timespec { + tv_sec: c_long, + tv_nsec: c_long, + } + + unsafe extern "C" { + fn clock_gettime(clock_id: c_int, tp: *mut Timespec) -> c_int; + } + + /// `CLOCK_THREAD_CPUTIME_ID` in Linux ``. + const CLOCK_THREAD_CPUTIME_ID: c_int = 3; + + /// This thread's CPU time. + pub fn thread_cpu() -> Duration { + let mut ts = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mut ts) }, 0, "clock_gettime"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// Spin this thread until it has used `d` of its own CPU time, or `wall_cap` + /// of wall time has passed. Returns the CPU it got. + pub fn burn(d: Duration, wall_cap: Duration) -> Duration { + let (started, wall) = (thread_cpu(), Instant::now()); + let mut acc: u64 = 0; + while thread_cpu() - started < d && wall.elapsed() < wall_cap { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + thread_cpu() - started + } + + /// One entry of the powercap directory. + #[derive(Debug, Clone)] + pub struct Zone { + /// The directory entry: `intel-rapl:0`, `intel-rapl:0:0`, `intel-rapl-mmio:0`, ... + pub dir: String, + pub name: Option, + pub energy_uj: Option, + pub max_uj: Option, + } + + fn text(p: PathBuf) -> Option { + std::fs::read_to_string(p).ok().map(|s| s.trim().to_string()) + } + + fn number(p: PathBuf) -> Option { + text(p)?.parse().ok() + } + + /// Every entry under `root`, in name order. + pub fn zones(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { return Vec::new() }; + let mut zones: Vec = entries + .flatten() + .map(|e| { + let dir = e.file_name().to_string_lossy().into_owned(); + let p = e.path(); + Zone { name: text(p.join("name")), energy_uj: number(p.join("energy_uj")), max_uj: number(p.join("max_energy_range_uj")), dir } + }) + .collect(); + zones.sort_by(|a, b| a.dir.cmp(&b.dir)); + zones + } + + fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) + } + + /// A top-level RAPL zone: `intel-rapl:`. Not a sub-zone, not MMIO. + pub fn top_level(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(digits) + } + + /// A package zone: top-level, named `package-`. + pub fn is_package(z: &Zone) -> bool { + top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + } + + /// The platform zone. + pub fn is_psys(z: &Zone) -> bool { + top_level(z) && z.name.as_deref() == Some("psys") + } + + /// A package's `core` sub-zone (`intel-rapl::` named `core`). + pub fn is_core(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(|r| r.contains(':')) && z.name.as_deref() == Some("core") + } + + /// One zone's energy between two reads, modulo its own range. + pub fn delta(e0: u64, e1: u64, max: Option) -> Option { + if e1 >= e0 { + Some(e1 - e0) + } else { + max.filter(|&m| m > 0)?.checked_sub(e0)?.checked_add(e1) + } + } + + /// `energy_uj` of each of `dirs` under `root`, in order; `None` if any fails. + pub fn read(root: &Path, dirs: &[&Zone]) -> Option> { + dirs.iter().map(|z| number(root.join(&z.dir).join("energy_uj"))).collect() + } + + /// The summed energy of `which` of `set` between two reads of `set`. + pub fn energy(set: &[&Zone], before: &[u64], after: &[u64], which: impl Fn(&Zone) -> bool) -> u64 { + set.iter() + .zip(before.iter().zip(after)) + .filter(|(z, _)| which(z)) + .map(|(z, (&b, &a))| delta(b, a, z.max_uj).expect("a zone fell with no readable range")) + .sum() + } + + /// What the oracle concluded about the package counters under a root. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No top-level zone's `energy_uj` reads (no powercap, a masked or + /// root-only tree). + Unreadable, + /// Top-level zones read, but none is a package. + NoPackage, + /// Every package read, but not every one kept advancing. + Frozen, + /// Every package advanced in at least five intervals, at least 80% of + /// them, over at least 300 ms. + Live, + } + + /// The oracle's sustained-advance run over the package zones under `root`. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub packages: Vec, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + } + + impl Watch { + /// The verdict and each package's advances, for the test's log line. + pub fn summary(&self) -> String { + let per: Vec = + self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); + format!("{:?} ({} over {} ms)", self.verdict, per.join(", "), self.window.as_millis()) + } + } + + /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU + /// between samples (capped at 500 ms of wall time), until every package + /// has advanced in at least five intervals, at least 80% of those + /// observed, over at least 300 ms; or one second has passed with at least + /// five intervals; or five seconds have. + pub fn watch(root: &Path) -> Watch { + let all = zones(root); + let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); + let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; + if packages.is_empty() { + let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); + return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + } + let started = Instant::now(); + let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; packages.len()]; + let mut intervals = 0u32; + let live = |adv: &[u32], n: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, el) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + burn(Duration::from_millis(25), Duration::from_millis(500)); + if let Some(now) = read(root, &packages) { + intervals += 1; + for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(n > p); + } + prev = now; + } + } + let window = started.elapsed(); + let verdict = if live(&advances, intervals, window) { Verdict::Live } else { Verdict::Frozen }; + Watch { verdict, packages: names, intervals, advances, window } + } + + /// The zones under `root`, as one line for the log. + pub fn listing(root: &Path) -> String { + zones(root) + .iter() + .map(|z| { + let kind = if is_package(z) { + "package" + } else if is_psys(z) { + "psys" + } else if is_core(z) { + "core sub-zone" + } else if top_level(z) { + "top-level" + } else { + "not summed" + }; + let e = z.energy_uj.map_or("unreadable".to_string(), |e| e.to_string()); + let m = z.max_uj.map_or("-".to_string(), |m| m.to_string()); + format!("{}={} [{kind}] energy_uj={e} max={m}", z.dir, z.name.as_deref().unwrap_or("?")) + }) + .collect::>() + .join("; ") + } + + /// The CI container's view of the host's powercap directory: the same + /// entries, each a symlink into `/sys/devices/virtual/powercap`, which + /// Docker masks with an empty tmpfs, so every target is missing. Built + /// from this host's own listing; removed when dropped. + pub struct MaskedView(pub PathBuf); + + impl MaskedView { + /// Build the masked view of this host's powercap entries in a + /// temporary directory named by `tag` and this process. + pub fn of_host(tag: &str) -> MaskedView { + let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create the masked view"); + let masked = root.join("masked-devices-virtual-powercap"); + for z in zones(Path::new(POWERCAP)) { + std::os::unix::fs::symlink(masked.join(&z.dir), root.join(&z.dir)).expect("symlink"); + } + MaskedView(root) + } + } + + impl Drop for MaskedView { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A bracket of reads around the meter's two reads: every package, psys + /// and core zone read at four points, outer and inner on each side. + pub struct Bracket<'a> { + pub set: Vec<&'a Zone>, + pub reads: [Vec; 4], + } + + impl Bracket<'_> { + /// The packages' energy over the outer bracket, in µJ: the most a + /// package-only figure read inside it can be. + pub fn package_outer(&self) -> u64 { + energy(&self.set, &self.reads[0], &self.reads[3], is_package) + } + /// The packages' energy over the inner bracket, in µJ: the least a + /// package-only figure read around it can be. + pub fn package_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_package) + } + /// The psys zone's energy over the inner bracket, in µJ. + pub fn psys_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_psys) + } + /// The core sub-zones' energy over the inner bracket, in µJ. + pub fn core_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_core) + } + /// Whether the bracket read a zone of the kind `which` picks. + pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { + self.set.iter().any(|z| which(z)) + } + } + + /// The family's rule on a figure `uj` the meter read between the + /// bracket's inner and outer reads: no less than the packages' energy + /// over the inner bracket, no more than over the outer one, and below + /// package+psys and package+core where those zones exist. A figure that + /// added a zone reads at least the packages' and that zone's energy over + /// the inner bracket, so `uj < inner + zone` is the double-count test; it + /// can only be put where the zone moved more over the burn than the + /// packages did in the gap between the outer and inner reads (else a + /// correct figure could reach it too), and the line says when it could + /// not. Returns the figures, for the log. + pub fn assert_package_only(b: &Bracket, uj: u64) -> String { + let (inner, outer, psys, core) = (b.package_inner(), b.package_outer(), b.psys_inner(), b.core_inner()); + let gap = outer - inner; + let mut line = format!( + "meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ (gap {gap}); psys inner {psys} µJ; \ + core inner {core} µJ" + ); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + for (zone, present, moved) in [("psys", b.has(is_psys), psys), ("core", b.has(is_core), core)] { + if !present { + continue; + } + assert!(moved > 0, "{zone} did not advance: {line}"); + if moved > gap { + assert!(uj < inner + moved, "the figure reaches package+{zone}: {zone} was added: {line}"); + line.push_str(&format!("; below package+{zone}")); + } else { + line.push_str(&format!("; {zone} moved less than the gap, so only the outer bound tests it")); + } + } + line + } + + /// Every package, psys and core zone of `all` whose counter reads. + pub fn bracket_set(all: &[Zone]) -> Vec<&Zone> { + all.iter().filter(|z| (is_package(z) || is_psys(z) || is_core(z)) && z.energy_uj.is_some()).collect() + } + + /// One oracle read of every zone of `set`; panics if one fails. + pub fn read_all(set: &[&Zone]) -> Vec { + read(Path::new(POWERCAP), set).expect("an oracle read of the bracket zones") + } + + /// Read the bracket around `meter_read` (called twice) and `work`: + /// outer, meter, inner, work, inner, meter, outer. + pub fn around<'a, M>(set: Vec<&'a Zone>, mut meter_read: impl FnMut() -> M, work: impl FnOnce()) -> (Bracket<'a>, M, M) { + let o0 = read_all(&set); + let m0 = meter_read(); + let o1 = read_all(&set); + work(); + let o2 = read_all(&set); + let m1 = meter_read(); + let o3 = read_all(&set); + (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) + } +} + +use oracle::Verdict; + +/// The status agrees with the oracle's own verdict, and only a live status +/// constructs the counter and gets the RAPL rung selected: a live counter's +/// advance is what eoc's consumers grade `Measured`, so the gate is the grade. +#[test] +fn the_rapl_status_agrees_with_the_oracle_and_gates_the_rung() { + let _one = serial(); + let status = eoc_meter::rapl_status(); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + let constructed = LinuxRaplCounter::try_detect().is_ok(); + let selected = eoc_meter::detect().name(); + eprintln!( + "rapl hardware: {}; oracle {}; status={status:?} constructed={constructed} detect()={selected}", + oracle::listing(Path::new(oracle::POWERCAP)), + watch.summary() + ); + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unreadable | Verdict::NoPackage => assert_eq!(status, CounterStatus::Unavailable), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Live => {} + } + assert_eq!(constructed, status == CounterStatus::Live, "construction disagrees with the status"); + assert_eq!(selected == "rapl", status == CounterStatus::Live, "detect() picked {selected} under {status:?}"); +} + +/// Over a 300 ms one-thread CPU burn the counter reads the packages' energy: +/// no less than the oracle's inner package bracket, no more than its outer +/// one, and clearly below package+psys and package+core (neither the +/// platform zone nor a package's own sub-zone is added). Where RAPL is not +/// live, no counter is constructed and there is no figure to grade. +#[test] +fn a_cpu_burn_reads_inside_the_package_only_bracket() { + let _one = serial(); + let Ok(counter) = LinuxRaplCounter::try_detect() else { + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + eprintln!("rapl burn: no counter; oracle {}", watch.summary()); + assert_ne!(watch.verdict, Verdict::Live, "the oracle saw live packages, yet no counter was constructed"); + assert_ne!(eoc_meter::rapl_status(), CounterStatus::Live); + return; + }; + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let wall = Instant::now(); + let mut cpu = Duration::ZERO; + let (b, m0, m1) = oracle::around( + oracle::bracket_set(&all), + || counter.read_microjoules().expect("a live counter reads"), + || cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)), + ); + let uj = m1 - m0; + eprintln!("rapl burn: {cpu:?} of thread CPU in {:?}; {}", wall.elapsed(), oracle::assert_package_only(&b, uj)); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); +} diff --git a/jcp/jcp-rs/crates/jcp-energy/src/lib.rs b/jcp/jcp-rs/crates/jcp-energy/src/lib.rs index af3782ac8..e72664012 100644 --- a/jcp/jcp-rs/crates/jcp-energy/src/lib.rs +++ b/jcp/jcp-rs/crates/jcp-energy/src/lib.rs @@ -26,8 +26,10 @@ //! 2. **NVML** (feature `nvml`) — GPU energy via `libnvidia-ml`, //! `Provenance::HwShunt` with `Coverage::Partial`: the counter reads the //! GPU, so the CPU's share of the work is unmetered and the figure never -//! claims the whole task. Only while the counter is live ([`nvml_status`]); -//! a span over which it did not advance is the model's whole-task figure. +//! claims the whole task. Only while the counter is live ([`nvml_status`]) +//! and plausibly counts the board's energy: over the probe it ran no +//! faster than the board can draw ([`CounterStatus::Implausible`]); a span +//! over which it did not advance is the model's whole-task figure. //! 3. **Per-process CPU energy** (macOS) — the kernel's count of this //! process's own CPU energy, `ri_energy_nj` from //! `proc_pid_rusage(getpid(), RUSAGE_INFO_V6)` (public libproc, no @@ -251,6 +253,17 @@ impl Nvml { Some(mj.saturating_mul(1_000)) } + /// The board's power draw, in mW, as NVML reports it. + fn draw_mw(&self) -> Option { + self.nvml.device_by_index(self.device_index).ok()?.power_usage().ok() + } + + /// The board's maximum power-management limit, in mW. + fn max_limit_mw(&self) -> Option { + let limits = self.nvml.device_by_index(self.device_index).ok()?.power_management_limit_constraints().ok()?; + (limits.max_limit > 0).then_some(limits.max_limit) + } + /// One liveness sample: the device's total-energy counter as its one /// rail, in its raw millijoules. `None` when the read fails, which the /// probe skips. @@ -631,6 +644,15 @@ pub enum CounterStatus { /// would be graded a hardware reading of that span's work, and a held rail /// beside live ones would be left out of a figure that claims it. Frozen, + /// The counter kept advancing, but a GPU counter (NVML's) ran faster than + /// its board can draw power over the probe: past 1.5 times the board's + /// maximum power limit, past twice the board's own power draw plus 5 W, + /// or with the board's draw reading itself past that limit. It is not + /// counting the board's energy, so a figure from it would debit energy + /// the device never spent. Seen on an RTX 4050 Laptop GPU (driver + /// 595.84), whose counter runs at 4 to 30 times the board's draw while it + /// is polled. Never selected. + Implausible, /// Every rail of the counter kept advancing while the probe ran: each in /// at least five separate sample intervals, making up at least 80% of /// those observed, over at least 300 ms. The rung is selected. @@ -738,27 +760,39 @@ const NVML_PROBE_WAIT: std::time::Duration = std::time::Duration::from_millis(10 /// /// [`Meter::detect`] selects the NVML rung only when this is /// [`CounterStatus::Live`]: device 0's total-energy counter advanced in at -/// least five sample intervals, at least 80% of them, over at least 300 ms. The -/// probe has no way to load the GPU, so this shows the counter is published -/// continuously, not that it tracks a load; a counter that holds or steps in -/// lumps is declined. The first call on a host where NVML opens waits about -/// half a second (one second if the counter is not live). +/// least five sample intervals, at least 80% of them, over at least 300 ms, +/// and over that window counted no faster than the board can draw +/// ([`plausible_gpu_counter`], with the board's power draw read at every +/// sample and its maximum power limit). The probe has no way to load the +/// GPU, so liveness shows the counter is published continuously, not that it +/// tracks a load; a counter that holds or steps in lumps is declined, and one +/// that runs faster than its board draws is [`CounterStatus::Implausible`]. +/// The first call on a host where NVML opens waits about half a second (one +/// second if the counter is not live). pub fn nvml_status() -> CounterStatus { #[cfg(feature = "nvml")] { static STATUS: std::sync::OnceLock = std::sync::OnceLock::new(); *STATUS.get_or_init(|| { NVML_PROBE_RUNS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - liveness::status_of( - liveness::probe( - Nvml::open, - Nvml::liveness_sample, - || std::thread::sleep(NVML_PROBE_WAIT), - Instant::now, - liveness::BUDGET, - ) - .as_ref(), - ) + let seen = std::cell::RefCell::new(GpuWatch::default()); + let probed = liveness::probe( + || { + let n = Nvml::open()?; + seen.borrow_mut().ceiling_mw = n.max_limit_mw(); + Some(n) + }, + |n: &Nvml| { + let sample = n.liveness_sample(); + let mj = sample.as_ref().and_then(|rails| rails.first()).map(|(_, raw)| *raw); + seen.borrow_mut().saw(mj.and_then(|mj| u64::try_from(mj).ok()), n.draw_mw(), Instant::now()); + sample + }, + || std::thread::sleep(NVML_PROBE_WAIT), + Instant::now, + liveness::BUDGET, + ); + nvml_status_of(probed.as_ref(), &seen.into_inner()) }) } #[cfg(not(feature = "nvml"))] @@ -767,6 +801,98 @@ pub fn nvml_status() -> CounterStatus { } } +/// What the NVML probe saw of the board beside its counter: the counter's +/// first and last reading (mJ) and when, the board's power draw read with +/// every sample, and its maximum power limit. +#[derive(Debug, Default, Clone)] +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] +struct GpuWatch { + first: Option<(u64, Instant)>, + last: Option<(u64, Instant)>, + draws_mw: Vec, + ceiling_mw: Option, +} + +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] +impl GpuWatch { + /// Fold in one sample: the counter's reading, if it read, and the + /// board's draw, if that read, at `at`. + fn saw(&mut self, mj: Option, draw_mw: Option, at: Instant) { + if let Some(mj) = mj { + self.first.get_or_insert((mj, at)); + self.last = Some((mj, at)); + } + self.draws_mw.extend(draw_mw); + } +} + +/// How far past its board's maximum power limit a GPU energy counter may run, +/// on average over a probe's window, as a fraction (3/2): a board can draw +/// past its limit for milliseconds, not on average over a third of a second. +const GPU_CEILING_MARGIN: (u128, u128) = (3, 2); + +/// How much faster than its board's own power draw a GPU energy counter may +/// run over a probe's window: twice the draw's mean over the samples, plus +/// [`GPU_DRAW_SLACK_MW`]. The draw is a one-second average, so a load that +/// changed during the window can leave it behind the counter, but not by a +/// factor of two at a probe. +const GPU_DRAW_FACTOR: u128 = 2; + +/// NVML's documented accuracy of a board's power draw, ±5 W, in mW. +const GPU_DRAW_SLACK_MW: u128 = 5_000; + +/// Whether a GPU energy counter that counted `energy_mj` over `window` +/// plausibly counted its board's energy, beside the board's power draw +/// `draws_mw` read with the samples and its maximum power limit `ceiling_mw`. +/// Not when the counter ran past [`GPU_CEILING_MARGIN`] of the limit, or a +/// draw reading did (the board's own telemetry then is not a reading either: +/// an RTX 4050 Laptop GPU reports 590 W for about a second after it wakes), +/// or the counter ran past [`GPU_DRAW_FACTOR`] times the mean draw plus +/// [`GPU_DRAW_SLACK_MW`]. A bound whose reading is missing is not applied. +/// +/// Liveness says the counter is republished continuously; this says what it +/// republishes is the board's energy. On the RTX 4050 Laptop GPU (driver +/// 595.84) the total-energy counter runs several times faster than the +/// board draws for as long as it is polled: at this probe's 100 ms wait it +/// ran at 58-305 W while the board drew 5-14 W against a 65 W limit. +fn plausible_gpu_counter(energy_mj: u64, window: std::time::Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { + let ns = window.as_nanos().max(1); + // mJ per second is mW. + let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; + if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { + let cap = u128::from(limit) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1; + if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { + return false; + } + } + if !draws_mw.is_empty() { + let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; + if rate_mw > GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW { + return false; + } + } + true +} + +/// The status an NVML probe reports: [`liveness::status_of`] its run, except +/// that a live counter which did not plausibly count its board's energy over +/// the run ([`plausible_gpu_counter`]) is [`CounterStatus::Implausible`]. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] +fn nvml_status_of(probed: Option<&liveness::Advances>, gpu: &GpuWatch) -> CounterStatus { + match liveness::status_of(probed) { + CounterStatus::Live => { + let plausible = match (gpu.first, gpu.last) { + (Some((a, t0)), Some((b, t1))) => { + plausible_gpu_counter(b.saturating_sub(a), t1.saturating_duration_since(t0), &gpu.draws_mw, gpu.ceiling_mw) + } + _ => true, + }; + if plausible { CounterStatus::Live } else { CounterStatus::Implausible } + } + status => status, + } +} + /// The byte layout of `struct rusage_info_v6`, which macOS's /// `proc_pid_rusage(pid, RUSAGE_INFO_V6, &info)` fills, as far as this crate /// reads it. From ``, measured with `sizeof` and `offsetof` by @@ -5231,15 +5357,24 @@ mod tests { nvml_status(), process_energy_status() ); - // Every `detect()` asks for the RAPL status first, then NVML's, then - // the per-process counter's: each probe ran once, on whichever thread - // asked first. Off macOS there is no per-process probe. + // Every `detect()` asks for the RAPL status first, then, only while + // no rung above was selected, NVML's, then the per-process counter's: + // each probe the ladder reached ran once, on whichever thread asked + // first, and one below the selected rung never ran. On a Linux host + // whose RAPL is live the ladder stops there, so NVML is never probed; + // the detection records which statuses were asked. Off macOS there + // is no per-process probe. + let (_, why) = Meter::detect_explained(); assert_eq!(rapl_runs, 1, "the RAPL liveness probe did not run exactly once in this process"); - assert_eq!(nvml_runs, usize::from(cfg!(feature = "nvml")), "the NVML liveness probe ran {nvml_runs} times"); + assert_eq!( + nvml_runs, + usize::from(cfg!(feature = "nvml") && why.nvml.is_some()), + "the NVML liveness probe ran {nvml_runs} times ({why:?})" + ); assert_eq!( process_runs, - usize::from(cfg!(target_os = "macos")), - "the per-process liveness probe ran {process_runs} times" + usize::from(cfg!(target_os = "macos") && why.process_energy.is_some()), + "the per-process liveness probe ran {process_runs} times ({why:?})" ); assert!(statuses.iter().all(|s| *s == statuses[0]), "threads saw different statuses: {statuses:?}"); assert_eq!(statuses[0], ioreport_status(), "the spawned threads and this one disagree"); @@ -5386,7 +5521,7 @@ mod tests { fn only_live_admits_a_counter_rung_and_nothing_else_is_opened() { use CounterStatus as S; assert!(admits_counter(S::Live), "Live does not admit the rung"); - for status in [S::Unavailable, S::Frozen] { + for status in [S::Unavailable, S::Frozen, S::Implausible] { assert!(!admits_counter(status), "{status:?} admits the rung"); let opened = Cell::new(0); let got = select(status, || { @@ -5431,6 +5566,56 @@ mod tests { assert_eq!(liveness::status_of(Some(&empty)), CounterStatus::Frozen); } + /// The plausibility rule on a GPU counter, with the readings of the RTX + /// 4050 Laptop GPU on jetson-hub (driver 595.84, 65 W maximum limit): + /// polled at the probe's 100 ms wait its counter ran at 58-305 W while the + /// board drew 5-14 W, which is no count of the board's energy. A counter + /// that tracks its board passes, a one-second draw lagging a load step by + /// up to twice still passes, and a bound whose reading is missing is not + /// applied. + #[test] + fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { + let w = Duration::from_millis(500); + assert!(!plausible_gpu_counter(59_750, w, &[12_970, 13_520], Some(65_000)), "119.5 W against a 13 W draw"); + assert!(!plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000)), "44 W against a 10.4 W draw"); + assert!(!plausible_gpu_counter(60_000, w, &[], Some(65_000)), "120 W on a 65 W board"); + assert!(!plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000)), "a 590 W draw reading"); + assert!(plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000)), "13 W against a 13 W draw"); + assert!(plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000)), "300 W against a 280 W draw"); + assert!(plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000)), "205 W is 2 x 100 W + 5 W"); + assert!(!plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000))); + assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); + assert!(!plausible_gpu_counter(48_751, w, &[], Some(65_000))); + assert!(plausible_gpu_counter(1_000_000, w, &[], None), "nothing to judge by"); + } + + /// The NVML probe's status: a live counter that ran past its board is + /// `Implausible`; one that tracked it stays `Live`; a counter that was not + /// live keeps its status, whatever its board drew. + #[test] + fn a_live_gpu_counter_past_its_board_is_implausible() { + let rail = |v| liveness::every_rail(&[("nvml device 0 total energy".to_string(), v)]); + let (mut live, mut held) = (liveness::Advances::default(), liveness::Advances::default()); + let t0 = Instant::now(); + let mut gpu = GpuWatch { ceiling_mw: Some(65_000), ..GpuWatch::default() }; + for k in 0..6u32 { + // 12 J every 100 ms: 120 W. + live.observe(rail(1_000_000 + 12_000 * i64::from(k)), Duration::from_millis(100) * k); + held.observe(rail(1_000_000), Duration::from_millis(100) * k); + gpu.saw(Some(1_000_000 + 12_000 * u64::from(k)), Some(13_000), t0 + Duration::from_millis(100) * k); + } + assert_eq!(liveness::status_of(Some(&live)), CounterStatus::Live); + assert_eq!(nvml_status_of(Some(&live), &gpu), CounterStatus::Implausible); + assert_eq!(nvml_status_of(Some(&held), &gpu), CounterStatus::Frozen); + assert_eq!(nvml_status_of(None, &gpu), CounterStatus::Unavailable); + let mut big = GpuWatch { ceiling_mw: Some(700_000), ..GpuWatch::default() }; + for k in 0..6u32 { + big.saw(Some(1_000_000 + 12_000 * u64::from(k)), Some(115_000), t0 + Duration::from_millis(100) * k); + } + assert_eq!(nvml_status_of(Some(&live), &big), CounterStatus::Live, "120 W against a 115 W draw on a 700 W board"); + assert_eq!(nvml_status_of(Some(&live), &GpuWatch::default()), CounterStatus::Live, "nothing to judge by"); + } + /// The tests' interval for a CPU-counter probe: a 25 ms tick and a short /// read. const COUNTER_STEP: Duration = Duration::from_millis(31); @@ -5692,7 +5877,7 @@ mod tests { #[cfg(target_os = "macos")] let _hw = ioreport_hw_lock(); let fake = FakePowercap::new(&["1000"]); - for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Live] { + for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible, CounterStatus::Live] { let opened = Cell::new(0); let open = || { opened.set(opened.get() + 1); diff --git a/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs b/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs new file mode 100644 index 000000000..76096fb1b --- /dev/null +++ b/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs @@ -0,0 +1,395 @@ +//! NVML against the hardware (feature `nvml`): GPU 0's total-energy counter, +//! read by the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the NVML probe behind `nvml_status()` +//! runs in this process, and the tests take one lock. The oracle opens +//! `libnvidia-ml` itself, measures how often the counter is republished, +//! samples it at the probe's own wait (100 ms) by the sustained-advance rule +//! and holds it to the board's power ceiling, which it asks of `nvidia-smi`. +//! None of these tests passes without asserting: where there is no NVML (a +//! CI container has no GPU), the status must say so and agree with the +//! oracle; where the counter is live, both must say so, and the NVML rung's +//! figure must be `HwShunt` with `Coverage::Partial` — it reads the GPU, not +//! the CPU's share of the work — and no faster than the board can draw. + +#![cfg(all(target_os = "linux", feature = "nvml"))] + +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use jcp_energy::{nvml_status, CounterStatus, Meter, Nvml, Rung}; +use jcp_schema::{Coverage, Provenance}; + +/// One test at a time: each reads machine-wide counters, and the oracle's +/// own sampling must not overlap the meter's. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The NVML probe's wait between samples (`NVML_PROBE_WAIT`). +const TICK: Duration = Duration::from_millis(100); + +/// The tests' own reading of NVIDIA's total-energy counter: `libnvidia-ml` +/// opened here with `dlopen` and called through symbols looked up here, not +/// through the crate under test or its bindings, and each GPU's power limit +/// and draw asked of `nvidia-smi`, a separate process. It measures how often +/// the counter is republished, and judges it by the family's rule: every +/// GPU's counter advanced in at least five intervals, at least 80% of them, +/// over at least 300 ms, sampled at the crate's own tick; and it is plausibly +/// counting its board's energy: over the window it ran no faster than 1.5 +/// times the board's maximum power limit, the board's own power draw (as +/// `nvidia-smi` reports it just before and just after) was itself within that +/// bound, and the counter ran no faster than twice that draw plus 5 W (the +/// draw's documented accuracy). A counter that runs several times faster than +/// its board draws is not counting that board's energy. +#[allow(dead_code)] +mod nvml_oracle { + use std::ffi::{c_char, c_int, c_uint, c_void, CString}; + use std::time::{Duration, Instant}; + + unsafe extern "C" { + fn dlopen(filename: *const c_char, flag: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; + } + + /// `RTLD_NOW` in ``. + const RTLD_NOW: c_int = 2; + + type Device = *mut c_void; + type EnergyFn = unsafe extern "C" fn(Device, *mut u64) -> c_int; + type LimitsFn = unsafe extern "C" fn(Device, *mut c_uint, *mut c_uint) -> c_int; + type MilliwattsFn = unsafe extern "C" fn(Device, *mut c_uint) -> c_int; + + /// NVML, opened by the oracle. + pub struct Nvml { + energy: EnergyFn, + limits: Option, + power: Option, + /// Every GPU whose total-energy counter reads, in index order. + pub devices: Vec, + } + + fn sym(lib: *mut c_void, name: &str) -> Option<*mut c_void> { + let name = CString::new(name).ok()?; + let p = unsafe { dlsym(lib, name.as_ptr()) }; + (!p.is_null()).then_some(p) + } + + /// `libnvidia-ml.so.1`, initialised, with every GPU whose counter reads; + /// `None` where there is no library, no GPU, or no GPU with the counter. + pub fn open() -> Option { + let file = CString::new("libnvidia-ml.so.1").ok()?; + let lib = unsafe { dlopen(file.as_ptr(), RTLD_NOW) }; + if lib.is_null() { + return None; + } + unsafe { + let init: unsafe extern "C" fn() -> c_int = std::mem::transmute(sym(lib, "nvmlInit_v2")?); + let count: unsafe extern "C" fn(*mut c_uint) -> c_int = std::mem::transmute(sym(lib, "nvmlDeviceGetCount_v2")?); + let by_index: unsafe extern "C" fn(c_uint, *mut Device) -> c_int = + std::mem::transmute(sym(lib, "nvmlDeviceGetHandleByIndex_v2")?); + let energy: EnergyFn = std::mem::transmute(sym(lib, "nvmlDeviceGetTotalEnergyConsumption")?); + let limits = sym(lib, "nvmlDeviceGetPowerManagementLimitConstraints").map(|p| std::mem::transmute::<_, LimitsFn>(p)); + let power = sym(lib, "nvmlDeviceGetPowerUsage").map(|p| std::mem::transmute::<_, MilliwattsFn>(p)); + if init() != 0 { + return None; + } + let mut n: c_uint = 0; + if count(&mut n) != 0 { + return None; + } + let mut devices = Vec::new(); + for i in 0..n { + let mut d: Device = std::ptr::null_mut(); + let mut mj = 0u64; + if by_index(i, &mut d) == 0 && !d.is_null() && energy(d, &mut mj) == 0 { + devices.push(d); + } + } + (!devices.is_empty()).then_some(Nvml { energy, limits, power, devices }) + } + } + + impl Nvml { + /// Every GPU's total-energy counter, in mJ; `None` if one fails. + pub fn energy_mj(&self) -> Option> { + self.devices + .iter() + .map(|&d| { + let mut mj = 0u64; + (unsafe { (self.energy)(d, &mut mj) } == 0).then_some(mj) + }) + .collect() + } + + /// GPU `i`'s maximum power-management limit, in mW, as NVML gives it. + pub fn max_limit_mw(&self, i: usize) -> Option { + let (mut lo, mut hi) = (0, 0); + (unsafe { (self.limits?)(self.devices[i], &mut lo, &mut hi) } == 0 && hi > 0).then_some(hi) + } + + /// GPU `i`'s power draw, in mW, as NVML gives it. + pub fn power_mw(&self, i: usize) -> Option { + let mut mw = 0; + (unsafe { (self.power?)(self.devices[i], &mut mw) } == 0).then_some(mw) + } + } + + /// `nvidia-smi`'s view of every GPU: (maximum power limit, enforced limit, + /// draw), in watts; `None` without `nvidia-smi`. + pub fn smi() -> Option, Option, Option)>> { + let out = std::process::Command::new("nvidia-smi") + .args(["--query-gpu=power.max_limit,enforced.power.limit,power.draw", "--format=csv,noheader,nounits"]) + .output() + .ok()?; + out.status.success().then(|| { + String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| { + let f: Vec> = l.split(',').map(|v| v.trim().parse().ok()).collect(); + (f.first().copied().flatten(), f.get(1).copied().flatten(), f.get(2).copied().flatten()) + }) + .collect() + }) + } + + /// GPU `i`'s ceiling in watts: `nvidia-smi`'s maximum power limit, else + /// NVML's. + pub fn ceiling_w(nvml: &Nvml, smi: &Option, Option, Option)>>, i: usize) -> Option { + smi.as_ref() + .and_then(|s| s.get(i)) + .and_then(|(max, _, _)| *max) + .or_else(|| nvml.max_limit_mw(i).map(|mw| f64::from(mw) / 1000.0)) + } + + /// How often GPU 0's counter is republished: the median gap between its + /// changes, polled every 2 ms for up to 1.5 s (`None` if it changed + /// fewer than 6 times). + pub fn update_period(nvml: &Nvml) -> Option { + let started = Instant::now(); + let mut last = nvml.energy_mj()?[0]; + let mut changes = Vec::new(); + while started.elapsed() < Duration::from_millis(1500) && changes.len() < 16 { + std::thread::sleep(Duration::from_millis(2)); + let now = nvml.energy_mj()?[0]; + if now != last { + changes.push(started.elapsed()); + last = now; + } + } + let mut gaps: Vec = changes.windows(2).map(|w| w[1] - w[0]).collect(); + if gaps.len() < 5 { + return None; + } + gaps.sort(); + Some(gaps[gaps.len() / 2]) + } + + /// The oracle's verdict on the counters. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No NVML, no GPU, or no GPU whose total-energy counter reads. + Unavailable, + /// A counter did not keep advancing. + Frozen, + /// Every counter kept advancing, but one is not plausibly its board's + /// energy (see the module doc). + Implausible, + /// Every counter kept advancing, plausibly its board's energy. + Live, + } + + /// The family's plausibility rule for one GPU over a window: its counter's + /// average `rate_w`, the board's draws `draws_w` around the window and + /// its maximum power limit `ceiling_w`. A bound whose input is missing + /// is not applied. + pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { + if let Some(c) = ceiling_w { + if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { + return false; + } + } + if !draws_w.is_empty() { + let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if rate_w > 2.0 * mean + 5.0 { + return false; + } + } + true + } + + /// What the oracle saw. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + /// Each GPU's average rate over the window, in watts. + pub rate_w: Vec, + /// Each GPU's draw just before and just after the window, in watts. + pub draws_w: Vec>, + /// Each GPU's ceiling, in watts. + pub ceiling_w: Vec>, + } + + impl Watch { + /// The verdict, each GPU's advances, its counter's rate, the board's + /// draw and its maximum limit, for the test's log line. + pub fn summary(&self) -> String { + format!( + "{:?} (advanced {:?} of {} intervals over {} ms; counter rate {:?} W, board draw {:?} W, max power limit {:?} W)", + self.verdict, + self.advances, + self.intervals, + self.window.as_millis(), + self.rate_w.iter().map(|w| (w * 10.0).round() / 10.0).collect::>(), + self.draws_w, + self.ceiling_w + ) + } + } + + /// The oracle's run over the first `gpus` GPUs (all, if more), sleeping + /// `tick` between samples, by the sustained-advance rule (stopping when + /// live, or after a second with at least five intervals, or five + /// seconds); then each counter's plausibility over the whole window. + pub fn watch(tick: Duration, gpus: usize) -> Watch { + let none = Watch { + verdict: Verdict::Unavailable, + intervals: 0, + advances: Vec::new(), + window: Duration::ZERO, + rate_w: Vec::new(), + draws_w: Vec::new(), + ceiling_w: Vec::new(), + }; + let Some(nvml) = open() else { return none }; + let n = nvml.devices.len().min(gpus); + let before = smi(); + let read = || nvml.energy_mj().map(|v| v[..n].to_vec()); + let started = Instant::now(); + let Some(first) = read() else { return none }; + let (mut prev, mut last_at) = (first.clone(), Duration::ZERO); + let mut advances = vec![0u32; n]; + let mut intervals = 0u32; + let live = |adv: &[u32], k: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(k) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, last_at) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + std::thread::sleep(tick); + if let Some(now) = read() { + last_at = started.elapsed(); + intervals += 1; + for ((a, x), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(x > p); + } + prev = now; + } + } + let after = smi(); + let window = last_at; + let rate_w: Vec = + first.iter().zip(&prev).map(|(a, b)| b.saturating_sub(*a) as f64 / 1000.0 / window.as_secs_f64().max(1e-9)).collect(); + let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); + let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); + let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + let verdict = if !live(&advances, intervals, window) { + Verdict::Frozen + } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + Verdict::Implausible + } else { + Verdict::Live + }; + Watch { verdict, intervals, advances, window, rate_w, draws_w, ceiling_w } + } + + /// One line on the GPUs for the log: count, counters, ceilings, draw, and + /// the counter's measured republication `period`. + pub fn listing(period: Option) -> String { + let Some(nvml) = open() else { return "no NVML device with a total-energy counter".to_string() }; + format!( + "{} GPU(s) with a total-energy counter, counters {:?} mJ; NVML max power limit {:?} mW, power draw {:?} mW; \ + nvidia-smi (max limit, enforced limit, draw) W {:?}; counter republished every {:?}", + nvml.devices.len(), + nvml.energy_mj(), + (0..nvml.devices.len()).map(|i| nvml.max_limit_mw(i)).collect::>(), + (0..nvml.devices.len()).map(|i| nvml.power_mw(i)).collect::>(), + smi(), + period + ) + } + + /// GPU 0's counter's republication period, where there is one. + pub fn period() -> Option { + open().and_then(|n| update_period(&n)) + } +} + +use nvml_oracle::Verdict; + +/// The status agrees with the oracle, judged at the probe's own tick, and the +/// tick is no faster than the counter is republished. +#[test] +fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { + let _one = serial(); + let period = nvml_oracle::period(); + let status = nvml_status(); + let watch = nvml_oracle::watch(TICK, 1); + let (meter, why) = Meter::detect_explained(); + eprintln!( + "nvml hardware: {}; oracle at a {TICK:?} tick {}; status={status:?} selected={:?} detection={why:?}", + nvml_oracle::listing(period), + watch.summary(), + meter.rung() + ); + if let Some(p) = period { + assert!(TICK >= p * 9 / 10, "the {TICK:?} tick is faster than the counter's {p:?} republication"); + } + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Implausible => assert_eq!(status, CounterStatus::Implausible), + Verdict::Live => {} + } + if meter.rung() == Rung::Nvml { + assert_eq!(status, CounterStatus::Live, "the NVML rung was selected on a counter that is not live"); + } +} + +/// Over a one-second span, the NVML rung's figure is `HwShunt` and +/// `Coverage::Partial`, never a whole-task figure: it advanced, sits inside +/// the oracle's own reads of the counter around the span, and ran no faster +/// than the board can draw. The rung is built here as `Meter::detect` builds +/// it — only on a live status — because on a host whose RAPL is live the +/// ladder stops above it. Where NVML is not live there is no such figure. +#[test] +fn a_live_gpu_counter_is_a_partial_hwshunt_figure() { + let _one = serial(); + if nvml_status() != CounterStatus::Live { + eprintln!("nvml span: status {:?}; no NVML figure", nvml_status()); + assert_ne!(Meter::detect().rung(), Rung::Nvml, "the NVML rung was selected on a counter that is not live"); + return; + } + let meter = Meter::Nvml(Box::new(Nvml::open().expect("a live counter opens"))); + let oracle = nvml_oracle::open().expect("the meter reads NVML, yet the oracle cannot open it"); + let o0 = oracle.energy_mj().expect("oracle read")[0]; + let started = std::time::Instant::now(); + let ((), e) = meter.measure(|| std::thread::sleep(Duration::from_secs(1))); + let wall = started.elapsed(); + let o1 = oracle.energy_mj().expect("oracle read")[0]; + let ceiling = nvml_oracle::ceiling_w(&oracle, &nvml_oracle::smi(), 0).unwrap_or(f64::INFINITY); + eprintln!("nvml span: {wall:?}; energy {e:?}; oracle {o0} -> {o1} mJ; ceiling {ceiling} W"); + assert_eq!((e.provenance, e.coverage), (Provenance::HwShunt, Coverage::Partial), "a live NVML span: {e:?}"); + let uj = e.measured_uj.expect("an HwShunt figure carries its measured figure"); + assert!(uj > 0 && uj <= (o1 - o0) * 1_000, "{uj} µJ outside the oracle's (0, {}] µJ", (o1 - o0) * 1_000); + let watts = uj as f64 / 1e6 / wall.as_secs_f64(); + assert!(watts <= 1.5 * ceiling, "the NVML figure reads {watts:.1} W, past the board's {ceiling} W ceiling"); +} diff --git a/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs b/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs new file mode 100644 index 000000000..7f2d8baf4 --- /dev/null +++ b/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs @@ -0,0 +1,437 @@ +//! Linux RAPL against the hardware: this host's powercap counters, read by +//! the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the RAPL probe behind `rapl_status()` +//! runs in this process, and the tests take one lock: each burns CPU and +//! reads machine-wide counters. The oracle reads `/sys/class/powercap` +//! itself — every zone's `name`, `energy_uj` and `max_energy_range_uj` — and +//! applies the sustained-advance rule to the top-level `package-` zones +//! on its own. None of these tests passes without asserting: where the +//! counters cannot be read (no powercap; a CI container, whose +//! `/sys/devices/virtual/powercap` Docker masks), the status must say so and +//! agree with the oracle; where they are live, both must say live, the RAPL +//! rung must be selected, and a span over a CPU burn must be `HwShunt`, +//! `Whole` (the package counters read the CPU work they meter), its figure +//! inside the oracle's package-only bracket and clearly below package+psys +//! and package+core. + +#![cfg(target_os = "linux")] + +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use jcp_energy::{rapl_status, CounterStatus, Meter, Rung}; +use jcp_schema::{Coverage, Provenance}; + +/// One test at a time: each burns CPU and reads machine-wide counters. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The tests' own reading of `/sys/class/powercap`: every zone listed, its +/// `name`, `energy_uj` and `max_energy_range_uj`, read here and nowhere else, +/// and the sustained-advance verdict over the package zones by the family's +/// rule. Nothing in it comes from the crate under test. +#[allow(dead_code)] +mod oracle { + use std::os::raw::{c_int, c_long}; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// Where the kernel lists its powercap zones. + pub const POWERCAP: &str = "/sys/class/powercap"; + + #[repr(C)] + struct Timespec { + tv_sec: c_long, + tv_nsec: c_long, + } + + unsafe extern "C" { + fn clock_gettime(clock_id: c_int, tp: *mut Timespec) -> c_int; + } + + /// `CLOCK_THREAD_CPUTIME_ID` in Linux ``. + const CLOCK_THREAD_CPUTIME_ID: c_int = 3; + + /// This thread's CPU time. + pub fn thread_cpu() -> Duration { + let mut ts = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mut ts) }, 0, "clock_gettime"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// Spin this thread until it has used `d` of its own CPU time, or `wall_cap` + /// of wall time has passed. Returns the CPU it got. + pub fn burn(d: Duration, wall_cap: Duration) -> Duration { + let (started, wall) = (thread_cpu(), Instant::now()); + let mut acc: u64 = 0; + while thread_cpu() - started < d && wall.elapsed() < wall_cap { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + thread_cpu() - started + } + + /// One entry of the powercap directory. + #[derive(Debug, Clone)] + pub struct Zone { + /// The directory entry: `intel-rapl:0`, `intel-rapl:0:0`, `intel-rapl-mmio:0`, ... + pub dir: String, + pub name: Option, + pub energy_uj: Option, + pub max_uj: Option, + } + + fn text(p: PathBuf) -> Option { + std::fs::read_to_string(p).ok().map(|s| s.trim().to_string()) + } + + fn number(p: PathBuf) -> Option { + text(p)?.parse().ok() + } + + /// Every entry under `root`, in name order. + pub fn zones(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { return Vec::new() }; + let mut zones: Vec = entries + .flatten() + .map(|e| { + let dir = e.file_name().to_string_lossy().into_owned(); + let p = e.path(); + Zone { name: text(p.join("name")), energy_uj: number(p.join("energy_uj")), max_uj: number(p.join("max_energy_range_uj")), dir } + }) + .collect(); + zones.sort_by(|a, b| a.dir.cmp(&b.dir)); + zones + } + + fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) + } + + /// A top-level RAPL zone: `intel-rapl:`. Not a sub-zone, not MMIO. + pub fn top_level(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(digits) + } + + /// A package zone: top-level, named `package-`. + pub fn is_package(z: &Zone) -> bool { + top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + } + + /// The platform zone. + pub fn is_psys(z: &Zone) -> bool { + top_level(z) && z.name.as_deref() == Some("psys") + } + + /// A package's `core` sub-zone (`intel-rapl::` named `core`). + pub fn is_core(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(|r| r.contains(':')) && z.name.as_deref() == Some("core") + } + + /// One zone's energy between two reads, modulo its own range. + pub fn delta(e0: u64, e1: u64, max: Option) -> Option { + if e1 >= e0 { + Some(e1 - e0) + } else { + max.filter(|&m| m > 0)?.checked_sub(e0)?.checked_add(e1) + } + } + + /// `energy_uj` of each of `dirs` under `root`, in order; `None` if any fails. + pub fn read(root: &Path, dirs: &[&Zone]) -> Option> { + dirs.iter().map(|z| number(root.join(&z.dir).join("energy_uj"))).collect() + } + + /// The summed energy of `which` of `set` between two reads of `set`. + pub fn energy(set: &[&Zone], before: &[u64], after: &[u64], which: impl Fn(&Zone) -> bool) -> u64 { + set.iter() + .zip(before.iter().zip(after)) + .filter(|(z, _)| which(z)) + .map(|(z, (&b, &a))| delta(b, a, z.max_uj).expect("a zone fell with no readable range")) + .sum() + } + + /// What the oracle concluded about the package counters under a root. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No top-level zone's `energy_uj` reads (no powercap, a masked or + /// root-only tree). + Unreadable, + /// Top-level zones read, but none is a package. + NoPackage, + /// Every package read, but not every one kept advancing. + Frozen, + /// Every package advanced in at least five intervals, at least 80% of + /// them, over at least 300 ms. + Live, + } + + /// The oracle's sustained-advance run over the package zones under `root`. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub packages: Vec, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + } + + impl Watch { + /// The verdict and each package's advances, for the test's log line. + pub fn summary(&self) -> String { + let per: Vec = + self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); + format!("{:?} ({} over {} ms)", self.verdict, per.join(", "), self.window.as_millis()) + } + } + + /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU + /// between samples (capped at 500 ms of wall time), until every package + /// has advanced in at least five intervals, at least 80% of those + /// observed, over at least 300 ms; or one second has passed with at least + /// five intervals; or five seconds have. + pub fn watch(root: &Path) -> Watch { + let all = zones(root); + let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); + let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; + if packages.is_empty() { + let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); + return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + } + let started = Instant::now(); + let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; packages.len()]; + let mut intervals = 0u32; + let live = |adv: &[u32], n: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, el) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + burn(Duration::from_millis(25), Duration::from_millis(500)); + if let Some(now) = read(root, &packages) { + intervals += 1; + for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(n > p); + } + prev = now; + } + } + let window = started.elapsed(); + let verdict = if live(&advances, intervals, window) { Verdict::Live } else { Verdict::Frozen }; + Watch { verdict, packages: names, intervals, advances, window } + } + + /// The zones under `root`, as one line for the log. + pub fn listing(root: &Path) -> String { + zones(root) + .iter() + .map(|z| { + let kind = if is_package(z) { + "package" + } else if is_psys(z) { + "psys" + } else if is_core(z) { + "core sub-zone" + } else if top_level(z) { + "top-level" + } else { + "not summed" + }; + let e = z.energy_uj.map_or("unreadable".to_string(), |e| e.to_string()); + let m = z.max_uj.map_or("-".to_string(), |m| m.to_string()); + format!("{}={} [{kind}] energy_uj={e} max={m}", z.dir, z.name.as_deref().unwrap_or("?")) + }) + .collect::>() + .join("; ") + } + + /// The CI container's view of the host's powercap directory: the same + /// entries, each a symlink into `/sys/devices/virtual/powercap`, which + /// Docker masks with an empty tmpfs, so every target is missing. Built + /// from this host's own listing; removed when dropped. + pub struct MaskedView(pub PathBuf); + + impl MaskedView { + /// Build the masked view of this host's powercap entries in a + /// temporary directory named by `tag` and this process. + pub fn of_host(tag: &str) -> MaskedView { + let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create the masked view"); + let masked = root.join("masked-devices-virtual-powercap"); + for z in zones(Path::new(POWERCAP)) { + std::os::unix::fs::symlink(masked.join(&z.dir), root.join(&z.dir)).expect("symlink"); + } + MaskedView(root) + } + } + + impl Drop for MaskedView { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A bracket of reads around the meter's two reads: every package, psys + /// and core zone read at four points, outer and inner on each side. + pub struct Bracket<'a> { + pub set: Vec<&'a Zone>, + pub reads: [Vec; 4], + } + + impl Bracket<'_> { + /// The packages' energy over the outer bracket, in µJ: the most a + /// package-only figure read inside it can be. + pub fn package_outer(&self) -> u64 { + energy(&self.set, &self.reads[0], &self.reads[3], is_package) + } + /// The packages' energy over the inner bracket, in µJ: the least a + /// package-only figure read around it can be. + pub fn package_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_package) + } + /// The psys zone's energy over the inner bracket, in µJ. + pub fn psys_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_psys) + } + /// The core sub-zones' energy over the inner bracket, in µJ. + pub fn core_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_core) + } + /// Whether the bracket read a zone of the kind `which` picks. + pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { + self.set.iter().any(|z| which(z)) + } + } + + /// The family's rule on a figure `uj` the meter read between the + /// bracket's inner and outer reads: no less than the packages' energy + /// over the inner bracket, no more than over the outer one, and below + /// package+psys and package+core where those zones exist. A figure that + /// added a zone reads at least the packages' and that zone's energy over + /// the inner bracket, so `uj < inner + zone` is the double-count test; it + /// can only be put where the zone moved more over the burn than the + /// packages did in the gap between the outer and inner reads (else a + /// correct figure could reach it too), and the line says when it could + /// not. Returns the figures, for the log. + pub fn assert_package_only(b: &Bracket, uj: u64) -> String { + let (inner, outer, psys, core) = (b.package_inner(), b.package_outer(), b.psys_inner(), b.core_inner()); + let gap = outer - inner; + let mut line = format!( + "meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ (gap {gap}); psys inner {psys} µJ; \ + core inner {core} µJ" + ); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + for (zone, present, moved) in [("psys", b.has(is_psys), psys), ("core", b.has(is_core), core)] { + if !present { + continue; + } + assert!(moved > 0, "{zone} did not advance: {line}"); + if moved > gap { + assert!(uj < inner + moved, "the figure reaches package+{zone}: {zone} was added: {line}"); + line.push_str(&format!("; below package+{zone}")); + } else { + line.push_str(&format!("; {zone} moved less than the gap, so only the outer bound tests it")); + } + } + line + } + + /// Every package, psys and core zone of `all` whose counter reads. + pub fn bracket_set(all: &[Zone]) -> Vec<&Zone> { + all.iter().filter(|z| (is_package(z) || is_psys(z) || is_core(z)) && z.energy_uj.is_some()).collect() + } + + /// One oracle read of every zone of `set`; panics if one fails. + pub fn read_all(set: &[&Zone]) -> Vec { + read(Path::new(POWERCAP), set).expect("an oracle read of the bracket zones") + } + + /// Read the bracket around `meter_read` (called twice) and `work`: + /// outer, meter, inner, work, inner, meter, outer. + pub fn around<'a, M>(set: Vec<&'a Zone>, mut meter_read: impl FnMut() -> M, work: impl FnOnce()) -> (Bracket<'a>, M, M) { + let o0 = read_all(&set); + let m0 = meter_read(); + let o1 = read_all(&set); + work(); + let o2 = read_all(&set); + let m1 = meter_read(); + let o3 = read_all(&set); + (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) + } +} + +use oracle::Verdict; + +/// The status agrees with the oracle's own verdict, and the RAPL rung is +/// selected exactly when it is live. +#[test] +fn the_rapl_status_agrees_with_the_oracle_and_gates_the_rung() { + let _one = serial(); + let status = rapl_status(); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + let (meter, why) = Meter::detect_explained(); + eprintln!( + "rapl hardware: {}; oracle {}; status={status:?} selected={:?} provenance={:?} detection={why:?}", + oracle::listing(Path::new(oracle::POWERCAP)), + watch.summary(), + meter.rung(), + meter.provenance() + ); + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unreadable | Verdict::NoPackage => assert_eq!(status, CounterStatus::Unavailable), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Live => {} + } + assert_eq!(why.rapl, Some(status), "the detection did not record RAPL's status"); + assert_eq!(meter.rung() == Rung::Rapl, status == CounterStatus::Live, "selection disagrees with the status"); + if meter.rung() == Rung::Rapl { + assert_eq!(meter.provenance(), Provenance::HwShunt); + } +} + +/// Over a 300 ms one-thread CPU burn, the RAPL rung's span is a whole +/// hardware reading, `HwShunt` and `Whole`, whose figure sits inside the +/// oracle's package-only bracket and clearly below package+psys and +/// package+core. Where RAPL is not live, the meter is another rung and no +/// span of it is a RAPL reading. +#[test] +fn a_cpu_burn_is_a_whole_hwshunt_reading_inside_the_package_only_bracket() { + let _one = serial(); + let meter = Meter::detect(); + if meter.rung() != Rung::Rapl { + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + eprintln!("rapl burn: rung {:?}; oracle {}", meter.rung(), watch.summary()); + assert_ne!(watch.verdict, Verdict::Live, "the oracle saw live packages, yet the RAPL rung was not selected"); + assert_ne!(rapl_status(), CounterStatus::Live); + return; + } + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let set = oracle::bracket_set(&all); + let wall = Instant::now(); + let o0 = oracle::read_all(&set); + let ((o1, cpu, o2), e) = meter.measure(|| { + let o1 = oracle::read_all(&set); + let cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)); + (o1, cpu, oracle::read_all(&set)) + }); + let o3 = oracle::read_all(&set); + let b = oracle::Bracket { set, reads: [o0, o1, o2, o3] }; + assert_eq!((e.provenance, e.coverage), (Provenance::HwShunt, Coverage::Whole), "a live RAPL span: {e:?}"); + let uj = e.measured_uj.expect("an HwShunt figure carries its measured figure"); + assert_eq!(e.estimated_uj, uj); + eprintln!("rapl burn: {cpu:?} of thread CPU in {:?}; energy {e:?}; {}", wall.elapsed(), oracle::assert_package_only(&b, uj)); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); +} diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs index 6c0746b64..d1e82f633 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs @@ -59,6 +59,15 @@ pub enum CounterLiveness { /// intervals, four in every five of those observed, over at least 300 ms, /// while the probing thread burned CPU. Live, + /// Every channel kept advancing, but a GPU counter (NVML's) ran faster + /// than its board can draw power over the probe: past 1.5 times the + /// board's maximum power limit, past twice the board's own power draw plus + /// 5 W, or beside a draw reading itself past that limit + /// ([`plausible_gpu_counter`]). It is not counting the board's energy, so + /// its figure would not be a reading of the GPU's work. Seen on an RTX 4050 + /// Laptop GPU (driver 595.84), whose counter runs at 4 to 30 times the + /// board's draw while it is polled. + Implausible, } /// In at least how many separate sample intervals a channel must advance to @@ -272,6 +281,106 @@ pub(crate) fn burn_thread_cpu(cpu: Duration, wall_cap: Duration) { std::hint::black_box(acc); } +/// How far past its board's maximum power limit a GPU energy counter may run, +/// on average over a probe's window, as a fraction (3/2): a board can draw +/// past its limit for milliseconds, not on average over a third of a second. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` +pub(crate) const GPU_CEILING_MARGIN: (u128, u128) = (3, 2); + +/// How much faster than its board's own power draw a GPU energy counter may +/// run over a probe's window: twice the draw's mean over the samples, plus +/// [`GPU_DRAW_SLACK_MW`]. The draw is a one-second average, so a load that +/// changed during the window can leave it behind the counter, but not by a +/// factor of two at a probe. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` +pub(crate) const GPU_DRAW_FACTOR: u128 = 2; + +/// NVML's documented accuracy of a board's power draw, ±5 W, in mW. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` +pub(crate) const GPU_DRAW_SLACK_MW: u128 = 5_000; + +/// Whether one GPU's energy counter, which counted `energy_mj` over `window`, +/// plausibly counted its board's energy, beside the board's power draw +/// `draws_mw` read with the samples and its maximum power limit `ceiling_mw`. +/// Not when the counter ran past [`GPU_CEILING_MARGIN`] of the limit, or a +/// draw reading did (the board's own telemetry then is not a reading either: +/// an RTX 4050 Laptop GPU reports 590 W for about a second after it wakes), +/// or the counter ran past [`GPU_DRAW_FACTOR`] times the mean draw plus +/// [`GPU_DRAW_SLACK_MW`]. A bound whose reading is missing is not applied. +/// +/// Liveness says the counter is republished continuously; this says what it +/// republishes is the board's energy. On the RTX 4050 Laptop GPU (driver +/// 595.84) the total-energy counter runs several times faster than the board +/// draws for as long as it is polled: at the NVML probe's 100 ms tick it ran +/// at 58-305 W while the board drew 5-14 W against a 65 W limit. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` +pub(crate) fn plausible_gpu_counter(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { + let ns = window.as_nanos().max(1); + // mJ per second is mW. + let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; + if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { + let cap = u128::from(limit) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1; + if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { + return false; + } + } + if !draws_mw.is_empty() { + let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; + if rate_mw > GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW { + return false; + } + } + true +} + +/// What a GPU counter's probe saw beside the rule's verdict: each GPU's +/// counter at the first and last sample (µJ) and when, and each GPU's power +/// draw read with every sample. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` +#[derive(Debug, Default, Clone)] +pub(crate) struct GpuWatch { + first: Option<(Vec, Instant)>, + last: Option<(Vec, Instant)>, + draws_mw: Vec>, +} + +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` +impl GpuWatch { + /// Fold in one sample: every GPU's counter (µJ), if the sample read, and + /// each GPU's draw (`None` where it did not read), at `at`. + pub(crate) fn saw(&mut self, uj: Option<&[u64]>, draws_mw: &[Option], at: Instant) { + if let Some(uj) = uj { + self.first.get_or_insert_with(|| (uj.to_vec(), at)); + self.last = Some((uj.to_vec(), at)); + } + if self.draws_mw.len() < draws_mw.len() { + self.draws_mw.resize(draws_mw.len(), Vec::new()); + } + for (gpu, d) in self.draws_mw.iter_mut().zip(draws_mw) { + gpu.extend(*d); + } + } + + /// The liveness a GPU counter's probe gives it: the rule's `verdict`, + /// except that a live counter one of whose GPUs did not plausibly count + /// its board's energy over the probe, beside each GPU's maximum power + /// limit `ceilings_mw`, is [`CounterLiveness::Implausible`]. + pub(crate) fn liveness(&self, verdict: CounterLiveness, ceilings_mw: &[Option]) -> CounterLiveness { + let (Some((first, t0)), Some((last, t1))) = (&self.first, &self.last) else { + return verdict; + }; + let window = t1.saturating_duration_since(*t0); + let plausible = first.iter().zip(last).enumerate().all(|(i, (a, b))| { + let draws = self.draws_mw.get(i).map(Vec::as_slice).unwrap_or(&[]); + plausible_gpu_counter(b.saturating_sub(*a) / 1_000, window, draws, ceilings_mw.get(i).copied().flatten()) + }); + match verdict { + CounterLiveness::Live if !plausible => CounterLiveness::Implausible, + other => other, + } + } +} + #[cfg(test)] pub(crate) mod tests { use super::*; @@ -285,6 +394,58 @@ pub(crate) mod tests { const TEST_MIN_ADVANCES: u32 = 5; const TEST_MIN_WINDOW: Duration = Duration::from_millis(300); + /// The plausibility rule on one GPU counter, with the readings of the RTX + /// 4050 Laptop GPU on jetson-hub (driver 595.84, 65 W maximum limit): + /// polled at the NVML probe's 100 ms tick its counter ran at 58-305 W while + /// the board drew 5-14 W, which is no count of the board's energy. A + /// counter that tracks its board passes, a one-second draw lagging a load + /// step by up to twice still passes, and a bound whose reading is missing + /// is not applied. + #[test] + fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { + let w = Duration::from_millis(500); + assert!(!plausible_gpu_counter(152_250, w, &[5_550, 7_660], Some(65_000)), "304.5 W against a 6.6 W draw"); + assert!(!plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000)), "44 W against a 10.4 W draw"); + assert!(!plausible_gpu_counter(60_000, w, &[], Some(65_000)), "120 W on a 65 W board"); + assert!(!plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000)), "a 590 W draw reading"); + assert!(plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000)), "13 W against a 13 W draw"); + assert!(plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000)), "300 W against a 280 W draw"); + assert!(plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000)), "205 W is 2 x 100 W + 5 W"); + assert!(!plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000))); + assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); + assert!(!plausible_gpu_counter(48_751, w, &[], Some(65_000))); + assert!(plausible_gpu_counter(1_000_000, w, &[], None), "nothing to judge by"); + } + + /// The liveness a GPU probe gives: each GPU's rise from the first sample + /// to the last (µJ), over the window between them, beside that GPU's own + /// draws and limit. One implausible GPU of two makes the counter + /// `Implausible`; a verdict that was not `Live` stands, whatever the draws. + #[test] + fn every_gpu_must_count_no_faster_than_its_board_draws() { + let t0 = Instant::now(); + let watch = |draws: [Option; 2]| { + let mut g = GpuWatch::default(); + for k in 0..6u64 { + // GPU 0 at 10 W, GPU 1 at 300 W: 1 J and 30 J a 100 ms interval. + let uj = [8_000_000_000 + 1_000_000 * k, 9_000_000_000 + 30_000_000 * k]; + g.saw(Some(&uj), &draws, t0 + Duration::from_millis(100) * k as u32); + } + g + }; + let ceilings = [Some(65_000), Some(700_000)]; + use CounterLiveness::*; + assert_eq!(watch([Some(10_000), Some(290_000)]).liveness(Live, &ceilings), Live, "each at its own draw"); + assert_eq!(watch([Some(1_000), Some(290_000)]).liveness(Live, &ceilings), Implausible, "GPU 0 at 10 x its draw"); + assert_eq!(watch([Some(10_000), Some(290_000)]).liveness(Live, &[Some(65_000); 2]), Implausible, "300 W on 65 W"); + assert_eq!(watch([Some(290_000), Some(10_000)]).liveness(Live, &[None, None]), Implausible, "draws swapped"); + assert_eq!(watch([None, None]).liveness(Live, &[None, None]), Live, "nothing to judge by"); + for verdict in [Frozen, Unavailable] { + assert_eq!(watch([Some(1_000), Some(1_000)]).liveness(verdict, &ceilings), verdict); + } + assert_eq!(GpuWatch::default().liveness(Live, &ceilings), Live, "no sample"); + } + /// Feed a fresh rule `samples`, the i-th at `i * every`; return it and /// whether it was live after each interval. fn feed(channels: usize, samples: &[Vec], every: Duration) -> (AdvanceWatch, Vec) { diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs index ba979e54b..778950b1f 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs @@ -8,10 +8,15 @@ //! claims (JouleClaw SPEC §1.3): every GPU's counter must keep advancing under //! the sustained-advance rule ([`crate::CounterLiveness`]) — in at least five //! sample intervals, four in every five, over at least 300 ms, while the -//! probing thread burns 100 ms of its own CPU time between samples (twice the -//! counter's ~50 ms sample period, so a live counter has moved in every -//! interval; a powered GPU draws tens of watts at idle, ~1 J per interval, -//! against a 1 mJ quantum). Probed once per process and cached. +//! probing thread burns 100 ms of its own CPU time between samples (an RTX +//! 4050 Laptop GPU, driver 595.84, republishes the counter every ~100 ms, so a +//! live counter has moved in nearly every interval; even at a 2 W idle that is +//! ~0.2 J an interval, against a 1 mJ quantum). And every GPU's counter must +//! plausibly count its board's energy over the probe: no faster than 1.5 +//! times the board's maximum power limit, or than twice the board's own power +//! draw plus 5 W ([`crate::CounterLiveness::Implausible`]). That RTX 4050's +//! counter runs at 4 to 30 times its board's draw while it is polled, so it +//! is declined there. Probed once per process and cached. //! //! Where that counter is unsupported (pre-Volta, some consumer parts) or not //! live, the counter integrates `power_usage()` over wall time instead. That @@ -58,7 +63,9 @@ enum Mode { fn mode_for(supported: bool, liveness: CounterLiveness) -> Mode { match (supported, liveness) { (true, CounterLiveness::Live) => Mode::Counter, - (_, CounterLiveness::Live | CounterLiveness::Frozen | CounterLiveness::Unavailable) => Mode::PowerIntegral, + (_, CounterLiveness::Live | CounterLiveness::Frozen | CounterLiveness::Unavailable | CounterLiveness::Implausible) => { + Mode::PowerIntegral + } } } @@ -115,12 +122,7 @@ impl NvmlCounter { let supported = total_energy_uj(&nvml, device_count).is_some(); let liveness = if supported { static LIVENESS: OnceLock = OnceLock::new(); - *LIVENESS.get_or_init(|| { - crate::liveness::probe( - || total_energy_uj(&nvml, device_count).map(|first| (first.len(), || total_energy_uj(&nvml, device_count))), - PROBE_TICK, - ) - }) + *LIVENESS.get_or_init(|| probe_liveness(&nvml, device_count)) } else { CounterLiveness::Unavailable }; @@ -169,6 +171,38 @@ impl NvmlCounter { /// Every GPU's cumulative energy, in µJ, in device order; `None` if any of them /// cannot be read. +/// The real liveness probe of every GPU's cumulative-energy counter: the +/// sustained-advance rule at [`PROBE_TICK`], with every GPU's power draw read +/// at each sample and its maximum power limit once, so that a live counter +/// which ran faster than its board can draw is +/// [`CounterLiveness::Implausible`] ([`crate::liveness::GpuWatch`]). +fn probe_liveness(nvml: &nvml_wrapper::Nvml, device_count: u32) -> CounterLiveness { + let ceilings: Vec> = (0..device_count) + .map(|i| { + let limits = nvml.device_by_index(i).ok()?.power_management_limit_constraints().ok()?; + (limits.max_limit > 0).then_some(limits.max_limit) + }) + .collect(); + let draws = move || -> Vec> { + (0..device_count).map(|i| nvml.device_by_index(i).ok()?.power_usage().ok()).collect() + }; + let watch = std::cell::RefCell::new(crate::liveness::GpuWatch::default()); + let seen = &watch; + let verdict = crate::liveness::probe( + move || { + total_energy_uj(nvml, device_count).map(|first| { + (first.len(), move || { + let sample = total_energy_uj(nvml, device_count); + seen.borrow_mut().saw(sample.as_deref(), &draws(), std::time::Instant::now()); + sample + }) + }) + }, + PROBE_TICK, + ); + watch.into_inner().liveness(verdict, &ceilings) +} + fn total_energy_uj(nvml: &nvml_wrapper::Nvml, device_count: u32) -> Option> { (0..device_count) .map(|i| { @@ -321,6 +355,7 @@ mod tests { (true, Unavailable, Mode::PowerIntegral), (false, Unavailable, Mode::PowerIntegral), (false, Live, Mode::PowerIntegral), + (true, Implausible, Mode::PowerIntegral), ] { assert_eq!(mode_for(supported, liveness), mode, "supported={supported} {liveness:?}"); let c = NvmlCounter::with(None, 1, mode, liveness); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs index 8440b8a77..d36d3be11 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs @@ -205,7 +205,7 @@ impl ProcessCpuCounter { fn admits(liveness: CounterLiveness) -> bool { match liveness { CounterLiveness::Live => true, - CounterLiveness::Unavailable | CounterLiveness::Frozen => false, + CounterLiveness::Unavailable | CounterLiveness::Frozen | CounterLiveness::Implausible => false, } } @@ -277,7 +277,7 @@ mod tests { /// measurement, and its work still runs. #[test] fn a_counter_that_is_not_live_meters_nothing() { - for liveness in [CounterLiveness::Frozen, CounterLiveness::Unavailable] { + for liveness in [CounterLiveness::Frozen, CounterLiveness::Implausible, CounterLiveness::Unavailable] { let c = ProcessCpuCounter::given(liveness); assert!(!c.is_available() && !EnergyMeter::is_available(&c), "{liveness:?}"); assert!(matches!(c.read(), Err(EnergyError::NoCounter(EnergyDomain::ProcessCpu))), "{liveness:?}"); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs index 5bc06cad5..accf2cea8 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs @@ -270,7 +270,7 @@ impl RaplCounter { fn admits(liveness: CounterLiveness) -> bool { match liveness { CounterLiveness::Live => true, - CounterLiveness::Unavailable | CounterLiveness::Frozen => false, + CounterLiveness::Unavailable | CounterLiveness::Frozen | CounterLiveness::Implausible => false, } } @@ -444,6 +444,7 @@ mod tests { for (liveness, available) in [ (CounterLiveness::Live, true), (CounterLiveness::Frozen, false), + (CounterLiveness::Implausible, false), (CounterLiveness::Unavailable, false), ] { let c = RaplCounter::given(Vec::new(), liveness); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs new file mode 100644 index 000000000..bca4a6ff2 --- /dev/null +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs @@ -0,0 +1,414 @@ +//! NVML against the hardware (feature `nvml`): the GPUs' total-energy +//! counters, read by the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the NVML probe runs in this process, and +//! the tests take one lock. The oracle opens `libnvidia-ml` itself, measures +//! how often the counter is republished, samples it at the probe's own tick +//! (100 ms) by the sustained-advance rule and holds each counter to its +//! board's power ceiling, which it asks of `nvidia-smi`. None of these tests +//! passes without asserting: where there is no NVML (a CI container has no +//! GPU), the counter must say so and agree with the oracle; where the +//! counter is live, both must say so, and its readings must be `HwShunt` over +//! the `Gpu` domain — the GPU, never a whole-host or CPU figure — no faster +//! than the board can draw; where it is not live, its readings are the +//! power-draw integral, `ModelBased`, never `HwShunt`. + +#![cfg(all(target_os = "linux", feature = "nvml"))] + +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use jouleclaw_energy::nvml::NvmlCounter; +use jouleclaw_energy::{CounterLiveness, EnergyCounter, EnergyDomain, Provenance}; + +/// One test at a time: each reads machine-wide counters, and the oracle's +/// own sampling must not overlap the meter's. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The NVML probe's tick (`PROBE_TICK` in `nvml.rs`): 100 ms of the probing +/// thread's CPU between samples, so at least 100 ms of wall time. +const TICK: Duration = Duration::from_millis(100); + +/// The tests' own reading of NVIDIA's total-energy counter: `libnvidia-ml` +/// opened here with `dlopen` and called through symbols looked up here, not +/// through the crate under test or its bindings, and each GPU's power limit +/// and draw asked of `nvidia-smi`, a separate process. It measures how often +/// the counter is republished, and judges it by the family's rule: every +/// GPU's counter advanced in at least five intervals, at least 80% of them, +/// over at least 300 ms, sampled at the crate's own tick; and it is plausibly +/// counting its board's energy: over the window it ran no faster than 1.5 +/// times the board's maximum power limit, the board's own power draw (as +/// `nvidia-smi` reports it just before and just after) was itself within that +/// bound, and the counter ran no faster than twice that draw plus 5 W (the +/// draw's documented accuracy). A counter that runs several times faster than +/// its board draws is not counting that board's energy. +#[allow(dead_code)] +mod nvml_oracle { + use std::ffi::{c_char, c_int, c_uint, c_void, CString}; + use std::time::{Duration, Instant}; + + unsafe extern "C" { + fn dlopen(filename: *const c_char, flag: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; + } + + /// `RTLD_NOW` in ``. + const RTLD_NOW: c_int = 2; + + type Device = *mut c_void; + type EnergyFn = unsafe extern "C" fn(Device, *mut u64) -> c_int; + type LimitsFn = unsafe extern "C" fn(Device, *mut c_uint, *mut c_uint) -> c_int; + type MilliwattsFn = unsafe extern "C" fn(Device, *mut c_uint) -> c_int; + + /// NVML, opened by the oracle. + pub struct Nvml { + energy: EnergyFn, + limits: Option, + power: Option, + /// Every GPU whose total-energy counter reads, in index order. + pub devices: Vec, + } + + fn sym(lib: *mut c_void, name: &str) -> Option<*mut c_void> { + let name = CString::new(name).ok()?; + let p = unsafe { dlsym(lib, name.as_ptr()) }; + (!p.is_null()).then_some(p) + } + + /// `libnvidia-ml.so.1`, initialised, with every GPU whose counter reads; + /// `None` where there is no library, no GPU, or no GPU with the counter. + pub fn open() -> Option { + let file = CString::new("libnvidia-ml.so.1").ok()?; + let lib = unsafe { dlopen(file.as_ptr(), RTLD_NOW) }; + if lib.is_null() { + return None; + } + unsafe { + let init: unsafe extern "C" fn() -> c_int = std::mem::transmute(sym(lib, "nvmlInit_v2")?); + let count: unsafe extern "C" fn(*mut c_uint) -> c_int = std::mem::transmute(sym(lib, "nvmlDeviceGetCount_v2")?); + let by_index: unsafe extern "C" fn(c_uint, *mut Device) -> c_int = + std::mem::transmute(sym(lib, "nvmlDeviceGetHandleByIndex_v2")?); + let energy: EnergyFn = std::mem::transmute(sym(lib, "nvmlDeviceGetTotalEnergyConsumption")?); + let limits = sym(lib, "nvmlDeviceGetPowerManagementLimitConstraints").map(|p| std::mem::transmute::<_, LimitsFn>(p)); + let power = sym(lib, "nvmlDeviceGetPowerUsage").map(|p| std::mem::transmute::<_, MilliwattsFn>(p)); + if init() != 0 { + return None; + } + let mut n: c_uint = 0; + if count(&mut n) != 0 { + return None; + } + let mut devices = Vec::new(); + for i in 0..n { + let mut d: Device = std::ptr::null_mut(); + let mut mj = 0u64; + if by_index(i, &mut d) == 0 && !d.is_null() && energy(d, &mut mj) == 0 { + devices.push(d); + } + } + (!devices.is_empty()).then_some(Nvml { energy, limits, power, devices }) + } + } + + impl Nvml { + /// Every GPU's total-energy counter, in mJ; `None` if one fails. + pub fn energy_mj(&self) -> Option> { + self.devices + .iter() + .map(|&d| { + let mut mj = 0u64; + (unsafe { (self.energy)(d, &mut mj) } == 0).then_some(mj) + }) + .collect() + } + + /// GPU `i`'s maximum power-management limit, in mW, as NVML gives it. + pub fn max_limit_mw(&self, i: usize) -> Option { + let (mut lo, mut hi) = (0, 0); + (unsafe { (self.limits?)(self.devices[i], &mut lo, &mut hi) } == 0 && hi > 0).then_some(hi) + } + + /// GPU `i`'s power draw, in mW, as NVML gives it. + pub fn power_mw(&self, i: usize) -> Option { + let mut mw = 0; + (unsafe { (self.power?)(self.devices[i], &mut mw) } == 0).then_some(mw) + } + } + + /// `nvidia-smi`'s view of every GPU: (maximum power limit, enforced limit, + /// draw), in watts; `None` without `nvidia-smi`. + pub fn smi() -> Option, Option, Option)>> { + let out = std::process::Command::new("nvidia-smi") + .args(["--query-gpu=power.max_limit,enforced.power.limit,power.draw", "--format=csv,noheader,nounits"]) + .output() + .ok()?; + out.status.success().then(|| { + String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| { + let f: Vec> = l.split(',').map(|v| v.trim().parse().ok()).collect(); + (f.first().copied().flatten(), f.get(1).copied().flatten(), f.get(2).copied().flatten()) + }) + .collect() + }) + } + + /// GPU `i`'s ceiling in watts: `nvidia-smi`'s maximum power limit, else + /// NVML's. + pub fn ceiling_w(nvml: &Nvml, smi: &Option, Option, Option)>>, i: usize) -> Option { + smi.as_ref() + .and_then(|s| s.get(i)) + .and_then(|(max, _, _)| *max) + .or_else(|| nvml.max_limit_mw(i).map(|mw| f64::from(mw) / 1000.0)) + } + + /// How often GPU 0's counter is republished: the median gap between its + /// changes, polled every 2 ms for up to 1.5 s (`None` if it changed + /// fewer than 6 times). + pub fn update_period(nvml: &Nvml) -> Option { + let started = Instant::now(); + let mut last = nvml.energy_mj()?[0]; + let mut changes = Vec::new(); + while started.elapsed() < Duration::from_millis(1500) && changes.len() < 16 { + std::thread::sleep(Duration::from_millis(2)); + let now = nvml.energy_mj()?[0]; + if now != last { + changes.push(started.elapsed()); + last = now; + } + } + let mut gaps: Vec = changes.windows(2).map(|w| w[1] - w[0]).collect(); + if gaps.len() < 5 { + return None; + } + gaps.sort(); + Some(gaps[gaps.len() / 2]) + } + + /// The oracle's verdict on the counters. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No NVML, no GPU, or no GPU whose total-energy counter reads. + Unavailable, + /// A counter did not keep advancing. + Frozen, + /// Every counter kept advancing, but one is not plausibly its board's + /// energy (see the module doc). + Implausible, + /// Every counter kept advancing, plausibly its board's energy. + Live, + } + + /// The family's plausibility rule for one GPU over a window: its counter's + /// average `rate_w`, the board's draws `draws_w` around the window and + /// its maximum power limit `ceiling_w`. A bound whose input is missing + /// is not applied. + pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { + if let Some(c) = ceiling_w { + if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { + return false; + } + } + if !draws_w.is_empty() { + let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if rate_w > 2.0 * mean + 5.0 { + return false; + } + } + true + } + + /// What the oracle saw. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + /// Each GPU's average rate over the window, in watts. + pub rate_w: Vec, + /// Each GPU's draw just before and just after the window, in watts. + pub draws_w: Vec>, + /// Each GPU's ceiling, in watts. + pub ceiling_w: Vec>, + } + + impl Watch { + /// The verdict, each GPU's advances, its counter's rate, the board's + /// draw and its maximum limit, for the test's log line. + pub fn summary(&self) -> String { + format!( + "{:?} (advanced {:?} of {} intervals over {} ms; counter rate {:?} W, board draw {:?} W, max power limit {:?} W)", + self.verdict, + self.advances, + self.intervals, + self.window.as_millis(), + self.rate_w.iter().map(|w| (w * 10.0).round() / 10.0).collect::>(), + self.draws_w, + self.ceiling_w + ) + } + } + + /// The oracle's run over the first `gpus` GPUs (all, if more), sleeping + /// `tick` between samples, by the sustained-advance rule (stopping when + /// live, or after a second with at least five intervals, or five + /// seconds); then each counter's plausibility over the whole window. + pub fn watch(tick: Duration, gpus: usize) -> Watch { + let none = Watch { + verdict: Verdict::Unavailable, + intervals: 0, + advances: Vec::new(), + window: Duration::ZERO, + rate_w: Vec::new(), + draws_w: Vec::new(), + ceiling_w: Vec::new(), + }; + let Some(nvml) = open() else { return none }; + let n = nvml.devices.len().min(gpus); + let before = smi(); + let read = || nvml.energy_mj().map(|v| v[..n].to_vec()); + let started = Instant::now(); + let Some(first) = read() else { return none }; + let (mut prev, mut last_at) = (first.clone(), Duration::ZERO); + let mut advances = vec![0u32; n]; + let mut intervals = 0u32; + let live = |adv: &[u32], k: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(k) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, last_at) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + std::thread::sleep(tick); + if let Some(now) = read() { + last_at = started.elapsed(); + intervals += 1; + for ((a, x), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(x > p); + } + prev = now; + } + } + let after = smi(); + let window = last_at; + let rate_w: Vec = + first.iter().zip(&prev).map(|(a, b)| b.saturating_sub(*a) as f64 / 1000.0 / window.as_secs_f64().max(1e-9)).collect(); + let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); + let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); + let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + let verdict = if !live(&advances, intervals, window) { + Verdict::Frozen + } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + Verdict::Implausible + } else { + Verdict::Live + }; + Watch { verdict, intervals, advances, window, rate_w, draws_w, ceiling_w } + } + + /// One line on the GPUs for the log: count, counters, ceilings, draw, and + /// the counter's measured republication `period`. + pub fn listing(period: Option) -> String { + let Some(nvml) = open() else { return "no NVML device with a total-energy counter".to_string() }; + format!( + "{} GPU(s) with a total-energy counter, counters {:?} mJ; NVML max power limit {:?} mW, power draw {:?} mW; \ + nvidia-smi (max limit, enforced limit, draw) W {:?}; counter republished every {:?}", + nvml.devices.len(), + nvml.energy_mj(), + (0..nvml.devices.len()).map(|i| nvml.max_limit_mw(i)).collect::>(), + (0..nvml.devices.len()).map(|i| nvml.power_mw(i)).collect::>(), + smi(), + period + ) + } + + /// GPU 0's counter's republication period, where there is one. + pub fn period() -> Option { + open().and_then(|n| update_period(&n)) + } +} + +use nvml_oracle::Verdict; + +/// The liveness agrees with the oracle, judged at the probe's own tick, and +/// the tick is no faster than the counter is republished; the grade follows +/// the liveness. +#[test] +fn the_nvml_liveness_agrees_with_the_oracle_at_its_own_tick() { + let _one = serial(); + let period = nvml_oracle::period(); + let counter = NvmlCounter::new(); + let watch = nvml_oracle::watch(TICK, usize::MAX); + let liveness = counter.liveness(); + eprintln!( + "nvml hardware: {}; oracle at a {TICK:?} tick {}; liveness={liveness:?} available={} true_counter={} grade={:?}", + nvml_oracle::listing(period), + watch.summary(), + counter.is_available(), + counter.uses_true_counter(), + counter.provenance() + ); + if let Some(p) = period { + assert!(TICK >= p * 9 / 10, "the {TICK:?} tick is faster than the counter's {p:?} republication"); + } + assert_eq!(liveness == CounterLiveness::Live, watch.verdict == Verdict::Live, "the liveness disagrees with the oracle"); + match watch.verdict { + Verdict::Unavailable => assert_eq!(liveness, CounterLiveness::Unavailable), + Verdict::Frozen => assert_eq!(liveness, CounterLiveness::Frozen), + Verdict::Implausible => assert_eq!(liveness, CounterLiveness::Implausible), + Verdict::Live => {} + } + assert_eq!(counter.uses_true_counter(), liveness == CounterLiveness::Live, "the counter mode disagrees"); + assert_eq!(counter.domain(), EnergyDomain::Gpu); + if counter.is_available() { + let want = if liveness == CounterLiveness::Live { Provenance::HwShunt } else { Provenance::ModelBased }; + assert_eq!(counter.provenance(), want, "the grade does not follow the liveness"); + } +} + +/// Over a one-second span the counter's readings are graded by its liveness +/// over the `Gpu` domain: a live counter's are `HwShunt`, advanced, inside +/// the oracle's own reads around the span, and no faster than the boards can +/// draw; any other's are `ModelBased` (the power-draw integral), never +/// `HwShunt`. +#[test] +fn a_gpu_counter_reads_the_gpu_alone_graded_by_its_liveness() { + let _one = serial(); + let counter = NvmlCounter::new(); + if !counter.is_available() { + eprintln!("nvml span: no NVML ({:?})", counter.liveness()); + assert!(counter.read().is_err()); + assert!(nvml_oracle::open().is_none(), "the oracle opens NVML, yet the counter is unavailable"); + return; + } + let oracle = nvml_oracle::open(); + let o0 = oracle.as_ref().and_then(|o| o.energy_mj()); + let r0 = counter.read().expect("an available counter reads"); + std::thread::sleep(Duration::from_secs(1)); + let r1 = counter.read().expect("an available counter reads"); + let o1 = oracle.as_ref().and_then(|o| o.energy_mj()); + eprintln!("nvml span: liveness {:?}; readings {r0:?} -> {r1:?}; oracle {o0:?} -> {o1:?} mJ", counter.liveness()); + for r in [&r0, &r1] { + assert_eq!(r.domain, EnergyDomain::Gpu, "{r:?}"); + } + if counter.liveness() != CounterLiveness::Live { + assert!([&r0, &r1].iter().all(|r| r.provenance == Provenance::ModelBased), "a counter that is not live read HwShunt"); + return; + } + assert!([&r0, &r1].iter().all(|r| r.provenance == Provenance::HwShunt)); + let oracle = oracle.expect("a live counter, yet the oracle cannot open NVML"); + let (o0, o1) = (o0.expect("oracle read"), o1.expect("oracle read")); + let bracket: u64 = o0.iter().zip(&o1).map(|(a, b)| b - a).sum::() * 1_000; + let uj = r1.uj - r0.uj; + assert!(uj > 0 && uj <= bracket, "{uj} µJ outside the oracle's (0, {bracket}] µJ"); + let smi = nvml_oracle::smi(); + let ceiling: f64 = (0..oracle.devices.len()).map(|i| nvml_oracle::ceiling_w(&oracle, &smi, i).unwrap_or(f64::INFINITY)).sum(); + let watts = uj as f64 / 1e6 / ((r1.timestamp_ns - r0.timestamp_ns) as f64 / 1e9); + assert!(watts <= 1.5 * ceiling, "the GPU readings run at {watts:.1} W, past the boards' {ceiling} W ceiling"); +} diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs new file mode 100644 index 000000000..b063a5dda --- /dev/null +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs @@ -0,0 +1,449 @@ +//! Linux RAPL against the hardware (feature `rapl`): this host's powercap +//! counters, read by the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the RAPL probe runs in this process, and +//! the tests take one lock: each burns CPU and reads machine-wide counters. +//! The oracle reads `/sys/class/powercap` itself — every zone's `name`, +//! `energy_uj` and `max_energy_range_uj` — and applies the sustained-advance +//! rule to the top-level `package-` zones on its own. None of these tests +//! passes without asserting: where the counters cannot be read (no powercap; +//! a CI container, whose `/sys/devices/virtual/powercap` Docker masks), the +//! liveness must say so and agree with the oracle; where they are live, both +//! must say live, the counter must be the meter's active one, and its +//! readings must be `HwShunt` over `CpuPkg` (the CPU package, not the GPU), +//! the figure over a CPU burn inside the oracle's package-only bracket and +//! clearly below package+psys and package+core. + +#![cfg(all(target_os = "linux", feature = "rapl"))] + +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use jouleclaw_energy::rapl::{rapl_liveness, RaplCounter}; +use jouleclaw_energy::{meter, CounterLiveness, EnergyCounter, EnergyDomain, Provenance}; + +/// One test at a time: each burns CPU and reads machine-wide counters. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The tests' own reading of `/sys/class/powercap`: every zone listed, its +/// `name`, `energy_uj` and `max_energy_range_uj`, read here and nowhere else, +/// and the sustained-advance verdict over the package zones by the family's +/// rule. Nothing in it comes from the crate under test. +#[allow(dead_code)] +mod oracle { + use std::os::raw::{c_int, c_long}; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// Where the kernel lists its powercap zones. + pub const POWERCAP: &str = "/sys/class/powercap"; + + #[repr(C)] + struct Timespec { + tv_sec: c_long, + tv_nsec: c_long, + } + + unsafe extern "C" { + fn clock_gettime(clock_id: c_int, tp: *mut Timespec) -> c_int; + } + + /// `CLOCK_THREAD_CPUTIME_ID` in Linux ``. + const CLOCK_THREAD_CPUTIME_ID: c_int = 3; + + /// This thread's CPU time. + pub fn thread_cpu() -> Duration { + let mut ts = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mut ts) }, 0, "clock_gettime"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// Spin this thread until it has used `d` of its own CPU time, or `wall_cap` + /// of wall time has passed. Returns the CPU it got. + pub fn burn(d: Duration, wall_cap: Duration) -> Duration { + let (started, wall) = (thread_cpu(), Instant::now()); + let mut acc: u64 = 0; + while thread_cpu() - started < d && wall.elapsed() < wall_cap { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + thread_cpu() - started + } + + /// One entry of the powercap directory. + #[derive(Debug, Clone)] + pub struct Zone { + /// The directory entry: `intel-rapl:0`, `intel-rapl:0:0`, `intel-rapl-mmio:0`, ... + pub dir: String, + pub name: Option, + pub energy_uj: Option, + pub max_uj: Option, + } + + fn text(p: PathBuf) -> Option { + std::fs::read_to_string(p).ok().map(|s| s.trim().to_string()) + } + + fn number(p: PathBuf) -> Option { + text(p)?.parse().ok() + } + + /// Every entry under `root`, in name order. + pub fn zones(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { return Vec::new() }; + let mut zones: Vec = entries + .flatten() + .map(|e| { + let dir = e.file_name().to_string_lossy().into_owned(); + let p = e.path(); + Zone { name: text(p.join("name")), energy_uj: number(p.join("energy_uj")), max_uj: number(p.join("max_energy_range_uj")), dir } + }) + .collect(); + zones.sort_by(|a, b| a.dir.cmp(&b.dir)); + zones + } + + fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) + } + + /// A top-level RAPL zone: `intel-rapl:`. Not a sub-zone, not MMIO. + pub fn top_level(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(digits) + } + + /// A package zone: top-level, named `package-`. + pub fn is_package(z: &Zone) -> bool { + top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + } + + /// The platform zone. + pub fn is_psys(z: &Zone) -> bool { + top_level(z) && z.name.as_deref() == Some("psys") + } + + /// A package's `core` sub-zone (`intel-rapl::` named `core`). + pub fn is_core(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(|r| r.contains(':')) && z.name.as_deref() == Some("core") + } + + /// One zone's energy between two reads, modulo its own range. + pub fn delta(e0: u64, e1: u64, max: Option) -> Option { + if e1 >= e0 { + Some(e1 - e0) + } else { + max.filter(|&m| m > 0)?.checked_sub(e0)?.checked_add(e1) + } + } + + /// `energy_uj` of each of `dirs` under `root`, in order; `None` if any fails. + pub fn read(root: &Path, dirs: &[&Zone]) -> Option> { + dirs.iter().map(|z| number(root.join(&z.dir).join("energy_uj"))).collect() + } + + /// The summed energy of `which` of `set` between two reads of `set`. + pub fn energy(set: &[&Zone], before: &[u64], after: &[u64], which: impl Fn(&Zone) -> bool) -> u64 { + set.iter() + .zip(before.iter().zip(after)) + .filter(|(z, _)| which(z)) + .map(|(z, (&b, &a))| delta(b, a, z.max_uj).expect("a zone fell with no readable range")) + .sum() + } + + /// What the oracle concluded about the package counters under a root. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No top-level zone's `energy_uj` reads (no powercap, a masked or + /// root-only tree). + Unreadable, + /// Top-level zones read, but none is a package. + NoPackage, + /// Every package read, but not every one kept advancing. + Frozen, + /// Every package advanced in at least five intervals, at least 80% of + /// them, over at least 300 ms. + Live, + } + + /// The oracle's sustained-advance run over the package zones under `root`. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub packages: Vec, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + } + + impl Watch { + /// The verdict and each package's advances, for the test's log line. + pub fn summary(&self) -> String { + let per: Vec = + self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); + format!("{:?} ({} over {} ms)", self.verdict, per.join(", "), self.window.as_millis()) + } + } + + /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU + /// between samples (capped at 500 ms of wall time), until every package + /// has advanced in at least five intervals, at least 80% of those + /// observed, over at least 300 ms; or one second has passed with at least + /// five intervals; or five seconds have. + pub fn watch(root: &Path) -> Watch { + let all = zones(root); + let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); + let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; + if packages.is_empty() { + let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); + return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + } + let started = Instant::now(); + let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; packages.len()]; + let mut intervals = 0u32; + let live = |adv: &[u32], n: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, el) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + burn(Duration::from_millis(25), Duration::from_millis(500)); + if let Some(now) = read(root, &packages) { + intervals += 1; + for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(n > p); + } + prev = now; + } + } + let window = started.elapsed(); + let verdict = if live(&advances, intervals, window) { Verdict::Live } else { Verdict::Frozen }; + Watch { verdict, packages: names, intervals, advances, window } + } + + /// The zones under `root`, as one line for the log. + pub fn listing(root: &Path) -> String { + zones(root) + .iter() + .map(|z| { + let kind = if is_package(z) { + "package" + } else if is_psys(z) { + "psys" + } else if is_core(z) { + "core sub-zone" + } else if top_level(z) { + "top-level" + } else { + "not summed" + }; + let e = z.energy_uj.map_or("unreadable".to_string(), |e| e.to_string()); + let m = z.max_uj.map_or("-".to_string(), |m| m.to_string()); + format!("{}={} [{kind}] energy_uj={e} max={m}", z.dir, z.name.as_deref().unwrap_or("?")) + }) + .collect::>() + .join("; ") + } + + /// The CI container's view of the host's powercap directory: the same + /// entries, each a symlink into `/sys/devices/virtual/powercap`, which + /// Docker masks with an empty tmpfs, so every target is missing. Built + /// from this host's own listing; removed when dropped. + pub struct MaskedView(pub PathBuf); + + impl MaskedView { + /// Build the masked view of this host's powercap entries in a + /// temporary directory named by `tag` and this process. + pub fn of_host(tag: &str) -> MaskedView { + let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create the masked view"); + let masked = root.join("masked-devices-virtual-powercap"); + for z in zones(Path::new(POWERCAP)) { + std::os::unix::fs::symlink(masked.join(&z.dir), root.join(&z.dir)).expect("symlink"); + } + MaskedView(root) + } + } + + impl Drop for MaskedView { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A bracket of reads around the meter's two reads: every package, psys + /// and core zone read at four points, outer and inner on each side. + pub struct Bracket<'a> { + pub set: Vec<&'a Zone>, + pub reads: [Vec; 4], + } + + impl Bracket<'_> { + /// The packages' energy over the outer bracket, in µJ: the most a + /// package-only figure read inside it can be. + pub fn package_outer(&self) -> u64 { + energy(&self.set, &self.reads[0], &self.reads[3], is_package) + } + /// The packages' energy over the inner bracket, in µJ: the least a + /// package-only figure read around it can be. + pub fn package_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_package) + } + /// The psys zone's energy over the inner bracket, in µJ. + pub fn psys_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_psys) + } + /// The core sub-zones' energy over the inner bracket, in µJ. + pub fn core_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_core) + } + /// Whether the bracket read a zone of the kind `which` picks. + pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { + self.set.iter().any(|z| which(z)) + } + } + + /// The family's rule on a figure `uj` the meter read between the + /// bracket's inner and outer reads: no less than the packages' energy + /// over the inner bracket, no more than over the outer one, and below + /// package+psys and package+core where those zones exist. A figure that + /// added a zone reads at least the packages' and that zone's energy over + /// the inner bracket, so `uj < inner + zone` is the double-count test; it + /// can only be put where the zone moved more over the burn than the + /// packages did in the gap between the outer and inner reads (else a + /// correct figure could reach it too), and the line says when it could + /// not. Returns the figures, for the log. + pub fn assert_package_only(b: &Bracket, uj: u64) -> String { + let (inner, outer, psys, core) = (b.package_inner(), b.package_outer(), b.psys_inner(), b.core_inner()); + let gap = outer - inner; + let mut line = format!( + "meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ (gap {gap}); psys inner {psys} µJ; \ + core inner {core} µJ" + ); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + for (zone, present, moved) in [("psys", b.has(is_psys), psys), ("core", b.has(is_core), core)] { + if !present { + continue; + } + assert!(moved > 0, "{zone} did not advance: {line}"); + if moved > gap { + assert!(uj < inner + moved, "the figure reaches package+{zone}: {zone} was added: {line}"); + line.push_str(&format!("; below package+{zone}")); + } else { + line.push_str(&format!("; {zone} moved less than the gap, so only the outer bound tests it")); + } + } + line + } + + /// Every package, psys and core zone of `all` whose counter reads. + pub fn bracket_set(all: &[Zone]) -> Vec<&Zone> { + all.iter().filter(|z| (is_package(z) || is_psys(z) || is_core(z)) && z.energy_uj.is_some()).collect() + } + + /// One oracle read of every zone of `set`; panics if one fails. + pub fn read_all(set: &[&Zone]) -> Vec { + read(Path::new(POWERCAP), set).expect("an oracle read of the bracket zones") + } + + /// Read the bracket around `meter_read` (called twice) and `work`: + /// outer, meter, inner, work, inner, meter, outer. + pub fn around<'a, M>(set: Vec<&'a Zone>, mut meter_read: impl FnMut() -> M, work: impl FnOnce()) -> (Bracket<'a>, M, M) { + let o0 = read_all(&set); + let m0 = meter_read(); + let o1 = read_all(&set); + work(); + let o2 = read_all(&set); + let m1 = meter_read(); + let o3 = read_all(&set); + (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) + } +} + +use oracle::Verdict; + +/// The liveness agrees with the oracle's own verdict, and only a live +/// counter is available and is the meter's active one. +#[test] +fn the_rapl_liveness_agrees_with_the_oracle_and_gates_the_counter() { + let _one = serial(); + let liveness = rapl_liveness(); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + let counter = RaplCounter::new(); + let composite = meter::detect_meter(); + eprintln!( + "rapl hardware: {}; oracle {}; liveness={liveness:?} available={} active={} meters={:?}", + oracle::listing(Path::new(oracle::POWERCAP)), + watch.summary(), + counter.is_available(), + composite.active_name(), + composite.names() + ); + assert_eq!(liveness == CounterLiveness::Live, watch.verdict == Verdict::Live, "the liveness disagrees with the oracle"); + match watch.verdict { + Verdict::Unreadable | Verdict::NoPackage => assert_eq!(liveness, CounterLiveness::Unavailable), + Verdict::Frozen => assert_eq!(liveness, CounterLiveness::Frozen), + Verdict::Live => {} + } + assert_eq!(counter.liveness(), liveness, "the counter was built on another verdict"); + assert_eq!(counter.is_available(), liveness == CounterLiveness::Live, "availability disagrees with the liveness"); + assert_eq!(composite.active_name() == "rapl", liveness == CounterLiveness::Live, "the meter's choice disagrees"); + if !counter.is_available() { + assert!(counter.read().is_err(), "a counter that is not live read"); + } +} + +/// Over a 300 ms one-thread CPU burn, the counter's readings are `HwShunt` +/// over `CpuPkg`, and its figure sits inside the oracle's package-only +/// bracket and clearly below package+psys and package+core. +#[test] +fn a_cpu_burn_reads_inside_the_package_only_bracket() { + let _one = serial(); + let counter = RaplCounter::new(); + if !counter.is_available() { + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + eprintln!("rapl burn: counter {:?}; oracle {}", counter.liveness(), watch.summary()); + assert_ne!(watch.verdict, Verdict::Live, "the oracle saw live packages, yet the counter is not available"); + assert!(counter.read().is_err()); + return; + } + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let mut cpu = Duration::ZERO; + let (b, r0, r1) = oracle::around( + oracle::bracket_set(&all), + || counter.read().expect("a live counter reads"), + || cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)), + ); + for r in [&r0, &r1] { + assert_eq!((r.domain, r.provenance), (EnergyDomain::CpuPkg, Provenance::HwShunt), "{r:?}"); + } + eprintln!("rapl burn: {cpu:?} of thread CPU; {}", oracle::assert_package_only(&b, r1.uj - r0.uj)); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); +} + +/// What a CI container sees: Docker masks `/sys/devices/virtual/powercap` +/// with an empty tmpfs, so every entry of `/sys/class/powercap` is a symlink +/// to nothing. On that view of this host's own listing the counter is +/// unavailable and reads nothing, and the oracle agrees. +#[test] +fn the_ci_containers_masked_view_is_unavailable() { + let _one = serial(); + let view = oracle::MaskedView::of_host("jouleclaw-energy"); + let watch = oracle::watch(&view.0); + let counter = RaplCounter::with_root(&view.0); + eprintln!("rapl masked view: oracle {}; counter {:?} available={}", watch.summary(), counter.liveness(), counter.is_available()); + assert!(matches!(watch.verdict, Verdict::Unreadable | Verdict::NoPackage), "{watch:?}"); + assert_eq!((counter.liveness(), counter.is_available()), (CounterLiveness::Unavailable, false)); + assert!(counter.read().is_err()); +} diff --git a/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs b/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs new file mode 100644 index 000000000..dd2c06df0 --- /dev/null +++ b/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs @@ -0,0 +1,481 @@ +//! Linux RAPL against the hardware: this host's powercap counters, read by +//! the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the RAPL probe behind `rapl_status()` +//! runs in this process, and the tests take one lock: each burns CPU and +//! reads machine-wide counters. The oracle reads `/sys/class/powercap` +//! itself — every zone's `name`, `energy_uj` and `max_energy_range_uj` — and +//! applies the sustained-advance rule to the top-level `package-` zones +//! on its own. None of these tests passes without asserting: where the +//! counters cannot be read (no powercap; a CI container, whose +//! `/sys/devices/virtual/powercap` Docker masks), the status must say so and +//! agree with the oracle; where they are live, both must say live, RAPL must +//! be the meter's source, and the marginal figure of a burn on every spare +//! core must be `HwShunt`, claim the CPU package and not the GPU, and, summed +//! over the calls, fit inside the oracle's package-only bracket around the +//! whole measurement (idle and busy windows), which a figure that added +//! psys or a package's core sub-zone overruns. + +#![cfg(target_os = "linux")] + +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use proof_cascade::meter::{self, rapl_status, CounterStatus, MeterSource}; +use proof_schema::Provenance; + +/// One test at a time: each burns CPU and reads machine-wide counters. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The tests' own reading of `/sys/class/powercap`: every zone listed, its +/// `name`, `energy_uj` and `max_energy_range_uj`, read here and nowhere else, +/// and the sustained-advance verdict over the package zones by the family's +/// rule. Nothing in it comes from the crate under test. +#[allow(dead_code)] +mod oracle { + use std::os::raw::{c_int, c_long}; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// Where the kernel lists its powercap zones. + pub const POWERCAP: &str = "/sys/class/powercap"; + + #[repr(C)] + struct Timespec { + tv_sec: c_long, + tv_nsec: c_long, + } + + unsafe extern "C" { + fn clock_gettime(clock_id: c_int, tp: *mut Timespec) -> c_int; + } + + /// `CLOCK_THREAD_CPUTIME_ID` in Linux ``. + const CLOCK_THREAD_CPUTIME_ID: c_int = 3; + + /// This thread's CPU time. + pub fn thread_cpu() -> Duration { + let mut ts = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mut ts) }, 0, "clock_gettime"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// Spin this thread until it has used `d` of its own CPU time, or `wall_cap` + /// of wall time has passed. Returns the CPU it got. + pub fn burn(d: Duration, wall_cap: Duration) -> Duration { + let (started, wall) = (thread_cpu(), Instant::now()); + let mut acc: u64 = 0; + while thread_cpu() - started < d && wall.elapsed() < wall_cap { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + thread_cpu() - started + } + + /// One entry of the powercap directory. + #[derive(Debug, Clone)] + pub struct Zone { + /// The directory entry: `intel-rapl:0`, `intel-rapl:0:0`, `intel-rapl-mmio:0`, ... + pub dir: String, + pub name: Option, + pub energy_uj: Option, + pub max_uj: Option, + } + + fn text(p: PathBuf) -> Option { + std::fs::read_to_string(p).ok().map(|s| s.trim().to_string()) + } + + fn number(p: PathBuf) -> Option { + text(p)?.parse().ok() + } + + /// Every entry under `root`, in name order. + pub fn zones(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { return Vec::new() }; + let mut zones: Vec = entries + .flatten() + .map(|e| { + let dir = e.file_name().to_string_lossy().into_owned(); + let p = e.path(); + Zone { name: text(p.join("name")), energy_uj: number(p.join("energy_uj")), max_uj: number(p.join("max_energy_range_uj")), dir } + }) + .collect(); + zones.sort_by(|a, b| a.dir.cmp(&b.dir)); + zones + } + + fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) + } + + /// A top-level RAPL zone: `intel-rapl:`. Not a sub-zone, not MMIO. + pub fn top_level(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(digits) + } + + /// A package zone: top-level, named `package-`. + pub fn is_package(z: &Zone) -> bool { + top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + } + + /// The platform zone. + pub fn is_psys(z: &Zone) -> bool { + top_level(z) && z.name.as_deref() == Some("psys") + } + + /// A package's `core` sub-zone (`intel-rapl::` named `core`). + pub fn is_core(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(|r| r.contains(':')) && z.name.as_deref() == Some("core") + } + + /// One zone's energy between two reads, modulo its own range. + pub fn delta(e0: u64, e1: u64, max: Option) -> Option { + if e1 >= e0 { + Some(e1 - e0) + } else { + max.filter(|&m| m > 0)?.checked_sub(e0)?.checked_add(e1) + } + } + + /// `energy_uj` of each of `dirs` under `root`, in order; `None` if any fails. + pub fn read(root: &Path, dirs: &[&Zone]) -> Option> { + dirs.iter().map(|z| number(root.join(&z.dir).join("energy_uj"))).collect() + } + + /// The summed energy of `which` of `set` between two reads of `set`. + pub fn energy(set: &[&Zone], before: &[u64], after: &[u64], which: impl Fn(&Zone) -> bool) -> u64 { + set.iter() + .zip(before.iter().zip(after)) + .filter(|(z, _)| which(z)) + .map(|(z, (&b, &a))| delta(b, a, z.max_uj).expect("a zone fell with no readable range")) + .sum() + } + + /// What the oracle concluded about the package counters under a root. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No top-level zone's `energy_uj` reads (no powercap, a masked or + /// root-only tree). + Unreadable, + /// Top-level zones read, but none is a package. + NoPackage, + /// Every package read, but not every one kept advancing. + Frozen, + /// Every package advanced in at least five intervals, at least 80% of + /// them, over at least 300 ms. + Live, + } + + /// The oracle's sustained-advance run over the package zones under `root`. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub packages: Vec, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + } + + impl Watch { + /// The verdict and each package's advances, for the test's log line. + pub fn summary(&self) -> String { + let per: Vec = + self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); + format!("{:?} ({} over {} ms)", self.verdict, per.join(", "), self.window.as_millis()) + } + } + + /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU + /// between samples (capped at 500 ms of wall time), until every package + /// has advanced in at least five intervals, at least 80% of those + /// observed, over at least 300 ms; or one second has passed with at least + /// five intervals; or five seconds have. + pub fn watch(root: &Path) -> Watch { + let all = zones(root); + let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); + let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; + if packages.is_empty() { + let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); + return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + } + let started = Instant::now(); + let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; packages.len()]; + let mut intervals = 0u32; + let live = |adv: &[u32], n: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, el) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + burn(Duration::from_millis(25), Duration::from_millis(500)); + if let Some(now) = read(root, &packages) { + intervals += 1; + for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(n > p); + } + prev = now; + } + } + let window = started.elapsed(); + let verdict = if live(&advances, intervals, window) { Verdict::Live } else { Verdict::Frozen }; + Watch { verdict, packages: names, intervals, advances, window } + } + + /// The zones under `root`, as one line for the log. + pub fn listing(root: &Path) -> String { + zones(root) + .iter() + .map(|z| { + let kind = if is_package(z) { + "package" + } else if is_psys(z) { + "psys" + } else if is_core(z) { + "core sub-zone" + } else if top_level(z) { + "top-level" + } else { + "not summed" + }; + let e = z.energy_uj.map_or("unreadable".to_string(), |e| e.to_string()); + let m = z.max_uj.map_or("-".to_string(), |m| m.to_string()); + format!("{}={} [{kind}] energy_uj={e} max={m}", z.dir, z.name.as_deref().unwrap_or("?")) + }) + .collect::>() + .join("; ") + } + + /// The CI container's view of the host's powercap directory: the same + /// entries, each a symlink into `/sys/devices/virtual/powercap`, which + /// Docker masks with an empty tmpfs, so every target is missing. Built + /// from this host's own listing; removed when dropped. + pub struct MaskedView(pub PathBuf); + + impl MaskedView { + /// Build the masked view of this host's powercap entries in a + /// temporary directory named by `tag` and this process. + pub fn of_host(tag: &str) -> MaskedView { + let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create the masked view"); + let masked = root.join("masked-devices-virtual-powercap"); + for z in zones(Path::new(POWERCAP)) { + std::os::unix::fs::symlink(masked.join(&z.dir), root.join(&z.dir)).expect("symlink"); + } + MaskedView(root) + } + } + + impl Drop for MaskedView { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A bracket of reads around the meter's two reads: every package, psys + /// and core zone read at four points, outer and inner on each side. + pub struct Bracket<'a> { + pub set: Vec<&'a Zone>, + pub reads: [Vec; 4], + } + + impl Bracket<'_> { + /// The packages' energy over the outer bracket, in µJ: the most a + /// package-only figure read inside it can be. + pub fn package_outer(&self) -> u64 { + energy(&self.set, &self.reads[0], &self.reads[3], is_package) + } + /// The packages' energy over the inner bracket, in µJ: the least a + /// package-only figure read around it can be. + pub fn package_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_package) + } + /// The psys zone's energy over the inner bracket, in µJ. + pub fn psys_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_psys) + } + /// The core sub-zones' energy over the inner bracket, in µJ. + pub fn core_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_core) + } + /// Whether the bracket read a zone of the kind `which` picks. + pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { + self.set.iter().any(|z| which(z)) + } + } + + /// The family's rule on a figure `uj` the meter read between the + /// bracket's inner and outer reads: no less than the packages' energy + /// over the inner bracket, no more than over the outer one, and below + /// package+psys and package+core where those zones exist. A figure that + /// added a zone reads at least the packages' and that zone's energy over + /// the inner bracket, so `uj < inner + zone` is the double-count test; it + /// can only be put where the zone moved more over the burn than the + /// packages did in the gap between the outer and inner reads (else a + /// correct figure could reach it too), and the line says when it could + /// not. Returns the figures, for the log. + pub fn assert_package_only(b: &Bracket, uj: u64) -> String { + let (inner, outer, psys, core) = (b.package_inner(), b.package_outer(), b.psys_inner(), b.core_inner()); + let gap = outer - inner; + let mut line = format!( + "meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ (gap {gap}); psys inner {psys} µJ; \ + core inner {core} µJ" + ); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + for (zone, present, moved) in [("psys", b.has(is_psys), psys), ("core", b.has(is_core), core)] { + if !present { + continue; + } + assert!(moved > 0, "{zone} did not advance: {line}"); + if moved > gap { + assert!(uj < inner + moved, "the figure reaches package+{zone}: {zone} was added: {line}"); + line.push_str(&format!("; below package+{zone}")); + } else { + line.push_str(&format!("; {zone} moved less than the gap, so only the outer bound tests it")); + } + } + line + } + + /// Every package, psys and core zone of `all` whose counter reads. + pub fn bracket_set(all: &[Zone]) -> Vec<&Zone> { + all.iter().filter(|z| (is_package(z) || is_psys(z) || is_core(z)) && z.energy_uj.is_some()).collect() + } + + /// One oracle read of every zone of `set`; panics if one fails. + pub fn read_all(set: &[&Zone]) -> Vec { + read(Path::new(POWERCAP), set).expect("an oracle read of the bracket zones") + } + + /// Read the bracket around `meter_read` (called twice) and `work`: + /// outer, meter, inner, work, inner, meter, outer. + pub fn around<'a, M>(set: Vec<&'a Zone>, mut meter_read: impl FnMut() -> M, work: impl FnOnce()) -> (Bracket<'a>, M, M) { + let o0 = read_all(&set); + let m0 = meter_read(); + let o1 = read_all(&set); + work(); + let o2 = read_all(&set); + let m1 = meter_read(); + let o3 = read_all(&set); + (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) + } +} + +use oracle::Verdict; + +/// The status agrees with the oracle's own verdict, and RAPL is the meter's +/// source exactly when it is live. +#[test] +fn the_rapl_status_agrees_with_the_oracle_and_gates_the_source() { + let _one = serial(); + let status = rapl_status(); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + let source = meter::meter_source(); + eprintln!( + "rapl hardware: {}; oracle {}; status={status:?} source={source:?}", + oracle::listing(Path::new(oracle::POWERCAP)), + watch.summary() + ); + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unreadable => assert_eq!(status, CounterStatus::Unavailable), + Verdict::NoPackage => assert_eq!(status, CounterStatus::Absent), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Live => {} + } + assert_eq!(source == Some(MeterSource::Rapl), status == CounterStatus::Live, "selection disagrees with the status"); + assert_eq!(meter::hardware_meter_available(), status == CounterStatus::Live); +} + +/// Calls of [`ten_ms_of_work`] completed in this process. +static CALLS: AtomicU64 = AtomicU64::new(0); + +/// About 10 ms of one thread's CPU work, counted. +fn ten_ms_of_work() { + let t = Instant::now(); + let mut acc: u64 = 0; + while t.elapsed() < Duration::from_millis(10) { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + CALLS.fetch_add(1, Ordering::SeqCst); +} + +/// A burn on every spare core meters as a hardware figure of the CPU +/// package: `HwShunt`, from RAPL, claiming package energy and not the GPU. +/// The figure is a marginal per call (busy power over idle, times the busy +/// window, over the calls), so the calls' total can be no more than the +/// packages' energy over the whole measurement, which the oracle reads +/// around it; on a quiet host a figure that also summed a package's core +/// sub-zone (or psys, where psys tracks load) is about twice the busy margin +/// and overruns it. A figure of 0 is the crate's "inside the idle noise", and +/// is what a host whose other work holds the package at its power limit +/// reads. Where RAPL is not live there is no figure, and `work` never runs. +#[test] +fn an_all_core_burn_meters_inside_the_package_only_bracket() { + let _one = serial(); + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let set = oracle::bracket_set(&all); + if meter::meter_source() != Some(MeterSource::Rapl) { + let before = CALLS.load(Ordering::SeqCst); + let m = meter::measure_marginal(ten_ms_of_work); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + eprintln!("rapl burn: no RAPL source; figure {m:?}; oracle {}", watch.summary()); + assert_eq!(m, None, "a figure with RAPL declined"); + assert_eq!(CALLS.load(Ordering::SeqCst), before, "the work ran with RAPL declined"); + assert_ne!(watch.verdict, Verdict::Live, "the oracle saw live packages, yet RAPL is not the source"); + return; + } + let wall = Instant::now(); + let c0 = CALLS.load(Ordering::SeqCst); + let o0 = oracle::read_all(&set); + let m = meter::measure_marginal(ten_ms_of_work).expect("a live RAPL source metered the burn"); + let o3 = oracle::read_all(&set); + let calls = CALLS.load(Ordering::SeqCst) - c0; + let outer = oracle::energy(&set, &o0, &o3, oracle::is_package); + let core = oracle::energy(&set, &o0, &o3, oracle::is_core); + let psys = oracle::energy(&set, &o0, &o3, oracle::is_psys); + let total = m.uj * calls; + let energy = m.energy(); + eprintln!( + "rapl burn: {calls} calls in {:?}; {} µJ a call, {total} µJ over the calls; oracle package-only {outer} µJ \ + around the measurement (idle and busy), core {core} µJ, psys {psys} µJ; energy {energy:?}; domain {:?}", + wall.elapsed(), + m.uj, + m.source.domain() + ); + assert_eq!(m.source, MeterSource::Rapl); + assert_eq!(energy.provenance, Provenance::HwShunt); + assert_eq!(energy.measured_uj, Some(m.uj)); + let domain = m.source.domain(); + assert!(domain.contains("package") && !domain.contains("GPU"), "the RAPL figure claims {domain}"); + assert!(calls > 0, "the work never ran"); + assert!(outer > 0, "the packages did not advance"); + // Each call's figure is rounded to the microjoule: half a microjoule a call. + assert!(total <= outer + calls / 2 + 1, "{total} µJ over the calls overruns the packages' {outer} µJ"); + if m.uj == 0 { + // The crate's contract: from a bracket, 0 means the busy window drew no + // more than the idle one. On a runner whose other jobs hold the package + // at its power limit through both windows, that is what it reads. + eprintln!("rapl burn: the busy window drew no more than the idle one (load {:?}): inside the idle noise", loadavg()); + } +} + +/// The host's load averages, for the log. +fn loadavg() -> Option { + std::fs::read_to_string("/proc/loadavg").ok().map(|s| s.trim().to_string()) +} diff --git a/sandbox/sandbox-rs/crates/sandbox-exec-microvm/tests/rapl_hardware.rs b/sandbox/sandbox-rs/crates/sandbox-exec-microvm/tests/rapl_hardware.rs new file mode 100644 index 000000000..42bc66680 --- /dev/null +++ b/sandbox/sandbox-rs/crates/sandbox-exec-microvm/tests/rapl_hardware.rs @@ -0,0 +1,446 @@ +//! The S5 executor's CPU energy meters against the hardware: RAPL through +//! powercap (`RaplMeter`, sandbox-meter's package counters) and through the +//! MSRs (`MsrRaplMeter`), read beside an oracle the tests declare here. +//! +//! In a test binary of its own, so the probes run in this process, and the +//! tests take one lock: each burns CPU and reads machine-wide counters. The +//! oracle reads `/sys/class/powercap` itself and applies the sustained- +//! advance rule to the top-level `package-` zones on its own; for the +//! MSRs it tries to read `MSR_PKG_ENERGY_STATUS` from `/dev/cpu/0/msr` +//! itself. None of these tests passes without asserting: where a counter +//! cannot be read the meter must say so and the oracle agree; where it is +//! live, the meter's figure over a CPU burn must sit inside the oracle's +//! package-only bracket. + +#![cfg(target_os = "linux")] + +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use sandbox_exec_microvm::power::{MsrRaplMeter, PowerMeter, RaplMeter}; + +/// One test at a time: each burns CPU and reads machine-wide counters. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The tests' own reading of `/sys/class/powercap`: every zone listed, its +/// `name`, `energy_uj` and `max_energy_range_uj`, read here and nowhere else, +/// and the sustained-advance verdict over the package zones by the family's +/// rule. Nothing in it comes from the crate under test. +#[allow(dead_code)] +mod oracle { + use std::os::raw::{c_int, c_long}; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// Where the kernel lists its powercap zones. + pub const POWERCAP: &str = "/sys/class/powercap"; + + #[repr(C)] + struct Timespec { + tv_sec: c_long, + tv_nsec: c_long, + } + + unsafe extern "C" { + fn clock_gettime(clock_id: c_int, tp: *mut Timespec) -> c_int; + } + + /// `CLOCK_THREAD_CPUTIME_ID` in Linux ``. + const CLOCK_THREAD_CPUTIME_ID: c_int = 3; + + /// This thread's CPU time. + pub fn thread_cpu() -> Duration { + let mut ts = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mut ts) }, 0, "clock_gettime"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// Spin this thread until it has used `d` of its own CPU time, or `wall_cap` + /// of wall time has passed. Returns the CPU it got. + pub fn burn(d: Duration, wall_cap: Duration) -> Duration { + let (started, wall) = (thread_cpu(), Instant::now()); + let mut acc: u64 = 0; + while thread_cpu() - started < d && wall.elapsed() < wall_cap { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + thread_cpu() - started + } + + /// One entry of the powercap directory. + #[derive(Debug, Clone)] + pub struct Zone { + /// The directory entry: `intel-rapl:0`, `intel-rapl:0:0`, `intel-rapl-mmio:0`, ... + pub dir: String, + pub name: Option, + pub energy_uj: Option, + pub max_uj: Option, + } + + fn text(p: PathBuf) -> Option { + std::fs::read_to_string(p).ok().map(|s| s.trim().to_string()) + } + + fn number(p: PathBuf) -> Option { + text(p)?.parse().ok() + } + + /// Every entry under `root`, in name order. + pub fn zones(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { return Vec::new() }; + let mut zones: Vec = entries + .flatten() + .map(|e| { + let dir = e.file_name().to_string_lossy().into_owned(); + let p = e.path(); + Zone { name: text(p.join("name")), energy_uj: number(p.join("energy_uj")), max_uj: number(p.join("max_energy_range_uj")), dir } + }) + .collect(); + zones.sort_by(|a, b| a.dir.cmp(&b.dir)); + zones + } + + fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) + } + + /// A top-level RAPL zone: `intel-rapl:`. Not a sub-zone, not MMIO. + pub fn top_level(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(digits) + } + + /// A package zone: top-level, named `package-`. + pub fn is_package(z: &Zone) -> bool { + top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + } + + /// The platform zone. + pub fn is_psys(z: &Zone) -> bool { + top_level(z) && z.name.as_deref() == Some("psys") + } + + /// A package's `core` sub-zone (`intel-rapl::` named `core`). + pub fn is_core(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(|r| r.contains(':')) && z.name.as_deref() == Some("core") + } + + /// One zone's energy between two reads, modulo its own range. + pub fn delta(e0: u64, e1: u64, max: Option) -> Option { + if e1 >= e0 { + Some(e1 - e0) + } else { + max.filter(|&m| m > 0)?.checked_sub(e0)?.checked_add(e1) + } + } + + /// `energy_uj` of each of `dirs` under `root`, in order; `None` if any fails. + pub fn read(root: &Path, dirs: &[&Zone]) -> Option> { + dirs.iter().map(|z| number(root.join(&z.dir).join("energy_uj"))).collect() + } + + /// The summed energy of `which` of `set` between two reads of `set`. + pub fn energy(set: &[&Zone], before: &[u64], after: &[u64], which: impl Fn(&Zone) -> bool) -> u64 { + set.iter() + .zip(before.iter().zip(after)) + .filter(|(z, _)| which(z)) + .map(|(z, (&b, &a))| delta(b, a, z.max_uj).expect("a zone fell with no readable range")) + .sum() + } + + /// What the oracle concluded about the package counters under a root. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No top-level zone's `energy_uj` reads (no powercap, a masked or + /// root-only tree). + Unreadable, + /// Top-level zones read, but none is a package. + NoPackage, + /// Every package read, but not every one kept advancing. + Frozen, + /// Every package advanced in at least five intervals, at least 80% of + /// them, over at least 300 ms. + Live, + } + + /// The oracle's sustained-advance run over the package zones under `root`. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub packages: Vec, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + } + + impl Watch { + /// The verdict and each package's advances, for the test's log line. + pub fn summary(&self) -> String { + let per: Vec = + self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); + format!("{:?} ({} over {} ms)", self.verdict, per.join(", "), self.window.as_millis()) + } + } + + /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU + /// between samples (capped at 500 ms of wall time), until every package + /// has advanced in at least five intervals, at least 80% of those + /// observed, over at least 300 ms; or one second has passed with at least + /// five intervals; or five seconds have. + pub fn watch(root: &Path) -> Watch { + let all = zones(root); + let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); + let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; + if packages.is_empty() { + let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); + return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + } + let started = Instant::now(); + let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; packages.len()]; + let mut intervals = 0u32; + let live = |adv: &[u32], n: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, el) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + burn(Duration::from_millis(25), Duration::from_millis(500)); + if let Some(now) = read(root, &packages) { + intervals += 1; + for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(n > p); + } + prev = now; + } + } + let window = started.elapsed(); + let verdict = if live(&advances, intervals, window) { Verdict::Live } else { Verdict::Frozen }; + Watch { verdict, packages: names, intervals, advances, window } + } + + /// The zones under `root`, as one line for the log. + pub fn listing(root: &Path) -> String { + zones(root) + .iter() + .map(|z| { + let kind = if is_package(z) { + "package" + } else if is_psys(z) { + "psys" + } else if is_core(z) { + "core sub-zone" + } else if top_level(z) { + "top-level" + } else { + "not summed" + }; + let e = z.energy_uj.map_or("unreadable".to_string(), |e| e.to_string()); + let m = z.max_uj.map_or("-".to_string(), |m| m.to_string()); + format!("{}={} [{kind}] energy_uj={e} max={m}", z.dir, z.name.as_deref().unwrap_or("?")) + }) + .collect::>() + .join("; ") + } + + /// The CI container's view of the host's powercap directory: the same + /// entries, each a symlink into `/sys/devices/virtual/powercap`, which + /// Docker masks with an empty tmpfs, so every target is missing. Built + /// from this host's own listing; removed when dropped. + pub struct MaskedView(pub PathBuf); + + impl MaskedView { + /// Build the masked view of this host's powercap entries in a + /// temporary directory named by `tag` and this process. + pub fn of_host(tag: &str) -> MaskedView { + let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create the masked view"); + let masked = root.join("masked-devices-virtual-powercap"); + for z in zones(Path::new(POWERCAP)) { + std::os::unix::fs::symlink(masked.join(&z.dir), root.join(&z.dir)).expect("symlink"); + } + MaskedView(root) + } + } + + impl Drop for MaskedView { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A bracket of reads around the meter's two reads: every package, psys + /// and core zone read at four points, outer and inner on each side. + pub struct Bracket<'a> { + pub set: Vec<&'a Zone>, + pub reads: [Vec; 4], + } + + impl Bracket<'_> { + /// The packages' energy over the outer bracket, in µJ: the most a + /// package-only figure read inside it can be. + pub fn package_outer(&self) -> u64 { + energy(&self.set, &self.reads[0], &self.reads[3], is_package) + } + /// The packages' energy over the inner bracket, in µJ: the least a + /// package-only figure read around it can be. + pub fn package_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_package) + } + /// The psys zone's energy over the inner bracket, in µJ. + pub fn psys_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_psys) + } + /// The core sub-zones' energy over the inner bracket, in µJ. + pub fn core_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_core) + } + /// Whether the bracket read a zone of the kind `which` picks. + pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { + self.set.iter().any(|z| which(z)) + } + } + + /// The family's rule on a figure `uj` the meter read between the + /// bracket's inner and outer reads: no less than the packages' energy + /// over the inner bracket, no more than over the outer one, and below + /// package+psys and package+core where those zones exist. A figure that + /// added a zone reads at least the packages' and that zone's energy over + /// the inner bracket, so `uj < inner + zone` is the double-count test; it + /// can only be put where the zone moved more over the burn than the + /// packages did in the gap between the outer and inner reads (else a + /// correct figure could reach it too), and the line says when it could + /// not. Returns the figures, for the log. + pub fn assert_package_only(b: &Bracket, uj: u64) -> String { + let (inner, outer, psys, core) = (b.package_inner(), b.package_outer(), b.psys_inner(), b.core_inner()); + let gap = outer - inner; + let mut line = format!( + "meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ (gap {gap}); psys inner {psys} µJ; \ + core inner {core} µJ" + ); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + for (zone, present, moved) in [("psys", b.has(is_psys), psys), ("core", b.has(is_core), core)] { + if !present { + continue; + } + assert!(moved > 0, "{zone} did not advance: {line}"); + if moved > gap { + assert!(uj < inner + moved, "the figure reaches package+{zone}: {zone} was added: {line}"); + line.push_str(&format!("; below package+{zone}")); + } else { + line.push_str(&format!("; {zone} moved less than the gap, so only the outer bound tests it")); + } + } + line + } + + /// Every package, psys and core zone of `all` whose counter reads. + pub fn bracket_set(all: &[Zone]) -> Vec<&Zone> { + all.iter().filter(|z| (is_package(z) || is_psys(z) || is_core(z)) && z.energy_uj.is_some()).collect() + } + + /// One oracle read of every zone of `set`; panics if one fails. + pub fn read_all(set: &[&Zone]) -> Vec { + read(Path::new(POWERCAP), set).expect("an oracle read of the bracket zones") + } + + /// Read the bracket around `meter_read` (called twice) and `work`: + /// outer, meter, inner, work, inner, meter, outer. + pub fn around<'a, M>(set: Vec<&'a Zone>, mut meter_read: impl FnMut() -> M, work: impl FnOnce()) -> (Bracket<'a>, M, M) { + let o0 = read_all(&set); + let m0 = meter_read(); + let o1 = read_all(&set); + work(); + let o2 = read_all(&set); + let m1 = meter_read(); + let o3 = read_all(&set); + (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) + } +} + +use oracle::Verdict; + +/// The powercap meter opens exactly where the oracle finds every package +/// live, and its figure over a 300 ms burn sits inside the oracle's +/// package-only bracket and clearly below package+psys and package+core. +#[test] +fn the_powercap_meter_opens_where_the_packages_are_live_and_reads_them() { + let _one = serial(); + let meter = RaplMeter::host(); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + eprintln!( + "rapl hardware: {}; oracle {}; RaplMeter unavailable={:?}", + oracle::listing(Path::new(oracle::POWERCAP)), + watch.summary(), + meter.unavailable() + ); + assert_eq!(meter.unavailable().is_none(), watch.verdict == Verdict::Live, "the meter disagrees with the oracle"); + if meter.unavailable().is_some() { + assert_eq!(meter.sample_uj(), None, "a meter that did not open sampled"); + return; + } + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let mut cpu = Duration::ZERO; + let (b, m0, m1) = oracle::around( + oracle::bracket_set(&all), + || meter.sample_uj().expect("an open meter samples"), + || cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)), + ); + eprintln!("rapl burn: {cpu:?} of thread CPU; {}", oracle::assert_package_only(&b, m1 - m0)); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); +} + +/// `MSR_PKG_ENERGY_STATUS` read directly, the oracle's own way: `pread` of +/// 8 bytes at offset 0x611 of `/dev/cpu/0/msr`. `None` where it cannot be +/// read (no `msr` module, or not root: `CAP_SYS_RAWIO`). +fn oracle_msr_reads() -> Option { + use std::os::unix::fs::FileExt; + let f = std::fs::File::open("/dev/cpu/0/msr").ok()?; + let mut buf = [0u8; 8]; + f.read_exact_at(&mut buf, 0x611).ok()?; + Some(u64::from_le_bytes(buf)) +} + +/// The MSR meter opens only where the MSRs read: where the oracle cannot read +/// `MSR_PKG_ENERGY_STATUS` (an unprivileged process), the meter says why and +/// samples nothing; where it can, the meter opens and its figure over a 300 ms +/// burn sits inside the powercap packages' bracket, to a count's rounding. +#[test] +fn the_msr_meter_opens_only_where_the_msrs_read() { + let _one = serial(); + let msr = oracle_msr_reads(); + let meter = MsrRaplMeter::host(); + eprintln!( + "rapl msr hardware: /dev/cpu/0/msr {:?}; oracle MSR_PKG_ENERGY_STATUS {msr:?}; MsrRaplMeter {:?}", + std::fs::metadata("/dev/cpu/0/msr").map(|m| m.len()).map_err(|e| e.kind()), + meter.as_ref().map(|m| m.domains()).map_err(String::as_str) + ); + let Some(_) = msr else { + assert!(meter.is_err(), "the MSR meter opened where the oracle cannot read the MSR"); + return; + }; + let meter = meter.expect("the MSRs read, yet the MSR meter did not open"); + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let (b, m0, m1) = oracle::around( + oracle::bracket_set(&all), + || meter.sample_uj().expect("an open MSR meter samples"), + || { + oracle::burn(Duration::from_millis(300), Duration::from_secs(10)); + }, + ); + let uj = m1 - m0; + let (inner, outer) = (b.package_inner(), b.package_outer()); + eprintln!("rapl msr burn: meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ"); + // One count of the energy unit (2^-14 J, ~61 µJ) either side. + assert!(inner.saturating_sub(123) <= uj && uj <= outer + 123, "{uj} µJ outside [{inner}, {outer}]"); +} diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs index b06beb9c0..7dc4361ac 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs @@ -466,6 +466,82 @@ pub(super) fn counter_admits(status: CounterStatus) -> bool { status == CounterStatus::Live } +/// How far past its board's maximum power limit a GPU energy counter may run, +/// on average over a probe's window, as a fraction (3/2): a board can draw +/// past its limit for milliseconds, not on average over a third of a second. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) const GPU_CEILING_MARGIN: (u128, u128) = (3, 2); + +/// How much faster than its board's own power draw a GPU energy counter may +/// run over a probe's window: twice the draw's mean over the samples, plus +/// [`GPU_DRAW_SLACK_MW`]. The draw is a one-second average, so a load that +/// changed during the window can leave it behind the counter, but not by a +/// factor of two at a probe. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) const GPU_DRAW_FACTOR: u128 = 2; + +/// NVML's documented accuracy of a board's power draw, ±5 W, in mW. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) const GPU_DRAW_SLACK_MW: u128 = 5_000; + +/// Whether one GPU's energy counter, which counted `energy_mj` over `window`, +/// plausibly counted its board's energy, beside the board's power draw +/// `draws_mw` read with the samples and its maximum power limit `ceiling_mw`. +/// Not when the counter ran past [`GPU_CEILING_MARGIN`] of the limit, or a +/// draw reading did (the board's own telemetry then is not a reading either: +/// an RTX 4050 Laptop GPU reports 590 W for about a second after it wakes), +/// or the counter ran past [`GPU_DRAW_FACTOR`] times the mean draw plus +/// [`GPU_DRAW_SLACK_MW`]. A bound whose reading is missing is not applied. +/// +/// Liveness says the counter is republished continuously; this says what it +/// republishes is the board's energy. A counter that runs several times +/// faster than its board draws is counting something else, and on the RTX +/// 4050 Laptop GPU (driver 595.84) the total-energy counter does exactly +/// that while it is polled: at the probes' 100-150 ms ticks it ran at +/// 44-305 W while the board drew 5-14 W against a 65 W limit, and polled +/// every 1-2 s it still read 11-23 W while the board drew 2-10 W. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) fn plausible_gpu_counter(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { + let ns = window.as_nanos().max(1); + // mJ per second is mW. + let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; + if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { + let cap = u128::from(limit) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1; + if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { + return false; + } + } + if !draws_mw.is_empty() { + let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; + if rate_mw > GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW { + return false; + } + } + true +} + +/// Whether every GPU counter of a probe run plausibly counted its board's +/// energy ([`plausible_gpu_counter`]): each rail's rise from the run's first +/// sample to its last, over the window between them, beside that GPU's draw +/// readings (`draws`, one row per sample call, one column per GPU, `None` +/// where a reading failed) and its maximum power limit (`ceilings`, one per +/// GPU). Rails and columns are in device order. A run with fewer than two +/// samples has nothing to judge. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) fn gpu_counters_plausible(watch: &RailWatch, draws: &[Vec>], ceilings: &[Option]) -> bool { + let (Some(first), Some(last), Some(t0), Some(t1)) = + (watch.observed.first(), watch.observed.last(), watch.at.first(), watch.at.last()) + else { + return true; + }; + let window = t1.saturating_sub(*t0); + first.iter().zip(last).enumerate().all(|(i, ((_, a), (_, b)))| { + let energy_mj = u64::try_from(b.saturating_sub(*a)).unwrap_or(0); + let gpu_draws: Vec = draws.iter().filter_map(|row| row.get(i).copied().flatten()).collect(); + plausible_gpu_counter(energy_mj, window, &gpu_draws, ceilings.get(i).copied().flatten()) + }) +} + /// A counter probe's constructor behind its gate: `open` runs only when /// [`counter_admits`] the `status`; otherwise `Unavailable`, naming the /// counter (`what`) and why, and the counter is never touched. Every counter @@ -1813,6 +1889,69 @@ mod tests { assert_eq!(slow.watch.verdict(), Some(CpuRail::Frozen), "{:?}", slow.watch); } + /// The plausibility rule on one GPU counter, with the readings of the + /// RTX 4050 Laptop GPU on jetson-hub (driver 595.84, 65 W maximum limit): + /// polled at the probes' ticks its counter ran at 44-305 W while the board + /// drew 5-14 W, which is no count of the board's energy. A counter that + /// tracks its board's draw passes, a 1 s average lagging a load step by up + /// to twice still passes, and a bound whose reading is missing is not + /// applied. + #[test] + fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { + let w = Duration::from_millis(500); + // 66 W counted against a 7.5 W draw: what jetson-hub's probe saw. + assert!(!plausible_gpu_counter(33_000, w, &[7_500, 7_700], Some(65_000))); + // 44 W at the 150 ms tick against 10.4 W: past twice the draw + 5 W. + assert!(!plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000))); + // 120 W, past 1.5 x the 65 W limit, even with no draw reading. + assert!(!plausible_gpu_counter(60_000, w, &[], Some(65_000))); + // A draw reading of 590 W (the board's first second after waking) is no + // reading: it would excuse any counter. + assert!(!plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000))); + // A counter tracking its board: 13 W against 13 W, and 300 W against a + // 280 W draw on a 700 W board. + assert!(plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000))); + assert!(plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000))); + // A load step the one-second draw has not caught up with, up to twice + // the draw plus 5 W: 205 W against 100 W. + assert!(plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000))); + assert!(!plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000))); + // The exact bounds. + assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); + assert!(!plausible_gpu_counter(48_751, w, &[], Some(65_000))); + assert!(plausible_gpu_counter(12_500, w, &[10_000], None), "25 W is 2 x 10 W + 5 W"); + assert!(!plausible_gpu_counter(12_501, w, &[10_000], None)); + // Nothing to judge it by: no bound applies. + assert!(plausible_gpu_counter(1_000_000, w, &[], None)); + } + + /// The rule over a probe run: each GPU's rise from the run's first sample + /// to its last, over the window between them, beside that GPU's own draws + /// and limit. One implausible GPU of two makes the run implausible, and a + /// GPU's readings are never judged against another's. + #[test] + fn every_gpu_of_a_run_must_count_no_faster_than_its_board_draws() { + // Two GPUs republished every 100 ms, sampled at the NVML tick: one at + // 10 W (1 J a 100 ms step), one at 300 W (30 J a step). + let two = |t| { + Some(vec![ + ("nvml:0", published(Duration::from_millis(100), 1_000, t)), + ("nvml:1", published(Duration::from_millis(100), 30_000, t)), + ]) + }; + let r = counter_scripted(true, two, NVML_PROBE_TICK, Duration::from_micros(200)); + assert_eq!(r.watch.verdict(), Some(CpuRail::Live), "{:?}", r.watch); + let rows = r.watch.observed.len(); + let draws = |a: u32, b: u32| vec![vec![Some(a), Some(b)]; rows]; + let ceilings = [Some(65_000), Some(700_000)]; + assert!(gpu_counters_plausible(&r.watch, &draws(10_000, 290_000), &ceilings), "each at its own draw"); + assert!(!gpu_counters_plausible(&r.watch, &draws(1_000, 290_000), &ceilings), "the first at 9 x its draw"); + assert!(!gpu_counters_plausible(&r.watch, &draws(10_000, 290_000), &[Some(65_000), Some(65_000)]), "300 W on a 65 W board"); + assert!(!gpu_counters_plausible(&r.watch, &draws(290_000, 10_000), &[None, None]), "draws swapped between the GPUs"); + assert!(gpu_counters_plausible(&r.watch, &[], &[None, None]), "no reading to judge by"); + assert!(gpu_counters_plausible(&RailWatch::default(), &[], &[]), "no samples"); + } + /// The tick a counter probe burns is the tick it was asked for. #[test] fn a_counter_tick_burns_its_own_length() { @@ -1837,7 +1976,7 @@ mod tests { /// its own error passes through. #[test] fn a_gated_counter_is_opened_only_when_live() { - for status in [CounterStatus::Unavailable, CounterStatus::Frozen] { + for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible] { let opened = Cell::new(false); let r = gated("RAPL", status, || { opened.set(true); @@ -1862,6 +2001,7 @@ mod tests { fn only_a_live_counter_admits_its_probe() { assert!(counter_admits(CounterStatus::Live)); assert!(!counter_admits(CounterStatus::Frozen)); + assert!(!counter_admits(CounterStatus::Implausible)); assert!(!counter_admits(CounterStatus::Unavailable)); assert_eq!(counter_status_of(None), CounterStatus::Unavailable); assert_eq!(counter_status_of(Some(CpuRail::Absent)), CounterStatus::Unavailable); diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs index ca2d932a8..1fad7e401 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs @@ -183,6 +183,15 @@ pub enum CounterStatus { /// A figure summed over it would claim a domain it does not read, or a /// stale refresh as a reading. Never used. Frozen, + /// Every rail it sums kept advancing, but a GPU counter (NVML's) ran + /// faster than its board can draw power: past 1.5 times the board's + /// maximum power limit, or past twice the board's own power draw plus + /// 5 W, or the board's draw reading was itself past that limit. Such a + /// counter is not counting the board's energy, so a figure from it would + /// seal a number the device never spent. Seen on an RTX 4050 Laptop GPU + /// (driver 595.84), whose counter runs at 4 to 30 times the board's draw + /// while it is polled. Never used. + Implausible, /// Every rail it sums kept advancing: in at least five sample intervals, /// at least four in five of those observed, over at least 300 ms of wall /// time, while the probing thread burned CPU. The probe is used. @@ -197,6 +206,10 @@ impl std::fmt::Display for CounterStatus { "not every rail of the counter kept advancing while this process burned CPU (one \ held, or stepped only in a burst), so it cannot meter the work" } + CounterStatus::Implausible => { + "the counter kept advancing, but faster than its device can draw power, so it does not \ + count the device's energy" + } CounterStatus::Live => "every rail of the counter keeps advancing under load", }) } @@ -264,9 +277,12 @@ pub fn counter_liveness_tick() { /// [`PowerMeter::detect`] uses the NVML probe only when this is /// [`CounterStatus::Live`]: every GPU's total-energy counter must keep /// advancing — the same rule, at a 150 ms tick, since a GPU energy counter is -/// published more coarsely than a CPU rail — while the probing thread burns -/// CPU. A host with no `libnvidia-ml`, or no GPU that supports the counter, is -/// `Unavailable` at once, with no burn. Always `Unavailable` off Linux. +/// published more coarsely than a CPU rail (about every 100 ms on an RTX 4050 +/// Laptop GPU) — while the probing thread burns CPU, and over the probe each +/// must have counted no faster than its board can draw +/// ([`CounterStatus::Implausible`]). A host with no `libnvidia-ml`, or no GPU +/// that supports the counter, is `Unavailable` at once, with no burn. Always +/// `Unavailable` off Linux. pub fn nvml_status() -> CounterStatus { #[cfg(target_os = "linux")] { diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs index af06a2d35..98c27ce13 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs @@ -16,13 +16,27 @@ //! and the run live in [`liveness`](super::liveness) ([`counter_run`]); this //! module only opens the devices and reads their raw integers. //! +//! **And only when every GPU's counter plausibly counts its board's energy.** +//! A counter can be republished continuously and still not count the board: +//! on an RTX 4050 Laptop GPU (driver 595.84) the total-energy counter runs at +//! 4 to 30 times the board's power draw for as long as it is polled (44-305 W +//! at the probe's tick, against a 5-14 W draw and a 65 W limit). So over the +//! probe each counter's rise is held to its board's maximum power limit and +//! to the board's own draw, read with every sample +//! ([`plausible_gpu_counter`](super::liveness::plausible_gpu_counter)); one +//! that runs past them is [`Implausible`](super::CounterStatus::Implausible) +//! and the meter has no GPU reading. +//! //! A GPU-only figure is the GPU's energy, not the work's: the meter's span //! rule (a CPU counter that held estimates the span) does not apply to a //! probe set with no CPU class, so a caller that grades a whole unit of work //! by one figure must not grade a span with no CPU reading as a reading of //! that work ([`Measurement::reads_cpu`](super::Measurement::reads_cpu)). -use super::liveness::{counter_run, counter_status_of, counter_tick, gated, RailWatch, Sample, NVML_PROBE_TICK}; +use super::liveness::{ + counter_run, counter_status_of, counter_tick, gated, gpu_counters_plausible, RailWatch, Sample, NVML_PROBE_TICK, +}; +use std::cell::RefCell; use super::{CounterStatus, Probe, ProbeError, CPU_RAIL_LIVENESS_BUDGET}; use libloading::{Library, Symbol}; use sandbox_schema::DeviceClass; @@ -37,6 +51,12 @@ pub struct NvmlProbe { _lib: Library, devices: Vec, get_energy: unsafe extern "C" fn(Device, *mut u64) -> NvmlReturn, + /// `nvmlDeviceGetPowerUsage`, the board's power draw in mW, where the + /// library has it: the liveness probe's plausibility check reads it. + get_power: Option NvmlReturn>, + /// `nvmlDeviceGetPowerManagementLimitConstraints`, the board's minimum + /// and maximum power limit in mW, where the library has it. + get_limits: Option NvmlReturn>, } // NVML device handles are opaque pointers valid for the library's lifetime; @@ -54,22 +74,55 @@ pub(super) static REAL_RUNS: AtomicUsize = AtomicUsize::new(0); /// map its verdict). Call [`nvml_status`](super::nvml_status), not this. pub(super) fn host_status() -> CounterStatus { REAL_RUNS.fetch_add(1, Ordering::SeqCst); - counter_status_of(real_run().verdict()) + let run = real_run(); + status_of_run(&run.watch, &run.draws, &run.ceilings) +} + +/// The status of an NVML probe run: its verdict's, except that a live run +/// whose counters did not plausibly count their boards' energy +/// ([`gpu_counters_plausible`]) is `Implausible`. +fn status_of_run(watch: &RailWatch, draws: &[Vec>], ceilings: &[Option]) -> CounterStatus { + match counter_status_of(watch.verdict()) { + CounterStatus::Live if !gpu_counters_plausible(watch, draws, ceilings) => CounterStatus::Implausible, + status => status, + } +} + +/// What the real probe run saw: the rule's watch of the counters, and each +/// GPU's power draw read with every sample and its maximum power limit. +struct Run { + watch: RailWatch, + draws: Vec>>, + ceilings: Vec>, } /// The real probe run: [`counter_run`] over every supported GPU's raw /// counter, a tick that burns [`NVML_PROBE_TICK`] of this thread's CPU time -/// and a monotonic clock, for [`CPU_RAIL_LIVENESS_BUDGET`]. A host with no -/// NVML, or no GPU that supports the counter, does not open, and is never -/// ticked. -fn real_run() -> RailWatch { +/// and a monotonic clock, for [`CPU_RAIL_LIVENESS_BUDGET`]; with each sample, +/// every GPU's power draw, and once, every GPU's maximum power limit. A host +/// with no NVML, or no GPU that supports the counter, does not open, and is +/// never ticked. +fn real_run() -> Run { let epoch = Instant::now(); - counter_run( - || NvmlProbe::open().ok().map(|p| move || p.raw_sample()), + let draws = RefCell::new(Vec::new()); + let ceilings = RefCell::new(Vec::new()); + let watch = counter_run( + || { + NvmlProbe::open().ok().map(|p| { + *ceilings.borrow_mut() = p.max_limits_mw(); + let draws = &draws; + move || { + let sample = p.raw_sample(); + draws.borrow_mut().push(p.draws_mw()); + sample + } + }) + }, || counter_tick(NVML_PROBE_TICK), move || epoch.elapsed(), CPU_RAIL_LIVENESS_BUDGET, - ) + ); + Run { watch, draws: draws.into_inner(), ceilings: ceilings.into_inner() } } impl NvmlProbe { @@ -126,10 +179,22 @@ impl NvmlProbe { )); } let get_energy = *get_energy; + let get_power = lib + .get:: NvmlReturn>(b"nvmlDeviceGetPowerUsage\0") + .ok() + .map(|f| *f); + let get_limits = lib + .get:: NvmlReturn>( + b"nvmlDeviceGetPowerManagementLimitConstraints\0", + ) + .ok() + .map(|f| *f); Ok(NvmlProbe { _lib: lib, devices, get_energy, + get_power, + get_limits, }) } } @@ -152,6 +217,37 @@ impl NvmlProbe { } } +impl NvmlProbe { + /// Every GPU's power draw in mW, in device order; `None` for one whose + /// reading failed, and for all where the library has no such call. + fn draws_mw(&self) -> Vec> { + self.devices + .iter() + .map(|d| { + let f = self.get_power?; + let mut mw = 0u32; + // SAFETY: `d` is a handle NVML returned for the library this + // probe keeps loaded, and `mw` is writable for the call. + (unsafe { f(*d, &mut mw) } == 0).then_some(mw) + }) + .collect() + } + + /// Every GPU's maximum power limit in mW, in device order; `None` where + /// it cannot be read (or reads as zero). + fn max_limits_mw(&self) -> Vec> { + self.devices + .iter() + .map(|d| { + let f = self.get_limits?; + let (mut min, mut max) = (0u32, 0u32); + // SAFETY: as in `draws_mw`; both out-pointers are writable. + (unsafe { f(*d, &mut min, &mut max) } == 0 && max > 0).then_some(max) + }) + .collect() + } +} + impl Probe for NvmlProbe { fn name(&self) -> &'static str { "nvml-total-energy" @@ -175,12 +271,33 @@ impl Probe for NvmlProbe { mod tests { use super::*; use crate::probe::nvml_status; + use std::time::Duration; + + /// A probe run's status: a live run whose counters ran past their boards + /// is `Implausible`; a run that was not live keeps its status, whatever the + /// draws. + #[test] + fn a_live_run_past_its_boards_is_implausible() { + let mut live = RailWatch::default(); + let mut frozen = RailWatch::default(); + for (i, t) in (0..8u64).map(|i| (i, Duration::from_millis(150 * i))) { + live.observe(vec![("nvml:0".to_string(), 1_000_000 + 15_000 * i as i64)], t); + frozen.observe(vec![("nvml:0".to_string(), 1_000_000)], t); + } + assert_eq!(counter_status_of(live.verdict()), CounterStatus::Live, "{live:?}"); + // 15 J a 150 ms interval is 100 W. + assert_eq!(status_of_run(&live, &[vec![Some(95_000)]], &[Some(350_000)]), CounterStatus::Live); + assert_eq!(status_of_run(&live, &[vec![Some(10_000)]], &[Some(65_000)]), CounterStatus::Implausible); + assert_eq!(status_of_run(&live, &[vec![Some(95_000)]], &[Some(65_000)]), CounterStatus::Implausible); + assert_eq!(status_of_run(&frozen, &[vec![Some(10_000)]], &[Some(65_000)]), CounterStatus::Frozen); + assert_eq!(status_of_run(&RailWatch::default(), &[], &[]), CounterStatus::Unavailable); + } /// The gate declines every status but `Live`, without loading NVML, and /// `Live` builds the probe wherever NVML opens with a supported GPU. #[test] fn only_a_live_status_builds_the_nvml_probe() { - for status in [CounterStatus::Unavailable, CounterStatus::Frozen] { + for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible] { match NvmlProbe::given(status) { Err(ProbeError::Unavailable(why)) => assert!(why.contains("NVML"), "{why}"), Err(e) => panic!("{status:?}: {e}"), diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs b/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs new file mode 100644 index 000000000..170745c5e --- /dev/null +++ b/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs @@ -0,0 +1,400 @@ +//! NVML against the hardware: the GPUs' total-energy counters, read by the +//! meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the NVML probe behind `nvml_status()` +//! runs in this process, and the tests take one lock. The oracle opens +//! `libnvidia-ml` itself, measures how often the counter is republished, +//! samples it at the probe's own tick (150 ms) by the sustained-advance rule +//! and holds each counter to its board's power ceiling, which it asks of +//! `nvidia-smi`. None of these tests passes without asserting: where there is +//! no NVML (a CI container has no GPU), the status must say so and agree with +//! the oracle; where the counter is live, both must say so, and a span's GPU +//! figure must be a `Gpu` record, `HwShunt` for that class alone, never a +//! CPU reading, and no faster than the board can draw. + +#![cfg(target_os = "linux")] + +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use sandbox_meter::probe::{nvml_status, CounterStatus, PowerMeter}; +use sandbox_meter::Provenance; +use sandbox_schema::DeviceClass; + +/// One test at a time: each reads machine-wide counters, and the oracle's +/// own sampling must not overlap the meter's. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The NVML probe's tick (`NVML_PROBE_TICK`): 150 ms of the probing thread's +/// CPU between samples, so at least 150 ms of wall time. +const TICK: Duration = Duration::from_millis(150); + +/// The tests' own reading of NVIDIA's total-energy counter: `libnvidia-ml` +/// opened here with `dlopen` and called through symbols looked up here, not +/// through the crate under test or its bindings, and each GPU's power limit +/// and draw asked of `nvidia-smi`, a separate process. It measures how often +/// the counter is republished, and judges it by the family's rule: every +/// GPU's counter advanced in at least five intervals, at least 80% of them, +/// over at least 300 ms, sampled at the crate's own tick; and it is plausibly +/// counting its board's energy: over the window it ran no faster than 1.5 +/// times the board's maximum power limit, the board's own power draw (as +/// `nvidia-smi` reports it just before and just after) was itself within that +/// bound, and the counter ran no faster than twice that draw plus 5 W (the +/// draw's documented accuracy). A counter that runs several times faster than +/// its board draws is not counting that board's energy. +#[allow(dead_code)] +mod nvml_oracle { + use std::ffi::{c_char, c_int, c_uint, c_void, CString}; + use std::time::{Duration, Instant}; + + unsafe extern "C" { + fn dlopen(filename: *const c_char, flag: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; + } + + /// `RTLD_NOW` in ``. + const RTLD_NOW: c_int = 2; + + type Device = *mut c_void; + type EnergyFn = unsafe extern "C" fn(Device, *mut u64) -> c_int; + type LimitsFn = unsafe extern "C" fn(Device, *mut c_uint, *mut c_uint) -> c_int; + type MilliwattsFn = unsafe extern "C" fn(Device, *mut c_uint) -> c_int; + + /// NVML, opened by the oracle. + pub struct Nvml { + energy: EnergyFn, + limits: Option, + power: Option, + /// Every GPU whose total-energy counter reads, in index order. + pub devices: Vec, + } + + fn sym(lib: *mut c_void, name: &str) -> Option<*mut c_void> { + let name = CString::new(name).ok()?; + let p = unsafe { dlsym(lib, name.as_ptr()) }; + (!p.is_null()).then_some(p) + } + + /// `libnvidia-ml.so.1`, initialised, with every GPU whose counter reads; + /// `None` where there is no library, no GPU, or no GPU with the counter. + pub fn open() -> Option { + let file = CString::new("libnvidia-ml.so.1").ok()?; + let lib = unsafe { dlopen(file.as_ptr(), RTLD_NOW) }; + if lib.is_null() { + return None; + } + unsafe { + let init: unsafe extern "C" fn() -> c_int = std::mem::transmute(sym(lib, "nvmlInit_v2")?); + let count: unsafe extern "C" fn(*mut c_uint) -> c_int = std::mem::transmute(sym(lib, "nvmlDeviceGetCount_v2")?); + let by_index: unsafe extern "C" fn(c_uint, *mut Device) -> c_int = + std::mem::transmute(sym(lib, "nvmlDeviceGetHandleByIndex_v2")?); + let energy: EnergyFn = std::mem::transmute(sym(lib, "nvmlDeviceGetTotalEnergyConsumption")?); + let limits = sym(lib, "nvmlDeviceGetPowerManagementLimitConstraints").map(|p| std::mem::transmute::<_, LimitsFn>(p)); + let power = sym(lib, "nvmlDeviceGetPowerUsage").map(|p| std::mem::transmute::<_, MilliwattsFn>(p)); + if init() != 0 { + return None; + } + let mut n: c_uint = 0; + if count(&mut n) != 0 { + return None; + } + let mut devices = Vec::new(); + for i in 0..n { + let mut d: Device = std::ptr::null_mut(); + let mut mj = 0u64; + if by_index(i, &mut d) == 0 && !d.is_null() && energy(d, &mut mj) == 0 { + devices.push(d); + } + } + (!devices.is_empty()).then_some(Nvml { energy, limits, power, devices }) + } + } + + impl Nvml { + /// Every GPU's total-energy counter, in mJ; `None` if one fails. + pub fn energy_mj(&self) -> Option> { + self.devices + .iter() + .map(|&d| { + let mut mj = 0u64; + (unsafe { (self.energy)(d, &mut mj) } == 0).then_some(mj) + }) + .collect() + } + + /// GPU `i`'s maximum power-management limit, in mW, as NVML gives it. + pub fn max_limit_mw(&self, i: usize) -> Option { + let (mut lo, mut hi) = (0, 0); + (unsafe { (self.limits?)(self.devices[i], &mut lo, &mut hi) } == 0 && hi > 0).then_some(hi) + } + + /// GPU `i`'s power draw, in mW, as NVML gives it. + pub fn power_mw(&self, i: usize) -> Option { + let mut mw = 0; + (unsafe { (self.power?)(self.devices[i], &mut mw) } == 0).then_some(mw) + } + } + + /// `nvidia-smi`'s view of every GPU: (maximum power limit, enforced limit, + /// draw), in watts; `None` without `nvidia-smi`. + pub fn smi() -> Option, Option, Option)>> { + let out = std::process::Command::new("nvidia-smi") + .args(["--query-gpu=power.max_limit,enforced.power.limit,power.draw", "--format=csv,noheader,nounits"]) + .output() + .ok()?; + out.status.success().then(|| { + String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| { + let f: Vec> = l.split(',').map(|v| v.trim().parse().ok()).collect(); + (f.first().copied().flatten(), f.get(1).copied().flatten(), f.get(2).copied().flatten()) + }) + .collect() + }) + } + + /// GPU `i`'s ceiling in watts: `nvidia-smi`'s maximum power limit, else + /// NVML's. + pub fn ceiling_w(nvml: &Nvml, smi: &Option, Option, Option)>>, i: usize) -> Option { + smi.as_ref() + .and_then(|s| s.get(i)) + .and_then(|(max, _, _)| *max) + .or_else(|| nvml.max_limit_mw(i).map(|mw| f64::from(mw) / 1000.0)) + } + + /// How often GPU 0's counter is republished: the median gap between its + /// changes, polled every 2 ms for up to 1.5 s (`None` if it changed + /// fewer than 6 times). + pub fn update_period(nvml: &Nvml) -> Option { + let started = Instant::now(); + let mut last = nvml.energy_mj()?[0]; + let mut changes = Vec::new(); + while started.elapsed() < Duration::from_millis(1500) && changes.len() < 16 { + std::thread::sleep(Duration::from_millis(2)); + let now = nvml.energy_mj()?[0]; + if now != last { + changes.push(started.elapsed()); + last = now; + } + } + let mut gaps: Vec = changes.windows(2).map(|w| w[1] - w[0]).collect(); + if gaps.len() < 5 { + return None; + } + gaps.sort(); + Some(gaps[gaps.len() / 2]) + } + + /// The oracle's verdict on the counters. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No NVML, no GPU, or no GPU whose total-energy counter reads. + Unavailable, + /// A counter did not keep advancing. + Frozen, + /// Every counter kept advancing, but one is not plausibly its board's + /// energy (see the module doc). + Implausible, + /// Every counter kept advancing, plausibly its board's energy. + Live, + } + + /// The family's plausibility rule for one GPU over a window: its counter's + /// average `rate_w`, the board's draws `draws_w` around the window and + /// its maximum power limit `ceiling_w`. A bound whose input is missing + /// is not applied. + pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { + if let Some(c) = ceiling_w { + if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { + return false; + } + } + if !draws_w.is_empty() { + let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if rate_w > 2.0 * mean + 5.0 { + return false; + } + } + true + } + + /// What the oracle saw. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + /// Each GPU's average rate over the window, in watts. + pub rate_w: Vec, + /// Each GPU's draw just before and just after the window, in watts. + pub draws_w: Vec>, + /// Each GPU's ceiling, in watts. + pub ceiling_w: Vec>, + } + + impl Watch { + /// The verdict, each GPU's advances, its counter's rate, the board's + /// draw and its maximum limit, for the test's log line. + pub fn summary(&self) -> String { + format!( + "{:?} (advanced {:?} of {} intervals over {} ms; counter rate {:?} W, board draw {:?} W, max power limit {:?} W)", + self.verdict, + self.advances, + self.intervals, + self.window.as_millis(), + self.rate_w.iter().map(|w| (w * 10.0).round() / 10.0).collect::>(), + self.draws_w, + self.ceiling_w + ) + } + } + + /// The oracle's run over the first `gpus` GPUs (all, if more), sleeping + /// `tick` between samples, by the sustained-advance rule (stopping when + /// live, or after a second with at least five intervals, or five + /// seconds); then each counter's plausibility over the whole window. + pub fn watch(tick: Duration, gpus: usize) -> Watch { + let none = Watch { + verdict: Verdict::Unavailable, + intervals: 0, + advances: Vec::new(), + window: Duration::ZERO, + rate_w: Vec::new(), + draws_w: Vec::new(), + ceiling_w: Vec::new(), + }; + let Some(nvml) = open() else { return none }; + let n = nvml.devices.len().min(gpus); + let before = smi(); + let read = || nvml.energy_mj().map(|v| v[..n].to_vec()); + let started = Instant::now(); + let Some(first) = read() else { return none }; + let (mut prev, mut last_at) = (first.clone(), Duration::ZERO); + let mut advances = vec![0u32; n]; + let mut intervals = 0u32; + let live = |adv: &[u32], k: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(k) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, last_at) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + std::thread::sleep(tick); + if let Some(now) = read() { + last_at = started.elapsed(); + intervals += 1; + for ((a, x), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(x > p); + } + prev = now; + } + } + let after = smi(); + let window = last_at; + let rate_w: Vec = + first.iter().zip(&prev).map(|(a, b)| b.saturating_sub(*a) as f64 / 1000.0 / window.as_secs_f64().max(1e-9)).collect(); + let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); + let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); + let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + let verdict = if !live(&advances, intervals, window) { + Verdict::Frozen + } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + Verdict::Implausible + } else { + Verdict::Live + }; + Watch { verdict, intervals, advances, window, rate_w, draws_w, ceiling_w } + } + + /// One line on the GPUs for the log: count, counters, ceilings, draw, and + /// the counter's measured republication `period`. + pub fn listing(period: Option) -> String { + let Some(nvml) = open() else { return "no NVML device with a total-energy counter".to_string() }; + format!( + "{} GPU(s) with a total-energy counter, counters {:?} mJ; NVML max power limit {:?} mW, power draw {:?} mW; \ + nvidia-smi (max limit, enforced limit, draw) W {:?}; counter republished every {:?}", + nvml.devices.len(), + nvml.energy_mj(), + (0..nvml.devices.len()).map(|i| nvml.max_limit_mw(i)).collect::>(), + (0..nvml.devices.len()).map(|i| nvml.power_mw(i)).collect::>(), + smi(), + period + ) + } + + /// GPU 0's counter's republication period, where there is one. + pub fn period() -> Option { + open().and_then(|n| update_period(&n)) + } +} + +use nvml_oracle::Verdict; + +/// The status agrees with the oracle, judged at the probe's own tick, and the +/// tick is no faster than the counter is republished; only a live status puts +/// the NVML probe in the meter. +#[test] +fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { + let _one = serial(); + let period = nvml_oracle::period(); + let status = nvml_status(); + let watch = nvml_oracle::watch(TICK, usize::MAX); + let sources = PowerMeter::detect().sources(); + eprintln!( + "nvml hardware: {}; oracle at a {TICK:?} tick {}; status={status:?} sources={sources:?}", + nvml_oracle::listing(period), + watch.summary() + ); + if let Some(p) = period { + assert!(TICK >= p * 9 / 10, "the {TICK:?} tick is faster than the counter's {p:?} republication"); + } + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Implausible => assert_eq!(status, CounterStatus::Implausible), + Verdict::Live => {} + } + assert_eq!(sources.contains(&"nvml-total-energy"), status == CounterStatus::Live, "selection disagrees with the status"); +} + +/// Over a one-second span, a meter with a live NVML probe gives one `Gpu` +/// record, `HwShunt` for the GPU class alone: it advanced, it sits inside the +/// oracle's own reads of the counter around the span, it ran no faster than +/// the board can draw, and the span's CPU reading comes from a CPU counter or +/// not at all. Where NVML is not live, no record is a GPU reading. +#[test] +fn a_live_gpu_counter_is_a_gpu_record_alone() { + let _one = serial(); + let mut meter = PowerMeter::detect(); + let oracle = nvml_oracle::open(); + if !meter.sources().contains(&"nvml-total-energy") { + let ((), m) = meter.measure(|| std::thread::sleep(Duration::from_millis(300))); + eprintln!("nvml span: no NVML probe ({:?}); records {:?}", meter.sources(), m.devices); + assert_ne!(nvml_status(), CounterStatus::Live); + assert!(!m.devices.iter().any(|d| d.class == DeviceClass::Gpu), "a GPU record with NVML declined: {:?}", m.devices); + return; + } + let oracle = oracle.expect("the meter reads NVML, yet the oracle cannot open it"); + let o0 = oracle.energy_mj().expect("oracle read"); + let ((), m) = meter.measure(|| std::thread::sleep(Duration::from_secs(1))); + let o1 = oracle.energy_mj().expect("oracle read"); + let smi = nvml_oracle::smi(); + let ceiling: f64 = (0..oracle.devices.len()).map(|i| nvml_oracle::ceiling_w(&oracle, &smi, i).unwrap_or(f64::INFINITY)).sum(); + let gpu: Vec<_> = m.devices.iter().filter(|d| d.class == DeviceClass::Gpu).collect(); + eprintln!("nvml span: {:?}; records {:?}; oracle {o0:?} -> {o1:?} mJ; ceiling {ceiling} W", m.wall, m.devices); + assert_eq!(gpu.len(), 1, "not one GPU record: {:?}", m.devices); + let d = gpu[0]; + assert_eq!(d.provenance, Provenance::HwShunt); + let uj = d.measured_uj.expect("an HwShunt record carries its measured figure"); + let bracket: u64 = o0.iter().zip(&o1).map(|(a, b)| b - a).sum::() * 1_000; + assert!(uj > 0 && uj <= bracket, "{uj} µJ outside the oracle's (0, {bracket}] µJ"); + let watts = uj as f64 / 1e6 / m.wall.as_secs_f64(); + assert!(watts <= 1.5 * ceiling, "the GPU record reads {watts:.1} W, past the boards' {ceiling} W ceiling"); +} diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/tests/rapl_hardware.rs b/sandbox/sandbox-rs/crates/sandbox-meter/tests/rapl_hardware.rs new file mode 100644 index 000000000..ed81a215a --- /dev/null +++ b/sandbox/sandbox-rs/crates/sandbox-meter/tests/rapl_hardware.rs @@ -0,0 +1,492 @@ +//! Linux RAPL against the hardware: this host's powercap counters, read by +//! the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the RAPL probe behind `rapl_status()` +//! runs in this process, and the tests take one lock: each burns CPU and +//! reads machine-wide counters. The oracle reads `/sys/class/powercap` +//! itself — every zone's `name`, `energy_uj` and `max_energy_range_uj` — and +//! applies the sustained-advance rule to the top-level `package-` zones +//! on its own. None of these tests passes without asserting: where the +//! counters cannot be read (no powercap; a CI container, whose +//! `/sys/devices/virtual/powercap` Docker masks), the status must say so and +//! agree with the oracle; where they are live, both must say live, and a +//! span over a CPU burn must be one `Cpu` record, `HwShunt`, whose figure +//! sits inside the oracle's package-only bracket and clearly below +//! package+psys and package+core. + +#![cfg(target_os = "linux")] + +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use sandbox_meter::probe::{rapl_status, CounterStatus, PowerMeter, RaplPackages}; +use sandbox_meter::Provenance; +use sandbox_schema::DeviceClass; + +/// One test at a time: each burns CPU and reads machine-wide counters. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The tests' own reading of `/sys/class/powercap`: every zone listed, its +/// `name`, `energy_uj` and `max_energy_range_uj`, read here and nowhere else, +/// and the sustained-advance verdict over the package zones by the family's +/// rule. Nothing in it comes from the crate under test. +#[allow(dead_code)] +mod oracle { + use std::os::raw::{c_int, c_long}; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// Where the kernel lists its powercap zones. + pub const POWERCAP: &str = "/sys/class/powercap"; + + #[repr(C)] + struct Timespec { + tv_sec: c_long, + tv_nsec: c_long, + } + + unsafe extern "C" { + fn clock_gettime(clock_id: c_int, tp: *mut Timespec) -> c_int; + } + + /// `CLOCK_THREAD_CPUTIME_ID` in Linux ``. + const CLOCK_THREAD_CPUTIME_ID: c_int = 3; + + /// This thread's CPU time. + pub fn thread_cpu() -> Duration { + let mut ts = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mut ts) }, 0, "clock_gettime"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// Spin this thread until it has used `d` of its own CPU time, or `wall_cap` + /// of wall time has passed. Returns the CPU it got. + pub fn burn(d: Duration, wall_cap: Duration) -> Duration { + let (started, wall) = (thread_cpu(), Instant::now()); + let mut acc: u64 = 0; + while thread_cpu() - started < d && wall.elapsed() < wall_cap { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + thread_cpu() - started + } + + /// One entry of the powercap directory. + #[derive(Debug, Clone)] + pub struct Zone { + /// The directory entry: `intel-rapl:0`, `intel-rapl:0:0`, `intel-rapl-mmio:0`, ... + pub dir: String, + pub name: Option, + pub energy_uj: Option, + pub max_uj: Option, + } + + fn text(p: PathBuf) -> Option { + std::fs::read_to_string(p).ok().map(|s| s.trim().to_string()) + } + + fn number(p: PathBuf) -> Option { + text(p)?.parse().ok() + } + + /// Every entry under `root`, in name order. + pub fn zones(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { return Vec::new() }; + let mut zones: Vec = entries + .flatten() + .map(|e| { + let dir = e.file_name().to_string_lossy().into_owned(); + let p = e.path(); + Zone { name: text(p.join("name")), energy_uj: number(p.join("energy_uj")), max_uj: number(p.join("max_energy_range_uj")), dir } + }) + .collect(); + zones.sort_by(|a, b| a.dir.cmp(&b.dir)); + zones + } + + fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) + } + + /// A top-level RAPL zone: `intel-rapl:`. Not a sub-zone, not MMIO. + pub fn top_level(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(digits) + } + + /// A package zone: top-level, named `package-`. + pub fn is_package(z: &Zone) -> bool { + top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + } + + /// The platform zone. + pub fn is_psys(z: &Zone) -> bool { + top_level(z) && z.name.as_deref() == Some("psys") + } + + /// A package's `core` sub-zone (`intel-rapl::` named `core`). + pub fn is_core(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(|r| r.contains(':')) && z.name.as_deref() == Some("core") + } + + /// One zone's energy between two reads, modulo its own range. + pub fn delta(e0: u64, e1: u64, max: Option) -> Option { + if e1 >= e0 { + Some(e1 - e0) + } else { + max.filter(|&m| m > 0)?.checked_sub(e0)?.checked_add(e1) + } + } + + /// `energy_uj` of each of `dirs` under `root`, in order; `None` if any fails. + pub fn read(root: &Path, dirs: &[&Zone]) -> Option> { + dirs.iter().map(|z| number(root.join(&z.dir).join("energy_uj"))).collect() + } + + /// The summed energy of `which` of `set` between two reads of `set`. + pub fn energy(set: &[&Zone], before: &[u64], after: &[u64], which: impl Fn(&Zone) -> bool) -> u64 { + set.iter() + .zip(before.iter().zip(after)) + .filter(|(z, _)| which(z)) + .map(|(z, (&b, &a))| delta(b, a, z.max_uj).expect("a zone fell with no readable range")) + .sum() + } + + /// What the oracle concluded about the package counters under a root. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No top-level zone's `energy_uj` reads (no powercap, a masked or + /// root-only tree). + Unreadable, + /// Top-level zones read, but none is a package. + NoPackage, + /// Every package read, but not every one kept advancing. + Frozen, + /// Every package advanced in at least five intervals, at least 80% of + /// them, over at least 300 ms. + Live, + } + + /// The oracle's sustained-advance run over the package zones under `root`. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub packages: Vec, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + } + + impl Watch { + /// The verdict and each package's advances, for the test's log line. + pub fn summary(&self) -> String { + let per: Vec = + self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); + format!("{:?} ({} over {} ms)", self.verdict, per.join(", "), self.window.as_millis()) + } + } + + /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU + /// between samples (capped at 500 ms of wall time), until every package + /// has advanced in at least five intervals, at least 80% of those + /// observed, over at least 300 ms; or one second has passed with at least + /// five intervals; or five seconds have. + pub fn watch(root: &Path) -> Watch { + let all = zones(root); + let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); + let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; + if packages.is_empty() { + let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); + return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + } + let started = Instant::now(); + let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; packages.len()]; + let mut intervals = 0u32; + let live = |adv: &[u32], n: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, el) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + burn(Duration::from_millis(25), Duration::from_millis(500)); + if let Some(now) = read(root, &packages) { + intervals += 1; + for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(n > p); + } + prev = now; + } + } + let window = started.elapsed(); + let verdict = if live(&advances, intervals, window) { Verdict::Live } else { Verdict::Frozen }; + Watch { verdict, packages: names, intervals, advances, window } + } + + /// The zones under `root`, as one line for the log. + pub fn listing(root: &Path) -> String { + zones(root) + .iter() + .map(|z| { + let kind = if is_package(z) { + "package" + } else if is_psys(z) { + "psys" + } else if is_core(z) { + "core sub-zone" + } else if top_level(z) { + "top-level" + } else { + "not summed" + }; + let e = z.energy_uj.map_or("unreadable".to_string(), |e| e.to_string()); + let m = z.max_uj.map_or("-".to_string(), |m| m.to_string()); + format!("{}={} [{kind}] energy_uj={e} max={m}", z.dir, z.name.as_deref().unwrap_or("?")) + }) + .collect::>() + .join("; ") + } + + /// The CI container's view of the host's powercap directory: the same + /// entries, each a symlink into `/sys/devices/virtual/powercap`, which + /// Docker masks with an empty tmpfs, so every target is missing. Built + /// from this host's own listing; removed when dropped. + pub struct MaskedView(pub PathBuf); + + impl MaskedView { + /// Build the masked view of this host's powercap entries in a + /// temporary directory named by `tag` and this process. + pub fn of_host(tag: &str) -> MaskedView { + let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create the masked view"); + let masked = root.join("masked-devices-virtual-powercap"); + for z in zones(Path::new(POWERCAP)) { + std::os::unix::fs::symlink(masked.join(&z.dir), root.join(&z.dir)).expect("symlink"); + } + MaskedView(root) + } + } + + impl Drop for MaskedView { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A bracket of reads around the meter's two reads: every package, psys + /// and core zone read at four points, outer and inner on each side. + pub struct Bracket<'a> { + pub set: Vec<&'a Zone>, + pub reads: [Vec; 4], + } + + impl Bracket<'_> { + /// The packages' energy over the outer bracket, in µJ: the most a + /// package-only figure read inside it can be. + pub fn package_outer(&self) -> u64 { + energy(&self.set, &self.reads[0], &self.reads[3], is_package) + } + /// The packages' energy over the inner bracket, in µJ: the least a + /// package-only figure read around it can be. + pub fn package_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_package) + } + /// The psys zone's energy over the inner bracket, in µJ. + pub fn psys_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_psys) + } + /// The core sub-zones' energy over the inner bracket, in µJ. + pub fn core_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_core) + } + /// Whether the bracket read a zone of the kind `which` picks. + pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { + self.set.iter().any(|z| which(z)) + } + } + + /// The family's rule on a figure `uj` the meter read between the + /// bracket's inner and outer reads: no less than the packages' energy + /// over the inner bracket, no more than over the outer one, and below + /// package+psys and package+core where those zones exist. A figure that + /// added a zone reads at least the packages' and that zone's energy over + /// the inner bracket, so `uj < inner + zone` is the double-count test; it + /// can only be put where the zone moved more over the burn than the + /// packages did in the gap between the outer and inner reads (else a + /// correct figure could reach it too), and the line says when it could + /// not. Returns the figures, for the log. + pub fn assert_package_only(b: &Bracket, uj: u64) -> String { + let (inner, outer, psys, core) = (b.package_inner(), b.package_outer(), b.psys_inner(), b.core_inner()); + let gap = outer - inner; + let mut line = format!( + "meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ (gap {gap}); psys inner {psys} µJ; \ + core inner {core} µJ" + ); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + for (zone, present, moved) in [("psys", b.has(is_psys), psys), ("core", b.has(is_core), core)] { + if !present { + continue; + } + assert!(moved > 0, "{zone} did not advance: {line}"); + if moved > gap { + assert!(uj < inner + moved, "the figure reaches package+{zone}: {zone} was added: {line}"); + line.push_str(&format!("; below package+{zone}")); + } else { + line.push_str(&format!("; {zone} moved less than the gap, so only the outer bound tests it")); + } + } + line + } + + /// Every package, psys and core zone of `all` whose counter reads. + pub fn bracket_set(all: &[Zone]) -> Vec<&Zone> { + all.iter().filter(|z| (is_package(z) || is_psys(z) || is_core(z)) && z.energy_uj.is_some()).collect() + } + + /// One oracle read of every zone of `set`; panics if one fails. + pub fn read_all(set: &[&Zone]) -> Vec { + read(Path::new(POWERCAP), set).expect("an oracle read of the bracket zones") + } + + /// Read the bracket around `meter_read` (called twice) and `work`: + /// outer, meter, inner, work, inner, meter, outer. + pub fn around<'a, M>(set: Vec<&'a Zone>, mut meter_read: impl FnMut() -> M, work: impl FnOnce()) -> (Bracket<'a>, M, M) { + let o0 = read_all(&set); + let m0 = meter_read(); + let o1 = read_all(&set); + work(); + let o2 = read_all(&set); + let m1 = meter_read(); + let o3 = read_all(&set); + (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) + } +} + +use oracle::Verdict; + +/// The status agrees with the oracle's own verdict, and only a live status +/// opens the package counters and puts the RAPL probe in the meter. +#[test] +fn the_rapl_status_agrees_with_the_oracle_and_gates_the_probe() { + let _one = serial(); + let status = rapl_status(); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + let opened = RaplPackages::host().map(|p| p.zones().iter().map(|z| z.to_string()).collect::>()); + let sources = PowerMeter::detect().sources(); + eprintln!( + "rapl hardware: {}; oracle {}; status={status:?} opened={opened:?} sources={sources:?}", + oracle::listing(Path::new(oracle::POWERCAP)), + watch.summary() + ); + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unreadable | Verdict::NoPackage => assert_eq!(status, CounterStatus::Unavailable), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Live => { + let zones = opened.as_ref().expect("live, yet the package counters did not open"); + assert_eq!(*zones, watch.packages, "the probe holds other zones than the oracle's packages"); + } + } + assert_eq!(opened.is_ok(), status == CounterStatus::Live, "opening disagrees with the status"); + assert_eq!(sources.contains(&"rapl-powercap"), status == CounterStatus::Live, "selection disagrees with the status"); +} + +/// Over a 300 ms one-thread CPU burn, a span of a meter whose only CPU source +/// is RAPL is one `Cpu` record, `HwShunt`, its measured figure inside the +/// oracle's package-only bracket and clearly below package+psys and +/// package+core. Where RAPL is not live, no span has a RAPL figure. +#[test] +fn a_cpu_burn_is_one_hwshunt_cpu_record_inside_the_package_only_bracket() { + let _one = serial(); + let mut meter = PowerMeter::detect(); + if !meter.sources().contains(&"rapl-powercap") { + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + eprintln!("rapl burn: no RAPL probe in {:?}; oracle {}", meter.sources(), watch.summary()); + assert_ne!(watch.verdict, Verdict::Live, "the oracle saw live packages, yet the meter has no RAPL probe"); + assert_ne!(rapl_status(), CounterStatus::Live); + let ((), m) = meter.measure(|| { + oracle::burn(Duration::from_millis(50), Duration::from_secs(5)); + }); + assert!( + !m.devices.iter().any(|d| d.class == DeviceClass::Cpu && d.provenance == Provenance::HwShunt), + "a CPU record was graded HwShunt with RAPL declined: {:?}", + m.devices + ); + return; + } + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let set = oracle::bracket_set(&all); + let wall = Instant::now(); + let o0 = oracle::read_all(&set); + let span = meter.start(); + let o1 = oracle::read_all(&set); + let cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)); + let o2 = oracle::read_all(&set); + let m = span.stop(&mut meter); + let o3 = oracle::read_all(&set); + let b = oracle::Bracket { set, reads: [o0, o1, o2, o3] }; + let cpu_records: Vec<_> = m.devices.iter().filter(|d| d.class == DeviceClass::Cpu).collect(); + assert_eq!(cpu_records.len(), 1, "not one CPU record: {:?}", m.devices); + let d = cpu_records[0]; + assert_eq!(d.provenance, Provenance::HwShunt, "a live RAPL span's CPU record: {d:?}"); + let uj = d.measured_uj.expect("an HwShunt record carries its measured figure"); + assert_eq!(d.estimated_uj, uj, "the estimate of a measured figure is the figure"); + assert!(m.reads_cpu(), "a span with a live RAPL CPU record does not read the CPU"); + eprintln!( + "rapl burn: {cpu:?} of thread CPU in {:?}; sources {:?}; records {:?}; {}", + wall.elapsed(), + meter.sources(), + m.devices, + oracle::assert_package_only(&b, uj) + ); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); +} + +/// The package counters, read through `RaplPackages` directly (the S5 +/// executor's RAPL meter), read the same packages: over the same burn, the +/// figure sits inside the same bracket. +#[test] +fn the_package_counters_read_inside_the_package_only_bracket() { + let _one = serial(); + let Ok(mut packages) = RaplPackages::host() else { + assert_ne!(rapl_status(), CounterStatus::Live, "live, yet the package counters did not open"); + assert_ne!(oracle::watch(Path::new(oracle::POWERCAP)).verdict, Verdict::Live); + return; + }; + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let mut cpu = Duration::ZERO; + let (b, m0, m1) = oracle::around( + oracle::bracket_set(&all), + || packages.cumulative_uj().expect("live package counters read"), + || cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)), + ); + eprintln!("rapl packages: zones {:?}; {}", packages.zones(), oracle::assert_package_only(&b, m1 - m0)); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); +} + +/// What a CI container sees: Docker masks `/sys/devices/virtual/powercap` +/// with an empty tmpfs, so every entry of `/sys/class/powercap` is a symlink +/// to nothing. On that view of this host's own listing, the probe opens no +/// package and says so, and so does the oracle. +#[test] +fn the_ci_containers_masked_view_is_unavailable() { + let _one = serial(); + let view = oracle::MaskedView::of_host("sandbox-meter"); + let watch = oracle::watch(&view.0); + let opened = RaplPackages::live_in(&view.0, || { + oracle::burn(Duration::from_millis(25), Duration::from_millis(500)); + }); + eprintln!("rapl masked view: oracle {}; probe {:?}", watch.summary(), opened.as_ref().map(|p| p.zones().len()).map_err(|e| e.to_string())); + assert!(matches!(watch.verdict, Verdict::Unreadable | Verdict::NoPackage), "{watch:?}"); + assert!(opened.is_err(), "the probe opened package counters on a masked tree"); +}