From 4438d12eaf6f0ee4548d2ae909b1a815f5f52104 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 1 Oct 2026 01:38:58 -0400 Subject: [PATCH 01/10] energy meters: Linux RAPL hardware tests against an oracle of their own Each of the five meter crates (and the S5 executor's power meters) gets a test binary, cfg(target_os = "linux"), that reads /sys/class/powercap itself -- every zone's name, energy_uj and max_energy_range_uj -- and applies the sustained-advance rule to the top-level package- zones on its own. The crate's RAPL status must agree with that verdict; where live, the meter's figure over a CPU burn must sit inside the oracle's package-only bracket (outer and inner reads around the meter's), below package+psys and package+core wherever the bracket can resolve them, and be graded as the crate's hardware grade with honest coverage. Where the counters cannot be read the status must say so; the CI container's view (Docker masks /sys/devices/virtual/powercap) is replayed where the crate takes a root. Run on jetson-hub (i9-13900H: intel-rapl:0 package-0, :0:0 core, :1 psys, intel-rapl-mmio:0 root-only): all live, all inside the bracket. Co-Authored-By: Claude Opus 5.5 --- .../crates/eoc-meter/tests/rapl_hardware.rs | 415 +++++++++++++++ .../crates/jcp-energy/tests/rapl_hardware.rs | 427 ++++++++++++++++ .../jouleclaw-energy/tests/rapl_hardware.rs | 439 ++++++++++++++++ .../proof-cascade/tests/rapl_hardware.rs | 458 +++++++++++++++++ .../tests/rapl_hardware.rs | 436 ++++++++++++++++ .../sandbox-meter/tests/rapl_hardware.rs | 482 ++++++++++++++++++ 6 files changed, 2657 insertions(+) create mode 100644 eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs create mode 100644 jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs create mode 100644 jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs create mode 100644 proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs create mode 100644 sandbox/sandbox-rs/crates/sandbox-exec-microvm/tests/rapl_hardware.rs create mode 100644 sandbox/sandbox-rs/crates/sandbox-meter/tests/rapl_hardware.rs diff --git a/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs b/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs new file mode 100644 index 000000000..25c860e5d --- /dev/null +++ b/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs @@ -0,0 +1,415 @@ +//! Linux RAPL against the hardware: this host's powercap counters, read by +//! the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the RAPL probe behind `rapl_status()` +//! runs in this process, and the tests take one lock: each burns CPU and +//! reads machine-wide counters. The oracle reads `/sys/class/powercap` +//! itself — every zone's `name`, `energy_uj` and `max_energy_range_uj` — and +//! applies the sustained-advance rule to the top-level `package-` zones +//! on its own. None of these tests passes without asserting: where the +//! counters cannot be read (no powercap; a CI container, whose +//! `/sys/devices/virtual/powercap` Docker masks), the status must say so and +//! agree with the oracle; where they are live, both must say live, and the +//! meter's figure over a CPU burn must sit inside the oracle's package-only +//! bracket and clearly below package+psys and package+core. + +#![cfg(target_os = "linux")] + +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use eoc_meter::{CounterStatus, JouleCounter, LinuxRaplCounter}; + +/// One test at a time: each burns CPU and reads machine-wide counters. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The tests' own reading of `/sys/class/powercap`: every zone listed, its +/// `name`, `energy_uj` and `max_energy_range_uj`, read here and nowhere else, +/// and the sustained-advance verdict over the package zones by the family's +/// rule. Nothing in it comes from the crate under test. +#[allow(dead_code)] +mod oracle { + use std::os::raw::{c_int, c_long}; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// Where the kernel lists its powercap zones. + pub const POWERCAP: &str = "/sys/class/powercap"; + + #[repr(C)] + struct Timespec { + tv_sec: c_long, + tv_nsec: c_long, + } + + unsafe extern "C" { + fn clock_gettime(clock_id: c_int, tp: *mut Timespec) -> c_int; + } + + /// `CLOCK_THREAD_CPUTIME_ID` in Linux ``. + const CLOCK_THREAD_CPUTIME_ID: c_int = 3; + + /// This thread's CPU time. + pub fn thread_cpu() -> Duration { + let mut ts = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mut ts) }, 0, "clock_gettime"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// Spin this thread until it has used `d` of its own CPU time, or `wall_cap` + /// of wall time has passed. Returns the CPU it got. + pub fn burn(d: Duration, wall_cap: Duration) -> Duration { + let (started, wall) = (thread_cpu(), Instant::now()); + let mut acc: u64 = 0; + while thread_cpu() - started < d && wall.elapsed() < wall_cap { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + thread_cpu() - started + } + + /// One entry of the powercap directory. + #[derive(Debug, Clone)] + pub struct Zone { + /// The directory entry: `intel-rapl:0`, `intel-rapl:0:0`, `intel-rapl-mmio:0`, ... + pub dir: String, + pub name: Option, + pub energy_uj: Option, + pub max_uj: Option, + } + + fn text(p: PathBuf) -> Option { + std::fs::read_to_string(p).ok().map(|s| s.trim().to_string()) + } + + fn number(p: PathBuf) -> Option { + text(p)?.parse().ok() + } + + /// Every entry under `root`, in name order. + pub fn zones(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { return Vec::new() }; + let mut zones: Vec = entries + .flatten() + .map(|e| { + let dir = e.file_name().to_string_lossy().into_owned(); + let p = e.path(); + Zone { name: text(p.join("name")), energy_uj: number(p.join("energy_uj")), max_uj: number(p.join("max_energy_range_uj")), dir } + }) + .collect(); + zones.sort_by(|a, b| a.dir.cmp(&b.dir)); + zones + } + + fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) + } + + /// A top-level RAPL zone: `intel-rapl:`. Not a sub-zone, not MMIO. + pub fn top_level(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(digits) + } + + /// A package zone: top-level, named `package-`. + pub fn is_package(z: &Zone) -> bool { + top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + } + + /// The platform zone. + pub fn is_psys(z: &Zone) -> bool { + top_level(z) && z.name.as_deref() == Some("psys") + } + + /// A package's `core` sub-zone (`intel-rapl::` named `core`). + pub fn is_core(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(|r| r.contains(':')) && z.name.as_deref() == Some("core") + } + + /// One zone's energy between two reads, modulo its own range. + pub fn delta(e0: u64, e1: u64, max: Option) -> Option { + if e1 >= e0 { + Some(e1 - e0) + } else { + max.filter(|&m| m > 0)?.checked_sub(e0)?.checked_add(e1) + } + } + + /// `energy_uj` of each of `dirs` under `root`, in order; `None` if any fails. + pub fn read(root: &Path, dirs: &[&Zone]) -> Option> { + dirs.iter().map(|z| number(root.join(&z.dir).join("energy_uj"))).collect() + } + + /// The summed energy of `which` of `set` between two reads of `set`. + pub fn energy(set: &[&Zone], before: &[u64], after: &[u64], which: impl Fn(&Zone) -> bool) -> u64 { + set.iter() + .zip(before.iter().zip(after)) + .filter(|(z, _)| which(z)) + .map(|(z, (&b, &a))| delta(b, a, z.max_uj).expect("a zone fell with no readable range")) + .sum() + } + + /// What the oracle concluded about the package counters under a root. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No top-level zone's `energy_uj` reads (no powercap, a masked or + /// root-only tree). + Unreadable, + /// Top-level zones read, but none is a package. + NoPackage, + /// Every package read, but not every one kept advancing. + Frozen, + /// Every package advanced in at least five intervals, at least 80% of + /// them, over at least 300 ms. + Live, + } + + /// The oracle's sustained-advance run over the package zones under `root`. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub packages: Vec, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + } + + impl Watch { + pub fn summary(&self) -> String { + let per: Vec = + self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); + format!("{:?} ({} over {} ms)", self.verdict, per.join(", "), self.window.as_millis()) + } + } + + /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU + /// between samples (capped at 500 ms of wall time), until every package + /// has advanced in at least five intervals, at least 80% of those + /// observed, over at least 300 ms; or one second has passed with at least + /// five intervals; or five seconds have. + pub fn watch(root: &Path) -> Watch { + let all = zones(root); + let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); + let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; + if packages.is_empty() { + let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); + return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + } + let started = Instant::now(); + let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; packages.len()]; + let mut intervals = 0u32; + let live = |adv: &[u32], n: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, el) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + burn(Duration::from_millis(25), Duration::from_millis(500)); + if let Some(now) = read(root, &packages) { + intervals += 1; + for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(n > p); + } + prev = now; + } + } + let window = started.elapsed(); + let verdict = if live(&advances, intervals, window) { Verdict::Live } else { Verdict::Frozen }; + Watch { verdict, packages: names, intervals, advances, window } + } + + /// The zones under `root`, as one line for the log. + pub fn listing(root: &Path) -> String { + zones(root) + .iter() + .map(|z| { + let kind = if is_package(z) { + "package" + } else if is_psys(z) { + "psys" + } else if is_core(z) { + "core sub-zone" + } else if top_level(z) { + "top-level" + } else { + "not summed" + }; + let e = z.energy_uj.map_or("unreadable".to_string(), |e| e.to_string()); + let m = z.max_uj.map_or("-".to_string(), |m| m.to_string()); + format!("{}={} [{kind}] energy_uj={e} max={m}", z.dir, z.name.as_deref().unwrap_or("?")) + }) + .collect::>() + .join("; ") + } + + /// The CI container's view of the host's powercap directory: the same + /// entries, each a symlink into `/sys/devices/virtual/powercap`, which + /// Docker masks with an empty tmpfs, so every target is missing. Built + /// from this host's own listing; removed when dropped. + pub struct MaskedView(pub PathBuf); + + impl MaskedView { + pub fn of_host(tag: &str) -> MaskedView { + let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create the masked view"); + let masked = root.join("masked-devices-virtual-powercap"); + for z in zones(Path::new(POWERCAP)) { + std::os::unix::fs::symlink(masked.join(&z.dir), root.join(&z.dir)).expect("symlink"); + } + MaskedView(root) + } + } + + impl Drop for MaskedView { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A bracket of reads around the meter's two reads: every package, psys + /// and core zone read at four points, outer and inner on each side. + pub struct Bracket<'a> { + pub set: Vec<&'a Zone>, + pub reads: [Vec; 4], + } + + impl Bracket<'_> { + pub fn package_outer(&self) -> u64 { + energy(&self.set, &self.reads[0], &self.reads[3], is_package) + } + pub fn package_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_package) + } + pub fn psys_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_psys) + } + pub fn core_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_core) + } + pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { + self.set.iter().any(|z| which(z)) + } + } + + /// The family's rule on a figure `uj` the meter read between the + /// bracket's inner and outer reads: no less than the packages' energy + /// over the inner bracket, no more than over the outer one, and below + /// package+psys and package+core where those zones exist. A figure that + /// added a zone reads at least the packages' and that zone's energy over + /// the inner bracket, so `uj < inner + zone` is the double-count test; it + /// can only be put where the zone moved more over the burn than the + /// packages did in the gap between the outer and inner reads (else a + /// correct figure could reach it too), and the line says when it could + /// not. Returns the figures, for the log. + pub fn assert_package_only(b: &Bracket, uj: u64) -> String { + let (inner, outer, psys, core) = (b.package_inner(), b.package_outer(), b.psys_inner(), b.core_inner()); + let gap = outer - inner; + let mut line = format!( + "meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ (gap {gap}); psys inner {psys} µJ; \ + core inner {core} µJ" + ); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + for (zone, present, moved) in [("psys", b.has(is_psys), psys), ("core", b.has(is_core), core)] { + if !present { + continue; + } + assert!(moved > 0, "{zone} did not advance: {line}"); + if moved > gap { + assert!(uj < inner + moved, "the figure reaches package+{zone}: {zone} was added: {line}"); + line.push_str(&format!("; below package+{zone}")); + } else { + line.push_str(&format!("; {zone} moved less than the gap, so only the outer bound tests it")); + } + } + line + } + + /// Every package, psys and core zone of `all` whose counter reads. + pub fn bracket_set(all: &[Zone]) -> Vec<&Zone> { + all.iter().filter(|z| (is_package(z) || is_psys(z) || is_core(z)) && z.energy_uj.is_some()).collect() + } + + /// One oracle read of every zone of `set`; panics if one fails. + pub fn read_all(set: &[&Zone]) -> Vec { + read(Path::new(POWERCAP), set).expect("an oracle read of the bracket zones") + } + + /// Read the bracket around `meter_read` (called twice) and `work`: + /// outer, meter, inner, work, inner, meter, outer. + pub fn around<'a, M>(set: Vec<&'a Zone>, mut meter_read: impl FnMut() -> M, work: impl FnOnce()) -> (Bracket<'a>, M, M) { + let o0 = read_all(&set); + let m0 = meter_read(); + let o1 = read_all(&set); + work(); + let o2 = read_all(&set); + let m1 = meter_read(); + let o3 = read_all(&set); + (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) + } +} + +use oracle::Verdict; + +/// The status agrees with the oracle's own verdict, and only a live status +/// constructs the counter and gets the RAPL rung selected: a live counter's +/// advance is what eoc's consumers grade `Measured`, so the gate is the grade. +#[test] +fn the_rapl_status_agrees_with_the_oracle_and_gates_the_rung() { + let _one = serial(); + let status = eoc_meter::rapl_status(); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + let constructed = LinuxRaplCounter::try_detect().is_ok(); + let selected = eoc_meter::detect().name(); + eprintln!( + "rapl hardware: {}; oracle {}; status={status:?} constructed={constructed} detect()={selected}", + oracle::listing(Path::new(oracle::POWERCAP)), + watch.summary() + ); + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unreadable | Verdict::NoPackage => assert_eq!(status, CounterStatus::Unavailable), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Live => {} + } + assert_eq!(constructed, status == CounterStatus::Live, "construction disagrees with the status"); + assert_eq!(selected == "rapl", status == CounterStatus::Live, "detect() picked {selected} under {status:?}"); +} + +/// Over a 300 ms one-thread CPU burn the counter reads the packages' energy: +/// no less than the oracle's inner package bracket, no more than its outer +/// one, and clearly below package+psys and package+core (neither the +/// platform zone nor a package's own sub-zone is added). Where RAPL is not +/// live, no counter is constructed and there is no figure to grade. +#[test] +fn a_cpu_burn_reads_inside_the_package_only_bracket() { + let _one = serial(); + let Ok(counter) = LinuxRaplCounter::try_detect() else { + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + eprintln!("rapl burn: no counter; oracle {}", watch.summary()); + assert_ne!(watch.verdict, Verdict::Live, "the oracle saw live packages, yet no counter was constructed"); + assert_ne!(eoc_meter::rapl_status(), CounterStatus::Live); + return; + }; + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let wall = Instant::now(); + let mut cpu = Duration::ZERO; + let (b, m0, m1) = oracle::around( + oracle::bracket_set(&all), + || counter.read_microjoules().expect("a live counter reads"), + || cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)), + ); + let uj = m1 - m0; + eprintln!("rapl burn: {cpu:?} of thread CPU in {:?}; {}", wall.elapsed(), oracle::assert_package_only(&b, uj)); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); +} diff --git a/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs b/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs new file mode 100644 index 000000000..95a9a3e9c --- /dev/null +++ b/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs @@ -0,0 +1,427 @@ +//! Linux RAPL against the hardware: this host's powercap counters, read by +//! the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the RAPL probe behind `rapl_status()` +//! runs in this process, and the tests take one lock: each burns CPU and +//! reads machine-wide counters. The oracle reads `/sys/class/powercap` +//! itself — every zone's `name`, `energy_uj` and `max_energy_range_uj` — and +//! applies the sustained-advance rule to the top-level `package-` zones +//! on its own. None of these tests passes without asserting: where the +//! counters cannot be read (no powercap; a CI container, whose +//! `/sys/devices/virtual/powercap` Docker masks), the status must say so and +//! agree with the oracle; where they are live, both must say live, the RAPL +//! rung must be selected, and a span over a CPU burn must be `HwShunt`, +//! `Whole` (the package counters read the CPU work they meter), its figure +//! inside the oracle's package-only bracket and clearly below package+psys +//! and package+core. + +#![cfg(target_os = "linux")] + +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use jcp_energy::{rapl_status, CounterStatus, Meter, Rung}; +use jcp_schema::{Coverage, Provenance}; + +/// One test at a time: each burns CPU and reads machine-wide counters. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The tests' own reading of `/sys/class/powercap`: every zone listed, its +/// `name`, `energy_uj` and `max_energy_range_uj`, read here and nowhere else, +/// and the sustained-advance verdict over the package zones by the family's +/// rule. Nothing in it comes from the crate under test. +#[allow(dead_code)] +mod oracle { + use std::os::raw::{c_int, c_long}; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// Where the kernel lists its powercap zones. + pub const POWERCAP: &str = "/sys/class/powercap"; + + #[repr(C)] + struct Timespec { + tv_sec: c_long, + tv_nsec: c_long, + } + + unsafe extern "C" { + fn clock_gettime(clock_id: c_int, tp: *mut Timespec) -> c_int; + } + + /// `CLOCK_THREAD_CPUTIME_ID` in Linux ``. + const CLOCK_THREAD_CPUTIME_ID: c_int = 3; + + /// This thread's CPU time. + pub fn thread_cpu() -> Duration { + let mut ts = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mut ts) }, 0, "clock_gettime"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// Spin this thread until it has used `d` of its own CPU time, or `wall_cap` + /// of wall time has passed. Returns the CPU it got. + pub fn burn(d: Duration, wall_cap: Duration) -> Duration { + let (started, wall) = (thread_cpu(), Instant::now()); + let mut acc: u64 = 0; + while thread_cpu() - started < d && wall.elapsed() < wall_cap { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + thread_cpu() - started + } + + /// One entry of the powercap directory. + #[derive(Debug, Clone)] + pub struct Zone { + /// The directory entry: `intel-rapl:0`, `intel-rapl:0:0`, `intel-rapl-mmio:0`, ... + pub dir: String, + pub name: Option, + pub energy_uj: Option, + pub max_uj: Option, + } + + fn text(p: PathBuf) -> Option { + std::fs::read_to_string(p).ok().map(|s| s.trim().to_string()) + } + + fn number(p: PathBuf) -> Option { + text(p)?.parse().ok() + } + + /// Every entry under `root`, in name order. + pub fn zones(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { return Vec::new() }; + let mut zones: Vec = entries + .flatten() + .map(|e| { + let dir = e.file_name().to_string_lossy().into_owned(); + let p = e.path(); + Zone { name: text(p.join("name")), energy_uj: number(p.join("energy_uj")), max_uj: number(p.join("max_energy_range_uj")), dir } + }) + .collect(); + zones.sort_by(|a, b| a.dir.cmp(&b.dir)); + zones + } + + fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) + } + + /// A top-level RAPL zone: `intel-rapl:`. Not a sub-zone, not MMIO. + pub fn top_level(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(digits) + } + + /// A package zone: top-level, named `package-`. + pub fn is_package(z: &Zone) -> bool { + top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + } + + /// The platform zone. + pub fn is_psys(z: &Zone) -> bool { + top_level(z) && z.name.as_deref() == Some("psys") + } + + /// A package's `core` sub-zone (`intel-rapl::` named `core`). + pub fn is_core(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(|r| r.contains(':')) && z.name.as_deref() == Some("core") + } + + /// One zone's energy between two reads, modulo its own range. + pub fn delta(e0: u64, e1: u64, max: Option) -> Option { + if e1 >= e0 { + Some(e1 - e0) + } else { + max.filter(|&m| m > 0)?.checked_sub(e0)?.checked_add(e1) + } + } + + /// `energy_uj` of each of `dirs` under `root`, in order; `None` if any fails. + pub fn read(root: &Path, dirs: &[&Zone]) -> Option> { + dirs.iter().map(|z| number(root.join(&z.dir).join("energy_uj"))).collect() + } + + /// The summed energy of `which` of `set` between two reads of `set`. + pub fn energy(set: &[&Zone], before: &[u64], after: &[u64], which: impl Fn(&Zone) -> bool) -> u64 { + set.iter() + .zip(before.iter().zip(after)) + .filter(|(z, _)| which(z)) + .map(|(z, (&b, &a))| delta(b, a, z.max_uj).expect("a zone fell with no readable range")) + .sum() + } + + /// What the oracle concluded about the package counters under a root. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No top-level zone's `energy_uj` reads (no powercap, a masked or + /// root-only tree). + Unreadable, + /// Top-level zones read, but none is a package. + NoPackage, + /// Every package read, but not every one kept advancing. + Frozen, + /// Every package advanced in at least five intervals, at least 80% of + /// them, over at least 300 ms. + Live, + } + + /// The oracle's sustained-advance run over the package zones under `root`. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub packages: Vec, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + } + + impl Watch { + pub fn summary(&self) -> String { + let per: Vec = + self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); + format!("{:?} ({} over {} ms)", self.verdict, per.join(", "), self.window.as_millis()) + } + } + + /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU + /// between samples (capped at 500 ms of wall time), until every package + /// has advanced in at least five intervals, at least 80% of those + /// observed, over at least 300 ms; or one second has passed with at least + /// five intervals; or five seconds have. + pub fn watch(root: &Path) -> Watch { + let all = zones(root); + let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); + let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; + if packages.is_empty() { + let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); + return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + } + let started = Instant::now(); + let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; packages.len()]; + let mut intervals = 0u32; + let live = |adv: &[u32], n: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, el) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + burn(Duration::from_millis(25), Duration::from_millis(500)); + if let Some(now) = read(root, &packages) { + intervals += 1; + for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(n > p); + } + prev = now; + } + } + let window = started.elapsed(); + let verdict = if live(&advances, intervals, window) { Verdict::Live } else { Verdict::Frozen }; + Watch { verdict, packages: names, intervals, advances, window } + } + + /// The zones under `root`, as one line for the log. + pub fn listing(root: &Path) -> String { + zones(root) + .iter() + .map(|z| { + let kind = if is_package(z) { + "package" + } else if is_psys(z) { + "psys" + } else if is_core(z) { + "core sub-zone" + } else if top_level(z) { + "top-level" + } else { + "not summed" + }; + let e = z.energy_uj.map_or("unreadable".to_string(), |e| e.to_string()); + let m = z.max_uj.map_or("-".to_string(), |m| m.to_string()); + format!("{}={} [{kind}] energy_uj={e} max={m}", z.dir, z.name.as_deref().unwrap_or("?")) + }) + .collect::>() + .join("; ") + } + + /// The CI container's view of the host's powercap directory: the same + /// entries, each a symlink into `/sys/devices/virtual/powercap`, which + /// Docker masks with an empty tmpfs, so every target is missing. Built + /// from this host's own listing; removed when dropped. + pub struct MaskedView(pub PathBuf); + + impl MaskedView { + pub fn of_host(tag: &str) -> MaskedView { + let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create the masked view"); + let masked = root.join("masked-devices-virtual-powercap"); + for z in zones(Path::new(POWERCAP)) { + std::os::unix::fs::symlink(masked.join(&z.dir), root.join(&z.dir)).expect("symlink"); + } + MaskedView(root) + } + } + + impl Drop for MaskedView { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A bracket of reads around the meter's two reads: every package, psys + /// and core zone read at four points, outer and inner on each side. + pub struct Bracket<'a> { + pub set: Vec<&'a Zone>, + pub reads: [Vec; 4], + } + + impl Bracket<'_> { + pub fn package_outer(&self) -> u64 { + energy(&self.set, &self.reads[0], &self.reads[3], is_package) + } + pub fn package_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_package) + } + pub fn psys_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_psys) + } + pub fn core_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_core) + } + pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { + self.set.iter().any(|z| which(z)) + } + } + + /// The family's rule on a figure `uj` the meter read between the + /// bracket's inner and outer reads: no less than the packages' energy + /// over the inner bracket, no more than over the outer one, and below + /// package+psys and package+core where those zones exist. A figure that + /// added a zone reads at least the packages' and that zone's energy over + /// the inner bracket, so `uj < inner + zone` is the double-count test; it + /// can only be put where the zone moved more over the burn than the + /// packages did in the gap between the outer and inner reads (else a + /// correct figure could reach it too), and the line says when it could + /// not. Returns the figures, for the log. + pub fn assert_package_only(b: &Bracket, uj: u64) -> String { + let (inner, outer, psys, core) = (b.package_inner(), b.package_outer(), b.psys_inner(), b.core_inner()); + let gap = outer - inner; + let mut line = format!( + "meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ (gap {gap}); psys inner {psys} µJ; \ + core inner {core} µJ" + ); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + for (zone, present, moved) in [("psys", b.has(is_psys), psys), ("core", b.has(is_core), core)] { + if !present { + continue; + } + assert!(moved > 0, "{zone} did not advance: {line}"); + if moved > gap { + assert!(uj < inner + moved, "the figure reaches package+{zone}: {zone} was added: {line}"); + line.push_str(&format!("; below package+{zone}")); + } else { + line.push_str(&format!("; {zone} moved less than the gap, so only the outer bound tests it")); + } + } + line + } + + /// Every package, psys and core zone of `all` whose counter reads. + pub fn bracket_set(all: &[Zone]) -> Vec<&Zone> { + all.iter().filter(|z| (is_package(z) || is_psys(z) || is_core(z)) && z.energy_uj.is_some()).collect() + } + + /// One oracle read of every zone of `set`; panics if one fails. + pub fn read_all(set: &[&Zone]) -> Vec { + read(Path::new(POWERCAP), set).expect("an oracle read of the bracket zones") + } + + /// Read the bracket around `meter_read` (called twice) and `work`: + /// outer, meter, inner, work, inner, meter, outer. + pub fn around<'a, M>(set: Vec<&'a Zone>, mut meter_read: impl FnMut() -> M, work: impl FnOnce()) -> (Bracket<'a>, M, M) { + let o0 = read_all(&set); + let m0 = meter_read(); + let o1 = read_all(&set); + work(); + let o2 = read_all(&set); + let m1 = meter_read(); + let o3 = read_all(&set); + (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) + } +} + +use oracle::Verdict; + +/// The status agrees with the oracle's own verdict, and the RAPL rung is +/// selected exactly when it is live. +#[test] +fn the_rapl_status_agrees_with_the_oracle_and_gates_the_rung() { + let _one = serial(); + let status = rapl_status(); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + let (meter, why) = Meter::detect_explained(); + eprintln!( + "rapl hardware: {}; oracle {}; status={status:?} selected={:?} provenance={:?} detection={why:?}", + oracle::listing(Path::new(oracle::POWERCAP)), + watch.summary(), + meter.rung(), + meter.provenance() + ); + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unreadable | Verdict::NoPackage => assert_eq!(status, CounterStatus::Unavailable), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Live => {} + } + assert_eq!(why.rapl, Some(status), "the detection did not record RAPL's status"); + assert_eq!(meter.rung() == Rung::Rapl, status == CounterStatus::Live, "selection disagrees with the status"); + if meter.rung() == Rung::Rapl { + assert_eq!(meter.provenance(), Provenance::HwShunt); + } +} + +/// Over a 300 ms one-thread CPU burn, the RAPL rung's span is a whole +/// hardware reading, `HwShunt` and `Whole`, whose figure sits inside the +/// oracle's package-only bracket and clearly below package+psys and +/// package+core. Where RAPL is not live, the meter is another rung and no +/// span of it is a RAPL reading. +#[test] +fn a_cpu_burn_is_a_whole_hwshunt_reading_inside_the_package_only_bracket() { + let _one = serial(); + let meter = Meter::detect(); + if meter.rung() != Rung::Rapl { + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + eprintln!("rapl burn: rung {:?}; oracle {}", meter.rung(), watch.summary()); + assert_ne!(watch.verdict, Verdict::Live, "the oracle saw live packages, yet the RAPL rung was not selected"); + assert_ne!(rapl_status(), CounterStatus::Live); + return; + } + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let set = oracle::bracket_set(&all); + let wall = Instant::now(); + let o0 = oracle::read_all(&set); + let ((o1, cpu, o2), e) = meter.measure(|| { + let o1 = oracle::read_all(&set); + let cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)); + (o1, cpu, oracle::read_all(&set)) + }); + let o3 = oracle::read_all(&set); + let b = oracle::Bracket { set, reads: [o0, o1, o2, o3] }; + assert_eq!((e.provenance, e.coverage), (Provenance::HwShunt, Coverage::Whole), "a live RAPL span: {e:?}"); + let uj = e.measured_uj.expect("an HwShunt figure carries its measured figure"); + assert_eq!(e.estimated_uj, uj); + eprintln!("rapl burn: {cpu:?} of thread CPU in {:?}; energy {e:?}; {}", wall.elapsed(), oracle::assert_package_only(&b, uj)); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); +} diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs new file mode 100644 index 000000000..6f81934b4 --- /dev/null +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs @@ -0,0 +1,439 @@ +//! Linux RAPL against the hardware (feature `rapl`): this host's powercap +//! counters, read by the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the RAPL probe runs in this process, and +//! the tests take one lock: each burns CPU and reads machine-wide counters. +//! The oracle reads `/sys/class/powercap` itself — every zone's `name`, +//! `energy_uj` and `max_energy_range_uj` — and applies the sustained-advance +//! rule to the top-level `package-` zones on its own. None of these tests +//! passes without asserting: where the counters cannot be read (no powercap; +//! a CI container, whose `/sys/devices/virtual/powercap` Docker masks), the +//! liveness must say so and agree with the oracle; where they are live, both +//! must say live, the counter must be the meter's active one, and its +//! readings must be `HwShunt` over `CpuPkg` (the CPU package, not the GPU), +//! the figure over a CPU burn inside the oracle's package-only bracket and +//! clearly below package+psys and package+core. + +#![cfg(all(target_os = "linux", feature = "rapl"))] + +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use jouleclaw_energy::rapl::{rapl_liveness, RaplCounter}; +use jouleclaw_energy::{meter, CounterLiveness, EnergyCounter, EnergyDomain, Provenance}; + +/// One test at a time: each burns CPU and reads machine-wide counters. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The tests' own reading of `/sys/class/powercap`: every zone listed, its +/// `name`, `energy_uj` and `max_energy_range_uj`, read here and nowhere else, +/// and the sustained-advance verdict over the package zones by the family's +/// rule. Nothing in it comes from the crate under test. +#[allow(dead_code)] +mod oracle { + use std::os::raw::{c_int, c_long}; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// Where the kernel lists its powercap zones. + pub const POWERCAP: &str = "/sys/class/powercap"; + + #[repr(C)] + struct Timespec { + tv_sec: c_long, + tv_nsec: c_long, + } + + unsafe extern "C" { + fn clock_gettime(clock_id: c_int, tp: *mut Timespec) -> c_int; + } + + /// `CLOCK_THREAD_CPUTIME_ID` in Linux ``. + const CLOCK_THREAD_CPUTIME_ID: c_int = 3; + + /// This thread's CPU time. + pub fn thread_cpu() -> Duration { + let mut ts = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mut ts) }, 0, "clock_gettime"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// Spin this thread until it has used `d` of its own CPU time, or `wall_cap` + /// of wall time has passed. Returns the CPU it got. + pub fn burn(d: Duration, wall_cap: Duration) -> Duration { + let (started, wall) = (thread_cpu(), Instant::now()); + let mut acc: u64 = 0; + while thread_cpu() - started < d && wall.elapsed() < wall_cap { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + thread_cpu() - started + } + + /// One entry of the powercap directory. + #[derive(Debug, Clone)] + pub struct Zone { + /// The directory entry: `intel-rapl:0`, `intel-rapl:0:0`, `intel-rapl-mmio:0`, ... + pub dir: String, + pub name: Option, + pub energy_uj: Option, + pub max_uj: Option, + } + + fn text(p: PathBuf) -> Option { + std::fs::read_to_string(p).ok().map(|s| s.trim().to_string()) + } + + fn number(p: PathBuf) -> Option { + text(p)?.parse().ok() + } + + /// Every entry under `root`, in name order. + pub fn zones(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { return Vec::new() }; + let mut zones: Vec = entries + .flatten() + .map(|e| { + let dir = e.file_name().to_string_lossy().into_owned(); + let p = e.path(); + Zone { name: text(p.join("name")), energy_uj: number(p.join("energy_uj")), max_uj: number(p.join("max_energy_range_uj")), dir } + }) + .collect(); + zones.sort_by(|a, b| a.dir.cmp(&b.dir)); + zones + } + + fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) + } + + /// A top-level RAPL zone: `intel-rapl:`. Not a sub-zone, not MMIO. + pub fn top_level(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(digits) + } + + /// A package zone: top-level, named `package-`. + pub fn is_package(z: &Zone) -> bool { + top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + } + + /// The platform zone. + pub fn is_psys(z: &Zone) -> bool { + top_level(z) && z.name.as_deref() == Some("psys") + } + + /// A package's `core` sub-zone (`intel-rapl::` named `core`). + pub fn is_core(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(|r| r.contains(':')) && z.name.as_deref() == Some("core") + } + + /// One zone's energy between two reads, modulo its own range. + pub fn delta(e0: u64, e1: u64, max: Option) -> Option { + if e1 >= e0 { + Some(e1 - e0) + } else { + max.filter(|&m| m > 0)?.checked_sub(e0)?.checked_add(e1) + } + } + + /// `energy_uj` of each of `dirs` under `root`, in order; `None` if any fails. + pub fn read(root: &Path, dirs: &[&Zone]) -> Option> { + dirs.iter().map(|z| number(root.join(&z.dir).join("energy_uj"))).collect() + } + + /// The summed energy of `which` of `set` between two reads of `set`. + pub fn energy(set: &[&Zone], before: &[u64], after: &[u64], which: impl Fn(&Zone) -> bool) -> u64 { + set.iter() + .zip(before.iter().zip(after)) + .filter(|(z, _)| which(z)) + .map(|(z, (&b, &a))| delta(b, a, z.max_uj).expect("a zone fell with no readable range")) + .sum() + } + + /// What the oracle concluded about the package counters under a root. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No top-level zone's `energy_uj` reads (no powercap, a masked or + /// root-only tree). + Unreadable, + /// Top-level zones read, but none is a package. + NoPackage, + /// Every package read, but not every one kept advancing. + Frozen, + /// Every package advanced in at least five intervals, at least 80% of + /// them, over at least 300 ms. + Live, + } + + /// The oracle's sustained-advance run over the package zones under `root`. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub packages: Vec, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + } + + impl Watch { + pub fn summary(&self) -> String { + let per: Vec = + self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); + format!("{:?} ({} over {} ms)", self.verdict, per.join(", "), self.window.as_millis()) + } + } + + /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU + /// between samples (capped at 500 ms of wall time), until every package + /// has advanced in at least five intervals, at least 80% of those + /// observed, over at least 300 ms; or one second has passed with at least + /// five intervals; or five seconds have. + pub fn watch(root: &Path) -> Watch { + let all = zones(root); + let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); + let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; + if packages.is_empty() { + let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); + return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + } + let started = Instant::now(); + let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; packages.len()]; + let mut intervals = 0u32; + let live = |adv: &[u32], n: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, el) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + burn(Duration::from_millis(25), Duration::from_millis(500)); + if let Some(now) = read(root, &packages) { + intervals += 1; + for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(n > p); + } + prev = now; + } + } + let window = started.elapsed(); + let verdict = if live(&advances, intervals, window) { Verdict::Live } else { Verdict::Frozen }; + Watch { verdict, packages: names, intervals, advances, window } + } + + /// The zones under `root`, as one line for the log. + pub fn listing(root: &Path) -> String { + zones(root) + .iter() + .map(|z| { + let kind = if is_package(z) { + "package" + } else if is_psys(z) { + "psys" + } else if is_core(z) { + "core sub-zone" + } else if top_level(z) { + "top-level" + } else { + "not summed" + }; + let e = z.energy_uj.map_or("unreadable".to_string(), |e| e.to_string()); + let m = z.max_uj.map_or("-".to_string(), |m| m.to_string()); + format!("{}={} [{kind}] energy_uj={e} max={m}", z.dir, z.name.as_deref().unwrap_or("?")) + }) + .collect::>() + .join("; ") + } + + /// The CI container's view of the host's powercap directory: the same + /// entries, each a symlink into `/sys/devices/virtual/powercap`, which + /// Docker masks with an empty tmpfs, so every target is missing. Built + /// from this host's own listing; removed when dropped. + pub struct MaskedView(pub PathBuf); + + impl MaskedView { + pub fn of_host(tag: &str) -> MaskedView { + let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create the masked view"); + let masked = root.join("masked-devices-virtual-powercap"); + for z in zones(Path::new(POWERCAP)) { + std::os::unix::fs::symlink(masked.join(&z.dir), root.join(&z.dir)).expect("symlink"); + } + MaskedView(root) + } + } + + impl Drop for MaskedView { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A bracket of reads around the meter's two reads: every package, psys + /// and core zone read at four points, outer and inner on each side. + pub struct Bracket<'a> { + pub set: Vec<&'a Zone>, + pub reads: [Vec; 4], + } + + impl Bracket<'_> { + pub fn package_outer(&self) -> u64 { + energy(&self.set, &self.reads[0], &self.reads[3], is_package) + } + pub fn package_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_package) + } + pub fn psys_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_psys) + } + pub fn core_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_core) + } + pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { + self.set.iter().any(|z| which(z)) + } + } + + /// The family's rule on a figure `uj` the meter read between the + /// bracket's inner and outer reads: no less than the packages' energy + /// over the inner bracket, no more than over the outer one, and below + /// package+psys and package+core where those zones exist. A figure that + /// added a zone reads at least the packages' and that zone's energy over + /// the inner bracket, so `uj < inner + zone` is the double-count test; it + /// can only be put where the zone moved more over the burn than the + /// packages did in the gap between the outer and inner reads (else a + /// correct figure could reach it too), and the line says when it could + /// not. Returns the figures, for the log. + pub fn assert_package_only(b: &Bracket, uj: u64) -> String { + let (inner, outer, psys, core) = (b.package_inner(), b.package_outer(), b.psys_inner(), b.core_inner()); + let gap = outer - inner; + let mut line = format!( + "meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ (gap {gap}); psys inner {psys} µJ; \ + core inner {core} µJ" + ); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + for (zone, present, moved) in [("psys", b.has(is_psys), psys), ("core", b.has(is_core), core)] { + if !present { + continue; + } + assert!(moved > 0, "{zone} did not advance: {line}"); + if moved > gap { + assert!(uj < inner + moved, "the figure reaches package+{zone}: {zone} was added: {line}"); + line.push_str(&format!("; below package+{zone}")); + } else { + line.push_str(&format!("; {zone} moved less than the gap, so only the outer bound tests it")); + } + } + line + } + + /// Every package, psys and core zone of `all` whose counter reads. + pub fn bracket_set(all: &[Zone]) -> Vec<&Zone> { + all.iter().filter(|z| (is_package(z) || is_psys(z) || is_core(z)) && z.energy_uj.is_some()).collect() + } + + /// One oracle read of every zone of `set`; panics if one fails. + pub fn read_all(set: &[&Zone]) -> Vec { + read(Path::new(POWERCAP), set).expect("an oracle read of the bracket zones") + } + + /// Read the bracket around `meter_read` (called twice) and `work`: + /// outer, meter, inner, work, inner, meter, outer. + pub fn around<'a, M>(set: Vec<&'a Zone>, mut meter_read: impl FnMut() -> M, work: impl FnOnce()) -> (Bracket<'a>, M, M) { + let o0 = read_all(&set); + let m0 = meter_read(); + let o1 = read_all(&set); + work(); + let o2 = read_all(&set); + let m1 = meter_read(); + let o3 = read_all(&set); + (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) + } +} + +use oracle::Verdict; + +/// The liveness agrees with the oracle's own verdict, and only a live +/// counter is available and is the meter's active one. +#[test] +fn the_rapl_liveness_agrees_with_the_oracle_and_gates_the_counter() { + let _one = serial(); + let liveness = rapl_liveness(); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + let counter = RaplCounter::new(); + let composite = meter::detect_meter(); + eprintln!( + "rapl hardware: {}; oracle {}; liveness={liveness:?} available={} active={} meters={:?}", + oracle::listing(Path::new(oracle::POWERCAP)), + watch.summary(), + counter.is_available(), + composite.active_name(), + composite.names() + ); + assert_eq!(liveness == CounterLiveness::Live, watch.verdict == Verdict::Live, "the liveness disagrees with the oracle"); + match watch.verdict { + Verdict::Unreadable | Verdict::NoPackage => assert_eq!(liveness, CounterLiveness::Unavailable), + Verdict::Frozen => assert_eq!(liveness, CounterLiveness::Frozen), + Verdict::Live => {} + } + assert_eq!(counter.liveness(), liveness, "the counter was built on another verdict"); + assert_eq!(counter.is_available(), liveness == CounterLiveness::Live, "availability disagrees with the liveness"); + assert_eq!(composite.active_name() == "rapl", liveness == CounterLiveness::Live, "the meter's choice disagrees"); + if !counter.is_available() { + assert!(counter.read().is_err(), "a counter that is not live read"); + } +} + +/// Over a 300 ms one-thread CPU burn, the counter's readings are `HwShunt` +/// over `CpuPkg`, and its figure sits inside the oracle's package-only +/// bracket and clearly below package+psys and package+core. +#[test] +fn a_cpu_burn_reads_inside_the_package_only_bracket() { + let _one = serial(); + let counter = RaplCounter::new(); + if !counter.is_available() { + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + eprintln!("rapl burn: counter {:?}; oracle {}", counter.liveness(), watch.summary()); + assert_ne!(watch.verdict, Verdict::Live, "the oracle saw live packages, yet the counter is not available"); + assert!(counter.read().is_err()); + return; + } + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let mut cpu = Duration::ZERO; + let (b, r0, r1) = oracle::around( + oracle::bracket_set(&all), + || counter.read().expect("a live counter reads"), + || cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)), + ); + for r in [&r0, &r1] { + assert_eq!((r.domain, r.provenance), (EnergyDomain::CpuPkg, Provenance::HwShunt), "{r:?}"); + } + eprintln!("rapl burn: {cpu:?} of thread CPU; {}", oracle::assert_package_only(&b, r1.uj - r0.uj)); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); +} + +/// What a CI container sees: Docker masks `/sys/devices/virtual/powercap` +/// with an empty tmpfs, so every entry of `/sys/class/powercap` is a symlink +/// to nothing. On that view of this host's own listing the counter is +/// unavailable and reads nothing, and the oracle agrees. +#[test] +fn the_ci_containers_masked_view_is_unavailable() { + let _one = serial(); + let view = oracle::MaskedView::of_host("jouleclaw-energy"); + let watch = oracle::watch(&view.0); + let counter = RaplCounter::with_root(&view.0); + eprintln!("rapl masked view: oracle {}; counter {:?} available={}", watch.summary(), counter.liveness(), counter.is_available()); + assert!(matches!(watch.verdict, Verdict::Unreadable | Verdict::NoPackage), "{watch:?}"); + assert_eq!((counter.liveness(), counter.is_available()), (CounterLiveness::Unavailable, false)); + assert!(counter.read().is_err()); +} diff --git a/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs b/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs new file mode 100644 index 000000000..3ccaa79ee --- /dev/null +++ b/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs @@ -0,0 +1,458 @@ +//! Linux RAPL against the hardware: this host's powercap counters, read by +//! the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the RAPL probe behind `rapl_status()` +//! runs in this process, and the tests take one lock: each burns CPU and +//! reads machine-wide counters. The oracle reads `/sys/class/powercap` +//! itself — every zone's `name`, `energy_uj` and `max_energy_range_uj` — and +//! applies the sustained-advance rule to the top-level `package-` zones +//! on its own. None of these tests passes without asserting: where the +//! counters cannot be read (no powercap; a CI container, whose +//! `/sys/devices/virtual/powercap` Docker masks), the status must say so and +//! agree with the oracle; where they are live, both must say live, RAPL must +//! be the meter's source, and the marginal figure of a burn on every spare +//! core must be `HwShunt`, claim the CPU package and not the GPU, and, summed +//! over the calls, fit inside the oracle's package-only bracket around the +//! whole measurement (idle and busy windows), which a figure that added +//! psys or a package's core sub-zone overruns. + +#![cfg(target_os = "linux")] + +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use proof_cascade::meter::{self, rapl_status, CounterStatus, MeterSource}; +use proof_schema::Provenance; + +/// One test at a time: each burns CPU and reads machine-wide counters. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The tests' own reading of `/sys/class/powercap`: every zone listed, its +/// `name`, `energy_uj` and `max_energy_range_uj`, read here and nowhere else, +/// and the sustained-advance verdict over the package zones by the family's +/// rule. Nothing in it comes from the crate under test. +#[allow(dead_code)] +mod oracle { + use std::os::raw::{c_int, c_long}; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// Where the kernel lists its powercap zones. + pub const POWERCAP: &str = "/sys/class/powercap"; + + #[repr(C)] + struct Timespec { + tv_sec: c_long, + tv_nsec: c_long, + } + + unsafe extern "C" { + fn clock_gettime(clock_id: c_int, tp: *mut Timespec) -> c_int; + } + + /// `CLOCK_THREAD_CPUTIME_ID` in Linux ``. + const CLOCK_THREAD_CPUTIME_ID: c_int = 3; + + /// This thread's CPU time. + pub fn thread_cpu() -> Duration { + let mut ts = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mut ts) }, 0, "clock_gettime"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// Spin this thread until it has used `d` of its own CPU time, or `wall_cap` + /// of wall time has passed. Returns the CPU it got. + pub fn burn(d: Duration, wall_cap: Duration) -> Duration { + let (started, wall) = (thread_cpu(), Instant::now()); + let mut acc: u64 = 0; + while thread_cpu() - started < d && wall.elapsed() < wall_cap { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + thread_cpu() - started + } + + /// One entry of the powercap directory. + #[derive(Debug, Clone)] + pub struct Zone { + /// The directory entry: `intel-rapl:0`, `intel-rapl:0:0`, `intel-rapl-mmio:0`, ... + pub dir: String, + pub name: Option, + pub energy_uj: Option, + pub max_uj: Option, + } + + fn text(p: PathBuf) -> Option { + std::fs::read_to_string(p).ok().map(|s| s.trim().to_string()) + } + + fn number(p: PathBuf) -> Option { + text(p)?.parse().ok() + } + + /// Every entry under `root`, in name order. + pub fn zones(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { return Vec::new() }; + let mut zones: Vec = entries + .flatten() + .map(|e| { + let dir = e.file_name().to_string_lossy().into_owned(); + let p = e.path(); + Zone { name: text(p.join("name")), energy_uj: number(p.join("energy_uj")), max_uj: number(p.join("max_energy_range_uj")), dir } + }) + .collect(); + zones.sort_by(|a, b| a.dir.cmp(&b.dir)); + zones + } + + fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) + } + + /// A top-level RAPL zone: `intel-rapl:`. Not a sub-zone, not MMIO. + pub fn top_level(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(digits) + } + + /// A package zone: top-level, named `package-`. + pub fn is_package(z: &Zone) -> bool { + top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + } + + /// The platform zone. + pub fn is_psys(z: &Zone) -> bool { + top_level(z) && z.name.as_deref() == Some("psys") + } + + /// A package's `core` sub-zone (`intel-rapl::` named `core`). + pub fn is_core(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(|r| r.contains(':')) && z.name.as_deref() == Some("core") + } + + /// One zone's energy between two reads, modulo its own range. + pub fn delta(e0: u64, e1: u64, max: Option) -> Option { + if e1 >= e0 { + Some(e1 - e0) + } else { + max.filter(|&m| m > 0)?.checked_sub(e0)?.checked_add(e1) + } + } + + /// `energy_uj` of each of `dirs` under `root`, in order; `None` if any fails. + pub fn read(root: &Path, dirs: &[&Zone]) -> Option> { + dirs.iter().map(|z| number(root.join(&z.dir).join("energy_uj"))).collect() + } + + /// The summed energy of `which` of `set` between two reads of `set`. + pub fn energy(set: &[&Zone], before: &[u64], after: &[u64], which: impl Fn(&Zone) -> bool) -> u64 { + set.iter() + .zip(before.iter().zip(after)) + .filter(|(z, _)| which(z)) + .map(|(z, (&b, &a))| delta(b, a, z.max_uj).expect("a zone fell with no readable range")) + .sum() + } + + /// What the oracle concluded about the package counters under a root. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No top-level zone's `energy_uj` reads (no powercap, a masked or + /// root-only tree). + Unreadable, + /// Top-level zones read, but none is a package. + NoPackage, + /// Every package read, but not every one kept advancing. + Frozen, + /// Every package advanced in at least five intervals, at least 80% of + /// them, over at least 300 ms. + Live, + } + + /// The oracle's sustained-advance run over the package zones under `root`. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub packages: Vec, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + } + + impl Watch { + pub fn summary(&self) -> String { + let per: Vec = + self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); + format!("{:?} ({} over {} ms)", self.verdict, per.join(", "), self.window.as_millis()) + } + } + + /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU + /// between samples (capped at 500 ms of wall time), until every package + /// has advanced in at least five intervals, at least 80% of those + /// observed, over at least 300 ms; or one second has passed with at least + /// five intervals; or five seconds have. + pub fn watch(root: &Path) -> Watch { + let all = zones(root); + let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); + let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; + if packages.is_empty() { + let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); + return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + } + let started = Instant::now(); + let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; packages.len()]; + let mut intervals = 0u32; + let live = |adv: &[u32], n: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, el) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + burn(Duration::from_millis(25), Duration::from_millis(500)); + if let Some(now) = read(root, &packages) { + intervals += 1; + for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(n > p); + } + prev = now; + } + } + let window = started.elapsed(); + let verdict = if live(&advances, intervals, window) { Verdict::Live } else { Verdict::Frozen }; + Watch { verdict, packages: names, intervals, advances, window } + } + + /// The zones under `root`, as one line for the log. + pub fn listing(root: &Path) -> String { + zones(root) + .iter() + .map(|z| { + let kind = if is_package(z) { + "package" + } else if is_psys(z) { + "psys" + } else if is_core(z) { + "core sub-zone" + } else if top_level(z) { + "top-level" + } else { + "not summed" + }; + let e = z.energy_uj.map_or("unreadable".to_string(), |e| e.to_string()); + let m = z.max_uj.map_or("-".to_string(), |m| m.to_string()); + format!("{}={} [{kind}] energy_uj={e} max={m}", z.dir, z.name.as_deref().unwrap_or("?")) + }) + .collect::>() + .join("; ") + } + + /// The CI container's view of the host's powercap directory: the same + /// entries, each a symlink into `/sys/devices/virtual/powercap`, which + /// Docker masks with an empty tmpfs, so every target is missing. Built + /// from this host's own listing; removed when dropped. + pub struct MaskedView(pub PathBuf); + + impl MaskedView { + pub fn of_host(tag: &str) -> MaskedView { + let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create the masked view"); + let masked = root.join("masked-devices-virtual-powercap"); + for z in zones(Path::new(POWERCAP)) { + std::os::unix::fs::symlink(masked.join(&z.dir), root.join(&z.dir)).expect("symlink"); + } + MaskedView(root) + } + } + + impl Drop for MaskedView { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A bracket of reads around the meter's two reads: every package, psys + /// and core zone read at four points, outer and inner on each side. + pub struct Bracket<'a> { + pub set: Vec<&'a Zone>, + pub reads: [Vec; 4], + } + + impl Bracket<'_> { + pub fn package_outer(&self) -> u64 { + energy(&self.set, &self.reads[0], &self.reads[3], is_package) + } + pub fn package_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_package) + } + pub fn psys_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_psys) + } + pub fn core_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_core) + } + pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { + self.set.iter().any(|z| which(z)) + } + } + + /// The family's rule on a figure `uj` the meter read between the + /// bracket's inner and outer reads: no less than the packages' energy + /// over the inner bracket, no more than over the outer one, and below + /// package+psys and package+core where those zones exist. A figure that + /// added a zone reads at least the packages' and that zone's energy over + /// the inner bracket, so `uj < inner + zone` is the double-count test; it + /// can only be put where the zone moved more over the burn than the + /// packages did in the gap between the outer and inner reads (else a + /// correct figure could reach it too), and the line says when it could + /// not. Returns the figures, for the log. + pub fn assert_package_only(b: &Bracket, uj: u64) -> String { + let (inner, outer, psys, core) = (b.package_inner(), b.package_outer(), b.psys_inner(), b.core_inner()); + let gap = outer - inner; + let mut line = format!( + "meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ (gap {gap}); psys inner {psys} µJ; \ + core inner {core} µJ" + ); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + for (zone, present, moved) in [("psys", b.has(is_psys), psys), ("core", b.has(is_core), core)] { + if !present { + continue; + } + assert!(moved > 0, "{zone} did not advance: {line}"); + if moved > gap { + assert!(uj < inner + moved, "the figure reaches package+{zone}: {zone} was added: {line}"); + line.push_str(&format!("; below package+{zone}")); + } else { + line.push_str(&format!("; {zone} moved less than the gap, so only the outer bound tests it")); + } + } + line + } + + /// Every package, psys and core zone of `all` whose counter reads. + pub fn bracket_set(all: &[Zone]) -> Vec<&Zone> { + all.iter().filter(|z| (is_package(z) || is_psys(z) || is_core(z)) && z.energy_uj.is_some()).collect() + } + + /// One oracle read of every zone of `set`; panics if one fails. + pub fn read_all(set: &[&Zone]) -> Vec { + read(Path::new(POWERCAP), set).expect("an oracle read of the bracket zones") + } + + /// Read the bracket around `meter_read` (called twice) and `work`: + /// outer, meter, inner, work, inner, meter, outer. + pub fn around<'a, M>(set: Vec<&'a Zone>, mut meter_read: impl FnMut() -> M, work: impl FnOnce()) -> (Bracket<'a>, M, M) { + let o0 = read_all(&set); + let m0 = meter_read(); + let o1 = read_all(&set); + work(); + let o2 = read_all(&set); + let m1 = meter_read(); + let o3 = read_all(&set); + (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) + } +} + +use oracle::Verdict; + +/// The status agrees with the oracle's own verdict, and RAPL is the meter's +/// source exactly when it is live. +#[test] +fn the_rapl_status_agrees_with_the_oracle_and_gates_the_source() { + let _one = serial(); + let status = rapl_status(); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + let source = meter::meter_source(); + eprintln!( + "rapl hardware: {}; oracle {}; status={status:?} source={source:?}", + oracle::listing(Path::new(oracle::POWERCAP)), + watch.summary() + ); + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unreadable => assert_eq!(status, CounterStatus::Unavailable), + Verdict::NoPackage => assert_eq!(status, CounterStatus::Absent), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Live => {} + } + assert_eq!(source == Some(MeterSource::Rapl), status == CounterStatus::Live, "selection disagrees with the status"); + assert_eq!(meter::hardware_meter_available(), status == CounterStatus::Live); +} + +/// Calls of [`ten_ms_of_work`] completed in this process. +static CALLS: AtomicU64 = AtomicU64::new(0); + +/// About 10 ms of one thread's CPU work, counted. +fn ten_ms_of_work() { + let t = Instant::now(); + let mut acc: u64 = 0; + while t.elapsed() < Duration::from_millis(10) { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + CALLS.fetch_add(1, Ordering::SeqCst); +} + +/// A burn on every spare core meters as a hardware figure of the CPU +/// package: `HwShunt`, from RAPL, claiming package energy and not the GPU. +/// The figure is a marginal per call (busy power over idle, times the busy +/// window, over the calls), so the calls' total can be no more than the +/// packages' energy over the whole measurement, which the oracle reads +/// around it; a figure that also summed a package's core sub-zone (or psys, +/// where psys tracks load) would be about twice the busy margin and overrun +/// it. Where RAPL is not live there is no figure, and `work` never runs. +#[test] +fn an_all_core_burn_meters_inside_the_package_only_bracket() { + let _one = serial(); + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let set = oracle::bracket_set(&all); + if meter::meter_source() != Some(MeterSource::Rapl) { + let before = CALLS.load(Ordering::SeqCst); + let m = meter::measure_marginal(ten_ms_of_work); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + eprintln!("rapl burn: no RAPL source; figure {m:?}; oracle {}", watch.summary()); + assert_eq!(m, None, "a figure with RAPL declined"); + assert_eq!(CALLS.load(Ordering::SeqCst), before, "the work ran with RAPL declined"); + assert_ne!(watch.verdict, Verdict::Live, "the oracle saw live packages, yet RAPL is not the source"); + return; + } + let wall = Instant::now(); + let c0 = CALLS.load(Ordering::SeqCst); + let o0 = oracle::read_all(&set); + let m = meter::measure_marginal(ten_ms_of_work).expect("a live RAPL source metered the burn"); + let o3 = oracle::read_all(&set); + let calls = CALLS.load(Ordering::SeqCst) - c0; + let outer = oracle::energy(&set, &o0, &o3, oracle::is_package); + let core = oracle::energy(&set, &o0, &o3, oracle::is_core); + let psys = oracle::energy(&set, &o0, &o3, oracle::is_psys); + let total = m.uj * calls; + let energy = m.energy(); + eprintln!( + "rapl burn: {calls} calls in {:?}; {} µJ a call, {total} µJ over the calls; oracle package-only {outer} µJ \ + around the measurement (idle and busy), core {core} µJ, psys {psys} µJ; energy {energy:?}; domain {:?}", + wall.elapsed(), + m.uj, + m.source.domain() + ); + assert_eq!(m.source, MeterSource::Rapl); + assert_eq!(energy.provenance, Provenance::HwShunt); + assert_eq!(energy.measured_uj, Some(m.uj)); + let domain = m.source.domain(); + assert!(domain.contains("package") && !domain.contains("GPU"), "the RAPL figure claims {domain}"); + assert!(calls > 0 && m.uj > 0, "an all-core burn metered {} µJ a call over {calls} calls", m.uj); + assert!(outer > 0, "the packages did not advance"); + // Each call's figure is rounded to the microjoule: half a microjoule a call. + assert!(total <= outer + calls / 2 + 1, "{total} µJ over the calls overruns the packages' {outer} µJ"); +} diff --git a/sandbox/sandbox-rs/crates/sandbox-exec-microvm/tests/rapl_hardware.rs b/sandbox/sandbox-rs/crates/sandbox-exec-microvm/tests/rapl_hardware.rs new file mode 100644 index 000000000..e8ebf7e79 --- /dev/null +++ b/sandbox/sandbox-rs/crates/sandbox-exec-microvm/tests/rapl_hardware.rs @@ -0,0 +1,436 @@ +//! The S5 executor's CPU energy meters against the hardware: RAPL through +//! powercap (`RaplMeter`, sandbox-meter's package counters) and through the +//! MSRs (`MsrRaplMeter`), read beside an oracle the tests declare here. +//! +//! In a test binary of its own, so the probes run in this process, and the +//! tests take one lock: each burns CPU and reads machine-wide counters. The +//! oracle reads `/sys/class/powercap` itself and applies the sustained- +//! advance rule to the top-level `package-` zones on its own; for the +//! MSRs it tries to read `MSR_PKG_ENERGY_STATUS` from `/dev/cpu/0/msr` +//! itself. None of these tests passes without asserting: where a counter +//! cannot be read the meter must say so and the oracle agree; where it is +//! live, the meter's figure over a CPU burn must sit inside the oracle's +//! package-only bracket. + +#![cfg(target_os = "linux")] + +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use sandbox_exec_microvm::power::{MsrRaplMeter, PowerMeter, RaplMeter}; + +/// One test at a time: each burns CPU and reads machine-wide counters. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The tests' own reading of `/sys/class/powercap`: every zone listed, its +/// `name`, `energy_uj` and `max_energy_range_uj`, read here and nowhere else, +/// and the sustained-advance verdict over the package zones by the family's +/// rule. Nothing in it comes from the crate under test. +#[allow(dead_code)] +mod oracle { + use std::os::raw::{c_int, c_long}; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// Where the kernel lists its powercap zones. + pub const POWERCAP: &str = "/sys/class/powercap"; + + #[repr(C)] + struct Timespec { + tv_sec: c_long, + tv_nsec: c_long, + } + + unsafe extern "C" { + fn clock_gettime(clock_id: c_int, tp: *mut Timespec) -> c_int; + } + + /// `CLOCK_THREAD_CPUTIME_ID` in Linux ``. + const CLOCK_THREAD_CPUTIME_ID: c_int = 3; + + /// This thread's CPU time. + pub fn thread_cpu() -> Duration { + let mut ts = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mut ts) }, 0, "clock_gettime"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// Spin this thread until it has used `d` of its own CPU time, or `wall_cap` + /// of wall time has passed. Returns the CPU it got. + pub fn burn(d: Duration, wall_cap: Duration) -> Duration { + let (started, wall) = (thread_cpu(), Instant::now()); + let mut acc: u64 = 0; + while thread_cpu() - started < d && wall.elapsed() < wall_cap { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + thread_cpu() - started + } + + /// One entry of the powercap directory. + #[derive(Debug, Clone)] + pub struct Zone { + /// The directory entry: `intel-rapl:0`, `intel-rapl:0:0`, `intel-rapl-mmio:0`, ... + pub dir: String, + pub name: Option, + pub energy_uj: Option, + pub max_uj: Option, + } + + fn text(p: PathBuf) -> Option { + std::fs::read_to_string(p).ok().map(|s| s.trim().to_string()) + } + + fn number(p: PathBuf) -> Option { + text(p)?.parse().ok() + } + + /// Every entry under `root`, in name order. + pub fn zones(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { return Vec::new() }; + let mut zones: Vec = entries + .flatten() + .map(|e| { + let dir = e.file_name().to_string_lossy().into_owned(); + let p = e.path(); + Zone { name: text(p.join("name")), energy_uj: number(p.join("energy_uj")), max_uj: number(p.join("max_energy_range_uj")), dir } + }) + .collect(); + zones.sort_by(|a, b| a.dir.cmp(&b.dir)); + zones + } + + fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) + } + + /// A top-level RAPL zone: `intel-rapl:`. Not a sub-zone, not MMIO. + pub fn top_level(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(digits) + } + + /// A package zone: top-level, named `package-`. + pub fn is_package(z: &Zone) -> bool { + top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + } + + /// The platform zone. + pub fn is_psys(z: &Zone) -> bool { + top_level(z) && z.name.as_deref() == Some("psys") + } + + /// A package's `core` sub-zone (`intel-rapl::` named `core`). + pub fn is_core(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(|r| r.contains(':')) && z.name.as_deref() == Some("core") + } + + /// One zone's energy between two reads, modulo its own range. + pub fn delta(e0: u64, e1: u64, max: Option) -> Option { + if e1 >= e0 { + Some(e1 - e0) + } else { + max.filter(|&m| m > 0)?.checked_sub(e0)?.checked_add(e1) + } + } + + /// `energy_uj` of each of `dirs` under `root`, in order; `None` if any fails. + pub fn read(root: &Path, dirs: &[&Zone]) -> Option> { + dirs.iter().map(|z| number(root.join(&z.dir).join("energy_uj"))).collect() + } + + /// The summed energy of `which` of `set` between two reads of `set`. + pub fn energy(set: &[&Zone], before: &[u64], after: &[u64], which: impl Fn(&Zone) -> bool) -> u64 { + set.iter() + .zip(before.iter().zip(after)) + .filter(|(z, _)| which(z)) + .map(|(z, (&b, &a))| delta(b, a, z.max_uj).expect("a zone fell with no readable range")) + .sum() + } + + /// What the oracle concluded about the package counters under a root. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No top-level zone's `energy_uj` reads (no powercap, a masked or + /// root-only tree). + Unreadable, + /// Top-level zones read, but none is a package. + NoPackage, + /// Every package read, but not every one kept advancing. + Frozen, + /// Every package advanced in at least five intervals, at least 80% of + /// them, over at least 300 ms. + Live, + } + + /// The oracle's sustained-advance run over the package zones under `root`. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub packages: Vec, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + } + + impl Watch { + pub fn summary(&self) -> String { + let per: Vec = + self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); + format!("{:?} ({} over {} ms)", self.verdict, per.join(", "), self.window.as_millis()) + } + } + + /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU + /// between samples (capped at 500 ms of wall time), until every package + /// has advanced in at least five intervals, at least 80% of those + /// observed, over at least 300 ms; or one second has passed with at least + /// five intervals; or five seconds have. + pub fn watch(root: &Path) -> Watch { + let all = zones(root); + let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); + let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; + if packages.is_empty() { + let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); + return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + } + let started = Instant::now(); + let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; packages.len()]; + let mut intervals = 0u32; + let live = |adv: &[u32], n: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, el) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + burn(Duration::from_millis(25), Duration::from_millis(500)); + if let Some(now) = read(root, &packages) { + intervals += 1; + for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(n > p); + } + prev = now; + } + } + let window = started.elapsed(); + let verdict = if live(&advances, intervals, window) { Verdict::Live } else { Verdict::Frozen }; + Watch { verdict, packages: names, intervals, advances, window } + } + + /// The zones under `root`, as one line for the log. + pub fn listing(root: &Path) -> String { + zones(root) + .iter() + .map(|z| { + let kind = if is_package(z) { + "package" + } else if is_psys(z) { + "psys" + } else if is_core(z) { + "core sub-zone" + } else if top_level(z) { + "top-level" + } else { + "not summed" + }; + let e = z.energy_uj.map_or("unreadable".to_string(), |e| e.to_string()); + let m = z.max_uj.map_or("-".to_string(), |m| m.to_string()); + format!("{}={} [{kind}] energy_uj={e} max={m}", z.dir, z.name.as_deref().unwrap_or("?")) + }) + .collect::>() + .join("; ") + } + + /// The CI container's view of the host's powercap directory: the same + /// entries, each a symlink into `/sys/devices/virtual/powercap`, which + /// Docker masks with an empty tmpfs, so every target is missing. Built + /// from this host's own listing; removed when dropped. + pub struct MaskedView(pub PathBuf); + + impl MaskedView { + pub fn of_host(tag: &str) -> MaskedView { + let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create the masked view"); + let masked = root.join("masked-devices-virtual-powercap"); + for z in zones(Path::new(POWERCAP)) { + std::os::unix::fs::symlink(masked.join(&z.dir), root.join(&z.dir)).expect("symlink"); + } + MaskedView(root) + } + } + + impl Drop for MaskedView { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A bracket of reads around the meter's two reads: every package, psys + /// and core zone read at four points, outer and inner on each side. + pub struct Bracket<'a> { + pub set: Vec<&'a Zone>, + pub reads: [Vec; 4], + } + + impl Bracket<'_> { + pub fn package_outer(&self) -> u64 { + energy(&self.set, &self.reads[0], &self.reads[3], is_package) + } + pub fn package_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_package) + } + pub fn psys_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_psys) + } + pub fn core_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_core) + } + pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { + self.set.iter().any(|z| which(z)) + } + } + + /// The family's rule on a figure `uj` the meter read between the + /// bracket's inner and outer reads: no less than the packages' energy + /// over the inner bracket, no more than over the outer one, and below + /// package+psys and package+core where those zones exist. A figure that + /// added a zone reads at least the packages' and that zone's energy over + /// the inner bracket, so `uj < inner + zone` is the double-count test; it + /// can only be put where the zone moved more over the burn than the + /// packages did in the gap between the outer and inner reads (else a + /// correct figure could reach it too), and the line says when it could + /// not. Returns the figures, for the log. + pub fn assert_package_only(b: &Bracket, uj: u64) -> String { + let (inner, outer, psys, core) = (b.package_inner(), b.package_outer(), b.psys_inner(), b.core_inner()); + let gap = outer - inner; + let mut line = format!( + "meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ (gap {gap}); psys inner {psys} µJ; \ + core inner {core} µJ" + ); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + for (zone, present, moved) in [("psys", b.has(is_psys), psys), ("core", b.has(is_core), core)] { + if !present { + continue; + } + assert!(moved > 0, "{zone} did not advance: {line}"); + if moved > gap { + assert!(uj < inner + moved, "the figure reaches package+{zone}: {zone} was added: {line}"); + line.push_str(&format!("; below package+{zone}")); + } else { + line.push_str(&format!("; {zone} moved less than the gap, so only the outer bound tests it")); + } + } + line + } + + /// Every package, psys and core zone of `all` whose counter reads. + pub fn bracket_set(all: &[Zone]) -> Vec<&Zone> { + all.iter().filter(|z| (is_package(z) || is_psys(z) || is_core(z)) && z.energy_uj.is_some()).collect() + } + + /// One oracle read of every zone of `set`; panics if one fails. + pub fn read_all(set: &[&Zone]) -> Vec { + read(Path::new(POWERCAP), set).expect("an oracle read of the bracket zones") + } + + /// Read the bracket around `meter_read` (called twice) and `work`: + /// outer, meter, inner, work, inner, meter, outer. + pub fn around<'a, M>(set: Vec<&'a Zone>, mut meter_read: impl FnMut() -> M, work: impl FnOnce()) -> (Bracket<'a>, M, M) { + let o0 = read_all(&set); + let m0 = meter_read(); + let o1 = read_all(&set); + work(); + let o2 = read_all(&set); + let m1 = meter_read(); + let o3 = read_all(&set); + (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) + } +} + +use oracle::Verdict; + +/// The powercap meter opens exactly where the oracle finds every package +/// live, and its figure over a 300 ms burn sits inside the oracle's +/// package-only bracket and clearly below package+psys and package+core. +#[test] +fn the_powercap_meter_opens_where_the_packages_are_live_and_reads_them() { + let _one = serial(); + let meter = RaplMeter::host(); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + eprintln!( + "rapl hardware: {}; oracle {}; RaplMeter unavailable={:?}", + oracle::listing(Path::new(oracle::POWERCAP)), + watch.summary(), + meter.unavailable() + ); + assert_eq!(meter.unavailable().is_none(), watch.verdict == Verdict::Live, "the meter disagrees with the oracle"); + if meter.unavailable().is_some() { + assert_eq!(meter.sample_uj(), None, "a meter that did not open sampled"); + return; + } + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let mut cpu = Duration::ZERO; + let (b, m0, m1) = oracle::around( + oracle::bracket_set(&all), + || meter.sample_uj().expect("an open meter samples"), + || cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)), + ); + eprintln!("rapl burn: {cpu:?} of thread CPU; {}", oracle::assert_package_only(&b, m1 - m0)); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); +} + +/// `MSR_PKG_ENERGY_STATUS` read directly, the oracle's own way: `pread` of +/// 8 bytes at offset 0x611 of `/dev/cpu/0/msr`. `None` where it cannot be +/// read (no `msr` module, or not root: `CAP_SYS_RAWIO`). +fn oracle_msr_reads() -> Option { + use std::os::unix::fs::FileExt; + let f = std::fs::File::open("/dev/cpu/0/msr").ok()?; + let mut buf = [0u8; 8]; + f.read_exact_at(&mut buf, 0x611).ok()?; + Some(u64::from_le_bytes(buf)) +} + +/// The MSR meter opens only where the MSRs read: where the oracle cannot read +/// `MSR_PKG_ENERGY_STATUS` (an unprivileged process), the meter says why and +/// samples nothing; where it can, the meter opens and its figure over a 300 ms +/// burn sits inside the powercap packages' bracket, to a count's rounding. +#[test] +fn the_msr_meter_opens_only_where_the_msrs_read() { + let _one = serial(); + let msr = oracle_msr_reads(); + let meter = MsrRaplMeter::host(); + eprintln!( + "rapl msr hardware: /dev/cpu/0/msr {:?}; oracle MSR_PKG_ENERGY_STATUS {msr:?}; MsrRaplMeter {:?}", + std::fs::metadata("/dev/cpu/0/msr").map(|m| m.len()).map_err(|e| e.kind()), + meter.as_ref().map(|m| m.domains()).map_err(String::as_str) + ); + let Some(_) = msr else { + assert!(meter.is_err(), "the MSR meter opened where the oracle cannot read the MSR"); + return; + }; + let meter = meter.expect("the MSRs read, yet the MSR meter did not open"); + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let (b, m0, m1) = oracle::around( + oracle::bracket_set(&all), + || meter.sample_uj().expect("an open MSR meter samples"), + || { + oracle::burn(Duration::from_millis(300), Duration::from_secs(10)); + }, + ); + let uj = m1 - m0; + let (inner, outer) = (b.package_inner(), b.package_outer()); + eprintln!("rapl msr burn: meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ"); + // One count of the energy unit (2^-14 J, ~61 µJ) either side. + assert!(inner.saturating_sub(123) <= uj && uj <= outer + 123, "{uj} µJ outside [{inner}, {outer}]"); +} diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/tests/rapl_hardware.rs b/sandbox/sandbox-rs/crates/sandbox-meter/tests/rapl_hardware.rs new file mode 100644 index 000000000..08dd0bd91 --- /dev/null +++ b/sandbox/sandbox-rs/crates/sandbox-meter/tests/rapl_hardware.rs @@ -0,0 +1,482 @@ +//! Linux RAPL against the hardware: this host's powercap counters, read by +//! the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the RAPL probe behind `rapl_status()` +//! runs in this process, and the tests take one lock: each burns CPU and +//! reads machine-wide counters. The oracle reads `/sys/class/powercap` +//! itself — every zone's `name`, `energy_uj` and `max_energy_range_uj` — and +//! applies the sustained-advance rule to the top-level `package-` zones +//! on its own. None of these tests passes without asserting: where the +//! counters cannot be read (no powercap; a CI container, whose +//! `/sys/devices/virtual/powercap` Docker masks), the status must say so and +//! agree with the oracle; where they are live, both must say live, and a +//! span over a CPU burn must be one `Cpu` record, `HwShunt`, whose figure +//! sits inside the oracle's package-only bracket and clearly below +//! package+psys and package+core. + +#![cfg(target_os = "linux")] + +use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use sandbox_meter::probe::{rapl_status, CounterStatus, PowerMeter, RaplPackages}; +use sandbox_meter::Provenance; +use sandbox_schema::DeviceClass; + +/// One test at a time: each burns CPU and reads machine-wide counters. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The tests' own reading of `/sys/class/powercap`: every zone listed, its +/// `name`, `energy_uj` and `max_energy_range_uj`, read here and nowhere else, +/// and the sustained-advance verdict over the package zones by the family's +/// rule. Nothing in it comes from the crate under test. +#[allow(dead_code)] +mod oracle { + use std::os::raw::{c_int, c_long}; + use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// Where the kernel lists its powercap zones. + pub const POWERCAP: &str = "/sys/class/powercap"; + + #[repr(C)] + struct Timespec { + tv_sec: c_long, + tv_nsec: c_long, + } + + unsafe extern "C" { + fn clock_gettime(clock_id: c_int, tp: *mut Timespec) -> c_int; + } + + /// `CLOCK_THREAD_CPUTIME_ID` in Linux ``. + const CLOCK_THREAD_CPUTIME_ID: c_int = 3; + + /// This thread's CPU time. + pub fn thread_cpu() -> Duration { + let mut ts = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(CLOCK_THREAD_CPUTIME_ID, &mut ts) }, 0, "clock_gettime"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// Spin this thread until it has used `d` of its own CPU time, or `wall_cap` + /// of wall time has passed. Returns the CPU it got. + pub fn burn(d: Duration, wall_cap: Duration) -> Duration { + let (started, wall) = (thread_cpu(), Instant::now()); + let mut acc: u64 = 0; + while thread_cpu() - started < d && wall.elapsed() < wall_cap { + for _ in 0..1_000 { + acc = std::hint::black_box(acc.wrapping_add(1)); + } + } + thread_cpu() - started + } + + /// One entry of the powercap directory. + #[derive(Debug, Clone)] + pub struct Zone { + /// The directory entry: `intel-rapl:0`, `intel-rapl:0:0`, `intel-rapl-mmio:0`, ... + pub dir: String, + pub name: Option, + pub energy_uj: Option, + pub max_uj: Option, + } + + fn text(p: PathBuf) -> Option { + std::fs::read_to_string(p).ok().map(|s| s.trim().to_string()) + } + + fn number(p: PathBuf) -> Option { + text(p)?.parse().ok() + } + + /// Every entry under `root`, in name order. + pub fn zones(root: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(root) else { return Vec::new() }; + let mut zones: Vec = entries + .flatten() + .map(|e| { + let dir = e.file_name().to_string_lossy().into_owned(); + let p = e.path(); + Zone { name: text(p.join("name")), energy_uj: number(p.join("energy_uj")), max_uj: number(p.join("max_energy_range_uj")), dir } + }) + .collect(); + zones.sort_by(|a, b| a.dir.cmp(&b.dir)); + zones + } + + fn digits(s: &str) -> bool { + !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()) + } + + /// A top-level RAPL zone: `intel-rapl:`. Not a sub-zone, not MMIO. + pub fn top_level(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(digits) + } + + /// A package zone: top-level, named `package-`. + pub fn is_package(z: &Zone) -> bool { + top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + } + + /// The platform zone. + pub fn is_psys(z: &Zone) -> bool { + top_level(z) && z.name.as_deref() == Some("psys") + } + + /// A package's `core` sub-zone (`intel-rapl::` named `core`). + pub fn is_core(z: &Zone) -> bool { + z.dir.strip_prefix("intel-rapl:").is_some_and(|r| r.contains(':')) && z.name.as_deref() == Some("core") + } + + /// One zone's energy between two reads, modulo its own range. + pub fn delta(e0: u64, e1: u64, max: Option) -> Option { + if e1 >= e0 { + Some(e1 - e0) + } else { + max.filter(|&m| m > 0)?.checked_sub(e0)?.checked_add(e1) + } + } + + /// `energy_uj` of each of `dirs` under `root`, in order; `None` if any fails. + pub fn read(root: &Path, dirs: &[&Zone]) -> Option> { + dirs.iter().map(|z| number(root.join(&z.dir).join("energy_uj"))).collect() + } + + /// The summed energy of `which` of `set` between two reads of `set`. + pub fn energy(set: &[&Zone], before: &[u64], after: &[u64], which: impl Fn(&Zone) -> bool) -> u64 { + set.iter() + .zip(before.iter().zip(after)) + .filter(|(z, _)| which(z)) + .map(|(z, (&b, &a))| delta(b, a, z.max_uj).expect("a zone fell with no readable range")) + .sum() + } + + /// What the oracle concluded about the package counters under a root. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No top-level zone's `energy_uj` reads (no powercap, a masked or + /// root-only tree). + Unreadable, + /// Top-level zones read, but none is a package. + NoPackage, + /// Every package read, but not every one kept advancing. + Frozen, + /// Every package advanced in at least five intervals, at least 80% of + /// them, over at least 300 ms. + Live, + } + + /// The oracle's sustained-advance run over the package zones under `root`. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub packages: Vec, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + } + + impl Watch { + pub fn summary(&self) -> String { + let per: Vec = + self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); + format!("{:?} ({} over {} ms)", self.verdict, per.join(", "), self.window.as_millis()) + } + } + + /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU + /// between samples (capped at 500 ms of wall time), until every package + /// has advanced in at least five intervals, at least 80% of those + /// observed, over at least 300 ms; or one second has passed with at least + /// five intervals; or five seconds have. + pub fn watch(root: &Path) -> Watch { + let all = zones(root); + let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); + let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; + if packages.is_empty() { + let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); + return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + } + let started = Instant::now(); + let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; packages.len()]; + let mut intervals = 0u32; + let live = |adv: &[u32], n: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, el) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + burn(Duration::from_millis(25), Duration::from_millis(500)); + if let Some(now) = read(root, &packages) { + intervals += 1; + for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(n > p); + } + prev = now; + } + } + let window = started.elapsed(); + let verdict = if live(&advances, intervals, window) { Verdict::Live } else { Verdict::Frozen }; + Watch { verdict, packages: names, intervals, advances, window } + } + + /// The zones under `root`, as one line for the log. + pub fn listing(root: &Path) -> String { + zones(root) + .iter() + .map(|z| { + let kind = if is_package(z) { + "package" + } else if is_psys(z) { + "psys" + } else if is_core(z) { + "core sub-zone" + } else if top_level(z) { + "top-level" + } else { + "not summed" + }; + let e = z.energy_uj.map_or("unreadable".to_string(), |e| e.to_string()); + let m = z.max_uj.map_or("-".to_string(), |m| m.to_string()); + format!("{}={} [{kind}] energy_uj={e} max={m}", z.dir, z.name.as_deref().unwrap_or("?")) + }) + .collect::>() + .join("; ") + } + + /// The CI container's view of the host's powercap directory: the same + /// entries, each a symlink into `/sys/devices/virtual/powercap`, which + /// Docker masks with an empty tmpfs, so every target is missing. Built + /// from this host's own listing; removed when dropped. + pub struct MaskedView(pub PathBuf); + + impl MaskedView { + pub fn of_host(tag: &str) -> MaskedView { + let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create the masked view"); + let masked = root.join("masked-devices-virtual-powercap"); + for z in zones(Path::new(POWERCAP)) { + std::os::unix::fs::symlink(masked.join(&z.dir), root.join(&z.dir)).expect("symlink"); + } + MaskedView(root) + } + } + + impl Drop for MaskedView { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A bracket of reads around the meter's two reads: every package, psys + /// and core zone read at four points, outer and inner on each side. + pub struct Bracket<'a> { + pub set: Vec<&'a Zone>, + pub reads: [Vec; 4], + } + + impl Bracket<'_> { + pub fn package_outer(&self) -> u64 { + energy(&self.set, &self.reads[0], &self.reads[3], is_package) + } + pub fn package_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_package) + } + pub fn psys_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_psys) + } + pub fn core_inner(&self) -> u64 { + energy(&self.set, &self.reads[1], &self.reads[2], is_core) + } + pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { + self.set.iter().any(|z| which(z)) + } + } + + /// The family's rule on a figure `uj` the meter read between the + /// bracket's inner and outer reads: no less than the packages' energy + /// over the inner bracket, no more than over the outer one, and below + /// package+psys and package+core where those zones exist. A figure that + /// added a zone reads at least the packages' and that zone's energy over + /// the inner bracket, so `uj < inner + zone` is the double-count test; it + /// can only be put where the zone moved more over the burn than the + /// packages did in the gap between the outer and inner reads (else a + /// correct figure could reach it too), and the line says when it could + /// not. Returns the figures, for the log. + pub fn assert_package_only(b: &Bracket, uj: u64) -> String { + let (inner, outer, psys, core) = (b.package_inner(), b.package_outer(), b.psys_inner(), b.core_inner()); + let gap = outer - inner; + let mut line = format!( + "meter {uj} µJ; oracle package-only inner {inner} outer {outer} µJ (gap {gap}); psys inner {psys} µJ; \ + core inner {core} µJ" + ); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + for (zone, present, moved) in [("psys", b.has(is_psys), psys), ("core", b.has(is_core), core)] { + if !present { + continue; + } + assert!(moved > 0, "{zone} did not advance: {line}"); + if moved > gap { + assert!(uj < inner + moved, "the figure reaches package+{zone}: {zone} was added: {line}"); + line.push_str(&format!("; below package+{zone}")); + } else { + line.push_str(&format!("; {zone} moved less than the gap, so only the outer bound tests it")); + } + } + line + } + + /// Every package, psys and core zone of `all` whose counter reads. + pub fn bracket_set(all: &[Zone]) -> Vec<&Zone> { + all.iter().filter(|z| (is_package(z) || is_psys(z) || is_core(z)) && z.energy_uj.is_some()).collect() + } + + /// One oracle read of every zone of `set`; panics if one fails. + pub fn read_all(set: &[&Zone]) -> Vec { + read(Path::new(POWERCAP), set).expect("an oracle read of the bracket zones") + } + + /// Read the bracket around `meter_read` (called twice) and `work`: + /// outer, meter, inner, work, inner, meter, outer. + pub fn around<'a, M>(set: Vec<&'a Zone>, mut meter_read: impl FnMut() -> M, work: impl FnOnce()) -> (Bracket<'a>, M, M) { + let o0 = read_all(&set); + let m0 = meter_read(); + let o1 = read_all(&set); + work(); + let o2 = read_all(&set); + let m1 = meter_read(); + let o3 = read_all(&set); + (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) + } +} + +use oracle::Verdict; + +/// The status agrees with the oracle's own verdict, and only a live status +/// opens the package counters and puts the RAPL probe in the meter. +#[test] +fn the_rapl_status_agrees_with_the_oracle_and_gates_the_probe() { + let _one = serial(); + let status = rapl_status(); + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + let opened = RaplPackages::host().map(|p| p.zones().iter().map(|z| z.to_string()).collect::>()); + let sources = PowerMeter::detect().sources(); + eprintln!( + "rapl hardware: {}; oracle {}; status={status:?} opened={opened:?} sources={sources:?}", + oracle::listing(Path::new(oracle::POWERCAP)), + watch.summary() + ); + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unreadable | Verdict::NoPackage => assert_eq!(status, CounterStatus::Unavailable), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Live => { + let zones = opened.as_ref().expect("live, yet the package counters did not open"); + assert_eq!(*zones, watch.packages, "the probe holds other zones than the oracle's packages"); + } + } + assert_eq!(opened.is_ok(), status == CounterStatus::Live, "opening disagrees with the status"); + assert_eq!(sources.contains(&"rapl-powercap"), status == CounterStatus::Live, "selection disagrees with the status"); +} + +/// Over a 300 ms one-thread CPU burn, a span of a meter whose only CPU source +/// is RAPL is one `Cpu` record, `HwShunt`, its measured figure inside the +/// oracle's package-only bracket and clearly below package+psys and +/// package+core. Where RAPL is not live, no span has a RAPL figure. +#[test] +fn a_cpu_burn_is_one_hwshunt_cpu_record_inside_the_package_only_bracket() { + let _one = serial(); + let mut meter = PowerMeter::detect(); + if !meter.sources().contains(&"rapl-powercap") { + let watch = oracle::watch(Path::new(oracle::POWERCAP)); + eprintln!("rapl burn: no RAPL probe in {:?}; oracle {}", meter.sources(), watch.summary()); + assert_ne!(watch.verdict, Verdict::Live, "the oracle saw live packages, yet the meter has no RAPL probe"); + assert_ne!(rapl_status(), CounterStatus::Live); + let ((), m) = meter.measure(|| { + oracle::burn(Duration::from_millis(50), Duration::from_secs(5)); + }); + assert!( + !m.devices.iter().any(|d| d.class == DeviceClass::Cpu && d.provenance == Provenance::HwShunt), + "a CPU record was graded HwShunt with RAPL declined: {:?}", + m.devices + ); + return; + } + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let set = oracle::bracket_set(&all); + let wall = Instant::now(); + let o0 = oracle::read_all(&set); + let span = meter.start(); + let o1 = oracle::read_all(&set); + let cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)); + let o2 = oracle::read_all(&set); + let m = span.stop(&mut meter); + let o3 = oracle::read_all(&set); + let b = oracle::Bracket { set, reads: [o0, o1, o2, o3] }; + let cpu_records: Vec<_> = m.devices.iter().filter(|d| d.class == DeviceClass::Cpu).collect(); + assert_eq!(cpu_records.len(), 1, "not one CPU record: {:?}", m.devices); + let d = cpu_records[0]; + assert_eq!(d.provenance, Provenance::HwShunt, "a live RAPL span's CPU record: {d:?}"); + let uj = d.measured_uj.expect("an HwShunt record carries its measured figure"); + assert_eq!(d.estimated_uj, uj, "the estimate of a measured figure is the figure"); + assert!(m.reads_cpu(), "a span with a live RAPL CPU record does not read the CPU"); + eprintln!( + "rapl burn: {cpu:?} of thread CPU in {:?}; sources {:?}; records {:?}; {}", + wall.elapsed(), + meter.sources(), + m.devices, + oracle::assert_package_only(&b, uj) + ); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); +} + +/// The package counters, read through `RaplPackages` directly (the S5 +/// executor's RAPL meter), read the same packages: over the same burn, the +/// figure sits inside the same bracket. +#[test] +fn the_package_counters_read_inside_the_package_only_bracket() { + let _one = serial(); + let Ok(mut packages) = RaplPackages::host() else { + assert_ne!(rapl_status(), CounterStatus::Live, "live, yet the package counters did not open"); + assert_ne!(oracle::watch(Path::new(oracle::POWERCAP)).verdict, Verdict::Live); + return; + }; + let all = oracle::zones(Path::new(oracle::POWERCAP)); + let mut cpu = Duration::ZERO; + let (b, m0, m1) = oracle::around( + oracle::bracket_set(&all), + || packages.cumulative_uj().expect("live package counters read"), + || cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)), + ); + eprintln!("rapl packages: zones {:?}; {}", packages.zones(), oracle::assert_package_only(&b, m1 - m0)); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); +} + +/// What a CI container sees: Docker masks `/sys/devices/virtual/powercap` +/// with an empty tmpfs, so every entry of `/sys/class/powercap` is a symlink +/// to nothing. On that view of this host's own listing, the probe opens no +/// package and says so, and so does the oracle. +#[test] +fn the_ci_containers_masked_view_is_unavailable() { + let _one = serial(); + let view = oracle::MaskedView::of_host("sandbox-meter"); + let watch = oracle::watch(&view.0); + let opened = RaplPackages::live_in(&view.0, || { + oracle::burn(Duration::from_millis(25), Duration::from_millis(500)); + }); + eprintln!("rapl masked view: oracle {}; probe {:?}", watch.summary(), opened.as_ref().map(|p| p.zones().len()).map_err(|e| e.to_string())); + assert!(matches!(watch.verdict, Verdict::Unreadable | Verdict::NoPackage), "{watch:?}"); + assert!(opened.is_err(), "the probe opened package counters on a masked tree"); +} -- 2.54.0 (Apple Git-157) From 5ed44d2b2d54b951f1722d48ae2afcc6efeb1548 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 1 Oct 2026 01:39:35 -0400 Subject: [PATCH 02/10] jcp-energy: the fresh-process pin expects only the probes the ladder reached On a Linux host whose RAPL is live (jetson-hub, i9-13900H) the ladder selects RAPL and never asks NVML or the per-process counter, so with the nvml feature the pin's 'NVML probe ran once' failed there: 'the NVML liveness probe ran 0 times' (left 0, right 1). The expectation now follows the detection's own record of which statuses were asked. Run on jetson-hub: 30 lib tests pass with and without nvml (29 + this one failed before). Co-Authored-By: Claude Opus 5.5 --- jcp/jcp-rs/crates/jcp-energy/src/lib.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/jcp/jcp-rs/crates/jcp-energy/src/lib.rs b/jcp/jcp-rs/crates/jcp-energy/src/lib.rs index af3782ac8..b17ead44c 100644 --- a/jcp/jcp-rs/crates/jcp-energy/src/lib.rs +++ b/jcp/jcp-rs/crates/jcp-energy/src/lib.rs @@ -5231,15 +5231,24 @@ mod tests { nvml_status(), process_energy_status() ); - // Every `detect()` asks for the RAPL status first, then NVML's, then - // the per-process counter's: each probe ran once, on whichever thread - // asked first. Off macOS there is no per-process probe. + // Every `detect()` asks for the RAPL status first, then, only while + // no rung above was selected, NVML's, then the per-process counter's: + // each probe the ladder reached ran once, on whichever thread asked + // first, and one below the selected rung never ran. On a Linux host + // whose RAPL is live the ladder stops there, so NVML is never probed; + // the detection records which statuses were asked. Off macOS there + // is no per-process probe. + let (_, why) = Meter::detect_explained(); assert_eq!(rapl_runs, 1, "the RAPL liveness probe did not run exactly once in this process"); - assert_eq!(nvml_runs, usize::from(cfg!(feature = "nvml")), "the NVML liveness probe ran {nvml_runs} times"); + assert_eq!( + nvml_runs, + usize::from(cfg!(feature = "nvml") && why.nvml.is_some()), + "the NVML liveness probe ran {nvml_runs} times ({why:?})" + ); assert_eq!( process_runs, - usize::from(cfg!(target_os = "macos")), - "the per-process liveness probe ran {process_runs} times" + usize::from(cfg!(target_os = "macos") && why.process_energy.is_some()), + "the per-process liveness probe ran {process_runs} times ({why:?})" ); assert!(statuses.iter().all(|s| *s == statuses[0]), "threads saw different statuses: {statuses:?}"); assert_eq!(statuses[0], ioreport_status(), "the spawned threads and this one disagree"); -- 2.54.0 (Apple Git-157) From 012a08be7f4419f0165335defae6ce4383346dc1 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 1 Oct 2026 01:45:47 -0400 Subject: [PATCH 03/10] energy meters: NVML hardware tests against an oracle of their own eoc-meter (cuda), sandbox-meter, jcp-energy (nvml) and jouleclaw-energy (nvml) each get a test binary that opens libnvidia-ml itself (dlopen and its own symbols, not the crate's bindings), measures how often the total-energy counter is republished (about every 100 ms on jetson-hub's RTX 4050 Laptop, driver 595.84), samples it at the crate's own tick by the sustained-advance rule, and holds it to its board through nvidia-smi: no faster than 1.5x the maximum power limit, the board's draw within that bound too, and no faster than twice the draw plus 5 W. The status must agree with that verdict; a live counter's figure must be graded as the family's rule says for a GPU-only figure. On jetson-hub all four fail as committed: the counter advances in every interval, so each crate calls it Live and grades it as hardware, but polled at the probes' 100-150 ms ticks it runs at 58-305 W while the board draws 5.5-13.6 W (max limit 65 W). The fixes follow. Co-Authored-By: Claude Opus 5.5 --- .../crates/eoc-meter/tests/nvml_hardware.rs | 390 +++++++++++++++++ .../crates/jcp-energy/tests/nvml_hardware.rs | 393 +++++++++++++++++ .../jouleclaw-energy/tests/nvml_hardware.rs | 412 ++++++++++++++++++ .../sandbox-meter/tests/nvml_hardware.rs | 398 +++++++++++++++++ 4 files changed, 1593 insertions(+) create mode 100644 eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs create mode 100644 jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs create mode 100644 jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs create mode 100644 sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs diff --git a/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs b/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs new file mode 100644 index 000000000..6f961a3d4 --- /dev/null +++ b/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs @@ -0,0 +1,390 @@ +//! NVML against the hardware (feature `cuda`): GPU 0's total-energy counter, +//! read by the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the NVML probe behind `nvml_status()` +//! runs in this process, and the tests take one lock. The oracle opens +//! `libnvidia-ml` itself, measures how often the counter is republished, +//! samples it at the probe's own wait (100 ms) by the sustained-advance rule +//! and holds it to the board's power ceiling, which it asks of `nvidia-smi`. +//! None of these tests passes without asserting: where there is no NVML (a +//! CI container has no GPU), the status must say so and agree with the +//! oracle; where the counter is live, both must say so. Either way the GPU +//! counter is never the counter `detect()` hands eoc's consumers, which +//! grade any advance of it `Measured` with no coverage field to say it read +//! only the GPU: by the family's rule a GPU-only figure in such a format is +//! an estimate, never a whole-work hardware reading. + +#![cfg(all(target_os = "linux", feature = "cuda"))] + +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use eoc_meter::{CounterStatus, JouleCounter, NvmlCounter}; + +/// One test at a time: each reads machine-wide counters, and the oracle's +/// own sampling must not overlap the meter's. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The NVML probe's wait between samples (`NVML_PROBE_WAIT`). +const TICK: Duration = Duration::from_millis(100); + +/// The tests' own reading of NVIDIA's total-energy counter: `libnvidia-ml` +/// opened here with `dlopen` and called through symbols looked up here, not +/// through the crate under test or its bindings, and each GPU's power limit +/// and draw asked of `nvidia-smi`, a separate process. It measures how often +/// the counter is republished, and judges it by the family's rule: every +/// GPU's counter advanced in at least five intervals, at least 80% of them, +/// over at least 300 ms, sampled at the crate's own tick; and it is plausibly +/// counting its board's energy: over the window it ran no faster than 1.5 +/// times the board's maximum power limit, the board's own power draw (as +/// `nvidia-smi` reports it just before and just after) was itself within that +/// bound, and the counter ran no faster than twice that draw plus 5 W (the +/// draw's documented accuracy). A counter that runs several times faster than +/// its board draws is not counting that board's energy. +#[allow(dead_code)] +mod nvml_oracle { + use std::ffi::{c_char, c_int, c_uint, c_void, CString}; + use std::time::{Duration, Instant}; + + unsafe extern "C" { + fn dlopen(filename: *const c_char, flag: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; + } + + /// `RTLD_NOW` in ``. + const RTLD_NOW: c_int = 2; + + type Device = *mut c_void; + type EnergyFn = unsafe extern "C" fn(Device, *mut u64) -> c_int; + type LimitsFn = unsafe extern "C" fn(Device, *mut c_uint, *mut c_uint) -> c_int; + type MilliwattsFn = unsafe extern "C" fn(Device, *mut c_uint) -> c_int; + + /// NVML, opened by the oracle. + pub struct Nvml { + energy: EnergyFn, + limits: Option, + power: Option, + /// Every GPU whose total-energy counter reads, in index order. + pub devices: Vec, + } + + fn sym(lib: *mut c_void, name: &str) -> Option<*mut c_void> { + let name = CString::new(name).ok()?; + let p = unsafe { dlsym(lib, name.as_ptr()) }; + (!p.is_null()).then_some(p) + } + + /// `libnvidia-ml.so.1`, initialised, with every GPU whose counter reads; + /// `None` where there is no library, no GPU, or no GPU with the counter. + pub fn open() -> Option { + let file = CString::new("libnvidia-ml.so.1").ok()?; + let lib = unsafe { dlopen(file.as_ptr(), RTLD_NOW) }; + if lib.is_null() { + return None; + } + unsafe { + let init: unsafe extern "C" fn() -> c_int = std::mem::transmute(sym(lib, "nvmlInit_v2")?); + let count: unsafe extern "C" fn(*mut c_uint) -> c_int = std::mem::transmute(sym(lib, "nvmlDeviceGetCount_v2")?); + let by_index: unsafe extern "C" fn(c_uint, *mut Device) -> c_int = + std::mem::transmute(sym(lib, "nvmlDeviceGetHandleByIndex_v2")?); + let energy: EnergyFn = std::mem::transmute(sym(lib, "nvmlDeviceGetTotalEnergyConsumption")?); + let limits = sym(lib, "nvmlDeviceGetPowerManagementLimitConstraints").map(|p| std::mem::transmute::<_, LimitsFn>(p)); + let power = sym(lib, "nvmlDeviceGetPowerUsage").map(|p| std::mem::transmute::<_, MilliwattsFn>(p)); + if init() != 0 { + return None; + } + let mut n: c_uint = 0; + if count(&mut n) != 0 { + return None; + } + let mut devices = Vec::new(); + for i in 0..n { + let mut d: Device = std::ptr::null_mut(); + let mut mj = 0u64; + if by_index(i, &mut d) == 0 && !d.is_null() && energy(d, &mut mj) == 0 { + devices.push(d); + } + } + (!devices.is_empty()).then_some(Nvml { energy, limits, power, devices }) + } + } + + impl Nvml { + /// Every GPU's total-energy counter, in mJ; `None` if one fails. + pub fn energy_mj(&self) -> Option> { + self.devices + .iter() + .map(|&d| { + let mut mj = 0u64; + (unsafe { (self.energy)(d, &mut mj) } == 0).then_some(mj) + }) + .collect() + } + + /// GPU `i`'s maximum power-management limit, in mW, as NVML gives it. + pub fn max_limit_mw(&self, i: usize) -> Option { + let (mut lo, mut hi) = (0, 0); + (unsafe { (self.limits?)(self.devices[i], &mut lo, &mut hi) } == 0 && hi > 0).then_some(hi) + } + + /// GPU `i`'s power draw, in mW, as NVML gives it. + pub fn power_mw(&self, i: usize) -> Option { + let mut mw = 0; + (unsafe { (self.power?)(self.devices[i], &mut mw) } == 0).then_some(mw) + } + } + + /// `nvidia-smi`'s view of every GPU: (maximum power limit, enforced limit, + /// draw), in watts; `None` without `nvidia-smi`. + pub fn smi() -> Option, Option, Option)>> { + let out = std::process::Command::new("nvidia-smi") + .args(["--query-gpu=power.max_limit,enforced.power.limit,power.draw", "--format=csv,noheader,nounits"]) + .output() + .ok()?; + out.status.success().then(|| { + String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| { + let f: Vec> = l.split(',').map(|v| v.trim().parse().ok()).collect(); + (f.first().copied().flatten(), f.get(1).copied().flatten(), f.get(2).copied().flatten()) + }) + .collect() + }) + } + + /// GPU `i`'s ceiling in watts: `nvidia-smi`'s maximum power limit, else + /// NVML's. + pub fn ceiling_w(nvml: &Nvml, smi: &Option, Option, Option)>>, i: usize) -> Option { + smi.as_ref() + .and_then(|s| s.get(i)) + .and_then(|(max, _, _)| *max) + .or_else(|| nvml.max_limit_mw(i).map(|mw| f64::from(mw) / 1000.0)) + } + + /// How often GPU 0's counter is republished: the median gap between its + /// changes, polled every 2 ms for up to 1.5 s (`None` if it changed + /// fewer than 6 times). + pub fn update_period(nvml: &Nvml) -> Option { + let started = Instant::now(); + let mut last = nvml.energy_mj()?[0]; + let mut changes = Vec::new(); + while started.elapsed() < Duration::from_millis(1500) && changes.len() < 16 { + std::thread::sleep(Duration::from_millis(2)); + let now = nvml.energy_mj()?[0]; + if now != last { + changes.push(started.elapsed()); + last = now; + } + } + let mut gaps: Vec = changes.windows(2).map(|w| w[1] - w[0]).collect(); + if gaps.len() < 5 { + return None; + } + gaps.sort(); + Some(gaps[gaps.len() / 2]) + } + + /// The oracle's verdict on the counters. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No NVML, no GPU, or no GPU whose total-energy counter reads. + Unavailable, + /// A counter did not keep advancing. + Frozen, + /// Every counter kept advancing, but one is not plausibly its board's + /// energy (see the module doc). + Implausible, + /// Every counter kept advancing, plausibly its board's energy. + Live, + } + + /// The family's plausibility rule for one GPU over a window: its counter's + /// average `rate_w`, the board's draws `draws_w` around the window and + /// its maximum power limit `ceiling_w`. A bound whose input is missing + /// is not applied. + pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { + if let Some(c) = ceiling_w { + if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { + return false; + } + } + if !draws_w.is_empty() { + let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if rate_w > 2.0 * mean + 5.0 { + return false; + } + } + true + } + + /// What the oracle saw. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + /// Each GPU's average rate over the window, in watts. + pub rate_w: Vec, + /// Each GPU's draw just before and just after the window, in watts. + pub draws_w: Vec>, + /// Each GPU's ceiling, in watts. + pub ceiling_w: Vec>, + } + + impl Watch { + pub fn summary(&self) -> String { + format!( + "{:?} (advanced {:?} of {} intervals over {} ms; counter rate {:?} W, board draw {:?} W, max power limit {:?} W)", + self.verdict, + self.advances, + self.intervals, + self.window.as_millis(), + self.rate_w.iter().map(|w| (w * 10.0).round() / 10.0).collect::>(), + self.draws_w, + self.ceiling_w + ) + } + } + + /// The oracle's run over the first `gpus` GPUs (all, if more), sleeping + /// `tick` between samples, by the sustained-advance rule (stopping when + /// live, or after a second with at least five intervals, or five + /// seconds); then each counter's plausibility over the whole window. + pub fn watch(tick: Duration, gpus: usize) -> Watch { + let none = Watch { + verdict: Verdict::Unavailable, + intervals: 0, + advances: Vec::new(), + window: Duration::ZERO, + rate_w: Vec::new(), + draws_w: Vec::new(), + ceiling_w: Vec::new(), + }; + let Some(nvml) = open() else { return none }; + let n = nvml.devices.len().min(gpus); + let before = smi(); + let read = || nvml.energy_mj().map(|v| v[..n].to_vec()); + let started = Instant::now(); + let Some(first) = read() else { return none }; + let (mut prev, mut last_at) = (first.clone(), Duration::ZERO); + let mut advances = vec![0u32; n]; + let mut intervals = 0u32; + let live = |adv: &[u32], k: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(k) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, last_at) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + std::thread::sleep(tick); + if let Some(now) = read() { + last_at = started.elapsed(); + intervals += 1; + for ((a, x), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(x > p); + } + prev = now; + } + } + let after = smi(); + let window = last_at; + let rate_w: Vec = + first.iter().zip(&prev).map(|(a, b)| b.saturating_sub(*a) as f64 / 1000.0 / window.as_secs_f64().max(1e-9)).collect(); + let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); + let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); + let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + let verdict = if !live(&advances, intervals, window) { + Verdict::Frozen + } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + Verdict::Implausible + } else { + Verdict::Live + }; + Watch { verdict, intervals, advances, window, rate_w, draws_w, ceiling_w } + } + + /// One line on the GPUs for the log: count, counters, ceilings, draw, and + /// the counter's measured republication `period`. + pub fn listing(period: Option) -> String { + let Some(nvml) = open() else { return "no NVML device with a total-energy counter".to_string() }; + format!( + "{} GPU(s) with a total-energy counter, counters {:?} mJ; NVML max power limit {:?} mW, power draw {:?} mW; \ + nvidia-smi (max limit, enforced limit, draw) W {:?}; counter republished every {:?}", + nvml.devices.len(), + nvml.energy_mj(), + (0..nvml.devices.len()).map(|i| nvml.max_limit_mw(i)).collect::>(), + (0..nvml.devices.len()).map(|i| nvml.power_mw(i)).collect::>(), + smi(), + period + ) + } + + /// GPU 0's counter's republication period, where there is one. + pub fn period() -> Option { + open().and_then(|n| update_period(&n)) + } +} + +use nvml_oracle::Verdict; + +/// The status agrees with the oracle, judged at the probe's own tick, and the +/// tick is no faster than the counter is republished; only a live status +/// constructs the counter, and `detect()` never selects it. +#[test] +fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { + let _one = serial(); + let period = nvml_oracle::period(); + let status = eoc_meter::nvml_status(); + let watch = nvml_oracle::watch(TICK, 1); + let constructed = NvmlCounter::try_detect().is_ok(); + let selected = eoc_meter::detect().name(); + eprintln!( + "nvml hardware: {}; oracle at a {TICK:?} tick {}; status={status:?} constructed={constructed} detect()={selected}", + nvml_oracle::listing(period), + watch.summary() + ); + if let Some(p) = period { + assert!(TICK >= p * 9 / 10, "the {TICK:?} tick is faster than the counter's {p:?} republication"); + } + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Implausible => assert_ne!(status, CounterStatus::Live), + Verdict::Live => {} + } + assert_eq!(constructed, status == CounterStatus::Live, "construction disagrees with the status"); + assert_ne!(selected, "nvml", "detect() handed eoc's consumers the GPU-only counter, which they grade Measured"); +} + +/// Over a one-second span a live GPU counter advances, inside the oracle's +/// own reads around the span, and no faster than the board can draw. +#[test] +fn a_live_gpu_counter_reads_inside_the_oracles_bracket() { + let _one = serial(); + let Ok(counter) = NvmlCounter::try_detect() else { + eprintln!("nvml span: status {:?}; no counter", eoc_meter::nvml_status()); + assert_ne!(eoc_meter::nvml_status(), CounterStatus::Live); + return; + }; + let oracle = nvml_oracle::open().expect("the meter reads NVML, yet the oracle cannot open it"); + let o0 = oracle.energy_mj().expect("oracle read")[0]; + let started = std::time::Instant::now(); + let m0 = counter.read_microjoules().expect("a live counter reads"); + std::thread::sleep(Duration::from_secs(1)); + let m1 = counter.read_microjoules().expect("a live counter reads"); + let wall = started.elapsed(); + let o1 = oracle.energy_mj().expect("oracle read")[0]; + let ceiling = nvml_oracle::ceiling_w(&oracle, &nvml_oracle::smi(), 0).unwrap_or(f64::INFINITY); + let uj = m1 - m0; + eprintln!("nvml span: {wall:?}; meter {uj} µJ; oracle {o0} -> {o1} mJ; ceiling {ceiling} W"); + assert!(uj > 0 && uj <= (o1 - o0) * 1_000, "{uj} µJ outside the oracle's (0, {}] µJ", (o1 - o0) * 1_000); + let watts = uj as f64 / 1e6 / wall.as_secs_f64(); + assert!(watts <= 1.5 * ceiling, "the GPU counter reads {watts:.1} W, past the board's {ceiling} W ceiling"); +} diff --git a/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs b/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs new file mode 100644 index 000000000..9169f8ef4 --- /dev/null +++ b/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs @@ -0,0 +1,393 @@ +//! NVML against the hardware (feature `nvml`): GPU 0's total-energy counter, +//! read by the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the NVML probe behind `nvml_status()` +//! runs in this process, and the tests take one lock. The oracle opens +//! `libnvidia-ml` itself, measures how often the counter is republished, +//! samples it at the probe's own wait (100 ms) by the sustained-advance rule +//! and holds it to the board's power ceiling, which it asks of `nvidia-smi`. +//! None of these tests passes without asserting: where there is no NVML (a +//! CI container has no GPU), the status must say so and agree with the +//! oracle; where the counter is live, both must say so, and the NVML rung's +//! figure must be `HwShunt` with `Coverage::Partial` — it reads the GPU, not +//! the CPU's share of the work — and no faster than the board can draw. + +#![cfg(all(target_os = "linux", feature = "nvml"))] + +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use jcp_energy::{nvml_status, CounterStatus, Meter, Nvml, Rung}; +use jcp_schema::{Coverage, Provenance}; + +/// One test at a time: each reads machine-wide counters, and the oracle's +/// own sampling must not overlap the meter's. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The NVML probe's wait between samples (`NVML_PROBE_WAIT`). +const TICK: Duration = Duration::from_millis(100); + +/// The tests' own reading of NVIDIA's total-energy counter: `libnvidia-ml` +/// opened here with `dlopen` and called through symbols looked up here, not +/// through the crate under test or its bindings, and each GPU's power limit +/// and draw asked of `nvidia-smi`, a separate process. It measures how often +/// the counter is republished, and judges it by the family's rule: every +/// GPU's counter advanced in at least five intervals, at least 80% of them, +/// over at least 300 ms, sampled at the crate's own tick; and it is plausibly +/// counting its board's energy: over the window it ran no faster than 1.5 +/// times the board's maximum power limit, the board's own power draw (as +/// `nvidia-smi` reports it just before and just after) was itself within that +/// bound, and the counter ran no faster than twice that draw plus 5 W (the +/// draw's documented accuracy). A counter that runs several times faster than +/// its board draws is not counting that board's energy. +#[allow(dead_code)] +mod nvml_oracle { + use std::ffi::{c_char, c_int, c_uint, c_void, CString}; + use std::time::{Duration, Instant}; + + unsafe extern "C" { + fn dlopen(filename: *const c_char, flag: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; + } + + /// `RTLD_NOW` in ``. + const RTLD_NOW: c_int = 2; + + type Device = *mut c_void; + type EnergyFn = unsafe extern "C" fn(Device, *mut u64) -> c_int; + type LimitsFn = unsafe extern "C" fn(Device, *mut c_uint, *mut c_uint) -> c_int; + type MilliwattsFn = unsafe extern "C" fn(Device, *mut c_uint) -> c_int; + + /// NVML, opened by the oracle. + pub struct Nvml { + energy: EnergyFn, + limits: Option, + power: Option, + /// Every GPU whose total-energy counter reads, in index order. + pub devices: Vec, + } + + fn sym(lib: *mut c_void, name: &str) -> Option<*mut c_void> { + let name = CString::new(name).ok()?; + let p = unsafe { dlsym(lib, name.as_ptr()) }; + (!p.is_null()).then_some(p) + } + + /// `libnvidia-ml.so.1`, initialised, with every GPU whose counter reads; + /// `None` where there is no library, no GPU, or no GPU with the counter. + pub fn open() -> Option { + let file = CString::new("libnvidia-ml.so.1").ok()?; + let lib = unsafe { dlopen(file.as_ptr(), RTLD_NOW) }; + if lib.is_null() { + return None; + } + unsafe { + let init: unsafe extern "C" fn() -> c_int = std::mem::transmute(sym(lib, "nvmlInit_v2")?); + let count: unsafe extern "C" fn(*mut c_uint) -> c_int = std::mem::transmute(sym(lib, "nvmlDeviceGetCount_v2")?); + let by_index: unsafe extern "C" fn(c_uint, *mut Device) -> c_int = + std::mem::transmute(sym(lib, "nvmlDeviceGetHandleByIndex_v2")?); + let energy: EnergyFn = std::mem::transmute(sym(lib, "nvmlDeviceGetTotalEnergyConsumption")?); + let limits = sym(lib, "nvmlDeviceGetPowerManagementLimitConstraints").map(|p| std::mem::transmute::<_, LimitsFn>(p)); + let power = sym(lib, "nvmlDeviceGetPowerUsage").map(|p| std::mem::transmute::<_, MilliwattsFn>(p)); + if init() != 0 { + return None; + } + let mut n: c_uint = 0; + if count(&mut n) != 0 { + return None; + } + let mut devices = Vec::new(); + for i in 0..n { + let mut d: Device = std::ptr::null_mut(); + let mut mj = 0u64; + if by_index(i, &mut d) == 0 && !d.is_null() && energy(d, &mut mj) == 0 { + devices.push(d); + } + } + (!devices.is_empty()).then_some(Nvml { energy, limits, power, devices }) + } + } + + impl Nvml { + /// Every GPU's total-energy counter, in mJ; `None` if one fails. + pub fn energy_mj(&self) -> Option> { + self.devices + .iter() + .map(|&d| { + let mut mj = 0u64; + (unsafe { (self.energy)(d, &mut mj) } == 0).then_some(mj) + }) + .collect() + } + + /// GPU `i`'s maximum power-management limit, in mW, as NVML gives it. + pub fn max_limit_mw(&self, i: usize) -> Option { + let (mut lo, mut hi) = (0, 0); + (unsafe { (self.limits?)(self.devices[i], &mut lo, &mut hi) } == 0 && hi > 0).then_some(hi) + } + + /// GPU `i`'s power draw, in mW, as NVML gives it. + pub fn power_mw(&self, i: usize) -> Option { + let mut mw = 0; + (unsafe { (self.power?)(self.devices[i], &mut mw) } == 0).then_some(mw) + } + } + + /// `nvidia-smi`'s view of every GPU: (maximum power limit, enforced limit, + /// draw), in watts; `None` without `nvidia-smi`. + pub fn smi() -> Option, Option, Option)>> { + let out = std::process::Command::new("nvidia-smi") + .args(["--query-gpu=power.max_limit,enforced.power.limit,power.draw", "--format=csv,noheader,nounits"]) + .output() + .ok()?; + out.status.success().then(|| { + String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| { + let f: Vec> = l.split(',').map(|v| v.trim().parse().ok()).collect(); + (f.first().copied().flatten(), f.get(1).copied().flatten(), f.get(2).copied().flatten()) + }) + .collect() + }) + } + + /// GPU `i`'s ceiling in watts: `nvidia-smi`'s maximum power limit, else + /// NVML's. + pub fn ceiling_w(nvml: &Nvml, smi: &Option, Option, Option)>>, i: usize) -> Option { + smi.as_ref() + .and_then(|s| s.get(i)) + .and_then(|(max, _, _)| *max) + .or_else(|| nvml.max_limit_mw(i).map(|mw| f64::from(mw) / 1000.0)) + } + + /// How often GPU 0's counter is republished: the median gap between its + /// changes, polled every 2 ms for up to 1.5 s (`None` if it changed + /// fewer than 6 times). + pub fn update_period(nvml: &Nvml) -> Option { + let started = Instant::now(); + let mut last = nvml.energy_mj()?[0]; + let mut changes = Vec::new(); + while started.elapsed() < Duration::from_millis(1500) && changes.len() < 16 { + std::thread::sleep(Duration::from_millis(2)); + let now = nvml.energy_mj()?[0]; + if now != last { + changes.push(started.elapsed()); + last = now; + } + } + let mut gaps: Vec = changes.windows(2).map(|w| w[1] - w[0]).collect(); + if gaps.len() < 5 { + return None; + } + gaps.sort(); + Some(gaps[gaps.len() / 2]) + } + + /// The oracle's verdict on the counters. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No NVML, no GPU, or no GPU whose total-energy counter reads. + Unavailable, + /// A counter did not keep advancing. + Frozen, + /// Every counter kept advancing, but one is not plausibly its board's + /// energy (see the module doc). + Implausible, + /// Every counter kept advancing, plausibly its board's energy. + Live, + } + + /// The family's plausibility rule for one GPU over a window: its counter's + /// average `rate_w`, the board's draws `draws_w` around the window and + /// its maximum power limit `ceiling_w`. A bound whose input is missing + /// is not applied. + pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { + if let Some(c) = ceiling_w { + if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { + return false; + } + } + if !draws_w.is_empty() { + let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if rate_w > 2.0 * mean + 5.0 { + return false; + } + } + true + } + + /// What the oracle saw. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + /// Each GPU's average rate over the window, in watts. + pub rate_w: Vec, + /// Each GPU's draw just before and just after the window, in watts. + pub draws_w: Vec>, + /// Each GPU's ceiling, in watts. + pub ceiling_w: Vec>, + } + + impl Watch { + pub fn summary(&self) -> String { + format!( + "{:?} (advanced {:?} of {} intervals over {} ms; counter rate {:?} W, board draw {:?} W, max power limit {:?} W)", + self.verdict, + self.advances, + self.intervals, + self.window.as_millis(), + self.rate_w.iter().map(|w| (w * 10.0).round() / 10.0).collect::>(), + self.draws_w, + self.ceiling_w + ) + } + } + + /// The oracle's run over the first `gpus` GPUs (all, if more), sleeping + /// `tick` between samples, by the sustained-advance rule (stopping when + /// live, or after a second with at least five intervals, or five + /// seconds); then each counter's plausibility over the whole window. + pub fn watch(tick: Duration, gpus: usize) -> Watch { + let none = Watch { + verdict: Verdict::Unavailable, + intervals: 0, + advances: Vec::new(), + window: Duration::ZERO, + rate_w: Vec::new(), + draws_w: Vec::new(), + ceiling_w: Vec::new(), + }; + let Some(nvml) = open() else { return none }; + let n = nvml.devices.len().min(gpus); + let before = smi(); + let read = || nvml.energy_mj().map(|v| v[..n].to_vec()); + let started = Instant::now(); + let Some(first) = read() else { return none }; + let (mut prev, mut last_at) = (first.clone(), Duration::ZERO); + let mut advances = vec![0u32; n]; + let mut intervals = 0u32; + let live = |adv: &[u32], k: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(k) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, last_at) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + std::thread::sleep(tick); + if let Some(now) = read() { + last_at = started.elapsed(); + intervals += 1; + for ((a, x), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(x > p); + } + prev = now; + } + } + let after = smi(); + let window = last_at; + let rate_w: Vec = + first.iter().zip(&prev).map(|(a, b)| b.saturating_sub(*a) as f64 / 1000.0 / window.as_secs_f64().max(1e-9)).collect(); + let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); + let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); + let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + let verdict = if !live(&advances, intervals, window) { + Verdict::Frozen + } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + Verdict::Implausible + } else { + Verdict::Live + }; + Watch { verdict, intervals, advances, window, rate_w, draws_w, ceiling_w } + } + + /// One line on the GPUs for the log: count, counters, ceilings, draw, and + /// the counter's measured republication `period`. + pub fn listing(period: Option) -> String { + let Some(nvml) = open() else { return "no NVML device with a total-energy counter".to_string() }; + format!( + "{} GPU(s) with a total-energy counter, counters {:?} mJ; NVML max power limit {:?} mW, power draw {:?} mW; \ + nvidia-smi (max limit, enforced limit, draw) W {:?}; counter republished every {:?}", + nvml.devices.len(), + nvml.energy_mj(), + (0..nvml.devices.len()).map(|i| nvml.max_limit_mw(i)).collect::>(), + (0..nvml.devices.len()).map(|i| nvml.power_mw(i)).collect::>(), + smi(), + period + ) + } + + /// GPU 0's counter's republication period, where there is one. + pub fn period() -> Option { + open().and_then(|n| update_period(&n)) + } +} + +use nvml_oracle::Verdict; + +/// The status agrees with the oracle, judged at the probe's own tick, and the +/// tick is no faster than the counter is republished. +#[test] +fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { + let _one = serial(); + let period = nvml_oracle::period(); + let status = nvml_status(); + let watch = nvml_oracle::watch(TICK, 1); + let (meter, why) = Meter::detect_explained(); + eprintln!( + "nvml hardware: {}; oracle at a {TICK:?} tick {}; status={status:?} selected={:?} detection={why:?}", + nvml_oracle::listing(period), + watch.summary(), + meter.rung() + ); + if let Some(p) = period { + assert!(TICK >= p * 9 / 10, "the {TICK:?} tick is faster than the counter's {p:?} republication"); + } + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Implausible => assert_ne!(status, CounterStatus::Live), + Verdict::Live => {} + } + if meter.rung() == Rung::Nvml { + assert_eq!(status, CounterStatus::Live, "the NVML rung was selected on a counter that is not live"); + } +} + +/// Over a one-second span, the NVML rung's figure is `HwShunt` and +/// `Coverage::Partial`, never a whole-task figure: it advanced, sits inside +/// the oracle's own reads of the counter around the span, and ran no faster +/// than the board can draw. The rung is built here as `Meter::detect` builds +/// it — only on a live status — because on a host whose RAPL is live the +/// ladder stops above it. Where NVML is not live there is no such figure. +#[test] +fn a_live_gpu_counter_is_a_partial_hwshunt_figure() { + let _one = serial(); + if nvml_status() != CounterStatus::Live { + eprintln!("nvml span: status {:?}; no NVML figure", nvml_status()); + assert_ne!(Meter::detect().rung(), Rung::Nvml, "the NVML rung was selected on a counter that is not live"); + return; + } + let meter = Meter::Nvml(Box::new(Nvml::open().expect("a live counter opens"))); + let oracle = nvml_oracle::open().expect("the meter reads NVML, yet the oracle cannot open it"); + let o0 = oracle.energy_mj().expect("oracle read")[0]; + let started = std::time::Instant::now(); + let ((), e) = meter.measure(|| std::thread::sleep(Duration::from_secs(1))); + let wall = started.elapsed(); + let o1 = oracle.energy_mj().expect("oracle read")[0]; + let ceiling = nvml_oracle::ceiling_w(&oracle, &nvml_oracle::smi(), 0).unwrap_or(f64::INFINITY); + eprintln!("nvml span: {wall:?}; energy {e:?}; oracle {o0} -> {o1} mJ; ceiling {ceiling} W"); + assert_eq!((e.provenance, e.coverage), (Provenance::HwShunt, Coverage::Partial), "a live NVML span: {e:?}"); + let uj = e.measured_uj.expect("an HwShunt figure carries its measured figure"); + assert!(uj > 0 && uj <= (o1 - o0) * 1_000, "{uj} µJ outside the oracle's (0, {}] µJ", (o1 - o0) * 1_000); + let watts = uj as f64 / 1e6 / wall.as_secs_f64(); + assert!(watts <= 1.5 * ceiling, "the NVML figure reads {watts:.1} W, past the board's {ceiling} W ceiling"); +} diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs new file mode 100644 index 000000000..780da7dd1 --- /dev/null +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs @@ -0,0 +1,412 @@ +//! NVML against the hardware (feature `nvml`): the GPUs' total-energy +//! counters, read by the meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the NVML probe runs in this process, and +//! the tests take one lock. The oracle opens `libnvidia-ml` itself, measures +//! how often the counter is republished, samples it at the probe's own tick +//! (100 ms) by the sustained-advance rule and holds each counter to its +//! board's power ceiling, which it asks of `nvidia-smi`. None of these tests +//! passes without asserting: where there is no NVML (a CI container has no +//! GPU), the counter must say so and agree with the oracle; where the +//! counter is live, both must say so, and its readings must be `HwShunt` over +//! the `Gpu` domain — the GPU, never a whole-host or CPU figure — no faster +//! than the board can draw; where it is not live, its readings are the +//! power-draw integral, `ModelBased`, never `HwShunt`. + +#![cfg(all(target_os = "linux", feature = "nvml"))] + +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use jouleclaw_energy::nvml::NvmlCounter; +use jouleclaw_energy::{CounterLiveness, EnergyCounter, EnergyDomain, Provenance}; + +/// One test at a time: each reads machine-wide counters, and the oracle's +/// own sampling must not overlap the meter's. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The NVML probe's tick (`PROBE_TICK` in `nvml.rs`): 100 ms of the probing +/// thread's CPU between samples, so at least 100 ms of wall time. +const TICK: Duration = Duration::from_millis(100); + +/// The tests' own reading of NVIDIA's total-energy counter: `libnvidia-ml` +/// opened here with `dlopen` and called through symbols looked up here, not +/// through the crate under test or its bindings, and each GPU's power limit +/// and draw asked of `nvidia-smi`, a separate process. It measures how often +/// the counter is republished, and judges it by the family's rule: every +/// GPU's counter advanced in at least five intervals, at least 80% of them, +/// over at least 300 ms, sampled at the crate's own tick; and it is plausibly +/// counting its board's energy: over the window it ran no faster than 1.5 +/// times the board's maximum power limit, the board's own power draw (as +/// `nvidia-smi` reports it just before and just after) was itself within that +/// bound, and the counter ran no faster than twice that draw plus 5 W (the +/// draw's documented accuracy). A counter that runs several times faster than +/// its board draws is not counting that board's energy. +#[allow(dead_code)] +mod nvml_oracle { + use std::ffi::{c_char, c_int, c_uint, c_void, CString}; + use std::time::{Duration, Instant}; + + unsafe extern "C" { + fn dlopen(filename: *const c_char, flag: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; + } + + /// `RTLD_NOW` in ``. + const RTLD_NOW: c_int = 2; + + type Device = *mut c_void; + type EnergyFn = unsafe extern "C" fn(Device, *mut u64) -> c_int; + type LimitsFn = unsafe extern "C" fn(Device, *mut c_uint, *mut c_uint) -> c_int; + type MilliwattsFn = unsafe extern "C" fn(Device, *mut c_uint) -> c_int; + + /// NVML, opened by the oracle. + pub struct Nvml { + energy: EnergyFn, + limits: Option, + power: Option, + /// Every GPU whose total-energy counter reads, in index order. + pub devices: Vec, + } + + fn sym(lib: *mut c_void, name: &str) -> Option<*mut c_void> { + let name = CString::new(name).ok()?; + let p = unsafe { dlsym(lib, name.as_ptr()) }; + (!p.is_null()).then_some(p) + } + + /// `libnvidia-ml.so.1`, initialised, with every GPU whose counter reads; + /// `None` where there is no library, no GPU, or no GPU with the counter. + pub fn open() -> Option { + let file = CString::new("libnvidia-ml.so.1").ok()?; + let lib = unsafe { dlopen(file.as_ptr(), RTLD_NOW) }; + if lib.is_null() { + return None; + } + unsafe { + let init: unsafe extern "C" fn() -> c_int = std::mem::transmute(sym(lib, "nvmlInit_v2")?); + let count: unsafe extern "C" fn(*mut c_uint) -> c_int = std::mem::transmute(sym(lib, "nvmlDeviceGetCount_v2")?); + let by_index: unsafe extern "C" fn(c_uint, *mut Device) -> c_int = + std::mem::transmute(sym(lib, "nvmlDeviceGetHandleByIndex_v2")?); + let energy: EnergyFn = std::mem::transmute(sym(lib, "nvmlDeviceGetTotalEnergyConsumption")?); + let limits = sym(lib, "nvmlDeviceGetPowerManagementLimitConstraints").map(|p| std::mem::transmute::<_, LimitsFn>(p)); + let power = sym(lib, "nvmlDeviceGetPowerUsage").map(|p| std::mem::transmute::<_, MilliwattsFn>(p)); + if init() != 0 { + return None; + } + let mut n: c_uint = 0; + if count(&mut n) != 0 { + return None; + } + let mut devices = Vec::new(); + for i in 0..n { + let mut d: Device = std::ptr::null_mut(); + let mut mj = 0u64; + if by_index(i, &mut d) == 0 && !d.is_null() && energy(d, &mut mj) == 0 { + devices.push(d); + } + } + (!devices.is_empty()).then_some(Nvml { energy, limits, power, devices }) + } + } + + impl Nvml { + /// Every GPU's total-energy counter, in mJ; `None` if one fails. + pub fn energy_mj(&self) -> Option> { + self.devices + .iter() + .map(|&d| { + let mut mj = 0u64; + (unsafe { (self.energy)(d, &mut mj) } == 0).then_some(mj) + }) + .collect() + } + + /// GPU `i`'s maximum power-management limit, in mW, as NVML gives it. + pub fn max_limit_mw(&self, i: usize) -> Option { + let (mut lo, mut hi) = (0, 0); + (unsafe { (self.limits?)(self.devices[i], &mut lo, &mut hi) } == 0 && hi > 0).then_some(hi) + } + + /// GPU `i`'s power draw, in mW, as NVML gives it. + pub fn power_mw(&self, i: usize) -> Option { + let mut mw = 0; + (unsafe { (self.power?)(self.devices[i], &mut mw) } == 0).then_some(mw) + } + } + + /// `nvidia-smi`'s view of every GPU: (maximum power limit, enforced limit, + /// draw), in watts; `None` without `nvidia-smi`. + pub fn smi() -> Option, Option, Option)>> { + let out = std::process::Command::new("nvidia-smi") + .args(["--query-gpu=power.max_limit,enforced.power.limit,power.draw", "--format=csv,noheader,nounits"]) + .output() + .ok()?; + out.status.success().then(|| { + String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| { + let f: Vec> = l.split(',').map(|v| v.trim().parse().ok()).collect(); + (f.first().copied().flatten(), f.get(1).copied().flatten(), f.get(2).copied().flatten()) + }) + .collect() + }) + } + + /// GPU `i`'s ceiling in watts: `nvidia-smi`'s maximum power limit, else + /// NVML's. + pub fn ceiling_w(nvml: &Nvml, smi: &Option, Option, Option)>>, i: usize) -> Option { + smi.as_ref() + .and_then(|s| s.get(i)) + .and_then(|(max, _, _)| *max) + .or_else(|| nvml.max_limit_mw(i).map(|mw| f64::from(mw) / 1000.0)) + } + + /// How often GPU 0's counter is republished: the median gap between its + /// changes, polled every 2 ms for up to 1.5 s (`None` if it changed + /// fewer than 6 times). + pub fn update_period(nvml: &Nvml) -> Option { + let started = Instant::now(); + let mut last = nvml.energy_mj()?[0]; + let mut changes = Vec::new(); + while started.elapsed() < Duration::from_millis(1500) && changes.len() < 16 { + std::thread::sleep(Duration::from_millis(2)); + let now = nvml.energy_mj()?[0]; + if now != last { + changes.push(started.elapsed()); + last = now; + } + } + let mut gaps: Vec = changes.windows(2).map(|w| w[1] - w[0]).collect(); + if gaps.len() < 5 { + return None; + } + gaps.sort(); + Some(gaps[gaps.len() / 2]) + } + + /// The oracle's verdict on the counters. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No NVML, no GPU, or no GPU whose total-energy counter reads. + Unavailable, + /// A counter did not keep advancing. + Frozen, + /// Every counter kept advancing, but one is not plausibly its board's + /// energy (see the module doc). + Implausible, + /// Every counter kept advancing, plausibly its board's energy. + Live, + } + + /// The family's plausibility rule for one GPU over a window: its counter's + /// average `rate_w`, the board's draws `draws_w` around the window and + /// its maximum power limit `ceiling_w`. A bound whose input is missing + /// is not applied. + pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { + if let Some(c) = ceiling_w { + if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { + return false; + } + } + if !draws_w.is_empty() { + let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if rate_w > 2.0 * mean + 5.0 { + return false; + } + } + true + } + + /// What the oracle saw. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + /// Each GPU's average rate over the window, in watts. + pub rate_w: Vec, + /// Each GPU's draw just before and just after the window, in watts. + pub draws_w: Vec>, + /// Each GPU's ceiling, in watts. + pub ceiling_w: Vec>, + } + + impl Watch { + pub fn summary(&self) -> String { + format!( + "{:?} (advanced {:?} of {} intervals over {} ms; counter rate {:?} W, board draw {:?} W, max power limit {:?} W)", + self.verdict, + self.advances, + self.intervals, + self.window.as_millis(), + self.rate_w.iter().map(|w| (w * 10.0).round() / 10.0).collect::>(), + self.draws_w, + self.ceiling_w + ) + } + } + + /// The oracle's run over the first `gpus` GPUs (all, if more), sleeping + /// `tick` between samples, by the sustained-advance rule (stopping when + /// live, or after a second with at least five intervals, or five + /// seconds); then each counter's plausibility over the whole window. + pub fn watch(tick: Duration, gpus: usize) -> Watch { + let none = Watch { + verdict: Verdict::Unavailable, + intervals: 0, + advances: Vec::new(), + window: Duration::ZERO, + rate_w: Vec::new(), + draws_w: Vec::new(), + ceiling_w: Vec::new(), + }; + let Some(nvml) = open() else { return none }; + let n = nvml.devices.len().min(gpus); + let before = smi(); + let read = || nvml.energy_mj().map(|v| v[..n].to_vec()); + let started = Instant::now(); + let Some(first) = read() else { return none }; + let (mut prev, mut last_at) = (first.clone(), Duration::ZERO); + let mut advances = vec![0u32; n]; + let mut intervals = 0u32; + let live = |adv: &[u32], k: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(k) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, last_at) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + std::thread::sleep(tick); + if let Some(now) = read() { + last_at = started.elapsed(); + intervals += 1; + for ((a, x), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(x > p); + } + prev = now; + } + } + let after = smi(); + let window = last_at; + let rate_w: Vec = + first.iter().zip(&prev).map(|(a, b)| b.saturating_sub(*a) as f64 / 1000.0 / window.as_secs_f64().max(1e-9)).collect(); + let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); + let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); + let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + let verdict = if !live(&advances, intervals, window) { + Verdict::Frozen + } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + Verdict::Implausible + } else { + Verdict::Live + }; + Watch { verdict, intervals, advances, window, rate_w, draws_w, ceiling_w } + } + + /// One line on the GPUs for the log: count, counters, ceilings, draw, and + /// the counter's measured republication `period`. + pub fn listing(period: Option) -> String { + let Some(nvml) = open() else { return "no NVML device with a total-energy counter".to_string() }; + format!( + "{} GPU(s) with a total-energy counter, counters {:?} mJ; NVML max power limit {:?} mW, power draw {:?} mW; \ + nvidia-smi (max limit, enforced limit, draw) W {:?}; counter republished every {:?}", + nvml.devices.len(), + nvml.energy_mj(), + (0..nvml.devices.len()).map(|i| nvml.max_limit_mw(i)).collect::>(), + (0..nvml.devices.len()).map(|i| nvml.power_mw(i)).collect::>(), + smi(), + period + ) + } + + /// GPU 0's counter's republication period, where there is one. + pub fn period() -> Option { + open().and_then(|n| update_period(&n)) + } +} + +use nvml_oracle::Verdict; + +/// The liveness agrees with the oracle, judged at the probe's own tick, and +/// the tick is no faster than the counter is republished; the grade follows +/// the liveness. +#[test] +fn the_nvml_liveness_agrees_with_the_oracle_at_its_own_tick() { + let _one = serial(); + let period = nvml_oracle::period(); + let counter = NvmlCounter::new(); + let watch = nvml_oracle::watch(TICK, usize::MAX); + let liveness = counter.liveness(); + eprintln!( + "nvml hardware: {}; oracle at a {TICK:?} tick {}; liveness={liveness:?} available={} true_counter={} grade={:?}", + nvml_oracle::listing(period), + watch.summary(), + counter.is_available(), + counter.uses_true_counter(), + counter.provenance() + ); + if let Some(p) = period { + assert!(TICK >= p * 9 / 10, "the {TICK:?} tick is faster than the counter's {p:?} republication"); + } + assert_eq!(liveness == CounterLiveness::Live, watch.verdict == Verdict::Live, "the liveness disagrees with the oracle"); + match watch.verdict { + Verdict::Unavailable => assert_eq!(liveness, CounterLiveness::Unavailable), + Verdict::Frozen => assert_eq!(liveness, CounterLiveness::Frozen), + Verdict::Implausible => assert_ne!(liveness, CounterLiveness::Live), + Verdict::Live => {} + } + assert_eq!(counter.uses_true_counter(), liveness == CounterLiveness::Live, "the counter mode disagrees"); + assert_eq!(counter.domain(), EnergyDomain::Gpu); + if counter.is_available() { + let want = if liveness == CounterLiveness::Live { Provenance::HwShunt } else { Provenance::ModelBased }; + assert_eq!(counter.provenance(), want, "the grade does not follow the liveness"); + } +} + +/// Over a one-second span the counter's readings are graded by its liveness +/// over the `Gpu` domain: a live counter's are `HwShunt`, advanced, inside +/// the oracle's own reads around the span, and no faster than the boards can +/// draw; any other's are `ModelBased` (the power-draw integral), never +/// `HwShunt`. +#[test] +fn a_gpu_counter_reads_the_gpu_alone_graded_by_its_liveness() { + let _one = serial(); + let counter = NvmlCounter::new(); + if !counter.is_available() { + eprintln!("nvml span: no NVML ({:?})", counter.liveness()); + assert!(counter.read().is_err()); + assert!(nvml_oracle::open().is_none(), "the oracle opens NVML, yet the counter is unavailable"); + return; + } + let oracle = nvml_oracle::open(); + let o0 = oracle.as_ref().and_then(|o| o.energy_mj()); + let r0 = counter.read().expect("an available counter reads"); + std::thread::sleep(Duration::from_secs(1)); + let r1 = counter.read().expect("an available counter reads"); + let o1 = oracle.as_ref().and_then(|o| o.energy_mj()); + eprintln!("nvml span: liveness {:?}; readings {r0:?} -> {r1:?}; oracle {o0:?} -> {o1:?} mJ", counter.liveness()); + for r in [&r0, &r1] { + assert_eq!(r.domain, EnergyDomain::Gpu, "{r:?}"); + } + if counter.liveness() != CounterLiveness::Live { + assert!([&r0, &r1].iter().all(|r| r.provenance == Provenance::ModelBased), "a counter that is not live read HwShunt"); + return; + } + assert!([&r0, &r1].iter().all(|r| r.provenance == Provenance::HwShunt)); + let oracle = oracle.expect("a live counter, yet the oracle cannot open NVML"); + let (o0, o1) = (o0.expect("oracle read"), o1.expect("oracle read")); + let bracket: u64 = o0.iter().zip(&o1).map(|(a, b)| b - a).sum::() * 1_000; + let uj = r1.uj - r0.uj; + assert!(uj > 0 && uj <= bracket, "{uj} µJ outside the oracle's (0, {bracket}] µJ"); + let smi = nvml_oracle::smi(); + let ceiling: f64 = (0..oracle.devices.len()).map(|i| nvml_oracle::ceiling_w(&oracle, &smi, i).unwrap_or(f64::INFINITY)).sum(); + let watts = uj as f64 / 1e6 / ((r1.timestamp_ns - r0.timestamp_ns) as f64 / 1e9); + assert!(watts <= 1.5 * ceiling, "the GPU readings run at {watts:.1} W, past the boards' {ceiling} W ceiling"); +} diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs b/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs new file mode 100644 index 000000000..f4b789ba8 --- /dev/null +++ b/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs @@ -0,0 +1,398 @@ +//! NVML against the hardware: the GPUs' total-energy counters, read by the +//! meter and by an oracle the tests declare here. +//! +//! In a test binary of its own, so the NVML probe behind `nvml_status()` +//! runs in this process, and the tests take one lock. The oracle opens +//! `libnvidia-ml` itself, measures how often the counter is republished, +//! samples it at the probe's own tick (150 ms) by the sustained-advance rule +//! and holds each counter to its board's power ceiling, which it asks of +//! `nvidia-smi`. None of these tests passes without asserting: where there is +//! no NVML (a CI container has no GPU), the status must say so and agree with +//! the oracle; where the counter is live, both must say so, and a span's GPU +//! figure must be a `Gpu` record, `HwShunt` for that class alone, never a +//! CPU reading, and no faster than the board can draw. + +#![cfg(target_os = "linux")] + +use std::sync::{Mutex, MutexGuard}; +use std::time::Duration; + +use sandbox_meter::probe::{nvml_status, CounterStatus, PowerMeter}; +use sandbox_meter::Provenance; +use sandbox_schema::DeviceClass; + +/// One test at a time: each reads machine-wide counters, and the oracle's +/// own sampling must not overlap the meter's. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn serial() -> MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// The NVML probe's tick (`NVML_PROBE_TICK`): 150 ms of the probing thread's +/// CPU between samples, so at least 150 ms of wall time. +const TICK: Duration = Duration::from_millis(150); + +/// The tests' own reading of NVIDIA's total-energy counter: `libnvidia-ml` +/// opened here with `dlopen` and called through symbols looked up here, not +/// through the crate under test or its bindings, and each GPU's power limit +/// and draw asked of `nvidia-smi`, a separate process. It measures how often +/// the counter is republished, and judges it by the family's rule: every +/// GPU's counter advanced in at least five intervals, at least 80% of them, +/// over at least 300 ms, sampled at the crate's own tick; and it is plausibly +/// counting its board's energy: over the window it ran no faster than 1.5 +/// times the board's maximum power limit, the board's own power draw (as +/// `nvidia-smi` reports it just before and just after) was itself within that +/// bound, and the counter ran no faster than twice that draw plus 5 W (the +/// draw's documented accuracy). A counter that runs several times faster than +/// its board draws is not counting that board's energy. +#[allow(dead_code)] +mod nvml_oracle { + use std::ffi::{c_char, c_int, c_uint, c_void, CString}; + use std::time::{Duration, Instant}; + + unsafe extern "C" { + fn dlopen(filename: *const c_char, flag: c_int) -> *mut c_void; + fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; + } + + /// `RTLD_NOW` in ``. + const RTLD_NOW: c_int = 2; + + type Device = *mut c_void; + type EnergyFn = unsafe extern "C" fn(Device, *mut u64) -> c_int; + type LimitsFn = unsafe extern "C" fn(Device, *mut c_uint, *mut c_uint) -> c_int; + type MilliwattsFn = unsafe extern "C" fn(Device, *mut c_uint) -> c_int; + + /// NVML, opened by the oracle. + pub struct Nvml { + energy: EnergyFn, + limits: Option, + power: Option, + /// Every GPU whose total-energy counter reads, in index order. + pub devices: Vec, + } + + fn sym(lib: *mut c_void, name: &str) -> Option<*mut c_void> { + let name = CString::new(name).ok()?; + let p = unsafe { dlsym(lib, name.as_ptr()) }; + (!p.is_null()).then_some(p) + } + + /// `libnvidia-ml.so.1`, initialised, with every GPU whose counter reads; + /// `None` where there is no library, no GPU, or no GPU with the counter. + pub fn open() -> Option { + let file = CString::new("libnvidia-ml.so.1").ok()?; + let lib = unsafe { dlopen(file.as_ptr(), RTLD_NOW) }; + if lib.is_null() { + return None; + } + unsafe { + let init: unsafe extern "C" fn() -> c_int = std::mem::transmute(sym(lib, "nvmlInit_v2")?); + let count: unsafe extern "C" fn(*mut c_uint) -> c_int = std::mem::transmute(sym(lib, "nvmlDeviceGetCount_v2")?); + let by_index: unsafe extern "C" fn(c_uint, *mut Device) -> c_int = + std::mem::transmute(sym(lib, "nvmlDeviceGetHandleByIndex_v2")?); + let energy: EnergyFn = std::mem::transmute(sym(lib, "nvmlDeviceGetTotalEnergyConsumption")?); + let limits = sym(lib, "nvmlDeviceGetPowerManagementLimitConstraints").map(|p| std::mem::transmute::<_, LimitsFn>(p)); + let power = sym(lib, "nvmlDeviceGetPowerUsage").map(|p| std::mem::transmute::<_, MilliwattsFn>(p)); + if init() != 0 { + return None; + } + let mut n: c_uint = 0; + if count(&mut n) != 0 { + return None; + } + let mut devices = Vec::new(); + for i in 0..n { + let mut d: Device = std::ptr::null_mut(); + let mut mj = 0u64; + if by_index(i, &mut d) == 0 && !d.is_null() && energy(d, &mut mj) == 0 { + devices.push(d); + } + } + (!devices.is_empty()).then_some(Nvml { energy, limits, power, devices }) + } + } + + impl Nvml { + /// Every GPU's total-energy counter, in mJ; `None` if one fails. + pub fn energy_mj(&self) -> Option> { + self.devices + .iter() + .map(|&d| { + let mut mj = 0u64; + (unsafe { (self.energy)(d, &mut mj) } == 0).then_some(mj) + }) + .collect() + } + + /// GPU `i`'s maximum power-management limit, in mW, as NVML gives it. + pub fn max_limit_mw(&self, i: usize) -> Option { + let (mut lo, mut hi) = (0, 0); + (unsafe { (self.limits?)(self.devices[i], &mut lo, &mut hi) } == 0 && hi > 0).then_some(hi) + } + + /// GPU `i`'s power draw, in mW, as NVML gives it. + pub fn power_mw(&self, i: usize) -> Option { + let mut mw = 0; + (unsafe { (self.power?)(self.devices[i], &mut mw) } == 0).then_some(mw) + } + } + + /// `nvidia-smi`'s view of every GPU: (maximum power limit, enforced limit, + /// draw), in watts; `None` without `nvidia-smi`. + pub fn smi() -> Option, Option, Option)>> { + let out = std::process::Command::new("nvidia-smi") + .args(["--query-gpu=power.max_limit,enforced.power.limit,power.draw", "--format=csv,noheader,nounits"]) + .output() + .ok()?; + out.status.success().then(|| { + String::from_utf8_lossy(&out.stdout) + .lines() + .map(|l| { + let f: Vec> = l.split(',').map(|v| v.trim().parse().ok()).collect(); + (f.first().copied().flatten(), f.get(1).copied().flatten(), f.get(2).copied().flatten()) + }) + .collect() + }) + } + + /// GPU `i`'s ceiling in watts: `nvidia-smi`'s maximum power limit, else + /// NVML's. + pub fn ceiling_w(nvml: &Nvml, smi: &Option, Option, Option)>>, i: usize) -> Option { + smi.as_ref() + .and_then(|s| s.get(i)) + .and_then(|(max, _, _)| *max) + .or_else(|| nvml.max_limit_mw(i).map(|mw| f64::from(mw) / 1000.0)) + } + + /// How often GPU 0's counter is republished: the median gap between its + /// changes, polled every 2 ms for up to 1.5 s (`None` if it changed + /// fewer than 6 times). + pub fn update_period(nvml: &Nvml) -> Option { + let started = Instant::now(); + let mut last = nvml.energy_mj()?[0]; + let mut changes = Vec::new(); + while started.elapsed() < Duration::from_millis(1500) && changes.len() < 16 { + std::thread::sleep(Duration::from_millis(2)); + let now = nvml.energy_mj()?[0]; + if now != last { + changes.push(started.elapsed()); + last = now; + } + } + let mut gaps: Vec = changes.windows(2).map(|w| w[1] - w[0]).collect(); + if gaps.len() < 5 { + return None; + } + gaps.sort(); + Some(gaps[gaps.len() / 2]) + } + + /// The oracle's verdict on the counters. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum Verdict { + /// No NVML, no GPU, or no GPU whose total-energy counter reads. + Unavailable, + /// A counter did not keep advancing. + Frozen, + /// Every counter kept advancing, but one is not plausibly its board's + /// energy (see the module doc). + Implausible, + /// Every counter kept advancing, plausibly its board's energy. + Live, + } + + /// The family's plausibility rule for one GPU over a window: its counter's + /// average `rate_w`, the board's draws `draws_w` around the window and + /// its maximum power limit `ceiling_w`. A bound whose input is missing + /// is not applied. + pub fn plausible(rate_w: f64, draws_w: &[f64], ceiling_w: Option) -> bool { + if let Some(c) = ceiling_w { + if rate_w > 1.5 * c || draws_w.iter().any(|d| *d > 1.5 * c) { + return false; + } + } + if !draws_w.is_empty() { + let mean = draws_w.iter().sum::() / draws_w.len() as f64; + if rate_w > 2.0 * mean + 5.0 { + return false; + } + } + true + } + + /// What the oracle saw. + #[derive(Debug)] + pub struct Watch { + pub verdict: Verdict, + pub intervals: u32, + pub advances: Vec, + pub window: Duration, + /// Each GPU's average rate over the window, in watts. + pub rate_w: Vec, + /// Each GPU's draw just before and just after the window, in watts. + pub draws_w: Vec>, + /// Each GPU's ceiling, in watts. + pub ceiling_w: Vec>, + } + + impl Watch { + pub fn summary(&self) -> String { + format!( + "{:?} (advanced {:?} of {} intervals over {} ms; counter rate {:?} W, board draw {:?} W, max power limit {:?} W)", + self.verdict, + self.advances, + self.intervals, + self.window.as_millis(), + self.rate_w.iter().map(|w| (w * 10.0).round() / 10.0).collect::>(), + self.draws_w, + self.ceiling_w + ) + } + } + + /// The oracle's run over the first `gpus` GPUs (all, if more), sleeping + /// `tick` between samples, by the sustained-advance rule (stopping when + /// live, or after a second with at least five intervals, or five + /// seconds); then each counter's plausibility over the whole window. + pub fn watch(tick: Duration, gpus: usize) -> Watch { + let none = Watch { + verdict: Verdict::Unavailable, + intervals: 0, + advances: Vec::new(), + window: Duration::ZERO, + rate_w: Vec::new(), + draws_w: Vec::new(), + ceiling_w: Vec::new(), + }; + let Some(nvml) = open() else { return none }; + let n = nvml.devices.len().min(gpus); + let before = smi(); + let read = || nvml.energy_mj().map(|v| v[..n].to_vec()); + let started = Instant::now(); + let Some(first) = read() else { return none }; + let (mut prev, mut last_at) = (first.clone(), Duration::ZERO); + let mut advances = vec![0u32; n]; + let mut intervals = 0u32; + let live = |adv: &[u32], k: u32, w: Duration| { + w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(k) * 4) + }; + loop { + let el = started.elapsed(); + if live(&advances, intervals, last_at) || el >= Duration::from_secs(5) || (el >= Duration::from_secs(1) && intervals >= 5) { + break; + } + std::thread::sleep(tick); + if let Some(now) = read() { + last_at = started.elapsed(); + intervals += 1; + for ((a, x), p) in advances.iter_mut().zip(&now).zip(&prev) { + *a += u32::from(x > p); + } + prev = now; + } + } + let after = smi(); + let window = last_at; + let rate_w: Vec = + first.iter().zip(&prev).map(|(a, b)| b.saturating_sub(*a) as f64 / 1000.0 / window.as_secs_f64().max(1e-9)).collect(); + let draw = |s: &Option, Option, Option)>>, i: usize| s.as_ref().and_then(|s| s.get(i)).and_then(|t| t.2); + let draws_w: Vec> = (0..n).map(|i| [draw(&before, i), draw(&after, i)].into_iter().flatten().collect()).collect(); + let ceiling_w: Vec> = (0..n).map(|i| ceiling_w(&nvml, &after, i)).collect(); + let verdict = if !live(&advances, intervals, window) { + Verdict::Frozen + } else if (0..n).any(|i| !plausible(rate_w[i], &draws_w[i], ceiling_w[i])) { + Verdict::Implausible + } else { + Verdict::Live + }; + Watch { verdict, intervals, advances, window, rate_w, draws_w, ceiling_w } + } + + /// One line on the GPUs for the log: count, counters, ceilings, draw, and + /// the counter's measured republication `period`. + pub fn listing(period: Option) -> String { + let Some(nvml) = open() else { return "no NVML device with a total-energy counter".to_string() }; + format!( + "{} GPU(s) with a total-energy counter, counters {:?} mJ; NVML max power limit {:?} mW, power draw {:?} mW; \ + nvidia-smi (max limit, enforced limit, draw) W {:?}; counter republished every {:?}", + nvml.devices.len(), + nvml.energy_mj(), + (0..nvml.devices.len()).map(|i| nvml.max_limit_mw(i)).collect::>(), + (0..nvml.devices.len()).map(|i| nvml.power_mw(i)).collect::>(), + smi(), + period + ) + } + + /// GPU 0's counter's republication period, where there is one. + pub fn period() -> Option { + open().and_then(|n| update_period(&n)) + } +} + +use nvml_oracle::Verdict; + +/// The status agrees with the oracle, judged at the probe's own tick, and the +/// tick is no faster than the counter is republished; only a live status puts +/// the NVML probe in the meter. +#[test] +fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { + let _one = serial(); + let period = nvml_oracle::period(); + let status = nvml_status(); + let watch = nvml_oracle::watch(TICK, usize::MAX); + let sources = PowerMeter::detect().sources(); + eprintln!( + "nvml hardware: {}; oracle at a {TICK:?} tick {}; status={status:?} sources={sources:?}", + nvml_oracle::listing(period), + watch.summary() + ); + if let Some(p) = period { + assert!(TICK >= p * 9 / 10, "the {TICK:?} tick is faster than the counter's {p:?} republication"); + } + assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); + match watch.verdict { + Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), + Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), + Verdict::Implausible => assert_ne!(status, CounterStatus::Live), + Verdict::Live => {} + } + assert_eq!(sources.contains(&"nvml-total-energy"), status == CounterStatus::Live, "selection disagrees with the status"); +} + +/// Over a one-second span, a meter with a live NVML probe gives one `Gpu` +/// record, `HwShunt` for the GPU class alone: it advanced, it sits inside the +/// oracle's own reads of the counter around the span, it ran no faster than +/// the board can draw, and the span's CPU reading comes from a CPU counter or +/// not at all. Where NVML is not live, no record is a GPU reading. +#[test] +fn a_live_gpu_counter_is_a_gpu_record_alone() { + let _one = serial(); + let mut meter = PowerMeter::detect(); + let oracle = nvml_oracle::open(); + if !meter.sources().contains(&"nvml-total-energy") { + let ((), m) = meter.measure(|| std::thread::sleep(Duration::from_millis(300))); + eprintln!("nvml span: no NVML probe ({:?}); records {:?}", meter.sources(), m.devices); + assert_ne!(nvml_status(), CounterStatus::Live); + assert!(!m.devices.iter().any(|d| d.class == DeviceClass::Gpu), "a GPU record with NVML declined: {:?}", m.devices); + return; + } + let oracle = oracle.expect("the meter reads NVML, yet the oracle cannot open it"); + let o0 = oracle.energy_mj().expect("oracle read"); + let ((), m) = meter.measure(|| std::thread::sleep(Duration::from_secs(1))); + let o1 = oracle.energy_mj().expect("oracle read"); + let smi = nvml_oracle::smi(); + let ceiling: f64 = (0..oracle.devices.len()).map(|i| nvml_oracle::ceiling_w(&oracle, &smi, i).unwrap_or(f64::INFINITY)).sum(); + let gpu: Vec<_> = m.devices.iter().filter(|d| d.class == DeviceClass::Gpu).collect(); + eprintln!("nvml span: {:?}; records {:?}; oracle {o0:?} -> {o1:?} mJ; ceiling {ceiling} W", m.wall, m.devices); + assert_eq!(gpu.len(), 1, "not one GPU record: {:?}", m.devices); + let d = gpu[0]; + assert_eq!(d.provenance, Provenance::HwShunt); + let uj = d.measured_uj.expect("an HwShunt record carries its measured figure"); + let bracket: u64 = o0.iter().zip(&o1).map(|(a, b)| b - a).sum::() * 1_000; + assert!(uj > 0 && uj <= bracket, "{uj} µJ outside the oracle's (0, {bracket}] µJ"); + let watts = uj as f64 / 1e6 / m.wall.as_secs_f64(); + assert!(watts <= 1.5 * ceiling, "the GPU record reads {watts:.1} W, past the boards' {ceiling} W ceiling"); +} -- 2.54.0 (Apple Git-157) From 96537ba519a8f42d21d463fff64f2bf4c2f70d2f Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 1 Oct 2026 01:49:22 -0400 Subject: [PATCH 04/10] sandbox-meter: an NVML counter that runs past its board is Implausible, not Live On jetson-hub's RTX 4050 Laptop GPU (driver 595.84) the total-energy counter is republished every ~100 ms and advances in every probe interval, so the probe called it Live and every span carried a Gpu record graded HwShunt. But the counter does not count the board's energy while it is polled: at the probe's 150 ms tick it ran at 44-103 W while the board drew 10-14 W against a 65 W maximum limit (304 W at a 100 ms tick; another process polling every 10 ms drove it to 415 W in an unpolled reader's span), and a 1 s span with the meter's two reads came out at 22-31 W. The probe now reads each GPU's power draw with every sample and its maximum power limit once, and a live run whose counter ran past 1.5x the limit, past twice the board's mean draw plus 5 W (NVML's stated draw accuracy), or whose draw reading itself was past 1.5x the limit (590 W for the first second after the GPU wakes) is CounterStatus::Implausible: the NVML probe is not built, and the meter has no GPU reading. A bound whose reading is missing is not applied. Before: tests/nvml_hardware.rs failed on jetson-hub, status=Live against an oracle verdict of Implausible (57.8 W counter, 13.1-13.6 W draw). After: status=Implausible, sources [rapl-powercap]; 104 lib tests (the new rule and run tests among them), the NVML and RAPL hardware tests and sandbox-exec-microvm's pass there. Co-Authored-By: Claude Opus 5.5 --- .../sandbox-meter/src/probe/liveness.rs | 142 +++++++++++++++++- .../crates/sandbox-meter/src/probe/mod.rs | 22 ++- .../crates/sandbox-meter/src/probe/nvml.rs | 137 +++++++++++++++-- .../sandbox-meter/tests/nvml_hardware.rs | 2 +- 4 files changed, 288 insertions(+), 15 deletions(-) diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs index b06beb9c0..7dc4361ac 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs @@ -466,6 +466,82 @@ pub(super) fn counter_admits(status: CounterStatus) -> bool { status == CounterStatus::Live } +/// How far past its board's maximum power limit a GPU energy counter may run, +/// on average over a probe's window, as a fraction (3/2): a board can draw +/// past its limit for milliseconds, not on average over a third of a second. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) const GPU_CEILING_MARGIN: (u128, u128) = (3, 2); + +/// How much faster than its board's own power draw a GPU energy counter may +/// run over a probe's window: twice the draw's mean over the samples, plus +/// [`GPU_DRAW_SLACK_MW`]. The draw is a one-second average, so a load that +/// changed during the window can leave it behind the counter, but not by a +/// factor of two at a probe. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) const GPU_DRAW_FACTOR: u128 = 2; + +/// NVML's documented accuracy of a board's power draw, ±5 W, in mW. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) const GPU_DRAW_SLACK_MW: u128 = 5_000; + +/// Whether one GPU's energy counter, which counted `energy_mj` over `window`, +/// plausibly counted its board's energy, beside the board's power draw +/// `draws_mw` read with the samples and its maximum power limit `ceiling_mw`. +/// Not when the counter ran past [`GPU_CEILING_MARGIN`] of the limit, or a +/// draw reading did (the board's own telemetry then is not a reading either: +/// an RTX 4050 Laptop GPU reports 590 W for about a second after it wakes), +/// or the counter ran past [`GPU_DRAW_FACTOR`] times the mean draw plus +/// [`GPU_DRAW_SLACK_MW`]. A bound whose reading is missing is not applied. +/// +/// Liveness says the counter is republished continuously; this says what it +/// republishes is the board's energy. A counter that runs several times +/// faster than its board draws is counting something else, and on the RTX +/// 4050 Laptop GPU (driver 595.84) the total-energy counter does exactly +/// that while it is polled: at the probes' 100-150 ms ticks it ran at +/// 44-305 W while the board drew 5-14 W against a 65 W limit, and polled +/// every 1-2 s it still read 11-23 W while the board drew 2-10 W. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) fn plausible_gpu_counter(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { + let ns = window.as_nanos().max(1); + // mJ per second is mW. + let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; + if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { + let cap = u128::from(limit) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1; + if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { + return false; + } + } + if !draws_mw.is_empty() { + let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; + if rate_mw > GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW { + return false; + } + } + true +} + +/// Whether every GPU counter of a probe run plausibly counted its board's +/// energy ([`plausible_gpu_counter`]): each rail's rise from the run's first +/// sample to its last, over the window between them, beside that GPU's draw +/// readings (`draws`, one row per sample call, one column per GPU, `None` +/// where a reading failed) and its maximum power limit (`ceilings`, one per +/// GPU). Rails and columns are in device order. A run with fewer than two +/// samples has nothing to judge. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `nvml` +pub(super) fn gpu_counters_plausible(watch: &RailWatch, draws: &[Vec>], ceilings: &[Option]) -> bool { + let (Some(first), Some(last), Some(t0), Some(t1)) = + (watch.observed.first(), watch.observed.last(), watch.at.first(), watch.at.last()) + else { + return true; + }; + let window = t1.saturating_sub(*t0); + first.iter().zip(last).enumerate().all(|(i, ((_, a), (_, b)))| { + let energy_mj = u64::try_from(b.saturating_sub(*a)).unwrap_or(0); + let gpu_draws: Vec = draws.iter().filter_map(|row| row.get(i).copied().flatten()).collect(); + plausible_gpu_counter(energy_mj, window, &gpu_draws, ceilings.get(i).copied().flatten()) + }) +} + /// A counter probe's constructor behind its gate: `open` runs only when /// [`counter_admits`] the `status`; otherwise `Unavailable`, naming the /// counter (`what`) and why, and the counter is never touched. Every counter @@ -1813,6 +1889,69 @@ mod tests { assert_eq!(slow.watch.verdict(), Some(CpuRail::Frozen), "{:?}", slow.watch); } + /// The plausibility rule on one GPU counter, with the readings of the + /// RTX 4050 Laptop GPU on jetson-hub (driver 595.84, 65 W maximum limit): + /// polled at the probes' ticks its counter ran at 44-305 W while the board + /// drew 5-14 W, which is no count of the board's energy. A counter that + /// tracks its board's draw passes, a 1 s average lagging a load step by up + /// to twice still passes, and a bound whose reading is missing is not + /// applied. + #[test] + fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { + let w = Duration::from_millis(500); + // 66 W counted against a 7.5 W draw: what jetson-hub's probe saw. + assert!(!plausible_gpu_counter(33_000, w, &[7_500, 7_700], Some(65_000))); + // 44 W at the 150 ms tick against 10.4 W: past twice the draw + 5 W. + assert!(!plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000))); + // 120 W, past 1.5 x the 65 W limit, even with no draw reading. + assert!(!plausible_gpu_counter(60_000, w, &[], Some(65_000))); + // A draw reading of 590 W (the board's first second after waking) is no + // reading: it would excuse any counter. + assert!(!plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000))); + // A counter tracking its board: 13 W against 13 W, and 300 W against a + // 280 W draw on a 700 W board. + assert!(plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000))); + assert!(plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000))); + // A load step the one-second draw has not caught up with, up to twice + // the draw plus 5 W: 205 W against 100 W. + assert!(plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000))); + assert!(!plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000))); + // The exact bounds. + assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); + assert!(!plausible_gpu_counter(48_751, w, &[], Some(65_000))); + assert!(plausible_gpu_counter(12_500, w, &[10_000], None), "25 W is 2 x 10 W + 5 W"); + assert!(!plausible_gpu_counter(12_501, w, &[10_000], None)); + // Nothing to judge it by: no bound applies. + assert!(plausible_gpu_counter(1_000_000, w, &[], None)); + } + + /// The rule over a probe run: each GPU's rise from the run's first sample + /// to its last, over the window between them, beside that GPU's own draws + /// and limit. One implausible GPU of two makes the run implausible, and a + /// GPU's readings are never judged against another's. + #[test] + fn every_gpu_of_a_run_must_count_no_faster_than_its_board_draws() { + // Two GPUs republished every 100 ms, sampled at the NVML tick: one at + // 10 W (1 J a 100 ms step), one at 300 W (30 J a step). + let two = |t| { + Some(vec![ + ("nvml:0", published(Duration::from_millis(100), 1_000, t)), + ("nvml:1", published(Duration::from_millis(100), 30_000, t)), + ]) + }; + let r = counter_scripted(true, two, NVML_PROBE_TICK, Duration::from_micros(200)); + assert_eq!(r.watch.verdict(), Some(CpuRail::Live), "{:?}", r.watch); + let rows = r.watch.observed.len(); + let draws = |a: u32, b: u32| vec![vec![Some(a), Some(b)]; rows]; + let ceilings = [Some(65_000), Some(700_000)]; + assert!(gpu_counters_plausible(&r.watch, &draws(10_000, 290_000), &ceilings), "each at its own draw"); + assert!(!gpu_counters_plausible(&r.watch, &draws(1_000, 290_000), &ceilings), "the first at 9 x its draw"); + assert!(!gpu_counters_plausible(&r.watch, &draws(10_000, 290_000), &[Some(65_000), Some(65_000)]), "300 W on a 65 W board"); + assert!(!gpu_counters_plausible(&r.watch, &draws(290_000, 10_000), &[None, None]), "draws swapped between the GPUs"); + assert!(gpu_counters_plausible(&r.watch, &[], &[None, None]), "no reading to judge by"); + assert!(gpu_counters_plausible(&RailWatch::default(), &[], &[]), "no samples"); + } + /// The tick a counter probe burns is the tick it was asked for. #[test] fn a_counter_tick_burns_its_own_length() { @@ -1837,7 +1976,7 @@ mod tests { /// its own error passes through. #[test] fn a_gated_counter_is_opened_only_when_live() { - for status in [CounterStatus::Unavailable, CounterStatus::Frozen] { + for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible] { let opened = Cell::new(false); let r = gated("RAPL", status, || { opened.set(true); @@ -1862,6 +2001,7 @@ mod tests { fn only_a_live_counter_admits_its_probe() { assert!(counter_admits(CounterStatus::Live)); assert!(!counter_admits(CounterStatus::Frozen)); + assert!(!counter_admits(CounterStatus::Implausible)); assert!(!counter_admits(CounterStatus::Unavailable)); assert_eq!(counter_status_of(None), CounterStatus::Unavailable); assert_eq!(counter_status_of(Some(CpuRail::Absent)), CounterStatus::Unavailable); diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs index ca2d932a8..1fad7e401 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs @@ -183,6 +183,15 @@ pub enum CounterStatus { /// A figure summed over it would claim a domain it does not read, or a /// stale refresh as a reading. Never used. Frozen, + /// Every rail it sums kept advancing, but a GPU counter (NVML's) ran + /// faster than its board can draw power: past 1.5 times the board's + /// maximum power limit, or past twice the board's own power draw plus + /// 5 W, or the board's draw reading was itself past that limit. Such a + /// counter is not counting the board's energy, so a figure from it would + /// seal a number the device never spent. Seen on an RTX 4050 Laptop GPU + /// (driver 595.84), whose counter runs at 4 to 30 times the board's draw + /// while it is polled. Never used. + Implausible, /// Every rail it sums kept advancing: in at least five sample intervals, /// at least four in five of those observed, over at least 300 ms of wall /// time, while the probing thread burned CPU. The probe is used. @@ -197,6 +206,10 @@ impl std::fmt::Display for CounterStatus { "not every rail of the counter kept advancing while this process burned CPU (one \ held, or stepped only in a burst), so it cannot meter the work" } + CounterStatus::Implausible => { + "the counter kept advancing, but faster than its device can draw power, so it does not \ + count the device's energy" + } CounterStatus::Live => "every rail of the counter keeps advancing under load", }) } @@ -264,9 +277,12 @@ pub fn counter_liveness_tick() { /// [`PowerMeter::detect`] uses the NVML probe only when this is /// [`CounterStatus::Live`]: every GPU's total-energy counter must keep /// advancing — the same rule, at a 150 ms tick, since a GPU energy counter is -/// published more coarsely than a CPU rail — while the probing thread burns -/// CPU. A host with no `libnvidia-ml`, or no GPU that supports the counter, is -/// `Unavailable` at once, with no burn. Always `Unavailable` off Linux. +/// published more coarsely than a CPU rail (about every 100 ms on an RTX 4050 +/// Laptop GPU) — while the probing thread burns CPU, and over the probe each +/// must have counted no faster than its board can draw +/// ([`CounterStatus::Implausible`]). A host with no `libnvidia-ml`, or no GPU +/// that supports the counter, is `Unavailable` at once, with no burn. Always +/// `Unavailable` off Linux. pub fn nvml_status() -> CounterStatus { #[cfg(target_os = "linux")] { diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs index af06a2d35..98c27ce13 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs @@ -16,13 +16,27 @@ //! and the run live in [`liveness`](super::liveness) ([`counter_run`]); this //! module only opens the devices and reads their raw integers. //! +//! **And only when every GPU's counter plausibly counts its board's energy.** +//! A counter can be republished continuously and still not count the board: +//! on an RTX 4050 Laptop GPU (driver 595.84) the total-energy counter runs at +//! 4 to 30 times the board's power draw for as long as it is polled (44-305 W +//! at the probe's tick, against a 5-14 W draw and a 65 W limit). So over the +//! probe each counter's rise is held to its board's maximum power limit and +//! to the board's own draw, read with every sample +//! ([`plausible_gpu_counter`](super::liveness::plausible_gpu_counter)); one +//! that runs past them is [`Implausible`](super::CounterStatus::Implausible) +//! and the meter has no GPU reading. +//! //! A GPU-only figure is the GPU's energy, not the work's: the meter's span //! rule (a CPU counter that held estimates the span) does not apply to a //! probe set with no CPU class, so a caller that grades a whole unit of work //! by one figure must not grade a span with no CPU reading as a reading of //! that work ([`Measurement::reads_cpu`](super::Measurement::reads_cpu)). -use super::liveness::{counter_run, counter_status_of, counter_tick, gated, RailWatch, Sample, NVML_PROBE_TICK}; +use super::liveness::{ + counter_run, counter_status_of, counter_tick, gated, gpu_counters_plausible, RailWatch, Sample, NVML_PROBE_TICK, +}; +use std::cell::RefCell; use super::{CounterStatus, Probe, ProbeError, CPU_RAIL_LIVENESS_BUDGET}; use libloading::{Library, Symbol}; use sandbox_schema::DeviceClass; @@ -37,6 +51,12 @@ pub struct NvmlProbe { _lib: Library, devices: Vec, get_energy: unsafe extern "C" fn(Device, *mut u64) -> NvmlReturn, + /// `nvmlDeviceGetPowerUsage`, the board's power draw in mW, where the + /// library has it: the liveness probe's plausibility check reads it. + get_power: Option NvmlReturn>, + /// `nvmlDeviceGetPowerManagementLimitConstraints`, the board's minimum + /// and maximum power limit in mW, where the library has it. + get_limits: Option NvmlReturn>, } // NVML device handles are opaque pointers valid for the library's lifetime; @@ -54,22 +74,55 @@ pub(super) static REAL_RUNS: AtomicUsize = AtomicUsize::new(0); /// map its verdict). Call [`nvml_status`](super::nvml_status), not this. pub(super) fn host_status() -> CounterStatus { REAL_RUNS.fetch_add(1, Ordering::SeqCst); - counter_status_of(real_run().verdict()) + let run = real_run(); + status_of_run(&run.watch, &run.draws, &run.ceilings) +} + +/// The status of an NVML probe run: its verdict's, except that a live run +/// whose counters did not plausibly count their boards' energy +/// ([`gpu_counters_plausible`]) is `Implausible`. +fn status_of_run(watch: &RailWatch, draws: &[Vec>], ceilings: &[Option]) -> CounterStatus { + match counter_status_of(watch.verdict()) { + CounterStatus::Live if !gpu_counters_plausible(watch, draws, ceilings) => CounterStatus::Implausible, + status => status, + } +} + +/// What the real probe run saw: the rule's watch of the counters, and each +/// GPU's power draw read with every sample and its maximum power limit. +struct Run { + watch: RailWatch, + draws: Vec>>, + ceilings: Vec>, } /// The real probe run: [`counter_run`] over every supported GPU's raw /// counter, a tick that burns [`NVML_PROBE_TICK`] of this thread's CPU time -/// and a monotonic clock, for [`CPU_RAIL_LIVENESS_BUDGET`]. A host with no -/// NVML, or no GPU that supports the counter, does not open, and is never -/// ticked. -fn real_run() -> RailWatch { +/// and a monotonic clock, for [`CPU_RAIL_LIVENESS_BUDGET`]; with each sample, +/// every GPU's power draw, and once, every GPU's maximum power limit. A host +/// with no NVML, or no GPU that supports the counter, does not open, and is +/// never ticked. +fn real_run() -> Run { let epoch = Instant::now(); - counter_run( - || NvmlProbe::open().ok().map(|p| move || p.raw_sample()), + let draws = RefCell::new(Vec::new()); + let ceilings = RefCell::new(Vec::new()); + let watch = counter_run( + || { + NvmlProbe::open().ok().map(|p| { + *ceilings.borrow_mut() = p.max_limits_mw(); + let draws = &draws; + move || { + let sample = p.raw_sample(); + draws.borrow_mut().push(p.draws_mw()); + sample + } + }) + }, || counter_tick(NVML_PROBE_TICK), move || epoch.elapsed(), CPU_RAIL_LIVENESS_BUDGET, - ) + ); + Run { watch, draws: draws.into_inner(), ceilings: ceilings.into_inner() } } impl NvmlProbe { @@ -126,10 +179,22 @@ impl NvmlProbe { )); } let get_energy = *get_energy; + let get_power = lib + .get:: NvmlReturn>(b"nvmlDeviceGetPowerUsage\0") + .ok() + .map(|f| *f); + let get_limits = lib + .get:: NvmlReturn>( + b"nvmlDeviceGetPowerManagementLimitConstraints\0", + ) + .ok() + .map(|f| *f); Ok(NvmlProbe { _lib: lib, devices, get_energy, + get_power, + get_limits, }) } } @@ -152,6 +217,37 @@ impl NvmlProbe { } } +impl NvmlProbe { + /// Every GPU's power draw in mW, in device order; `None` for one whose + /// reading failed, and for all where the library has no such call. + fn draws_mw(&self) -> Vec> { + self.devices + .iter() + .map(|d| { + let f = self.get_power?; + let mut mw = 0u32; + // SAFETY: `d` is a handle NVML returned for the library this + // probe keeps loaded, and `mw` is writable for the call. + (unsafe { f(*d, &mut mw) } == 0).then_some(mw) + }) + .collect() + } + + /// Every GPU's maximum power limit in mW, in device order; `None` where + /// it cannot be read (or reads as zero). + fn max_limits_mw(&self) -> Vec> { + self.devices + .iter() + .map(|d| { + let f = self.get_limits?; + let (mut min, mut max) = (0u32, 0u32); + // SAFETY: as in `draws_mw`; both out-pointers are writable. + (unsafe { f(*d, &mut min, &mut max) } == 0 && max > 0).then_some(max) + }) + .collect() + } +} + impl Probe for NvmlProbe { fn name(&self) -> &'static str { "nvml-total-energy" @@ -175,12 +271,33 @@ impl Probe for NvmlProbe { mod tests { use super::*; use crate::probe::nvml_status; + use std::time::Duration; + + /// A probe run's status: a live run whose counters ran past their boards + /// is `Implausible`; a run that was not live keeps its status, whatever the + /// draws. + #[test] + fn a_live_run_past_its_boards_is_implausible() { + let mut live = RailWatch::default(); + let mut frozen = RailWatch::default(); + for (i, t) in (0..8u64).map(|i| (i, Duration::from_millis(150 * i))) { + live.observe(vec![("nvml:0".to_string(), 1_000_000 + 15_000 * i as i64)], t); + frozen.observe(vec![("nvml:0".to_string(), 1_000_000)], t); + } + assert_eq!(counter_status_of(live.verdict()), CounterStatus::Live, "{live:?}"); + // 15 J a 150 ms interval is 100 W. + assert_eq!(status_of_run(&live, &[vec![Some(95_000)]], &[Some(350_000)]), CounterStatus::Live); + assert_eq!(status_of_run(&live, &[vec![Some(10_000)]], &[Some(65_000)]), CounterStatus::Implausible); + assert_eq!(status_of_run(&live, &[vec![Some(95_000)]], &[Some(65_000)]), CounterStatus::Implausible); + assert_eq!(status_of_run(&frozen, &[vec![Some(10_000)]], &[Some(65_000)]), CounterStatus::Frozen); + assert_eq!(status_of_run(&RailWatch::default(), &[], &[]), CounterStatus::Unavailable); + } /// The gate declines every status but `Live`, without loading NVML, and /// `Live` builds the probe wherever NVML opens with a supported GPU. #[test] fn only_a_live_status_builds_the_nvml_probe() { - for status in [CounterStatus::Unavailable, CounterStatus::Frozen] { + for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible] { match NvmlProbe::given(status) { Err(ProbeError::Unavailable(why)) => assert!(why.contains("NVML"), "{why}"), Err(e) => panic!("{status:?}: {e}"), diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs b/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs index f4b789ba8..c5ab0ab2c 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs @@ -356,7 +356,7 @@ fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { match watch.verdict { Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), - Verdict::Implausible => assert_ne!(status, CounterStatus::Live), + Verdict::Implausible => assert_eq!(status, CounterStatus::Implausible), Verdict::Live => {} } assert_eq!(sources.contains(&"nvml-total-energy"), status == CounterStatus::Live, "selection disagrees with the status"); -- 2.54.0 (Apple Git-157) From 274a72e00d61ee05c7f87c215dcb112b2d06aaa3 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 1 Oct 2026 01:51:12 -0400 Subject: [PATCH 05/10] jcp-energy: an NVML counter that runs past its board is Implausible, not Live On jetson-hub's RTX 4050 Laptop GPU (driver 595.84) the total-energy counter advances in every 100 ms probe interval, so nvml_status() was Live and the NVML rung would grade its spans HwShunt (Partial). But the counter does not count the board's energy while it is polled: at the probe's 100 ms wait it ran at 72-136 W while the board drew 12-14 W against a 65 W maximum limit. The probe now reads the board's power draw with every sample and its maximum power limit once, and a live counter that ran past 1.5x the limit, past twice the board's mean draw plus 5 W, or beside a draw reading itself past 1.5x the limit (590 W in the first second after the GPU wakes) is CounterStatus::Implausible, which no rung admits. A bound whose reading is missing is not applied. Before: tests/nvml_hardware.rs failed on jetson-hub, status=Live against an oracle verdict of Implausible (135.8 W counter, 13.5 W draw). After: status=Implausible; 32 lib tests (the rule's among them), the NVML and RAPL hardware tests pass there with --features nvml. Co-Authored-By: Claude Opus 5.5 --- jcp/jcp-rs/crates/jcp-energy/src/lib.rs | 214 ++++++++++++++++-- .../crates/jcp-energy/tests/nvml_hardware.rs | 2 +- 2 files changed, 196 insertions(+), 20 deletions(-) diff --git a/jcp/jcp-rs/crates/jcp-energy/src/lib.rs b/jcp/jcp-rs/crates/jcp-energy/src/lib.rs index b17ead44c..e72664012 100644 --- a/jcp/jcp-rs/crates/jcp-energy/src/lib.rs +++ b/jcp/jcp-rs/crates/jcp-energy/src/lib.rs @@ -26,8 +26,10 @@ //! 2. **NVML** (feature `nvml`) — GPU energy via `libnvidia-ml`, //! `Provenance::HwShunt` with `Coverage::Partial`: the counter reads the //! GPU, so the CPU's share of the work is unmetered and the figure never -//! claims the whole task. Only while the counter is live ([`nvml_status`]); -//! a span over which it did not advance is the model's whole-task figure. +//! claims the whole task. Only while the counter is live ([`nvml_status`]) +//! and plausibly counts the board's energy: over the probe it ran no +//! faster than the board can draw ([`CounterStatus::Implausible`]); a span +//! over which it did not advance is the model's whole-task figure. //! 3. **Per-process CPU energy** (macOS) — the kernel's count of this //! process's own CPU energy, `ri_energy_nj` from //! `proc_pid_rusage(getpid(), RUSAGE_INFO_V6)` (public libproc, no @@ -251,6 +253,17 @@ impl Nvml { Some(mj.saturating_mul(1_000)) } + /// The board's power draw, in mW, as NVML reports it. + fn draw_mw(&self) -> Option { + self.nvml.device_by_index(self.device_index).ok()?.power_usage().ok() + } + + /// The board's maximum power-management limit, in mW. + fn max_limit_mw(&self) -> Option { + let limits = self.nvml.device_by_index(self.device_index).ok()?.power_management_limit_constraints().ok()?; + (limits.max_limit > 0).then_some(limits.max_limit) + } + /// One liveness sample: the device's total-energy counter as its one /// rail, in its raw millijoules. `None` when the read fails, which the /// probe skips. @@ -631,6 +644,15 @@ pub enum CounterStatus { /// would be graded a hardware reading of that span's work, and a held rail /// beside live ones would be left out of a figure that claims it. Frozen, + /// The counter kept advancing, but a GPU counter (NVML's) ran faster than + /// its board can draw power over the probe: past 1.5 times the board's + /// maximum power limit, past twice the board's own power draw plus 5 W, + /// or with the board's draw reading itself past that limit. It is not + /// counting the board's energy, so a figure from it would debit energy + /// the device never spent. Seen on an RTX 4050 Laptop GPU (driver + /// 595.84), whose counter runs at 4 to 30 times the board's draw while it + /// is polled. Never selected. + Implausible, /// Every rail of the counter kept advancing while the probe ran: each in /// at least five separate sample intervals, making up at least 80% of /// those observed, over at least 300 ms. The rung is selected. @@ -738,27 +760,39 @@ const NVML_PROBE_WAIT: std::time::Duration = std::time::Duration::from_millis(10 /// /// [`Meter::detect`] selects the NVML rung only when this is /// [`CounterStatus::Live`]: device 0's total-energy counter advanced in at -/// least five sample intervals, at least 80% of them, over at least 300 ms. The -/// probe has no way to load the GPU, so this shows the counter is published -/// continuously, not that it tracks a load; a counter that holds or steps in -/// lumps is declined. The first call on a host where NVML opens waits about -/// half a second (one second if the counter is not live). +/// least five sample intervals, at least 80% of them, over at least 300 ms, +/// and over that window counted no faster than the board can draw +/// ([`plausible_gpu_counter`], with the board's power draw read at every +/// sample and its maximum power limit). The probe has no way to load the +/// GPU, so liveness shows the counter is published continuously, not that it +/// tracks a load; a counter that holds or steps in lumps is declined, and one +/// that runs faster than its board draws is [`CounterStatus::Implausible`]. +/// The first call on a host where NVML opens waits about half a second (one +/// second if the counter is not live). pub fn nvml_status() -> CounterStatus { #[cfg(feature = "nvml")] { static STATUS: std::sync::OnceLock = std::sync::OnceLock::new(); *STATUS.get_or_init(|| { NVML_PROBE_RUNS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - liveness::status_of( - liveness::probe( - Nvml::open, - Nvml::liveness_sample, - || std::thread::sleep(NVML_PROBE_WAIT), - Instant::now, - liveness::BUDGET, - ) - .as_ref(), - ) + let seen = std::cell::RefCell::new(GpuWatch::default()); + let probed = liveness::probe( + || { + let n = Nvml::open()?; + seen.borrow_mut().ceiling_mw = n.max_limit_mw(); + Some(n) + }, + |n: &Nvml| { + let sample = n.liveness_sample(); + let mj = sample.as_ref().and_then(|rails| rails.first()).map(|(_, raw)| *raw); + seen.borrow_mut().saw(mj.and_then(|mj| u64::try_from(mj).ok()), n.draw_mw(), Instant::now()); + sample + }, + || std::thread::sleep(NVML_PROBE_WAIT), + Instant::now, + liveness::BUDGET, + ); + nvml_status_of(probed.as_ref(), &seen.into_inner()) }) } #[cfg(not(feature = "nvml"))] @@ -767,6 +801,98 @@ pub fn nvml_status() -> CounterStatus { } } +/// What the NVML probe saw of the board beside its counter: the counter's +/// first and last reading (mJ) and when, the board's power draw read with +/// every sample, and its maximum power limit. +#[derive(Debug, Default, Clone)] +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] +struct GpuWatch { + first: Option<(u64, Instant)>, + last: Option<(u64, Instant)>, + draws_mw: Vec, + ceiling_mw: Option, +} + +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] +impl GpuWatch { + /// Fold in one sample: the counter's reading, if it read, and the + /// board's draw, if that read, at `at`. + fn saw(&mut self, mj: Option, draw_mw: Option, at: Instant) { + if let Some(mj) = mj { + self.first.get_or_insert((mj, at)); + self.last = Some((mj, at)); + } + self.draws_mw.extend(draw_mw); + } +} + +/// How far past its board's maximum power limit a GPU energy counter may run, +/// on average over a probe's window, as a fraction (3/2): a board can draw +/// past its limit for milliseconds, not on average over a third of a second. +const GPU_CEILING_MARGIN: (u128, u128) = (3, 2); + +/// How much faster than its board's own power draw a GPU energy counter may +/// run over a probe's window: twice the draw's mean over the samples, plus +/// [`GPU_DRAW_SLACK_MW`]. The draw is a one-second average, so a load that +/// changed during the window can leave it behind the counter, but not by a +/// factor of two at a probe. +const GPU_DRAW_FACTOR: u128 = 2; + +/// NVML's documented accuracy of a board's power draw, ±5 W, in mW. +const GPU_DRAW_SLACK_MW: u128 = 5_000; + +/// Whether a GPU energy counter that counted `energy_mj` over `window` +/// plausibly counted its board's energy, beside the board's power draw +/// `draws_mw` read with the samples and its maximum power limit `ceiling_mw`. +/// Not when the counter ran past [`GPU_CEILING_MARGIN`] of the limit, or a +/// draw reading did (the board's own telemetry then is not a reading either: +/// an RTX 4050 Laptop GPU reports 590 W for about a second after it wakes), +/// or the counter ran past [`GPU_DRAW_FACTOR`] times the mean draw plus +/// [`GPU_DRAW_SLACK_MW`]. A bound whose reading is missing is not applied. +/// +/// Liveness says the counter is republished continuously; this says what it +/// republishes is the board's energy. On the RTX 4050 Laptop GPU (driver +/// 595.84) the total-energy counter runs several times faster than the +/// board draws for as long as it is polled: at this probe's 100 ms wait it +/// ran at 58-305 W while the board drew 5-14 W against a 65 W limit. +fn plausible_gpu_counter(energy_mj: u64, window: std::time::Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { + let ns = window.as_nanos().max(1); + // mJ per second is mW. + let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; + if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { + let cap = u128::from(limit) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1; + if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { + return false; + } + } + if !draws_mw.is_empty() { + let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; + if rate_mw > GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW { + return false; + } + } + true +} + +/// The status an NVML probe reports: [`liveness::status_of`] its run, except +/// that a live counter which did not plausibly count its board's energy over +/// the run ([`plausible_gpu_counter`]) is [`CounterStatus::Implausible`]. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] +fn nvml_status_of(probed: Option<&liveness::Advances>, gpu: &GpuWatch) -> CounterStatus { + match liveness::status_of(probed) { + CounterStatus::Live => { + let plausible = match (gpu.first, gpu.last) { + (Some((a, t0)), Some((b, t1))) => { + plausible_gpu_counter(b.saturating_sub(a), t1.saturating_duration_since(t0), &gpu.draws_mw, gpu.ceiling_mw) + } + _ => true, + }; + if plausible { CounterStatus::Live } else { CounterStatus::Implausible } + } + status => status, + } +} + /// The byte layout of `struct rusage_info_v6`, which macOS's /// `proc_pid_rusage(pid, RUSAGE_INFO_V6, &info)` fills, as far as this crate /// reads it. From ``, measured with `sizeof` and `offsetof` by @@ -5395,7 +5521,7 @@ mod tests { fn only_live_admits_a_counter_rung_and_nothing_else_is_opened() { use CounterStatus as S; assert!(admits_counter(S::Live), "Live does not admit the rung"); - for status in [S::Unavailable, S::Frozen] { + for status in [S::Unavailable, S::Frozen, S::Implausible] { assert!(!admits_counter(status), "{status:?} admits the rung"); let opened = Cell::new(0); let got = select(status, || { @@ -5440,6 +5566,56 @@ mod tests { assert_eq!(liveness::status_of(Some(&empty)), CounterStatus::Frozen); } + /// The plausibility rule on a GPU counter, with the readings of the RTX + /// 4050 Laptop GPU on jetson-hub (driver 595.84, 65 W maximum limit): + /// polled at the probe's 100 ms wait its counter ran at 58-305 W while the + /// board drew 5-14 W, which is no count of the board's energy. A counter + /// that tracks its board passes, a one-second draw lagging a load step by + /// up to twice still passes, and a bound whose reading is missing is not + /// applied. + #[test] + fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { + let w = Duration::from_millis(500); + assert!(!plausible_gpu_counter(59_750, w, &[12_970, 13_520], Some(65_000)), "119.5 W against a 13 W draw"); + assert!(!plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000)), "44 W against a 10.4 W draw"); + assert!(!plausible_gpu_counter(60_000, w, &[], Some(65_000)), "120 W on a 65 W board"); + assert!(!plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000)), "a 590 W draw reading"); + assert!(plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000)), "13 W against a 13 W draw"); + assert!(plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000)), "300 W against a 280 W draw"); + assert!(plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000)), "205 W is 2 x 100 W + 5 W"); + assert!(!plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000))); + assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); + assert!(!plausible_gpu_counter(48_751, w, &[], Some(65_000))); + assert!(plausible_gpu_counter(1_000_000, w, &[], None), "nothing to judge by"); + } + + /// The NVML probe's status: a live counter that ran past its board is + /// `Implausible`; one that tracked it stays `Live`; a counter that was not + /// live keeps its status, whatever its board drew. + #[test] + fn a_live_gpu_counter_past_its_board_is_implausible() { + let rail = |v| liveness::every_rail(&[("nvml device 0 total energy".to_string(), v)]); + let (mut live, mut held) = (liveness::Advances::default(), liveness::Advances::default()); + let t0 = Instant::now(); + let mut gpu = GpuWatch { ceiling_mw: Some(65_000), ..GpuWatch::default() }; + for k in 0..6u32 { + // 12 J every 100 ms: 120 W. + live.observe(rail(1_000_000 + 12_000 * i64::from(k)), Duration::from_millis(100) * k); + held.observe(rail(1_000_000), Duration::from_millis(100) * k); + gpu.saw(Some(1_000_000 + 12_000 * u64::from(k)), Some(13_000), t0 + Duration::from_millis(100) * k); + } + assert_eq!(liveness::status_of(Some(&live)), CounterStatus::Live); + assert_eq!(nvml_status_of(Some(&live), &gpu), CounterStatus::Implausible); + assert_eq!(nvml_status_of(Some(&held), &gpu), CounterStatus::Frozen); + assert_eq!(nvml_status_of(None, &gpu), CounterStatus::Unavailable); + let mut big = GpuWatch { ceiling_mw: Some(700_000), ..GpuWatch::default() }; + for k in 0..6u32 { + big.saw(Some(1_000_000 + 12_000 * u64::from(k)), Some(115_000), t0 + Duration::from_millis(100) * k); + } + assert_eq!(nvml_status_of(Some(&live), &big), CounterStatus::Live, "120 W against a 115 W draw on a 700 W board"); + assert_eq!(nvml_status_of(Some(&live), &GpuWatch::default()), CounterStatus::Live, "nothing to judge by"); + } + /// The tests' interval for a CPU-counter probe: a 25 ms tick and a short /// read. const COUNTER_STEP: Duration = Duration::from_millis(31); @@ -5701,7 +5877,7 @@ mod tests { #[cfg(target_os = "macos")] let _hw = ioreport_hw_lock(); let fake = FakePowercap::new(&["1000"]); - for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Live] { + for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible, CounterStatus::Live] { let opened = Cell::new(0); let open = || { opened.set(opened.get() + 1); diff --git a/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs b/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs index 9169f8ef4..da15b36eb 100644 --- a/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs +++ b/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs @@ -354,7 +354,7 @@ fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { match watch.verdict { Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), - Verdict::Implausible => assert_ne!(status, CounterStatus::Live), + Verdict::Implausible => assert_eq!(status, CounterStatus::Implausible), Verdict::Live => {} } if meter.rung() == Rung::Nvml { -- 2.54.0 (Apple Git-157) From 7739d5c113ee1cf8245e3369aabccd3792310c79 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 1 Oct 2026 01:53:36 -0400 Subject: [PATCH 06/10] jouleclaw-energy: an NVML counter that runs past its board is Implausible, not HwShunt On jetson-hub's RTX 4050 Laptop GPU (driver 595.84) the cumulative-energy counter advances in every 100 ms probe interval, so the counter was Live and every reading HwShunt over Gpu. But it does not count the board's energy while it is polled: at the probe's 100 ms tick it ran at 127-305 W while the board drew 5.5-13.4 W against a 65 W maximum limit, and a 1 s span read 44.8 W of HwShunt. The probe now reads every GPU's power draw with each sample and its maximum power limit once (liveness::GpuWatch), and a live counter one of whose GPUs ran past 1.5x its limit, past twice its board's mean draw plus 5 W, or beside a draw reading itself past 1.5x the limit, is CounterLiveness::Implausible: the counter falls to the power-draw integral, graded ModelBased, as for a counter that is not live. A bound whose reading is missing is not applied. The module doc's 50 ms republication period is corrected to the ~100 ms measured. Before: tests/nvml_hardware.rs failed on jetson-hub, liveness=Live against an oracle verdict of Implausible (304.5 W counter, 5.5-7.7 W draw). After: liveness=Implausible, grade ModelBased; 56 lib tests (the rule's among them) and the NVML, RAPL and conformance tests pass there with --features rapl,nvml. Co-Authored-By: Claude Opus 5.5 --- .../crates/jouleclaw-energy/src/liveness.rs | 155 ++++++++++++++++++ .../crates/jouleclaw-energy/src/nvml.rs | 57 +++++-- .../crates/jouleclaw-energy/src/process.rs | 4 +- .../crates/jouleclaw-energy/src/rapl.rs | 3 +- .../jouleclaw-energy/tests/nvml_hardware.rs | 2 +- 5 files changed, 206 insertions(+), 15 deletions(-) diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs index 6c0746b64..6eb2f547c 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs @@ -59,6 +59,15 @@ pub enum CounterLiveness { /// intervals, four in every five of those observed, over at least 300 ms, /// while the probing thread burned CPU. Live, + /// Every channel kept advancing, but a GPU counter (NVML's) ran faster + /// than its board can draw power over the probe: past 1.5 times the + /// board's maximum power limit, past twice the board's own power draw plus + /// 5 W, or beside a draw reading itself past that limit + /// ([`plausible_gpu_counter`]). It is not counting the board's energy, so + /// its figure would not be a reading of the GPU's work. Seen on an RTX 4050 + /// Laptop GPU (driver 595.84), whose counter runs at 4 to 30 times the + /// board's draw while it is polled. + Implausible, } /// In at least how many separate sample intervals a channel must advance to @@ -272,6 +281,100 @@ pub(crate) fn burn_thread_cpu(cpu: Duration, wall_cap: Duration) { std::hint::black_box(acc); } +/// How far past its board's maximum power limit a GPU energy counter may run, +/// on average over a probe's window, as a fraction (3/2): a board can draw +/// past its limit for milliseconds, not on average over a third of a second. +pub(crate) const GPU_CEILING_MARGIN: (u128, u128) = (3, 2); + +/// How much faster than its board's own power draw a GPU energy counter may +/// run over a probe's window: twice the draw's mean over the samples, plus +/// [`GPU_DRAW_SLACK_MW`]. The draw is a one-second average, so a load that +/// changed during the window can leave it behind the counter, but not by a +/// factor of two at a probe. +pub(crate) const GPU_DRAW_FACTOR: u128 = 2; + +/// NVML's documented accuracy of a board's power draw, ±5 W, in mW. +pub(crate) const GPU_DRAW_SLACK_MW: u128 = 5_000; + +/// Whether one GPU's energy counter, which counted `energy_mj` over `window`, +/// plausibly counted its board's energy, beside the board's power draw +/// `draws_mw` read with the samples and its maximum power limit `ceiling_mw`. +/// Not when the counter ran past [`GPU_CEILING_MARGIN`] of the limit, or a +/// draw reading did (the board's own telemetry then is not a reading either: +/// an RTX 4050 Laptop GPU reports 590 W for about a second after it wakes), +/// or the counter ran past [`GPU_DRAW_FACTOR`] times the mean draw plus +/// [`GPU_DRAW_SLACK_MW`]. A bound whose reading is missing is not applied. +/// +/// Liveness says the counter is republished continuously; this says what it +/// republishes is the board's energy. On the RTX 4050 Laptop GPU (driver +/// 595.84) the total-energy counter runs several times faster than the board +/// draws for as long as it is polled: at the NVML probe's 100 ms tick it ran +/// at 58-305 W while the board drew 5-14 W against a 65 W limit. +pub(crate) fn plausible_gpu_counter(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { + let ns = window.as_nanos().max(1); + // mJ per second is mW. + let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; + if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { + let cap = u128::from(limit) * GPU_CEILING_MARGIN.0 / GPU_CEILING_MARGIN.1; + if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { + return false; + } + } + if !draws_mw.is_empty() { + let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; + if rate_mw > GPU_DRAW_FACTOR * mean + GPU_DRAW_SLACK_MW { + return false; + } + } + true +} + +/// What a GPU counter's probe saw beside the rule's verdict: each GPU's +/// counter at the first and last sample (µJ) and when, and each GPU's power +/// draw read with every sample. +#[derive(Debug, Default, Clone)] +pub(crate) struct GpuWatch { + first: Option<(Vec, Instant)>, + last: Option<(Vec, Instant)>, + draws_mw: Vec>, +} + +impl GpuWatch { + /// Fold in one sample: every GPU's counter (µJ), if the sample read, and + /// each GPU's draw (`None` where it did not read), at `at`. + pub(crate) fn saw(&mut self, uj: Option<&[u64]>, draws_mw: &[Option], at: Instant) { + if let Some(uj) = uj { + self.first.get_or_insert_with(|| (uj.to_vec(), at)); + self.last = Some((uj.to_vec(), at)); + } + if self.draws_mw.len() < draws_mw.len() { + self.draws_mw.resize(draws_mw.len(), Vec::new()); + } + for (gpu, d) in self.draws_mw.iter_mut().zip(draws_mw) { + gpu.extend(*d); + } + } + + /// The liveness a GPU counter's probe gives it: the rule's `verdict`, + /// except that a live counter one of whose GPUs did not plausibly count + /// its board's energy over the probe, beside each GPU's maximum power + /// limit `ceilings_mw`, is [`CounterLiveness::Implausible`]. + pub(crate) fn liveness(&self, verdict: CounterLiveness, ceilings_mw: &[Option]) -> CounterLiveness { + let (Some((first, t0)), Some((last, t1))) = (&self.first, &self.last) else { + return verdict; + }; + let window = t1.saturating_duration_since(*t0); + let plausible = first.iter().zip(last).enumerate().all(|(i, (a, b))| { + let draws = self.draws_mw.get(i).map(Vec::as_slice).unwrap_or(&[]); + plausible_gpu_counter(b.saturating_sub(*a) / 1_000, window, draws, ceilings_mw.get(i).copied().flatten()) + }); + match verdict { + CounterLiveness::Live if !plausible => CounterLiveness::Implausible, + other => other, + } + } +} + #[cfg(test)] pub(crate) mod tests { use super::*; @@ -285,6 +388,58 @@ pub(crate) mod tests { const TEST_MIN_ADVANCES: u32 = 5; const TEST_MIN_WINDOW: Duration = Duration::from_millis(300); + /// The plausibility rule on one GPU counter, with the readings of the RTX + /// 4050 Laptop GPU on jetson-hub (driver 595.84, 65 W maximum limit): + /// polled at the NVML probe's 100 ms tick its counter ran at 58-305 W while + /// the board drew 5-14 W, which is no count of the board's energy. A + /// counter that tracks its board passes, a one-second draw lagging a load + /// step by up to twice still passes, and a bound whose reading is missing + /// is not applied. + #[test] + fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { + let w = Duration::from_millis(500); + assert!(!plausible_gpu_counter(152_250, w, &[5_550, 7_660], Some(65_000)), "304.5 W against a 6.6 W draw"); + assert!(!plausible_gpu_counter(22_000, w, &[10_400; 6], Some(65_000)), "44 W against a 10.4 W draw"); + assert!(!plausible_gpu_counter(60_000, w, &[], Some(65_000)), "120 W on a 65 W board"); + assert!(!plausible_gpu_counter(44_000, w, &[590_000, 10_300], Some(65_000)), "a 590 W draw reading"); + assert!(plausible_gpu_counter(6_500, w, &[13_000, 13_500], Some(65_000)), "13 W against a 13 W draw"); + assert!(plausible_gpu_counter(150_000, w, &[280_000; 5], Some(700_000)), "300 W against a 280 W draw"); + assert!(plausible_gpu_counter(102_500, w, &[100_000; 5], Some(700_000)), "205 W is 2 x 100 W + 5 W"); + assert!(!plausible_gpu_counter(102_501, w, &[100_000; 5], Some(700_000))); + assert!(plausible_gpu_counter(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); + assert!(!plausible_gpu_counter(48_751, w, &[], Some(65_000))); + assert!(plausible_gpu_counter(1_000_000, w, &[], None), "nothing to judge by"); + } + + /// The liveness a GPU probe gives: each GPU's rise from the first sample + /// to the last (µJ), over the window between them, beside that GPU's own + /// draws and limit. One implausible GPU of two makes the counter + /// `Implausible`; a verdict that was not `Live` stands, whatever the draws. + #[test] + fn every_gpu_must_count_no_faster_than_its_board_draws() { + let t0 = Instant::now(); + let watch = |draws: [Option; 2]| { + let mut g = GpuWatch::default(); + for k in 0..6u64 { + // GPU 0 at 10 W, GPU 1 at 300 W: 1 J and 30 J a 100 ms interval. + let uj = [8_000_000_000 + 1_000_000 * k, 9_000_000_000 + 30_000_000 * k]; + g.saw(Some(&uj), &draws, t0 + Duration::from_millis(100) * k as u32); + } + g + }; + let ceilings = [Some(65_000), Some(700_000)]; + use CounterLiveness::*; + assert_eq!(watch([Some(10_000), Some(290_000)]).liveness(Live, &ceilings), Live, "each at its own draw"); + assert_eq!(watch([Some(1_000), Some(290_000)]).liveness(Live, &ceilings), Implausible, "GPU 0 at 10 x its draw"); + assert_eq!(watch([Some(10_000), Some(290_000)]).liveness(Live, &[Some(65_000); 2]), Implausible, "300 W on 65 W"); + assert_eq!(watch([Some(290_000), Some(10_000)]).liveness(Live, &[None, None]), Implausible, "draws swapped"); + assert_eq!(watch([None, None]).liveness(Live, &[None, None]), Live, "nothing to judge by"); + for verdict in [Frozen, Unavailable] { + assert_eq!(watch([Some(1_000), Some(1_000)]).liveness(verdict, &ceilings), verdict); + } + assert_eq!(GpuWatch::default().liveness(Live, &ceilings), Live, "no sample"); + } + /// Feed a fresh rule `samples`, the i-th at `i * every`; return it and /// whether it was live after each interval. fn feed(channels: usize, samples: &[Vec], every: Duration) -> (AdvanceWatch, Vec) { diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs index ba979e54b..778950b1f 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/nvml.rs @@ -8,10 +8,15 @@ //! claims (JouleClaw SPEC §1.3): every GPU's counter must keep advancing under //! the sustained-advance rule ([`crate::CounterLiveness`]) — in at least five //! sample intervals, four in every five, over at least 300 ms, while the -//! probing thread burns 100 ms of its own CPU time between samples (twice the -//! counter's ~50 ms sample period, so a live counter has moved in every -//! interval; a powered GPU draws tens of watts at idle, ~1 J per interval, -//! against a 1 mJ quantum). Probed once per process and cached. +//! probing thread burns 100 ms of its own CPU time between samples (an RTX +//! 4050 Laptop GPU, driver 595.84, republishes the counter every ~100 ms, so a +//! live counter has moved in nearly every interval; even at a 2 W idle that is +//! ~0.2 J an interval, against a 1 mJ quantum). And every GPU's counter must +//! plausibly count its board's energy over the probe: no faster than 1.5 +//! times the board's maximum power limit, or than twice the board's own power +//! draw plus 5 W ([`crate::CounterLiveness::Implausible`]). That RTX 4050's +//! counter runs at 4 to 30 times its board's draw while it is polled, so it +//! is declined there. Probed once per process and cached. //! //! Where that counter is unsupported (pre-Volta, some consumer parts) or not //! live, the counter integrates `power_usage()` over wall time instead. That @@ -58,7 +63,9 @@ enum Mode { fn mode_for(supported: bool, liveness: CounterLiveness) -> Mode { match (supported, liveness) { (true, CounterLiveness::Live) => Mode::Counter, - (_, CounterLiveness::Live | CounterLiveness::Frozen | CounterLiveness::Unavailable) => Mode::PowerIntegral, + (_, CounterLiveness::Live | CounterLiveness::Frozen | CounterLiveness::Unavailable | CounterLiveness::Implausible) => { + Mode::PowerIntegral + } } } @@ -115,12 +122,7 @@ impl NvmlCounter { let supported = total_energy_uj(&nvml, device_count).is_some(); let liveness = if supported { static LIVENESS: OnceLock = OnceLock::new(); - *LIVENESS.get_or_init(|| { - crate::liveness::probe( - || total_energy_uj(&nvml, device_count).map(|first| (first.len(), || total_energy_uj(&nvml, device_count))), - PROBE_TICK, - ) - }) + *LIVENESS.get_or_init(|| probe_liveness(&nvml, device_count)) } else { CounterLiveness::Unavailable }; @@ -169,6 +171,38 @@ impl NvmlCounter { /// Every GPU's cumulative energy, in µJ, in device order; `None` if any of them /// cannot be read. +/// The real liveness probe of every GPU's cumulative-energy counter: the +/// sustained-advance rule at [`PROBE_TICK`], with every GPU's power draw read +/// at each sample and its maximum power limit once, so that a live counter +/// which ran faster than its board can draw is +/// [`CounterLiveness::Implausible`] ([`crate::liveness::GpuWatch`]). +fn probe_liveness(nvml: &nvml_wrapper::Nvml, device_count: u32) -> CounterLiveness { + let ceilings: Vec> = (0..device_count) + .map(|i| { + let limits = nvml.device_by_index(i).ok()?.power_management_limit_constraints().ok()?; + (limits.max_limit > 0).then_some(limits.max_limit) + }) + .collect(); + let draws = move || -> Vec> { + (0..device_count).map(|i| nvml.device_by_index(i).ok()?.power_usage().ok()).collect() + }; + let watch = std::cell::RefCell::new(crate::liveness::GpuWatch::default()); + let seen = &watch; + let verdict = crate::liveness::probe( + move || { + total_energy_uj(nvml, device_count).map(|first| { + (first.len(), move || { + let sample = total_energy_uj(nvml, device_count); + seen.borrow_mut().saw(sample.as_deref(), &draws(), std::time::Instant::now()); + sample + }) + }) + }, + PROBE_TICK, + ); + watch.into_inner().liveness(verdict, &ceilings) +} + fn total_energy_uj(nvml: &nvml_wrapper::Nvml, device_count: u32) -> Option> { (0..device_count) .map(|i| { @@ -321,6 +355,7 @@ mod tests { (true, Unavailable, Mode::PowerIntegral), (false, Unavailable, Mode::PowerIntegral), (false, Live, Mode::PowerIntegral), + (true, Implausible, Mode::PowerIntegral), ] { assert_eq!(mode_for(supported, liveness), mode, "supported={supported} {liveness:?}"); let c = NvmlCounter::with(None, 1, mode, liveness); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs index 8440b8a77..d36d3be11 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/process.rs @@ -205,7 +205,7 @@ impl ProcessCpuCounter { fn admits(liveness: CounterLiveness) -> bool { match liveness { CounterLiveness::Live => true, - CounterLiveness::Unavailable | CounterLiveness::Frozen => false, + CounterLiveness::Unavailable | CounterLiveness::Frozen | CounterLiveness::Implausible => false, } } @@ -277,7 +277,7 @@ mod tests { /// measurement, and its work still runs. #[test] fn a_counter_that_is_not_live_meters_nothing() { - for liveness in [CounterLiveness::Frozen, CounterLiveness::Unavailable] { + for liveness in [CounterLiveness::Frozen, CounterLiveness::Implausible, CounterLiveness::Unavailable] { let c = ProcessCpuCounter::given(liveness); assert!(!c.is_available() && !EnergyMeter::is_available(&c), "{liveness:?}"); assert!(matches!(c.read(), Err(EnergyError::NoCounter(EnergyDomain::ProcessCpu))), "{liveness:?}"); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs index 5bc06cad5..accf2cea8 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs @@ -270,7 +270,7 @@ impl RaplCounter { fn admits(liveness: CounterLiveness) -> bool { match liveness { CounterLiveness::Live => true, - CounterLiveness::Unavailable | CounterLiveness::Frozen => false, + CounterLiveness::Unavailable | CounterLiveness::Frozen | CounterLiveness::Implausible => false, } } @@ -444,6 +444,7 @@ mod tests { for (liveness, available) in [ (CounterLiveness::Live, true), (CounterLiveness::Frozen, false), + (CounterLiveness::Implausible, false), (CounterLiveness::Unavailable, false), ] { let c = RaplCounter::given(Vec::new(), liveness); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs index 780da7dd1..49c200912 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs @@ -359,7 +359,7 @@ fn the_nvml_liveness_agrees_with_the_oracle_at_its_own_tick() { match watch.verdict { Verdict::Unavailable => assert_eq!(liveness, CounterLiveness::Unavailable), Verdict::Frozen => assert_eq!(liveness, CounterLiveness::Frozen), - Verdict::Implausible => assert_ne!(liveness, CounterLiveness::Live), + Verdict::Implausible => assert_eq!(liveness, CounterLiveness::Implausible), Verdict::Live => {} } assert_eq!(counter.uses_true_counter(), liveness == CounterLiveness::Live, "the counter mode disagrees"); -- 2.54.0 (Apple Git-157) From 76ce401fe93738288b3b8a33b63b6a7f8325ab54 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 1 Oct 2026 01:56:27 -0400 Subject: [PATCH 07/10] eoc-meter: the GPU counter is never the cascade's counter, and one past its board is Implausible Two defects of eoc's NVML path (feature cuda), both seen on jetson-hub's RTX 4050 Laptop GPU (driver 595.84). detect() ranked NVML first, so on a host with a live NVML counter every consumer (eoc-cascade, eoc-local, eoc-streaming, eoc-agent) was handed the GPU's counter and graded its advance Measured, with no coverage field to say the CPU's share of the work went unread: on jetson-hub detect() returned nvml over a live RAPL. By the family's rule a figure that does not cover the work it is booked to is an estimate in a format without coverage. detect() now has no NVML rung, and NvmlCounter is no longer a JouleCounter (read_gpu_microjoules() reads it as GPU telemetry), so no consumer can grade it Measured. JouleSource and the README say so. And the counter does not count the board's energy while it is polled: at the probe's 100 ms wait it ran at 81-136 W while the board drew 13-15 W against a 65 W maximum limit. The probe now reads the board's draw with each sample and its maximum power limit once (gpu_plausibility), and a live counter past 1.5x the limit, past twice the mean draw plus 5 W, or beside a draw reading past 1.5x the limit, is CounterStatus::Implausible and NvmlCounter is not constructed. Before: tests/nvml_hardware.rs failed on jetson-hub (status=Live against an oracle verdict of Implausible, 119.4 W counter, 13-13.5 W draw; detect()=nvml). After: status=Implausible, constructed=false, detect()=rapl; 58 lib tests with cuda (57 without), the NVML and RAPL hardware tests pass there with and without cuda. Co-Authored-By: Claude Opus 5.5 --- eoc/eoc-rs/README.md | 4 +- eoc/eoc-rs/crates/eoc-core/src/joule_cost.rs | 12 +- eoc/eoc-rs/crates/eoc-meter/src/lib.rs | 306 ++++++++++++++---- .../crates/eoc-meter/tests/nvml_hardware.rs | 10 +- 4 files changed, 261 insertions(+), 71 deletions(-) diff --git a/eoc/eoc-rs/README.md b/eoc/eoc-rs/README.md index 10ac02428..e194e041d 100644 --- a/eoc/eoc-rs/README.md +++ b/eoc/eoc-rs/README.md @@ -9,7 +9,7 @@ EOC is a four-stage memoizing cascade. Every query is tried at each stage in tur 3. **Graph** (`eoc-graph`) — triple-store retrieval. Reference impl is a small in-memory matcher; production deployments swap in a real graph backend driven by DCY (EOC-4). 4. **Neural** (`eoc-neural`) — last-resort inference. Reference impl ships an `EchoBackend`; production deployments wire in llama.cpp / Ollama / Anthropic API / etc. -Joule cost is the unit of accounting. The `eoc-meter` crate reads cumulative micro-joules from whatever hardware counter is available (Linux RAPL, macOS `powermetrics`, NVIDIA NVML behind the `cuda` feature) and falls back to a `StubCounter` everywhere else — including WASM. +Joule cost is the unit of accounting. The `eoc-meter` crate reads cumulative micro-joules from whatever whole-work hardware counter is available and live (Linux RAPL, Apple Silicon IOReport; NVIDIA NVML behind the `cuda` feature is GPU telemetry, never the cascade's counter) and falls back to a `StubCounter` everywhere else — including WASM. ## Workspace layout @@ -46,7 +46,7 @@ receipt: <64-char hex> ## CUDA / NVML -NVIDIA NVML support is feature-gated and **off by default** so the workspace builds without `libnvidia-ml`. To enable: +NVIDIA NVML support is feature-gated and **off by default** so the workspace builds without `libnvidia-ml`. NVML's total-energy counter reads the GPU alone, so `detect()` never hands it to the cascade, whose costs carry no field to say a figure covers part of the work: `NvmlCounter` is GPU telemetry, read beside the whole-work counter, and it is constructed only while its counter is live and counts no faster than the board can draw. To enable: ```sh cargo build -p eoc-meter --features cuda diff --git a/eoc/eoc-rs/crates/eoc-core/src/joule_cost.rs b/eoc/eoc-rs/crates/eoc-core/src/joule_cost.rs index be5be856d..aa458a6e1 100644 --- a/eoc/eoc-rs/crates/eoc-core/src/joule_cost.rs +++ b/eoc/eoc-rs/crates/eoc-core/src/joule_cost.rs @@ -5,8 +5,11 @@ use serde::{Deserialize, Serialize}; /// Where the joule reading came from. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum JouleSource { - /// Read from a live hardware energy counter (RAPL, NVML, or IOReport - /// while its CPU rail is live): the counter advanced over the span. + /// Read from a live hardware energy counter of the work's whole domain + /// (RAPL's packages, or IOReport while its CPU rail is live): the counter + /// advanced over the span. Never a GPU-only counter (NVML's): with no + /// field to say a figure covers part of the work, a GPU-only figure is an + /// estimate of the work, not a reading of it. Measured, /// Anything else: a synthesized estimate, or a figure no live counter /// read (no counter, a failed read, a counter that did not advance). @@ -47,8 +50,9 @@ impl JouleCost { } } - /// A measured cost: the delta a live hardware counter (RAPL, NVML, a live - /// IOReport rail) read over the span of the work, and nothing else. + /// A measured cost: the delta a live hardware counter of the work's whole + /// domain (RAPL's packages, a live IOReport rail) read over the span of the + /// work, and nothing else. pub fn measured(microjoules: u64) -> Self { Self { microjoules, diff --git a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs index 984a7aef5..33d01e17e 100644 --- a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs +++ b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs @@ -53,8 +53,16 @@ //! reference wrapper does not parse it, so every read returns //! `Err(NoCounter)`: no figure, and callers keep their own estimate. //! - NVIDIA NVML is gated behind the `cuda` feature (off by default), and -//! selected only when GPU 0's total-energy counter is live -//! ([`nvml_status`]). +//! **never selected by [`detect`]**: [`NvmlCounter`] reads GPU 0's +//! total-energy counter, which is the GPU's energy and not the work's (the +//! CPU's share of it goes unread), and eoc's consumers grade any advance of +//! the counter they are handed `Measured`, with no coverage field to say a +//! figure covers part of the work. By the family's rule a figure that does +//! not cover the work it is booked to is an estimate in such a format, so +//! the GPU counter is not a [`JouleCounter`]: it is GPU telemetry, read +//! beside one. It is constructed only while its counter is live and +//! plausibly counts the board's energy ([`nvml_status`], +//! [`CounterStatus::Implausible`]). //! - WASM and unknown targets fall through to `StubCounter`. // `deny` not `forbid`: the whole crate is safe except three audited FFI @@ -2899,7 +2907,7 @@ mod macos_ioreport { let before = script::counter_opens(); let given = MacosIoReportCounter::try_detect_given(status); assert!(matches!(given, Err(Error::NoCounter)), "try_detect_given({status:?}) did not decline with NoCounter"); - let picked = crate::detect_with(|| CounterStatus::Unavailable, || CounterStatus::Unavailable, || status).name(); + let picked = crate::detect_with(|| CounterStatus::Unavailable, || status).name(); assert_ne!(picked, "ioreport", "detect_with({status:?}) selected the IOReport counter"); let (built, picked) = script::with_status(status, || (MacosIoReportCounter::try_detect().is_ok(), crate::detect().name())); @@ -2934,7 +2942,7 @@ mod macos_ioreport { if let Err(e) = given { assert!(matches!(e, Error::NoCounter), "try_detect_given({status:?}) declined with {e:?}"); } - let picked = crate::detect_with(|| CounterStatus::Unavailable, || CounterStatus::Unavailable, || status).name(); + let picked = crate::detect_with(|| CounterStatus::Unavailable, || status).name(); assert_eq!(picked == "ioreport", admitted, "detect_with({status:?}) selected {picked}"); let (built, picked) = script::with_status(status, || { assert_eq!(crate::ioreport_status(), status, "the installed status"); @@ -3654,6 +3662,14 @@ pub enum CounterStatus { /// held, or was republished in lumps. Never selected: a lump landing in /// one caller's span would be sealed as that span's reading. Frozen, + /// The counter kept advancing, but a GPU counter (NVML's) ran faster than + /// its board can draw power over the probe: past 1.5 times the board's + /// maximum power limit, past twice the board's own power draw plus 5 W, + /// or beside a draw reading itself past that limit. It is not counting the + /// board's energy. Seen on an RTX 4050 Laptop GPU (driver 595.84), whose + /// counter runs at 4 to 30 times the board's draw while it is polled. + /// Never constructed. + Implausible, /// The counter kept advancing while the probe ran: in at least five /// separate sample intervals, at least 80% of those observed, over at /// least 300 ms. The rung is selected. @@ -3696,14 +3712,17 @@ pub fn rapl_status() -> CounterStatus { /// This host's NVML status, probed once per process and cached. Always /// [`CounterStatus::Unavailable`] without the `cuda` feature. /// -/// [`detect`] selects the NVML rung only when this is [`CounterStatus::Live`]: +/// [`NvmlCounter`] is constructed only when this is [`CounterStatus::Live`]: /// GPU 0's total-energy counter kept advancing, in at least five sample -/// intervals, at least 80% of them, over at least 300 ms. The probe cannot -/// load the GPU, so it waits between samples rather than burning CPU, and a -/// powered GPU's own draw advances a live counter; this shows the counter is -/// published continuously, not that it tracks a load. One that holds or steps -/// in lumps is declined. The first call where NVML opens waits about half a -/// second (a second if the counter is not live). +/// intervals, at least 80% of them, over at least 300 ms, and over that window +/// counted no faster than the board can draw (with the board's power draw read +/// at every sample and its maximum power limit; otherwise +/// [`CounterStatus::Implausible`]). The probe cannot load the GPU, so it waits +/// between samples rather than burning CPU, and a powered GPU's own draw +/// advances a live counter; this shows the counter is published continuously, +/// not that it tracks a load. One that holds or steps in lumps is declined. +/// [`detect`] never selects the GPU counter, live or not. The first call where +/// NVML opens waits about half a second (a second if the counter is not live). pub fn nvml_status() -> CounterStatus { #[cfg(feature = "cuda")] { @@ -4719,17 +4738,169 @@ pub(crate) fn detect_in_process_with( fallback() } +/// Whether a GPU energy counter that kept advancing over its probe plausibly +/// counted its board's energy. Kept apart from NVML so it runs under test in +/// every build; used by the NVML probe (`cuda`). +#[cfg_attr(not(any(test, feature = "cuda")), allow(dead_code))] +mod gpu_plausibility { + use crate::CounterStatus; + use std::time::Duration; + + /// How far past its board's maximum power limit a GPU energy counter may + /// run, on average over a probe's window, as a fraction (3/2): a board can + /// draw past its limit for milliseconds, not on average over a third of a + /// second. + pub(crate) const CEILING_MARGIN: (u128, u128) = (3, 2); + + /// How much faster than its board's own power draw a GPU energy counter + /// may run over a probe's window: twice the draw's mean over the samples, + /// plus [`DRAW_SLACK_MW`]. The draw is a one-second average, so a load that + /// changed during the window can leave it behind the counter, but not by a + /// factor of two at a probe. + pub(crate) const DRAW_FACTOR: u128 = 2; + + /// NVML's documented accuracy of a board's power draw, ±5 W, in mW. + pub(crate) const DRAW_SLACK_MW: u128 = 5_000; + + /// Whether a GPU energy counter that counted `energy_mj` over `window` + /// plausibly counted its board's energy, beside the board's power draw + /// `draws_mw` read with the samples and its maximum power limit + /// `ceiling_mw`. Not when the counter ran past [`CEILING_MARGIN`] of the + /// limit, or a draw reading did (the board's own telemetry then is not a + /// reading either: an RTX 4050 Laptop GPU reports 590 W for about a second + /// after it wakes), or the counter ran past [`DRAW_FACTOR`] times the mean + /// draw plus [`DRAW_SLACK_MW`]. A bound whose reading is missing is not + /// applied. + /// + /// Liveness says the counter is republished continuously; this says what + /// it republishes is the board's energy. On the RTX 4050 Laptop GPU + /// (driver 595.84) the total-energy counter runs several times faster than + /// the board draws for as long as it is polled: at the probe's 100 ms wait + /// it ran at 58-305 W while the board drew 5-14 W against a 65 W limit. + pub(crate) fn plausible(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { + let ns = window.as_nanos().max(1); + // mJ per second is mW. + let rate_mw = u128::from(energy_mj) * 1_000_000_000 / ns; + if let Some(limit) = ceiling_mw.filter(|&l| l > 0) { + let cap = u128::from(limit) * CEILING_MARGIN.0 / CEILING_MARGIN.1; + if rate_mw > cap || draws_mw.iter().any(|&d| u128::from(d) > cap) { + return false; + } + } + if !draws_mw.is_empty() { + let mean = draws_mw.iter().map(|&d| u128::from(d)).sum::() / draws_mw.len() as u128; + if rate_mw > DRAW_FACTOR * mean + DRAW_SLACK_MW { + return false; + } + } + true + } + + /// What the GPU probe saw of the board beside its counter: the counter's + /// first and last reading (mJ) and when, the board's power draw read with + /// every sample, and its maximum power limit. + #[derive(Debug, Default, Clone)] + pub(crate) struct GpuWatch { + first: Option<(u64, Duration)>, + last: Option<(u64, Duration)>, + draws_mw: Vec, + pub(crate) ceiling_mw: Option, + } + + impl GpuWatch { + /// Fold in one sample: the counter's reading, if it read, and the + /// board's draw, if that read, at `at`. + pub(crate) fn saw(&mut self, mj: Option, draw_mw: Option, at: Duration) { + if let Some(mj) = mj { + self.first.get_or_insert((mj, at)); + self.last = Some((mj, at)); + } + self.draws_mw.extend(draw_mw); + } + } + + /// The status a GPU probe gives: the liveness rule's `status`, except + /// that a live counter which did not plausibly count its board's energy + /// over the probe is [`CounterStatus::Implausible`]. + pub(crate) fn status_of(status: CounterStatus, gpu: &GpuWatch) -> CounterStatus { + let plausible = match (gpu.first, gpu.last) { + (Some((a, t0)), Some((b, t1))) => { + plausible(b.saturating_sub(a), t1.saturating_sub(t0), &gpu.draws_mw, gpu.ceiling_mw) + } + _ => true, + }; + match status { + CounterStatus::Live if !plausible => CounterStatus::Implausible, + other => other, + } + } + + #[cfg(test)] + mod tests { + use super::*; + + /// The rule with the readings of the RTX 4050 Laptop GPU on + /// jetson-hub (driver 595.84, 65 W maximum limit): polled at the + /// probe's 100 ms wait its counter ran at 58-305 W while the board drew + /// 5-14 W, which is no count of the board's energy. A counter that + /// tracks its board passes, a one-second draw lagging a load step by up + /// to twice still passes, and a bound whose reading is missing is not + /// applied. + #[test] + fn a_gpu_counter_must_count_no_faster_than_its_board_draws() { + let w = Duration::from_millis(500); + assert!(!plausible(59_700, w, &[12_970, 13_520], Some(65_000)), "119.4 W against a 13 W draw"); + assert!(!plausible(22_000, w, &[10_400; 6], Some(65_000)), "44 W against a 10.4 W draw"); + assert!(!plausible(60_000, w, &[], Some(65_000)), "120 W on a 65 W board"); + assert!(!plausible(44_000, w, &[590_000, 10_300], Some(65_000)), "a 590 W draw reading"); + assert!(plausible(6_500, w, &[13_000, 13_500], Some(65_000)), "13 W against a 13 W draw"); + assert!(plausible(150_000, w, &[280_000; 5], Some(700_000)), "300 W against a 280 W draw"); + assert!(plausible(102_500, w, &[100_000; 5], Some(700_000)), "205 W is 2 x 100 W + 5 W"); + assert!(!plausible(102_501, w, &[100_000; 5], Some(700_000))); + assert!(plausible(48_750, w, &[], Some(65_000)), "97.5 W is 1.5 x 65 W"); + assert!(!plausible(48_751, w, &[], Some(65_000))); + assert!(plausible(1_000_000, w, &[], None), "nothing to judge by"); + } + + /// A live counter that ran past its board is `Implausible`; one that + /// tracked it stays `Live`; a status that was not `Live` stands. + #[test] + fn a_live_gpu_counter_past_its_board_is_implausible() { + let watch = |draw_mw, ceiling_mw| { + let mut g = GpuWatch { ceiling_mw, ..GpuWatch::default() }; + for k in 0..6u64 { + // 12 J every 100 ms: 120 W. + g.saw(Some(1_000_000 + 12_000 * k), Some(draw_mw), Duration::from_millis(100 * k)); + } + g + }; + use CounterStatus as S; + assert_eq!(status_of(S::Live, &watch(13_000, Some(65_000))), S::Implausible); + assert_eq!(status_of(S::Live, &watch(115_000, Some(700_000))), S::Live); + assert_eq!(status_of(S::Live, &GpuWatch::default()), S::Live, "nothing to judge by"); + for status in [S::Unavailable, S::Frozen] { + assert_eq!(status_of(status, &watch(13_000, Some(65_000))), status); + } + } + } +} + #[cfg(feature = "cuda")] mod nvml { use super::*; use crate::rail_liveness; use nvml_wrapper::Nvml; - /// Reads cumulative GPU energy via NVML. + /// Reads GPU 0's cumulative energy via NVML: GPU telemetry, not a + /// [`JouleCounter`]. /// /// Constructed only on a host whose total-energy counter the liveness - /// probe found live ([`crate::nvml_status`]); a live NVML counter is a - /// vendor energy counter, so a delta it reads is a measurement of GPU 0. + /// probe found live and plausibly counting the board's energy + /// ([`crate::nvml_status`]); a delta it reads is a measurement of GPU 0, + /// and of nothing else. It is not a [`JouleCounter`] because eoc's + /// consumers grade any advance of a `JouleCounter` `Measured` with no + /// coverage field: a GPU-only delta graded so would claim the CPU's share + /// of the work, which it does not read. [`crate::detect`] never selects it. pub struct NvmlCounter { _nvml: Nvml, device_index: u32, @@ -4786,21 +4957,35 @@ mod nvml { const NVML_PROBE_WAIT: std::time::Duration = std::time::Duration::from_millis(100); /// The real NVML probe. Cached by [`crate::nvml_status`], its only caller. + /// With each sample it reads the board's power draw, and once its maximum + /// power limit, so that a live counter that ran faster than the board can + /// draw is [`CounterStatus::Implausible`] ([`crate::gpu_plausibility`]). pub(crate) fn host_status() -> CounterStatus { - rail_liveness::counter_status( - rail_liveness::probe_counter( - NvmlCounter::open, - NvmlCounter::sample, - || std::thread::sleep(NVML_PROBE_WAIT), - crate::nvml_wall_now, - crate::VENDOR_COUNTER_LIVENESS_BUDGET, - ) - .as_ref(), - ) + let seen = std::cell::RefCell::new(crate::gpu_plausibility::GpuWatch::default()); + let watch = rail_liveness::probe_counter( + || { + let c = NvmlCounter::open()?; + seen.borrow_mut().ceiling_mw = c.max_limit_mw(); + Some(c) + }, + |c: &NvmlCounter| { + let sample = c.sample(); + let mj = sample.as_ref().and_then(|rails| rails.first()).and_then(|(_, raw)| u64::try_from(*raw).ok()); + seen.borrow_mut().saw(mj, c.draw_mw(), crate::nvml_wall_now()); + sample + }, + || std::thread::sleep(NVML_PROBE_WAIT), + crate::nvml_wall_now, + crate::VENDOR_COUNTER_LIVENESS_BUDGET, + ); + crate::gpu_plausibility::status_of(rail_liveness::counter_status(watch.as_ref()), &seen.into_inner()) } - impl JouleCounter for NvmlCounter { - fn read_microjoules(&self) -> Result { + impl NvmlCounter { + /// GPU 0's cumulative energy, in µJ (NVML counts millijoules). The + /// GPU's energy alone: never grade a delta of it as a figure for work + /// that also ran on the CPU. + pub fn read_gpu_microjoules(&self) -> Result { let device = self ._nvml .device_by_index(self.device_index) @@ -4811,8 +4996,16 @@ mod nvml { // NVML reports millijoules; convert to microjoules. Ok(mj.saturating_mul(1000)) } - fn name(&self) -> &'static str { - "nvml" + + /// The board's power draw, in mW, as NVML reports it. + fn draw_mw(&self) -> Option { + self._nvml.device_by_index(self.device_index).ok()?.power_usage().ok() + } + + /// The board's maximum power-management limit, in mW. + fn max_limit_mw(&self) -> Option { + let limits = self._nvml.device_by_index(self.device_index).ok()?.power_management_limit_constraints().ok()?; + (limits.max_limit > 0).then_some(limits.max_limit) } } @@ -4827,7 +5020,7 @@ mod nvml { #[test] fn only_a_live_nvml_status_constructs_the_counter() { let attempts = || OPEN_ATTEMPTS.with(|n| n.get()); - for status in [CounterStatus::Unavailable, CounterStatus::Frozen] { + for status in [CounterStatus::Unavailable, CounterStatus::Frozen, CounterStatus::Implausible] { let before = attempts(); assert!( matches!(NvmlCounter::try_detect_given(status), Err(Error::NoCounter)), @@ -4860,20 +5053,24 @@ pub use nvml::NvmlCounter; /// Runtime-detect the best available counter on this host. /// -/// Preference order: NVML (if compiled with `cuda`, present, and its counter -/// live — see [`nvml_status`]) → RAPL on Linux (only when every package +/// Preference order: RAPL on Linux (only when every package /// counter is live — see [`rapl_status`]) → IOReport on macOS (Energy-Model counters, /// no sudo; only when every CPU energy rail is live — see [`ioreport_status`]) /// → powermetrics on macOS → `StubCounter`. Every hardware rung is held to /// the same sustained-advance rule; a rung whose counter is not live is /// skipped, so callers estimate rather than seal what it read. /// +/// It never selects NVML's GPU counter ([`NvmlCounter`], feature `cuda`), +/// which reads the GPU and not the CPU's share of the work: eoc's consumers +/// grade whatever this returns `Measured`, with no field to say a figure +/// covers part of the work, so a GPU-only figure would claim the whole. +/// /// It never selects this process's own energy counter /// ([`MacosProcessEnergyCounter`]), whose domain stops at this process's /// CPU threads: what eoc meters may run in a subprocess or on the GPU. A /// caller whose work is in-process asks [`detect_in_process`]. pub fn detect() -> Box { - detect_with(nvml_status, rapl_status, ioreport_status) + detect_with(rapl_status, ioreport_status) } /// The selection behind [`detect`], given where each rung's status comes from @@ -4883,14 +5080,9 @@ pub fn detect() -> Box { // and build some of them go unused. #[allow(unused_variables)] pub(crate) fn detect_with( - nvml: impl FnOnce() -> CounterStatus, rapl: impl FnOnce() -> CounterStatus, ioreport: impl FnOnce() -> IoReportStatus, ) -> Box { - #[cfg(feature = "cuda")] - if let Ok(c) = NvmlCounter::try_detect_given(nvml()) { - return Box::new(c); - } #[cfg(target_os = "linux")] if let Ok(c) = LinuxRaplCounter::try_detect_given(rapl()) { return Box::new(c); @@ -4932,27 +5124,23 @@ mod tests { assert_eq!(ioreport_status(), IoReportStatus::Unavailable); } - /// RAPL and NVML are selected only when their counter is live, and only - /// where their rung exists: RAPL on Linux, NVML with `cuda` where NVML - /// opens. A frozen or unavailable counter is never selected, whatever - /// the other rungs say. + /// RAPL is selected only when its counter is live, and only where its + /// rung exists, on Linux. A frozen, implausible or unavailable counter is + /// never selected, whatever the other rungs say. NVML's GPU counter is + /// never selected at all: it is not a `JouleCounter`, and the ladder has + /// no rung for it, whatever its status. #[test] fn a_vendor_counter_is_selected_only_when_live() { use CounterStatus as S; let declined_ioreport = || IoReportStatus::CpuRailFrozen; - for status in [S::Unavailable, S::Frozen] { - let picked = detect_with(|| status, || status, declined_ioreport).name(); - assert!(!matches!(picked, "rapl" | "nvml"), "a {status:?} counter was selected: {picked}"); - let picked = detect_with(|| S::Unavailable, || status, declined_ioreport).name(); + for status in [S::Unavailable, S::Frozen, S::Implausible] { + let picked = detect_with(|| status, declined_ioreport).name(); assert_ne!(picked, "rapl", "a {status:?} RAPL counter was selected"); } - let picked = detect_with(|| S::Unavailable, || S::Live, declined_ioreport).name(); + let picked = detect_with(|| S::Live, declined_ioreport).name(); assert_eq!(picked == "rapl", cfg!(target_os = "linux"), "a live RAPL status selected {picked}"); - #[cfg(feature = "cuda")] - { - let opens = nvml::NvmlCounter::try_detect_given(S::Live).is_ok(); - let picked = detect_with(|| S::Live, || S::Unavailable, declined_ioreport).name(); - assert_eq!(picked == "nvml", opens, "a live NVML status selected {picked}"); + for status in [S::Unavailable, S::Live] { + assert_ne!(detect_with(|| status, declined_ioreport).name(), "nvml"); } } @@ -4969,12 +5157,10 @@ mod tests { IoReportStatus::CpuRailFrozen, IoReportStatus::Live, ]; - for nvml in [S::Unavailable, S::Frozen, S::Live] { - for rapl in [S::Unavailable, S::Frozen, S::Live] { - for io in ioreport { - let picked = detect_with(|| nvml, || rapl, || io).name(); - assert_ne!(picked, "process-energy", "detect_with({nvml:?}, {rapl:?}, {io:?})"); - } + for rapl in [S::Unavailable, S::Frozen, S::Implausible, S::Live] { + for io in ioreport { + let picked = detect_with(|| rapl, || io).name(); + assert_ne!(picked, "process-energy", "detect_with({rapl:?}, {io:?})"); } } let picked = detect().name(); @@ -5008,8 +5194,8 @@ mod tests { /// Against this host's RAPL package counters, where there are any: the /// status the probe cached agrees with what they do now. Live: every one - /// advances across 200 ms of CPU work, and `detect()` selects RAPL unless - /// NVML is live. Not live: `detect()` never selects it. Logs what it saw. + /// advances across 200 ms of CPU work, and `detect()` selects RAPL. Not + /// live: `detect()` never selects it. Logs what it saw. #[cfg(target_os = "linux")] #[test] fn the_rapl_rung_agrees_with_the_counter() { @@ -5030,7 +5216,7 @@ mod tests { for ((p, b), a) in host.packages.iter().zip(&b).zip(&a) { assert!(a != b, "live package zone intel-rapl:{} did not move across 200 ms of CPU work: {b} -> {a}", p.index); } - assert!(picked == "rapl" || nvml_status() == CounterStatus::Live, "detect() picked {picked}"); + assert_eq!(picked, "rapl", "detect() picked {picked}"); } else { assert_ne!(picked, "rapl", "detect() selected a {status:?} RAPL counter"); } diff --git a/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs b/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs index 6f961a3d4..0ce3d2e26 100644 --- a/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs +++ b/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs @@ -19,7 +19,7 @@ use std::sync::{Mutex, MutexGuard}; use std::time::Duration; -use eoc_meter::{CounterStatus, JouleCounter, NvmlCounter}; +use eoc_meter::{CounterStatus, NvmlCounter}; /// One test at a time: each reads machine-wide counters, and the oracle's /// own sampling must not overlap the meter's. @@ -335,7 +335,7 @@ use nvml_oracle::Verdict; /// The status agrees with the oracle, judged at the probe's own tick, and the /// tick is no faster than the counter is republished; only a live status -/// constructs the counter, and `detect()` never selects it. +/// constructs the counter, and `detect()` never selects it, live or not. #[test] fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { let _one = serial(); @@ -356,7 +356,7 @@ fn the_nvml_status_agrees_with_the_oracle_at_its_own_tick() { match watch.verdict { Verdict::Unavailable => assert_eq!(status, CounterStatus::Unavailable), Verdict::Frozen => assert_eq!(status, CounterStatus::Frozen), - Verdict::Implausible => assert_ne!(status, CounterStatus::Live), + Verdict::Implausible => assert_eq!(status, CounterStatus::Implausible), Verdict::Live => {} } assert_eq!(constructed, status == CounterStatus::Live, "construction disagrees with the status"); @@ -376,9 +376,9 @@ fn a_live_gpu_counter_reads_inside_the_oracles_bracket() { let oracle = nvml_oracle::open().expect("the meter reads NVML, yet the oracle cannot open it"); let o0 = oracle.energy_mj().expect("oracle read")[0]; let started = std::time::Instant::now(); - let m0 = counter.read_microjoules().expect("a live counter reads"); + let m0 = counter.read_gpu_microjoules().expect("a live counter reads"); std::thread::sleep(Duration::from_secs(1)); - let m1 = counter.read_microjoules().expect("a live counter reads"); + let m1 = counter.read_gpu_microjoules().expect("a live counter reads"); let wall = started.elapsed(); let o1 = oracle.energy_mj().expect("oracle read")[0]; let ceiling = nvml_oracle::ceiling_w(&oracle, &nvml_oracle::smi(), 0).unwrap_or(f64::INFINITY); -- 2.54.0 (Apple Git-157) From ef4aa3cd3a4103772d9e73b70e699a57e554c582 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 1 Oct 2026 02:00:54 -0400 Subject: [PATCH 08/10] proof-cascade: the RAPL hardware test takes 0 as the crate's 'inside the idle noise' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mutation run on jetson-hub while its CI jobs held the package near its power limit (load average 11) read 0 µJ a call through both windows: the busy window drew no more than the idle one, which the crate documents as a bracket's 'inside the idle noise', not a defect. The test asserted a non-zero figure; it now logs the load and keeps the assertions that hold whatever the load: the calls' total within the oracle's package-only energy around the measurement, the HwShunt grade and the package domain. Co-Authored-By: Claude Opus 5.5 --- .../proof-cascade/tests/rapl_hardware.rs | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs b/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs index 3ccaa79ee..f08722273 100644 --- a/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs +++ b/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs @@ -410,9 +410,11 @@ fn ten_ms_of_work() { /// The figure is a marginal per call (busy power over idle, times the busy /// window, over the calls), so the calls' total can be no more than the /// packages' energy over the whole measurement, which the oracle reads -/// around it; a figure that also summed a package's core sub-zone (or psys, -/// where psys tracks load) would be about twice the busy margin and overrun -/// it. Where RAPL is not live there is no figure, and `work` never runs. +/// around it; on a quiet host a figure that also summed a package's core +/// sub-zone (or psys, where psys tracks load) is about twice the busy margin +/// and overruns it. A figure of 0 is the crate's "inside the idle noise", and +/// is what a host whose other work holds the package at its power limit +/// reads. Where RAPL is not live there is no figure, and `work` never runs. #[test] fn an_all_core_burn_meters_inside_the_package_only_bracket() { let _one = serial(); @@ -451,8 +453,19 @@ fn an_all_core_burn_meters_inside_the_package_only_bracket() { assert_eq!(energy.measured_uj, Some(m.uj)); let domain = m.source.domain(); assert!(domain.contains("package") && !domain.contains("GPU"), "the RAPL figure claims {domain}"); - assert!(calls > 0 && m.uj > 0, "an all-core burn metered {} µJ a call over {calls} calls", m.uj); + assert!(calls > 0, "the work never ran"); assert!(outer > 0, "the packages did not advance"); // Each call's figure is rounded to the microjoule: half a microjoule a call. assert!(total <= outer + calls / 2 + 1, "{total} µJ over the calls overruns the packages' {outer} µJ"); + if m.uj == 0 { + // The crate's contract: from a bracket, 0 means the busy window drew no + // more than the idle one. On a runner whose other jobs hold the package + // at its power limit through both windows, that is what it reads. + eprintln!("rapl burn: the busy window drew no more than the idle one (load {:?}): inside the idle noise", loadavg()); + } +} + +/// The host's load averages, for the log. +fn loadavg() -> Option { + std::fs::read_to_string("/proc/loadavg").ok().map(|s| s.trim().to_string()) } -- 2.54.0 (Apple Git-157) From c57053658442ec4ce9a0f6239fafa2a1b25abc6c Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 1 Oct 2026 02:05:19 -0400 Subject: [PATCH 09/10] jouleclaw-energy: the GPU plausibility rule is dead code without nvml, and says so A build without the nvml feature (the workspace's own cargo check) warned that GpuWatch, plausible_gpu_counter and its constants are never used. Co-Authored-By: Claude Opus 5.5 --- .../jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs index 6eb2f547c..d1e82f633 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/liveness.rs @@ -284,6 +284,7 @@ pub(crate) fn burn_thread_cpu(cpu: Duration, wall_cap: Duration) { /// How far past its board's maximum power limit a GPU energy counter may run, /// on average over a probe's window, as a fraction (3/2): a board can draw /// past its limit for milliseconds, not on average over a third of a second. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` pub(crate) const GPU_CEILING_MARGIN: (u128, u128) = (3, 2); /// How much faster than its board's own power draw a GPU energy counter may @@ -291,9 +292,11 @@ pub(crate) const GPU_CEILING_MARGIN: (u128, u128) = (3, 2); /// [`GPU_DRAW_SLACK_MW`]. The draw is a one-second average, so a load that /// changed during the window can leave it behind the counter, but not by a /// factor of two at a probe. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` pub(crate) const GPU_DRAW_FACTOR: u128 = 2; /// NVML's documented accuracy of a board's power draw, ±5 W, in mW. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` pub(crate) const GPU_DRAW_SLACK_MW: u128 = 5_000; /// Whether one GPU's energy counter, which counted `energy_mj` over `window`, @@ -310,6 +313,7 @@ pub(crate) const GPU_DRAW_SLACK_MW: u128 = 5_000; /// 595.84) the total-energy counter runs several times faster than the board /// draws for as long as it is polled: at the NVML probe's 100 ms tick it ran /// at 58-305 W while the board drew 5-14 W against a 65 W limit. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` pub(crate) fn plausible_gpu_counter(energy_mj: u64, window: Duration, draws_mw: &[u32], ceiling_mw: Option) -> bool { let ns = window.as_nanos().max(1); // mJ per second is mW. @@ -332,6 +336,7 @@ pub(crate) fn plausible_gpu_counter(energy_mj: u64, window: Duration, draws_mw: /// What a GPU counter's probe saw beside the rule's verdict: each GPU's /// counter at the first and last sample (µJ) and when, and each GPU's power /// draw read with every sample. +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` #[derive(Debug, Default, Clone)] pub(crate) struct GpuWatch { first: Option<(Vec, Instant)>, @@ -339,6 +344,7 @@ pub(crate) struct GpuWatch { draws_mw: Vec>, } +#[cfg_attr(not(any(test, feature = "nvml")), allow(dead_code))] // used by `nvml` impl GpuWatch { /// Fold in one sample: every GPU's counter (µJ), if the sample read, and /// each GPU's draw (`None` where it did not read), at `at`. -- 2.54.0 (Apple Git-157) From 1d85821aa8741197ab530d3ab08cd4166ce253ee Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 1 Oct 2026 05:48:10 -0400 Subject: [PATCH 10/10] energy meters: document the hardware tests' helper functions jouleclaw's public-surface audit failed #129 on seven helpers in jouleclaw-energy's rapl_hardware.rs that had no doc comment: the bracket's zone sums, the masked view, and the watch summary. Document them, and the same helpers in the other crates' copies of the module, so the copies stay alike. Co-Authored-By: Claude Opus 5.5 --- eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs | 2 ++ eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs | 10 ++++++++++ jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs | 2 ++ jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs | 10 ++++++++++ .../crates/jouleclaw-energy/tests/nvml_hardware.rs | 2 ++ .../crates/jouleclaw-energy/tests/rapl_hardware.rs | 10 ++++++++++ .../crates/proof-cascade/tests/rapl_hardware.rs | 10 ++++++++++ .../crates/sandbox-exec-microvm/tests/rapl_hardware.rs | 10 ++++++++++ .../crates/sandbox-meter/tests/nvml_hardware.rs | 2 ++ .../crates/sandbox-meter/tests/rapl_hardware.rs | 10 ++++++++++ 10 files changed, 68 insertions(+) diff --git a/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs b/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs index 0ce3d2e26..9a8980170 100644 --- a/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs +++ b/eoc/eoc-rs/crates/eoc-meter/tests/nvml_hardware.rs @@ -237,6 +237,8 @@ mod nvml_oracle { } impl Watch { + /// The verdict, each GPU's advances, its counter's rate, the board's + /// draw and its maximum limit, for the test's log line. pub fn summary(&self) -> String { format!( "{:?} (advanced {:?} of {} intervals over {} ms; counter rate {:?} W, board draw {:?} W, max power limit {:?} W)", diff --git a/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs b/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs index 25c860e5d..5fb841e47 100644 --- a/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs +++ b/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs @@ -180,6 +180,7 @@ mod oracle { } impl Watch { + /// The verdict and each package's advances, for the test's log line. pub fn summary(&self) -> String { let per: Vec = self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); @@ -258,6 +259,8 @@ mod oracle { pub struct MaskedView(pub PathBuf); impl MaskedView { + /// Build the masked view of this host's powercap entries in a + /// temporary directory named by `tag` and this process. pub fn of_host(tag: &str) -> MaskedView { let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); let _ = std::fs::remove_dir_all(&root); @@ -284,18 +287,25 @@ mod oracle { } impl Bracket<'_> { + /// The packages' energy over the outer bracket, in µJ: the most a + /// package-only figure read inside it can be. pub fn package_outer(&self) -> u64 { energy(&self.set, &self.reads[0], &self.reads[3], is_package) } + /// The packages' energy over the inner bracket, in µJ: the least a + /// package-only figure read around it can be. pub fn package_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_package) } + /// The psys zone's energy over the inner bracket, in µJ. pub fn psys_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_psys) } + /// The core sub-zones' energy over the inner bracket, in µJ. pub fn core_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_core) } + /// Whether the bracket read a zone of the kind `which` picks. pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { self.set.iter().any(|z| which(z)) } diff --git a/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs b/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs index da15b36eb..76096fb1b 100644 --- a/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs +++ b/jcp/jcp-rs/crates/jcp-energy/tests/nvml_hardware.rs @@ -236,6 +236,8 @@ mod nvml_oracle { } impl Watch { + /// The verdict, each GPU's advances, its counter's rate, the board's + /// draw and its maximum limit, for the test's log line. pub fn summary(&self) -> String { format!( "{:?} (advanced {:?} of {} intervals over {} ms; counter rate {:?} W, board draw {:?} W, max power limit {:?} W)", diff --git a/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs b/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs index 95a9a3e9c..7f2d8baf4 100644 --- a/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs +++ b/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs @@ -183,6 +183,7 @@ mod oracle { } impl Watch { + /// The verdict and each package's advances, for the test's log line. pub fn summary(&self) -> String { let per: Vec = self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); @@ -261,6 +262,8 @@ mod oracle { pub struct MaskedView(pub PathBuf); impl MaskedView { + /// Build the masked view of this host's powercap entries in a + /// temporary directory named by `tag` and this process. pub fn of_host(tag: &str) -> MaskedView { let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); let _ = std::fs::remove_dir_all(&root); @@ -287,18 +290,25 @@ mod oracle { } impl Bracket<'_> { + /// The packages' energy over the outer bracket, in µJ: the most a + /// package-only figure read inside it can be. pub fn package_outer(&self) -> u64 { energy(&self.set, &self.reads[0], &self.reads[3], is_package) } + /// The packages' energy over the inner bracket, in µJ: the least a + /// package-only figure read around it can be. pub fn package_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_package) } + /// The psys zone's energy over the inner bracket, in µJ. pub fn psys_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_psys) } + /// The core sub-zones' energy over the inner bracket, in µJ. pub fn core_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_core) } + /// Whether the bracket read a zone of the kind `which` picks. pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { self.set.iter().any(|z| which(z)) } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs index 49c200912..bca4a6ff2 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/nvml_hardware.rs @@ -238,6 +238,8 @@ mod nvml_oracle { } impl Watch { + /// The verdict, each GPU's advances, its counter's rate, the board's + /// draw and its maximum limit, for the test's log line. pub fn summary(&self) -> String { format!( "{:?} (advanced {:?} of {} intervals over {} ms; counter rate {:?} W, board draw {:?} W, max power limit {:?} W)", diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs index 6f81934b4..b063a5dda 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs @@ -182,6 +182,7 @@ mod oracle { } impl Watch { + /// The verdict and each package's advances, for the test's log line. pub fn summary(&self) -> String { let per: Vec = self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); @@ -260,6 +261,8 @@ mod oracle { pub struct MaskedView(pub PathBuf); impl MaskedView { + /// Build the masked view of this host's powercap entries in a + /// temporary directory named by `tag` and this process. pub fn of_host(tag: &str) -> MaskedView { let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); let _ = std::fs::remove_dir_all(&root); @@ -286,18 +289,25 @@ mod oracle { } impl Bracket<'_> { + /// The packages' energy over the outer bracket, in µJ: the most a + /// package-only figure read inside it can be. pub fn package_outer(&self) -> u64 { energy(&self.set, &self.reads[0], &self.reads[3], is_package) } + /// The packages' energy over the inner bracket, in µJ: the least a + /// package-only figure read around it can be. pub fn package_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_package) } + /// The psys zone's energy over the inner bracket, in µJ. pub fn psys_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_psys) } + /// The core sub-zones' energy over the inner bracket, in µJ. pub fn core_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_core) } + /// Whether the bracket read a zone of the kind `which` picks. pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { self.set.iter().any(|z| which(z)) } diff --git a/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs b/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs index f08722273..dd2c06df0 100644 --- a/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs +++ b/proof/proof-rs/crates/proof-cascade/tests/rapl_hardware.rs @@ -185,6 +185,7 @@ mod oracle { } impl Watch { + /// The verdict and each package's advances, for the test's log line. pub fn summary(&self) -> String { let per: Vec = self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); @@ -263,6 +264,8 @@ mod oracle { pub struct MaskedView(pub PathBuf); impl MaskedView { + /// Build the masked view of this host's powercap entries in a + /// temporary directory named by `tag` and this process. pub fn of_host(tag: &str) -> MaskedView { let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); let _ = std::fs::remove_dir_all(&root); @@ -289,18 +292,25 @@ mod oracle { } impl Bracket<'_> { + /// The packages' energy over the outer bracket, in µJ: the most a + /// package-only figure read inside it can be. pub fn package_outer(&self) -> u64 { energy(&self.set, &self.reads[0], &self.reads[3], is_package) } + /// The packages' energy over the inner bracket, in µJ: the least a + /// package-only figure read around it can be. pub fn package_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_package) } + /// The psys zone's energy over the inner bracket, in µJ. pub fn psys_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_psys) } + /// The core sub-zones' energy over the inner bracket, in µJ. pub fn core_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_core) } + /// Whether the bracket read a zone of the kind `which` picks. pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { self.set.iter().any(|z| which(z)) } diff --git a/sandbox/sandbox-rs/crates/sandbox-exec-microvm/tests/rapl_hardware.rs b/sandbox/sandbox-rs/crates/sandbox-exec-microvm/tests/rapl_hardware.rs index e8ebf7e79..42bc66680 100644 --- a/sandbox/sandbox-rs/crates/sandbox-exec-microvm/tests/rapl_hardware.rs +++ b/sandbox/sandbox-rs/crates/sandbox-exec-microvm/tests/rapl_hardware.rs @@ -179,6 +179,7 @@ mod oracle { } impl Watch { + /// The verdict and each package's advances, for the test's log line. pub fn summary(&self) -> String { let per: Vec = self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); @@ -257,6 +258,8 @@ mod oracle { pub struct MaskedView(pub PathBuf); impl MaskedView { + /// Build the masked view of this host's powercap entries in a + /// temporary directory named by `tag` and this process. pub fn of_host(tag: &str) -> MaskedView { let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); let _ = std::fs::remove_dir_all(&root); @@ -283,18 +286,25 @@ mod oracle { } impl Bracket<'_> { + /// The packages' energy over the outer bracket, in µJ: the most a + /// package-only figure read inside it can be. pub fn package_outer(&self) -> u64 { energy(&self.set, &self.reads[0], &self.reads[3], is_package) } + /// The packages' energy over the inner bracket, in µJ: the least a + /// package-only figure read around it can be. pub fn package_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_package) } + /// The psys zone's energy over the inner bracket, in µJ. pub fn psys_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_psys) } + /// The core sub-zones' energy over the inner bracket, in µJ. pub fn core_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_core) } + /// Whether the bracket read a zone of the kind `which` picks. pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { self.set.iter().any(|z| which(z)) } diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs b/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs index c5ab0ab2c..170745c5e 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/tests/nvml_hardware.rs @@ -238,6 +238,8 @@ mod nvml_oracle { } impl Watch { + /// The verdict, each GPU's advances, its counter's rate, the board's + /// draw and its maximum limit, for the test's log line. pub fn summary(&self) -> String { format!( "{:?} (advanced {:?} of {} intervals over {} ms; counter rate {:?} W, board draw {:?} W, max power limit {:?} W)", diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/tests/rapl_hardware.rs b/sandbox/sandbox-rs/crates/sandbox-meter/tests/rapl_hardware.rs index 08dd0bd91..ed81a215a 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/tests/rapl_hardware.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/tests/rapl_hardware.rs @@ -183,6 +183,7 @@ mod oracle { } impl Watch { + /// The verdict and each package's advances, for the test's log line. pub fn summary(&self) -> String { let per: Vec = self.packages.iter().zip(&self.advances).map(|(p, a)| format!("{p} advanced {a}/{}", self.intervals)).collect(); @@ -261,6 +262,8 @@ mod oracle { pub struct MaskedView(pub PathBuf); impl MaskedView { + /// Build the masked view of this host's powercap entries in a + /// temporary directory named by `tag` and this process. pub fn of_host(tag: &str) -> MaskedView { let root = std::env::temp_dir().join(format!("{tag}-masked-powercap-{}", std::process::id())); let _ = std::fs::remove_dir_all(&root); @@ -287,18 +290,25 @@ mod oracle { } impl Bracket<'_> { + /// The packages' energy over the outer bracket, in µJ: the most a + /// package-only figure read inside it can be. pub fn package_outer(&self) -> u64 { energy(&self.set, &self.reads[0], &self.reads[3], is_package) } + /// The packages' energy over the inner bracket, in µJ: the least a + /// package-only figure read around it can be. pub fn package_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_package) } + /// The psys zone's energy over the inner bracket, in µJ. pub fn psys_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_psys) } + /// The core sub-zones' energy over the inner bracket, in µJ. pub fn core_inner(&self) -> u64 { energy(&self.set, &self.reads[1], &self.reads[2], is_core) } + /// Whether the bracket read a zone of the kind `which` picks. pub fn has(&self, which: impl Fn(&Zone) -> bool) -> bool { self.set.iter().any(|z| which(z)) } -- 2.54.0 (Apple Git-157)