diff --git a/jcp/jcp-rs/crates/jcp-energy/src/lib.rs b/jcp/jcp-rs/crates/jcp-energy/src/lib.rs index 1a67640a9..629a2c36d 100644 --- a/jcp/jcp-rs/crates/jcp-energy/src/lib.rs +++ b/jcp/jcp-rs/crates/jcp-energy/src/lib.rs @@ -18,11 +18,31 @@ //! 1. **RAPL** (`/sys/class/powercap`, Linux) — hardware package energy, //! `Provenance::HwShunt`, **only while every package counter is live** //! under the probe's CPU burn ([`rapl_status`]). The figure sums the -//! top-level zones whose `name` is `package-`, every socket's, each +//! top-level zones whose `name` is `package-`, every socket's (or +//! `package--die-`, every die's, where the kernel lists a multi-die +//! Intel package one zone per die), each //! wraparound-corrected at its own range; never `psys` (the platform zone, //! which already includes the packages), a top-level `dram` zone, or a //! package's subzones. A span over which no package counter advanced is the //! default-power estimate (rung 6). +//! +//! Some kernels ship without the powercap RAPL driver (Ubuntu without +//! `linux-modules-extra`, as on some rented bare-metal servers), so powercap +//! lists no package while the counters are there. Where it lists none, the +//! reader takes the same counters directly: the energy-status MSR +//! (`MSR_PKG_ENERGY_STATUS`, 0x611, on Intel family 6; 0xC001029B on AMD Zen +//! and Hygon) of each counter domain, read on the domain's first CPU through +//! the `msr` driver's `/dev/cpu//msr` (the `msr` module, `CAP_SYS_RAWIO` +//! and read access to the file). A domain is a die on Intel and a package on +//! AMD and Hygon, the Linux RAPL driver's own rule (`arch/x86/events/rapl.c` +//! gives only AMD and Hygon a per-package counter; a multi-die Intel part's +//! RAPL MSRs are die-scope), so a multi-die Intel package is read on each +//! die's first CPU and the dies summed. Each 32-bit count is converted at the +//! domain's own unit (bits +//! 12:8 of the power-unit MSR, 0x606 or 0xC0010299). It wraps at 2^32 counts +//! and resolves one count (about 61 µJ at the common unit of 2^-14 J). A +//! power-unit MSR that reads zero is no counter: some hypervisors let a guest +//! read these MSRs and return zeros. The liveness rule is the same. //! 2. **NVML** (feature `nvml`) — GPU energy via `libnvidia-ml`, //! `Provenance::HwShunt` with `Coverage::Partial`: the counter reads the //! GPU, so the CPU's share of the work is unmetered and the figure never @@ -114,10 +134,15 @@ use jcp_schema::{Energy, Provenance}; /// ([`Meter::time_power`]). pub const DEFAULT_ACTIVE_MILLIWATTS: u64 = 15_000; +/// Where the kernel lists its powercap zones. +const POWERCAP_ROOT: &str = "/sys/class/powercap"; + /// RAPL package-energy reader (Linux `powercap`). Reads `energy_uj` for every /// top-level zone (`intel-rapl:N`) whose `name` file reads `package-` — every -/// socket's package — and corrects each counter's wraparound at that zone's own -/// `max_energy_range_uj`. +/// socket's package — or `package--die-` — every die's, on a multi-die +/// Intel package — and corrects each counter's wraparound at that zone's own +/// `max_energy_range_uj`. Where powercap lists no package zone, it reads each +/// package's energy-status MSR instead (see the crate docs, rung 1). /// /// The figure is the packages' energy and nothing else. A top-level zone is not /// necessarily a package: many Intel hosts expose `psys`, the platform zone, @@ -128,19 +153,252 @@ pub const DEFAULT_ACTIVE_MILLIWATTS: u64 = 15_000; /// interface to the same packages (`intel-rapl-mmio:N`) is not read either. #[derive(Debug)] pub struct Rapl { - /// `(energy_uj path, max_energy_range_uj)` per package, in zone order. The - /// range is `None` where it could not be read, or read as zero: a counter - /// that falls across a span then has no known wrap, and gives no figure. - packages: Vec<(std::path::PathBuf, Option)>, + /// Every package, in package order. + packages: Vec, } -/// Whether a powercap zone's `name` file names a CPU package: `package-`, -/// `` one or more ASCII digits, trailing newline allowed. Not `psys`, `dram`, -/// `core`, `uncore`, or anything else. +/// One package a [`Rapl`] reader sums. +#[derive(Debug, Clone, PartialEq, Eq)] +struct RaplPackage { + /// The rail the liveness probe names it by: its powercap directory + /// (`intel-rapl:0`, ...), or for one read from its MSR `package- msr`, + /// or `package--die- msr` for each die of a multi-die Intel package. + rail: String, + /// Where its count is read: the zone's `energy_uj`, or the `msr` device + /// file of the domain's (package's, or on Intel die's) first CPU, + /// `/dev/cpu//msr`. + path: std::path::PathBuf, + /// The value its microjoule count wraps at: the zone's + /// `max_energy_range_uj`, `None` where that could not be read or read as + /// zero (a counter that falls across a span then has no known wrap, and + /// gives no figure); for an MSR package, 2^32 counts at its unit. + range: Option, + /// How to read it from its MSR; `None` for a powercap zone. + msr: Option, +} + +impl RaplPackage { + /// Its count in µJ: the zone's `energy_uj`, or its MSR's 32-bit count at + /// its unit. `None` when it cannot be read. + fn read_uj(&self) -> Option { + match self.msr { + None => std::fs::read_to_string(&self.path).ok()?.trim().parse::().ok(), + Some(msr) => read_msr(&self.path, msr.register).ok().map(|raw| msr.uj(raw)), + } + } +} + +/// A package read from its energy-status MSR, where powercap lists no zone +/// for it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Msr { + /// The energy-status MSR: [`MSR_PKG_ENERGY_STATUS`] or + /// [`MSR_AMD_PKG_ENERGY_STATUS`]. + register: u64, + /// The energy-status unit, 2^-`unit_shift` J per count: bits 12:8 of the + /// power-unit MSR. Never zero. + unit_shift: u32, +} + +/// Intel's power-unit MSR (family 6). +const MSR_RAPL_POWER_UNIT: u64 = 0x606; +/// Intel's package energy-status MSR (family 6). +const MSR_PKG_ENERGY_STATUS: u64 = 0x611; +/// AMD Zen's (and Hygon's) power-unit MSR. +const MSR_AMD_RAPL_POWER_UNIT: u64 = 0xC001_0299; +/// AMD Zen's (and Hygon's) package energy-status MSR. +const MSR_AMD_PKG_ENERGY_STATUS: u64 = 0xC001_029B; + +impl Msr { + /// The package's energy in µJ at a raw read of its energy-status MSR: the + /// low 32 bits are the count. + fn uj(self, raw: u64) -> u64 { + ((u128::from(raw & 0xFFFF_FFFF) * 1_000_000) >> self.unit_shift) as u64 + } + + /// The µJ figure's wrap: 2^32 counts. + fn range_uj(self) -> u64 { + (((1u128 << 32) * 1_000_000) >> self.unit_shift) as u64 + } +} + +/// The energy-status unit's shift, bits 12:8 of a power-unit MSR; `None` for a +/// zero unit, which is no counter (a hypervisor's stand-in). +fn msr_unit_shift(unit: u64) -> Option { + let shift = ((unit >> 8) & 0x1F) as u32; + (shift != 0).then_some(shift) +} + +/// The 64-bit MSR `register`, read through the `msr` driver's device file +/// `device`, where the register number is the file offset. An error when the +/// file cannot be opened (no `msr` module; or no `CAP_SYS_RAWIO` or no read +/// access, both `PermissionDenied`) or the register cannot be read (the CPU +/// does not implement it). +#[cfg(unix)] +fn read_msr(device: &std::path::Path, register: u64) -> std::io::Result { + use std::os::unix::fs::FileExt; + let file = std::fs::File::open(device)?; + let mut raw = [0u8; 8]; + file.read_exact_at(&mut raw, register)?; + Ok(u64::from_le_bytes(raw)) +} + +#[cfg(not(unix))] +fn read_msr(_device: &std::path::Path, _register: u64) -> std::io::Result { + Err(std::io::ErrorKind::Unsupported.into()) +} + +/// The CPU's vendor and family from `/proc/cpuinfo` text: the first CPU's +/// `vendor_id` and `cpu family`. +fn cpu_vendor_family(cpuinfo: &str) -> Option<(&str, u32)> { + let field = |key: &str| { + cpuinfo.lines().find_map(|line| { + let (k, v) = line.split_once(':')?; + (k.trim() == key).then(|| v.trim()) + }) + }; + Some((field("vendor_id")?, field("cpu family")?.parse().ok()?)) +} + +/// A CPU's RAPL MSRs, and the scope of its energy-status counter. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct MsrRegisters { + /// The power-unit MSR. + unit: u64, + /// The energy-status MSR. + energy: u64, + /// Whether the counter counts one die (Intel) rather than the whole + /// package (AMD, Hygon). + per_die: bool, +} + +/// The RAPL MSRs for a CPU's vendor and family, or `None` where this crate +/// knows none: Intel family 6, AMD Zen (family 0x17 and later) and Hygon. +/// +/// Intel's counter is per die, AMD's and Hygon's per package. That is the +/// Linux RAPL driver's own rule: `arch/x86/events/rapl.c` scopes the package +/// counter per package only on AMD and Hygon ("AMD/HYGON platforms have a +/// per-PKG package energy counter"), and per die on Intel, where a multi-die +/// part such as Cascade Lake-AP has die-scope RAPL MSRs; `intel_rapl_common.c` +/// names such a part's powercap zones `package--die-`. +fn msr_registers(vendor: &str, family: u32) -> Option { + match vendor { + "GenuineIntel" if family == 6 => { + Some(MsrRegisters { unit: MSR_RAPL_POWER_UNIT, energy: MSR_PKG_ENERGY_STATUS, per_die: true }) + } + "AuthenticAMD" | "HygonGenuine" if family >= 0x17 => { + Some(MsrRegisters { unit: MSR_AMD_RAPL_POWER_UNIT, energy: MSR_AMD_PKG_ENERGY_STATUS, per_die: false }) + } + _ => None, + } +} + +/// A host's RAPL MSRs, and each counter domain's lowest-numbered CPU by +/// domain, `(package, die)`; the die is 0 where the counter is per package. +type MsrHost = (MsrRegisters, std::collections::BTreeMap<(u32, u32), u32>); + +/// The CPU's RAPL MSRs under the filesystem root `root` (from its +/// `/proc/cpuinfo`), and each counter domain's lowest-numbered CPU, in domain +/// order. A domain is a package (`topology/physical_package_id`), and on Intel +/// a die of it (`topology/die_id`, 0 where the file is absent, as on kernels +/// that list no dies): see [`msr_registers`]. `None` when the CPU is not one +/// whose MSRs this crate knows, or no CPU topology is listed. +fn msr_host(root: &std::path::Path) -> Option { + let cpuinfo = std::fs::read_to_string(root.join("proc/cpuinfo")).ok()?; + let registers = cpu_vendor_family(&cpuinfo).and_then(|(vendor, family)| msr_registers(vendor, family))?; + let number = |p: std::path::PathBuf| std::fs::read_to_string(p).ok().and_then(|s| s.trim().parse::().ok()); + let mut first = std::collections::BTreeMap::new(); + for entry in std::fs::read_dir(root.join("sys/devices/system/cpu")).ok()?.flatten() { + let Some(cpu) = entry.file_name().to_str().and_then(|n| n.strip_prefix("cpu")?.parse::().ok()) else { + continue; + }; + let topology = entry.path().join("topology"); + let Some(package) = number(topology.join("physical_package_id")) else { + continue; + }; + let die = if registers.per_die { number(topology.join("die_id")).unwrap_or(0) } else { 0 }; + let lowest = first.entry((package, die)).or_insert(cpu); + *lowest = (*lowest).min(cpu); + } + (!first.is_empty()).then_some((registers, first)) +} + +/// The package zones of the powercap tree at `root`, in zone order. +/// `Ok(empty)` where the tree lists no package zone, or is not there; `Err` +/// where a zone that is or may be a package cannot be read — its `name`, or +/// its `energy_uj` — or the tree cannot be listed for want of privilege, since +/// a reader over the rest would leave that package out. +fn powercap_packages(root: &std::path::Path) -> Result, ()> { + let dir = match std::fs::read_dir(root) { + Ok(dir) => dir, + Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => return Err(()), + Err(_) => return Ok(Vec::new()), + }; + let mut packages = Vec::new(); + for entry in dir.flatten() { + let dir_name = entry.file_name().to_string_lossy().into_owned(); + let Some(index) = rapl_top_level_zone(&dir_name) else { + continue; + }; + let name = std::fs::read_to_string(entry.path().join("name")).map_err(|_| ())?; + if !is_rapl_package_name(&name) { + continue; + } + let energy = entry.path().join("energy_uj"); + // must actually be readable, not just present + std::fs::read_to_string(&energy).ok().and_then(|s| s.trim().parse::().ok()).ok_or(())?; + let range = std::fs::read_to_string(entry.path().join("max_energy_range_uj")) + .ok() + .and_then(|s| s.trim().parse::().ok()) + .filter(|&r| r > 0); + packages.push((index, RaplPackage { rail: dir_name, path: energy, range, msr: None })); + } + packages.sort_by_key(|(index, _)| *index); + Ok(packages.into_iter().map(|(_, package)| package).collect()) +} + +/// Every counter domain's energy-status MSR under the filesystem root `root` +/// — each die on Intel, each package on AMD and Hygon ([`msr_registers`]) — in +/// domain order, each read through the device file of the domain's first CPU +/// (`/dev/cpu//msr`) at the unit its own power-unit MSR reads. `None` +/// when the CPU is not one whose MSRs this crate knows, no CPU topology is +/// listed, or any domain's power-unit MSR or energy-status MSR cannot be read +/// or its unit is zero: a figure that left a die or a package out would miss +/// its energy, and a zero unit is a hypervisor's stand-in, not a counter. +fn msr_packages(root: &std::path::Path) -> Option> { + let (registers, first) = msr_host(root)?; + let dies = |package: u32| first.keys().filter(|(p, _)| *p == package).count(); + first + .iter() + .map(|(&(package, die), &cpu)| { + let path = root.join(format!("dev/cpu/{cpu}/msr")); + let unit_shift = msr_unit_shift(read_msr(&path, registers.unit).ok()?)?; + read_msr(&path, registers.energy).ok()?; + let msr = Msr { register: registers.energy, unit_shift }; + let rail = + if dies(package) > 1 { format!("package-{package}-die-{die} msr") } else { format!("package-{package} msr") }; + Some(RaplPackage { rail, path, range: Some(msr.range_uj()), msr: Some(msr) }) + }) + .collect() +} + +/// Whether a powercap zone's `name` file names a CPU package's counter: +/// `package-`, or `package--die-`, `` and `` one or more ASCII +/// digits, trailing newline allowed. The kernel names one top-level zone per +/// die `package--die-` on a multi-die Intel package, whose RAPL MSRs are +/// die-scope (`intel_rapl_common.c`: `"package-%d-die-%d"` where the MSRs are +/// not package-scope and a package has more than one die), so every die zone +/// is a package zone here and all are summed. Not `psys`, `dram`, `core`, +/// `uncore`, or anything else. fn is_rapl_package_name(name: &str) -> bool { - name.trim() - .strip_prefix("package-") - .is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit())) + let digits = |n: &str| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit()); + let Some(rest) = name.trim().strip_prefix("package-") else { + return false; + }; + match rest.split_once("-die-") { + Some((package, die)) => digits(package) && digits(die), + None => digits(rest), + } } /// The zone index of a top-level RAPL zone directory, `intel-rapl:`; `None` @@ -155,95 +413,85 @@ fn rapl_top_level_zone(dir_name: &str) -> Option { } impl Rapl { - /// Open the RAPL interface. `None` when the platform has no readable - /// powercap tree (non-Linux, locked-down sysfs, no Intel/AMD RAPL), no - /// package zone, or a package zone whose counter cannot be read. + /// Open the RAPL interface: the powercap package zones, or, on Linux where + /// powercap lists none, every package's energy-status MSR. `None` when the + /// platform has neither (non-Linux, locked-down sysfs, no Intel/AMD RAPL, + /// no `msr` module or no `CAP_SYS_RAWIO` on a host without the powercap + /// driver), or a package whose counter cannot be read. /// Opening is not selection: [`Meter::detect`] selects the rung only while /// [`rapl_status`] is [`CounterStatus::Live`]. pub fn open() -> Option { - Self::open_in(std::path::Path::new("/sys/class/powercap")) + Self::open_from(std::path::Path::new(POWERCAP_ROOT), host_root()) } - /// [`Rapl::open`] on the powercap tree at `root`, so the probe runs on fake - /// trees in tests. + /// The powercap tree at `powercap`, or, where it lists no package zone, + /// the MSRs under the filesystem root `root` (none when `root` is `None`). + /// A listed package zone that cannot be read is no reader, and no reason to + /// read the MSRs instead: powercap is there, and this process may not read it. + fn open_from(powercap: &std::path::Path, root: Option<&std::path::Path>) -> Option { + match powercap_packages(powercap) { + Ok(packages) if !packages.is_empty() => Some(Self { packages }), + Ok(_) => root.and_then(msr_packages).map(|packages| Self { packages }), + Err(()) => None, + } + } + + /// [`Rapl::open`] on the powercap tree at `root` alone, so the probe runs + /// on fake trees in tests. /// /// Every package zone or none: the figure claims the whole CPU, so a /// package whose `energy_uj` cannot be read is not left out of it — the /// reader does not open. Nor does it open when a top-level zone's `name` /// cannot be read, since that zone may be a package. + #[cfg(test)] fn open_in(root: &std::path::Path) -> Option { - let mut packages = Vec::new(); - let dir = std::fs::read_dir(root).ok()?; - for entry in dir.flatten() { - let dir_name = entry.file_name(); - let Some(index) = rapl_top_level_zone(&dir_name.to_string_lossy()) else { - continue; - }; - let name = std::fs::read_to_string(entry.path().join("name")).ok()?; - if !is_rapl_package_name(&name) { - continue; - } - let energy = entry.path().join("energy_uj"); - // must actually be readable, not just present - std::fs::read_to_string(&energy).ok()?.trim().parse::().ok()?; - let range = std::fs::read_to_string(entry.path().join("max_energy_range_uj")) - .ok() - .and_then(|s| s.trim().parse::().ok()) - .filter(|&r| r > 0); - packages.push((index, energy, range)); - } - packages.sort_by_key(|(index, _, _)| *index); - if packages.is_empty() { - None - } else { - Some(Self { packages: packages.into_iter().map(|(_, energy, range)| (energy, range)).collect() }) - } + Self::open_from(root, None) } /// Current package counters, in µJ, one per package. fn read(&self) -> Option> { - self.packages - .iter() - .map(|(p, _)| std::fs::read_to_string(p).ok()?.trim().parse::().ok()) - .collect() + self.packages.iter().map(RaplPackage::read_uj).collect() } /// Energy spent between two counter snapshots: the sum of each package's - /// own delta, each wraparound-corrected at that package's own - /// `max_energy_range_uj` ([`rapl_zone_delta`]). `None` — no figure, never - /// an invented one — if any package's delta is unknown. + /// own delta, each wraparound-corrected at that package's own range + /// ([`rapl_zone_delta`]). `None` — no figure, never an invented one — if + /// any package's delta is unknown. fn delta(&self, before: &[u64], after: &[u64]) -> Option { if before.len() != self.packages.len() || after.len() != self.packages.len() { return None; } let mut total = 0u64; - for ((_, range), (b, a)) in self.packages.iter().zip(before.iter().zip(after)) { - total = total.checked_add(rapl_zone_delta(*b, *a, *range)?)?; + for (package, (b, a)) in self.packages.iter().zip(before.iter().zip(after)) { + total = total.checked_add(rapl_zone_delta(*b, *a, package.range)?)?; } Some(total) } /// One liveness sample: each package counter as a rail, named by its - /// powercap directory (`intel-rapl:0`, ...), with its raw `energy_uj`. - /// Only the package zones the figure sums: `psys`, `dram` and subzones are - /// neither summed nor probed. `None` when a package cannot be read, which - /// the probe skips. A wrap shows as a fall, which the rule does not count - /// as an advance. + /// powercap directory (`intel-rapl:0`, ...) or as `package- msr`, with + /// its count in µJ. Only the packages the figure sums: `psys`, `dram` and + /// subzones are neither summed nor probed. `None` when a package cannot be + /// read, which the probe skips. A wrap shows as a fall, which the rule does + /// not count as an advance. fn liveness_sample(&self) -> Option> { let values = self.read()?; let named: Vec<(String, i64)> = self .packages .iter() .zip(values) - .map(|((path, _), uj)| { - let package = path.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().into_owned()); - (package.unwrap_or_default(), i64::try_from(uj).unwrap_or(i64::MAX)) - }) + .map(|(package, uj)| (package.rail.clone(), i64::try_from(uj).unwrap_or(i64::MAX))) .collect(); Some(liveness::every_rail(&named)) } } +/// The filesystem root the MSR fallback reads under: `/` on Linux, where the +/// `msr` driver lives; none elsewhere. +fn host_root() -> Option<&'static std::path::Path> { + cfg!(target_os = "linux").then_some(std::path::Path::new("/")) +} + /// One RAPL zone's energy between two reads of its `energy_uj`, modulo its own /// wrap. A counter that went down wrapped at `max_range` (its /// `max_energy_range_uj`): the energy is what it had left to the wrap plus what @@ -741,7 +989,8 @@ fn rapl_probe_runs() -> usize { /// /// [`Meter::detect`] selects the RAPL rung only when this is /// [`CounterStatus::Live`]: every package counter (the `energy_uj` of each -/// top-level `intel-rapl:N` zone whose `name` is `package-`) kept advancing +/// top-level `intel-rapl:N` zone whose `name` is `package-`, or +/// `package--die-` for each die of a multi-die Intel package) kept advancing /// while the probing thread burned CPU, each in at least five sample /// intervals, at least 80% of them, over at least 300 ms — the rule IOReport's /// CPU rails are held to. Every package, because a span's figure @@ -750,10 +999,12 @@ fn rapl_probe_runs() -> usize { /// republished in lumps (a virtualised powercap tree can be either), is /// declined to the default-power estimate. Only the package zones are probed, as /// only they are summed: a live `psys` or `dram` zone beside a held package -/// does not make the counter live. Where there is no readable powercap tree — -/// not Linux, no RAPL, no package zone (a host exposing only `psys`), or an -/// `energy_uj` only root may read — it is [`CounterStatus::Unavailable`] -/// without burning anything, as RAPL was declined before. The first call on a +/// does not make the counter live. Where powercap lists no package zone, the +/// packages are their energy-status MSRs ([`Rapl::open`]), probed by the same +/// rule. Where neither reads — not Linux, no RAPL, an `energy_uj` only root may +/// read, or no package zone and no MSR this process may read (no `msr` module, +/// no `CAP_SYS_RAWIO`) — it is [`CounterStatus::Unavailable`] without burning +/// anything, as RAPL was declined before ([`rapl_access`] says which). The first call on a /// host with a readable tree burns about 300 ms of the calling thread's CPU /// (one second if the counter is not live). pub fn rapl_status() -> CounterStatus { @@ -805,10 +1056,59 @@ pub enum RaplAccess { NoInterface, } -/// This host's RAPL accessibility. +/// This host's RAPL accessibility: its powercap package zones', or, on Linux +/// where powercap exposes none, its packages' energy-status MSRs' +/// ([`Rapl::open`]). An MSR this process may not open — the `msr` driver's +/// device files need `CAP_SYS_RAWIO` and read access — is +/// [`RaplAccess::NeedsPrivilege`]; no `msr` module, a CPU whose MSRs this crate +/// does not know, or a power-unit MSR that reads zero (a hypervisor's +/// stand-in) is [`RaplAccess::NoInterface`]. #[must_use] pub fn rapl_access() -> RaplAccess { - rapl_access_in(std::path::Path::new("/sys/class/powercap")) + rapl_access_from(std::path::Path::new(POWERCAP_ROOT), host_root(), read_msr) +} + +/// [`rapl_access`] on the powercap tree at `powercap` and, where it exposes no +/// package zone, the MSRs under the filesystem root `root`, read with +/// `read_msr`, so tests run on fake trees and inject permission errors. +fn rapl_access_from( + powercap: &std::path::Path, + root: Option<&std::path::Path>, + read_msr: impl Fn(&std::path::Path, u64) -> std::io::Result, +) -> RaplAccess { + match (rapl_access_in(powercap), root) { + (RaplAccess::NoInterface, Some(root)) => msr_access_with(root, read_msr), + (access, _) => access, + } +} + +/// Whether this process can read every counter domain's energy-status MSR +/// under the filesystem root `root` — each die on Intel, each package on AMD +/// and Hygon, the domains [`Rapl::open`] reads — each on the domain's first +/// CPU, with `read` standing in for [`read_msr`]. As for powercap, a permission +/// failure on any domain decides the answer, even beside a readable one; any +/// other failure (no device file, a register the CPU does not implement) or a +/// zero unit is no interface. +fn msr_access_with(root: &std::path::Path, read: impl Fn(&std::path::Path, u64) -> std::io::Result) -> RaplAccess { + let Some((registers, first)) = msr_host(root) else { + return RaplAccess::NoInterface; + }; + let denied = |e: &std::io::Error| e.kind() == std::io::ErrorKind::PermissionDenied; + let mut readable = true; + for cpu in first.values() { + let device = root.join(format!("dev/cpu/{cpu}/msr")); + match read(&device, registers.unit) { + Ok(unit) if msr_unit_shift(unit).is_some() => match read(&device, registers.energy) { + Ok(_) => {} + Err(e) if denied(&e) => return RaplAccess::NeedsPrivilege, + Err(_) => readable = false, + }, + Ok(_) => readable = false, + Err(e) if denied(&e) => return RaplAccess::NeedsPrivilege, + Err(_) => readable = false, + } + } + if readable { RaplAccess::Readable } else { RaplAccess::NoInterface } } /// [`rapl_access`] on the powercap tree at `root`, so tests run on fake trees. @@ -6433,7 +6733,8 @@ pub fn survey() -> Vec { } else { FabricStatus::Absent }, - "/sys/class/powercap/intel-rapl:*/energy_uj, package zones only", + "/sys/class/powercap/intel-rapl:*/energy_uj, package zones only; where powercap lists none, each package's \ + energy-status MSR through /dev/cpu//msr (needs the msr module and CAP_SYS_RAWIO)", ); push( Fabric::Dram, @@ -8883,11 +9184,22 @@ mod tests { }); } + impl Rapl { + /// A reader over powercap packages that wrap at `ranges`, read from + /// nowhere: for the delta arithmetic. + fn with_ranges(ranges: &[Option]) -> Self { + let packages = ranges + .iter() + .enumerate() + .map(|(i, &range)| RaplPackage { rail: format!("intel-rapl:{i}"), path: "/dev/null".into(), range, msr: None }) + .collect(); + Self { packages } + } + } + #[test] fn rapl_wraparound_delta() { - let r = Rapl { - packages: vec![(std::path::PathBuf::from("/dev/null"), Some(1_000))], - }; + let r = Rapl::with_ranges(&[Some(1_000)]); assert_eq!(r.delta(&[100], &[400]), Some(300)); // counter wrapped at 1_000: 900 → 50 means 150 spent assert_eq!(r.delta(&[900], &[50]), Some(150)); @@ -8908,30 +9220,40 @@ mod tests { assert_eq!(rapl_zone_delta(2_000, 50, Some(1_000)), None, "a fall from above the range gave a figure"); // Each package at its own range, summed: package 0 rose 1,000 µJ, // package 1 wrapped at its own (smaller) range. - let r = Rapl { - packages: vec![ - (std::path::PathBuf::from("/dev/null"), Some(262_143_328_850)), - (std::path::PathBuf::from("/dev/null"), Some(65_532_610_987)), - ], - }; + let r = Rapl::with_ranges(&[Some(262_143_328_850), Some(65_532_610_987)]); assert_eq!(r.delta(&[5_000, 65_532_600_000], &[6_000, 500]), Some(12_487)); // One package's delta unknown: no figure for the span. - let r = Rapl { - packages: vec![ - (std::path::PathBuf::from("/dev/null"), Some(262_143_328_850)), - (std::path::PathBuf::from("/dev/null"), None), - ], - }; + let r = Rapl::with_ranges(&[Some(262_143_328_850), None]); assert_eq!(r.delta(&[5_000, 900], &[6_000, 50]), None); } - /// The package zones by their `name` file, and no other: `package-`. + /// The package zones by their `name` file, and no other: `package-`, + /// and `package--die-`, one zone per die of a multi-die Intel + /// package. #[test] fn a_rapl_package_is_a_zone_named_package_n() { - for name in ["package-0", "package-1\n", "package-12", " package-3 \n"] { + for name in ["package-0", "package-1\n", "package-12", " package-3 \n", "package-0-die-0", "package-0-die-1\n", "package-12-die-3"] { assert!(is_rapl_package_name(name), "{name:?}"); } - for name in ["psys", "dram", "core", "uncore", "package-", "package-x", "package-0a", "Package-0", "package0", ""] { + for name in [ + "psys", + "dram", + "core", + "uncore", + "package-", + "package-x", + "package-0a", + "Package-0", + "package0", + "", + "package-0-die-", + "package--die-0", + "package-0-die-x", + "package-0-die-1-die-2", + "package-0-core-0", + "package-0-die", + "package-0-", + ] { assert!(!is_rapl_package_name(name), "{name:?}"); } assert_eq!(rapl_top_level_zone("intel-rapl:0"), Some(0)); @@ -8941,6 +9263,32 @@ mod tests { } } + /// A multi-die Intel package, as the kernel lists it: one top-level zone + /// per die, `package-0-die-0` and `package-0-die-1`. Both are package + /// zones; the reader holds both, probes both, and a span's figure is both + /// dies' rise — reading one die would leave the other's energy out. + #[test] + fn a_multi_die_package_sums_every_die_zone() { + let tree = FakePowercap::with_zones(&[ + ("intel-rapl:0", Some("package-0-die-0"), "1000", Some("262143328850")), + ("intel-rapl:1", Some("package-0-die-1"), "2000", Some("262143328850")), + ("intel-rapl:0:0", Some("core"), "700", Some("262143328850")), + ("intel-rapl:2", Some("psys"), "90000", Some("262143328850")), + ]); + let r = Rapl::open_in(&tree.root).expect("a tree with two die zones opens"); + let rails: Vec<&str> = r.packages.iter().map(|p| p.rail.as_str()).collect(); + assert_eq!(rails, ["intel-rapl:0", "intel-rapl:1"], "the reader did not hold both die zones and only them"); + let meter = Meter::Rapl(r); + let (_, e) = meter.measure(|| { + tree.write_zone("intel-rapl:0", "4000"); + tree.write_zone("intel-rapl:1", "7000"); + tree.write_zone("intel-rapl:0:0", "2700"); + tree.write_zone("intel-rapl:2", "190000"); + }); + assert_eq!((e.provenance, e.measured_uj), (Provenance::HwShunt, Some(8_000)), "not both dies' rise: {e:?}"); + assert_eq!(rapl_access_in(&tree.root), RaplAccess::Readable); + } + /// A tree laid out as a two-socket Intel host with a platform zone: two /// packages (`intel-rapl:0`, `intel-rapl:2`), `psys` (`intel-rapl:1`), a /// top-level `dram` zone (`intel-rapl:3`), each package's `core` and @@ -8974,7 +9322,7 @@ mod tests { let _hw = ioreport_hw_lock(); let tree = two_sockets_with_psys_and_dram(); let r = Rapl::open_in(&tree.root).expect("a tree with two package zones opens"); - let opened: Vec<_> = r.packages.iter().map(|(p, range)| (p.strip_prefix(&tree.root).unwrap().to_owned(), *range)).collect(); + let opened: Vec<_> = r.packages.iter().map(|p| (p.path.strip_prefix(&tree.root).unwrap().to_owned(), p.range)).collect(); assert_eq!( opened, [ @@ -9236,6 +9584,342 @@ mod tests { assert_eq!((e.provenance, e.measured_uj), (Provenance::Estimator, None), "a fall with no range: {e:?}"); e.check_provenance().unwrap(); } + + /// The MSR path, over fake device files: a regular file whose bytes at a + /// register's offset are that register is read exactly as the `msr` + /// driver's device file is. + #[cfg(unix)] + mod msr { + use super::*; + use std::os::unix::fs::FileExt; + + const INTEL: &str = "processor\t: 0\nvendor_id\t: GenuineIntel\ncpu family\t: 6\nmodel\t\t: 85\n"; + const AMD: &str = "processor\t: 0\nvendor_id\t: AuthenticAMD\ncpu family\t: 26\nmodel\t\t: 112\n"; + /// A power-unit MSR whose energy-status unit (bits 12:8) is 2^-14 J, + /// the common Intel value (0xA0E03). + const UNIT_14: u64 = 0xA_0E03; + /// 2^-16 J, as on AMD Zen. + const UNIT_16: u64 = 0x10 << 8; + + /// A fake filesystem root, removed when dropped: its `proc/cpuinfo`, + /// each CPU's `physical_package_id`, and the CPUs' `dev/cpu//msr` + /// files. + struct MsrRoot(std::path::PathBuf); + + impl MsrRoot { + /// `package_of_cpu[n]` is CPU `n`'s package. + fn new(cpuinfo: &str, package_of_cpu: &[u32]) -> Self { + static N: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let n = N.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let root = std::env::temp_dir().join(format!("jcp-energy-msr-{}-{n}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("proc")).expect("create proc"); + std::fs::write(root.join("proc/cpuinfo"), cpuinfo).expect("write cpuinfo"); + for (cpu, package) in package_of_cpu.iter().enumerate() { + let topology = root.join(format!("sys/devices/system/cpu/cpu{cpu}/topology")); + std::fs::create_dir_all(&topology).expect("create a CPU's topology"); + std::fs::write(topology.join("physical_package_id"), format!("{package}\n")).expect("write its package"); + } + Self(root) + } + + /// List CPU `cpu` on die `die` of its package (`topology/die_id`). + fn die(&self, cpu: u32, die: u32) -> &Self { + let topology = self.0.join(format!("sys/devices/system/cpu/cpu{cpu}/topology")); + std::fs::write(topology.join("die_id"), format!("{die}\n")).expect("write its die"); + self + } + + /// Set CPU `cpu`'s MSR `register` to `value`. + fn set(&self, cpu: u32, register: u64, value: u64) -> &Self { + let dir = self.0.join(format!("dev/cpu/{cpu}")); + std::fs::create_dir_all(&dir).expect("create dev/cpu/"); + let file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(dir.join("msr")) + .expect("open the fake msr file"); + file.write_all_at(&value.to_le_bytes(), register).expect("write the register"); + self + } + + /// An Intel host whose CPUs `cpus` each have the 2^-14 J unit and + /// the energy-status count `count`. + fn intel(package_of_cpu: &[u32], cpus: &[u32], count: u64) -> Self { + let host = Self::new(INTEL, package_of_cpu); + for &cpu in cpus { + host.set(cpu, MSR_RAPL_POWER_UNIT, UNIT_14).set(cpu, MSR_PKG_ENERGY_STATUS, count); + } + host + } + } + + impl Drop for MsrRoot { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A powercap tree that lists no package zone: only a subzone, which + /// is never a package. + fn no_package_zone() -> FakePowercap { + FakePowercap::new(&[]) + } + + /// A two-socket Intel host (CPUs 0–1 on package 0, 2–3 on package 1) + /// with no powercap package zone: each package read on its first CPU, + /// at its unit, summed, a span over it a whole hardware reading. + #[test] + fn a_two_socket_host_without_powercap_reads_each_packages_msr() { + let host = MsrRoot::new(INTEL, &[0, 0, 1, 1]); + host.set(0, MSR_RAPL_POWER_UNIT, UNIT_14).set(0, MSR_PKG_ENERGY_STATUS, 1_000); + host.set(2, MSR_RAPL_POWER_UNIT, UNIT_14).set(2, MSR_PKG_ENERGY_STATUS, 2_000); + let powercap = no_package_zone(); + let r = Rapl::open_from(&powercap.root, Some(&host.0)).expect("the MSRs open where powercap lists no package"); + let devices: Vec<_> = r.packages.iter().map(|p| (p.rail.as_str(), p.path.clone())).collect(); + assert_eq!( + devices, + vec![("package-0 msr", host.0.join("dev/cpu/0/msr")), ("package-1 msr", host.0.join("dev/cpu/2/msr"))], + "package 1 is read on its first CPU, 2" + ); + assert!(r.packages.iter().all(|p| p.msr == Some(Msr { register: MSR_PKG_ENERGY_STATUS, unit_shift: 14 }))); + assert!(r.packages.iter().all(|p| p.range == Some(262_144_000_000)), "2^32 counts of 2^-14 J"); + // 1,000 and 2,000 counts of 61.035 µJ, each floored. + assert_eq!(r.read(), Some(vec![61_035, 122_070])); + let sample = r.liveness_sample().expect("the packages read"); + let probed: Vec<(&str, i64)> = sample.iter().map(|(k, v)| (k.name.as_str(), *v)).collect(); + assert_eq!(probed, [("package-0 msr", 61_035), ("package-1 msr", 122_070)]); + + let meter = Meter::Rapl(r); + let (_, e) = meter.measure(|| { + host.set(0, MSR_PKG_ENERGY_STATUS, 3_000).set(2, MSR_PKG_ENERGY_STATUS, 2_100); + }); + // 183,105 - 61,035 on package 0, 128,173 - 122,070 on package 1. + assert_eq!((e.provenance, e.measured_uj, e.is_whole()), (Provenance::HwShunt, Some(128_173), true), "{e:?}"); + e.check_provenance().unwrap(); + } + + /// Intel's RAPL MSRs are die-scope: a package with two dies is two + /// counter domains, each read on its own first CPU, and summed; a + /// package with one die stays `package- msr`. Every domain or none, + /// and the access answer is over the same domains. + #[test] + fn an_intel_package_with_two_dies_is_read_on_each_die() { + // Package 0: CPUs 0–1 on die 0, 2–3 on die 1. Package 1: CPUs 4–5, one die. + let host = MsrRoot::new(INTEL, &[0, 0, 0, 0, 1, 1]); + for (cpu, die) in [(0, 0), (1, 0), (2, 1), (3, 1), (4, 0), (5, 0)] { + host.die(cpu, die); + } + for (cpu, count) in [(0, 1_000), (2, 2_000), (4, 4_000)] { + host.set(cpu, MSR_RAPL_POWER_UNIT, UNIT_14).set(cpu, MSR_PKG_ENERGY_STATUS, count); + } + let r = Rapl::open_from(&no_package_zone().root, Some(&host.0)).expect("every die's MSR reads"); + let domains: Vec<_> = r.packages.iter().map(|p| (p.rail.as_str(), p.path.clone())).collect(); + assert_eq!( + domains, + vec![ + ("package-0-die-0 msr", host.0.join("dev/cpu/0/msr")), + ("package-0-die-1 msr", host.0.join("dev/cpu/2/msr")), + ("package-1 msr", host.0.join("dev/cpu/4/msr")), + ], + "die 1 is read on its own first CPU, 2" + ); + assert_eq!(r.read(), Some(vec![61_035, 122_070, 244_140])); + let meter = Meter::Rapl(r); + let (_, e) = meter.measure(|| { + host.set(0, MSR_PKG_ENERGY_STATUS, 2_000).set(2, MSR_PKG_ENERGY_STATUS, 3_000); + }); + // 61,035 on die 0 and 61,035 on die 1 (122,070 -> 183,105). + assert_eq!((e.provenance, e.measured_uj), (Provenance::HwShunt, Some(122_070)), "both dies summed: {e:?}"); + + let die_1_denied = |p: &std::path::Path, r: u64| -> std::io::Result { + if p.ends_with("cpu/2/msr") { + Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "EPERM")) + } else { + read_msr(p, r) + } + }; + assert_eq!(msr_access_with(&host.0, read_msr), RaplAccess::Readable); + assert_eq!(msr_access_with(&host.0, die_1_denied), RaplAccess::NeedsPrivilege, "die 1 is a domain of its own"); + + let one_die_missing = MsrRoot::new(INTEL, &[0, 0]); + one_die_missing.die(0, 0).die(1, 1); + one_die_missing.set(0, MSR_RAPL_POWER_UNIT, UNIT_14).set(0, MSR_PKG_ENERGY_STATUS, 5); + assert!( + Rapl::open_from(&no_package_zone().root, Some(&one_die_missing.0)).is_none(), + "die 1 has no device file: a figure without it would under-count the package" + ); + } + + /// AMD's and Hygon's counter is per package: CPUs on different dies of + /// one package are one domain, read on the package's first CPU. + #[test] + fn an_amd_package_is_one_domain_whatever_its_dies() { + let host = MsrRoot::new(AMD, &[0, 0, 0, 0]); + host.die(0, 0).die(1, 0).die(2, 1).die(3, 1); + host.set(0, MSR_AMD_RAPL_POWER_UNIT, UNIT_16).set(0, MSR_AMD_PKG_ENERGY_STATUS, 65_536); + let r = Rapl::open_from(&no_package_zone().root, Some(&host.0)).expect("the package's MSR reads"); + let domains: Vec<_> = r.packages.iter().map(|p| (p.rail.as_str(), p.path.clone())).collect(); + assert_eq!(domains, vec![("package-0 msr", host.0.join("dev/cpu/0/msr"))], "one per-package counter"); + assert_eq!(msr_access_with(&host.0, read_msr), RaplAccess::Readable, "CPU 2's die is not a domain"); + } + + /// An Intel host whose kernel lists no `die_id` is read per package, as + /// before: each package one domain, `package- msr`. + #[test] + fn an_intel_host_without_die_ids_is_read_per_package() { + let host = MsrRoot::intel(&[0, 0, 1, 1], &[0, 2], 1_000); + let r = Rapl::open_from(&no_package_zone().root, Some(&host.0)).expect("opens"); + let rails: Vec<&str> = r.packages.iter().map(|p| p.rail.as_str()).collect(); + assert_eq!(rails, ["package-0 msr", "package-1 msr"]); + assert_eq!(r.read(), Some(vec![61_035, 61_035])); + } + + /// The 32-bit count wraps at 2^32 counts, so the µJ figure wraps at + /// that range, losing at most a microjoule; bits above the count are + /// not energy. + #[test] + fn the_count_wraps_at_2_to_the_32_counts() { + let host = MsrRoot::intel(&[0], &[0], 0xFFFF_FFF0); + let r = Rapl::open_from(&no_package_zone().root, Some(&host.0)).expect("opens"); + let before = r.read().expect("reads"); + host.set(0, MSR_PKG_ENERGY_STATUS, 0xABCD_0000_0000_0010); + let after = r.read().expect("reads"); + // 32 counts of 61.035 µJ is 1,953.125 µJ. + assert_eq!(r.delta(&before, &after), Some(1_953)); + } + + /// AMD Zen reads its own power-unit and energy-status MSRs. + #[test] + fn amd_zen_reads_its_own_msrs() { + let host = MsrRoot::new(AMD, &[0, 0]); + host.set(0, MSR_AMD_RAPL_POWER_UNIT, UNIT_16).set(0, MSR_AMD_PKG_ENERGY_STATUS, 65_536); + let r = Rapl::open_from(&no_package_zone().root, Some(&host.0)).expect("opens"); + assert_eq!(r.packages.len(), 1); + assert_eq!(r.packages[0].msr, Some(Msr { register: MSR_AMD_PKG_ENERGY_STATUS, unit_shift: 16 })); + assert_eq!(r.read(), Some(vec![1_000_000]), "65,536 counts of 2^-16 J is one joule"); + } + + /// No MSR reader where the power-unit MSR reads a zero unit (a + /// hypervisor's stand-in, seen under KVM), where the CPU is not one + /// whose MSRs are known, where a package's device file is missing, or + /// where the energy-status register does not read. + #[test] + fn a_zero_unit_an_unknown_cpu_or_a_missing_device_is_no_counter() { + let powercap = no_package_zone(); + let opens = |host: &MsrRoot| Rapl::open_from(&powercap.root, Some(&host.0)).is_some(); + + let zero = MsrRoot::new(INTEL, &[0]); + zero.set(0, MSR_RAPL_POWER_UNIT, 0).set(0, MSR_PKG_ENERGY_STATUS, 0); + assert!(!opens(&zero), "a zero unit"); + + let arm = MsrRoot::new("processor\t: 0\nCPU implementer\t: 0x41\n", &[0]); + assert!(!opens(&arm), "no vendor_id"); + + let netburst = MsrRoot::new("vendor_id\t: GenuineIntel\ncpu family\t: 15\n", &[0]); + netburst.set(0, MSR_RAPL_POWER_UNIT, UNIT_14).set(0, MSR_PKG_ENERGY_STATUS, 5); + assert!(!opens(&netburst), "not family 6"); + + let missing = MsrRoot::intel(&[0, 1], &[0], 5); + assert!(!opens(&missing), "package 1 has no device file"); + + let no_counter = MsrRoot::new(INTEL, &[0]); + no_counter.set(0, MSR_RAPL_POWER_UNIT, UNIT_14); + assert!(!opens(&no_counter), "the energy-status register does not read"); + + let no_topology = MsrRoot::new(INTEL, &[]); + assert!(!opens(&no_topology), "no CPU topology"); + } + + /// Powercap zones are preferred, and the MSRs are read only where + /// powercap lists no package: not where it lists one this process + /// cannot read, and not off Linux (no root to read under). + #[test] + fn powercap_is_preferred_and_the_msrs_are_the_fallback() { + let host = MsrRoot::intel(&[0], &[0], 5); + let zones = FakePowercap::new(&["1000", "2000"]); + let preferred = Rapl::open_from(&zones.root, Some(&host.0)).expect("powercap opens"); + assert_eq!(preferred.packages.len(), 2); + assert!(preferred.packages.iter().all(|p| p.msr.is_none()), "powercap's zones were listed"); + + let fallback = Rapl::open_from(&no_package_zone().root, Some(&host.0)).expect("the MSR opens"); + assert!(fallback.packages[0].msr.is_some(), "powercap listed no package, so the MSR is read"); + let absent = no_package_zone(); + assert!(Rapl::open_from(&absent.root.join("no-such-dir"), Some(&host.0)).is_some(), "no powercap tree at all"); + + let garbled = FakePowercap::new(&["not a number"]); + assert!(Rapl::open_from(&garbled.root, Some(&host.0)).is_none(), "a listed package that does not read fell back"); + assert!(Rapl::open_from(&no_package_zone().root, None).is_none(), "no root to read MSRs under"); + } + + /// The probe over the production opener, the CPU-burning tick and the + /// real clock on MSR packages: live when every package's count keeps + /// advancing, frozen when one holds. + #[test] + fn the_production_probe_judges_msr_packages_by_the_same_rule() { + #[cfg(target_os = "macos")] + let _hw = ioreport_hw_lock(); + let run = |moving: &[u32]| { + let host = MsrRoot::intel(&[0, 1], &[0, 1], 1_000_000); + let powercap = no_package_zone(); + let mut count = 1_000_000u64; + crate::liveness::status_of( + crate::liveness::probe( + || Rapl::open_from(&powercap.root, Some(&host.0)), + Rapl::liveness_sample, + || { + crate::liveness::burn_tick(); + count += 18_432; // 45 W over a 25 ms tick, in 61.035 µJ counts + for &cpu in moving { + host.set(cpu, MSR_PKG_ENERGY_STATUS, count); + } + }, + std::time::Instant::now, + crate::liveness::BUDGET, + ) + .as_ref(), + ) + }; + assert_eq!(run(&[0, 1]), CounterStatus::Live); + assert_eq!(run(&[0]), CounterStatus::Frozen, "package 1 held its count"); + } + + /// An MSR this process may not open is a privilege gap, not absent + /// hardware, and one such package decides the answer beside a readable + /// one. No device file, a zero unit or an unknown CPU is no interface. + /// Where powercap has a package zone, its answer stands and no MSR is + /// read. The permission is injected, as for powercap: `chmod` proves + /// nothing under root. + #[test] + fn an_msr_host_needing_privilege_is_not_reported_absent() { + let denied = |_: &std::path::Path, _: u64| -> std::io::Result { + Err(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "EPERM")) + }; + let none = no_package_zone(); + let host = MsrRoot::intel(&[0, 0, 1, 1], &[0, 2], 1_000); + assert_eq!(rapl_access_from(&none.root, Some(&host.0), read_msr), RaplAccess::Readable); + assert_eq!(rapl_access_from(&none.root, Some(&host.0), denied), RaplAccess::NeedsPrivilege); + let package_1_denied = |p: &std::path::Path, r: u64| { + if p.ends_with("cpu/2/msr") { denied(p, r) } else { read_msr(p, r) } + }; + assert_eq!(msr_access_with(&host.0, package_1_denied), RaplAccess::NeedsPrivilege, "one package decides"); + assert_eq!(rapl_access_from(&none.root, None, denied), RaplAccess::NoInterface, "no root, no MSR"); + + let missing = MsrRoot::intel(&[0, 1], &[0], 1_000); + assert_eq!(msr_access_with(&missing.0, read_msr), RaplAccess::NoInterface, "no device file for package 1"); + let zero = MsrRoot::new(INTEL, &[0]); + zero.set(0, MSR_RAPL_POWER_UNIT, 0).set(0, MSR_PKG_ENERGY_STATUS, 0); + assert_eq!(msr_access_with(&zero.0, read_msr), RaplAccess::NoInterface, "a zero unit"); + let arm = MsrRoot::new("processor\t: 0\n", &[0]); + assert_eq!(msr_access_with(&arm.0, denied), RaplAccess::NoInterface, "a CPU whose MSRs are unknown"); + + let zones = FakePowercap::new(&["1000"]); + let untouched = |_: &std::path::Path, _: u64| -> std::io::Result { panic!("an MSR was read beside a package zone") }; + assert_eq!(rapl_access_from(&zones.root, Some(&host.0), untouched), RaplAccess::Readable); + assert_eq!(rapl_fabric_status(RaplAccess::NeedsPrivilege, CounterStatus::Unavailable), FabricStatus::NeedsPrivilege); + } + } } #[cfg(test)] diff --git a/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs b/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs index 7f2d8baf4..244166d48 100644 --- a/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs +++ b/jcp/jcp-rs/crates/jcp-energy/tests/rapl_hardware.rs @@ -13,7 +13,9 @@ //! rung must be selected, and a span over a CPU burn must be `HwShunt`, //! `Whole` (the package counters read the CPU work they meter), its figure //! inside the oracle's package-only bracket and clearly below package+psys -//! and package+core. +//! and package+core. Where powercap lists no package zone (a kernel without +//! its RAPL driver), the oracle reads each package's energy-status MSR itself +//! (`oracle::msr`), and the same verdict and bracket are taken over those. #![cfg(target_os = "linux")] @@ -119,9 +121,14 @@ mod oracle { z.dir.strip_prefix("intel-rapl:").is_some_and(digits) } - /// A package zone: top-level, named `package-`. + /// A package zone: top-level, named `package-`, or `package--die-` + /// (one zone per die, as the kernel lists a multi-die Intel package). pub fn is_package(z: &Zone) -> bool { - top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + top_level(z) + && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(|rest| match rest.split_once("-die-") { + Some((package, die)) => digits(package) && digits(die), + None => digits(rest), + }) } /// The platform zone. @@ -195,19 +202,34 @@ mod oracle { /// between samples (capped at 500 ms of wall time), until every package /// has advanced in at least five intervals, at least 80% of those /// observed, over at least 300 ms; or one second has passed with at least - /// five intervals; or five seconds have. + /// five intervals; or five seconds have. Where this host's powercap lists + /// no package zone and its packages' energy-status MSRs read + /// ([`msr::packages`]), the same run over those. pub fn watch(root: &Path) -> Watch { let all = zones(root); let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; if packages.is_empty() { + if root == Path::new(POWERCAP) { + if let Some(msrs) = msr::packages() { + let names = msrs.iter().map(|p| p.name.clone()).collect(); + return watch_reads(names, || msr::read(&msrs)); + } + } let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); } + watch_reads(names, || read(root, &packages)) + } + + /// The sustained-advance run of [`watch`] over the packages `names`, + /// sampled by `read`. + fn watch_reads(names: Vec, read: impl Fn() -> Option>) -> Watch { + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; let started = Instant::now(); - let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; - let mut advances = vec![0u32; packages.len()]; + let Some(mut prev) = read() else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; names.len()]; let mut intervals = 0u32; let live = |adv: &[u32], n: u32, w: Duration| { w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) @@ -218,7 +240,7 @@ mod oracle { break; } burn(Duration::from_millis(25), Duration::from_millis(500)); - if let Some(now) = read(root, &packages) { + if let Some(now) = read() { intervals += 1; for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { *a += u32::from(n > p); @@ -370,6 +392,110 @@ mod oracle { let o3 = read_all(&set); (Bracket { set, reads: [o0, o1, o2, o3] }, m0, m1) } + + /// The tests' own reading of the packages' energy-status MSRs, for a host + /// whose powercap lists no package zone (a kernel without the powercap + /// RAPL driver): the CPU's vendor and family from `/proc/cpuinfo`, each + /// package's first CPU from `/sys/devices/system/cpu`, each read through + /// `/dev/cpu//msr` at the register's offset. Nothing in it comes from + /// the crate under test. + pub mod msr { + use std::os::unix::fs::FileExt; + use std::path::PathBuf; + + /// One counter domain's energy-status MSR: a die on Intel, a package on + /// AMD and Hygon (the kernel's RAPL scope). + #[derive(Debug, Clone)] + pub struct Package { + /// Its name: `package- msr`, or `package--die- msr` for a + /// die of a multi-die Intel package. + pub name: String, + /// The `msr` device file of its first CPU. + pub device: PathBuf, + /// The energy-status register. + pub register: u64, + /// The energy-status unit, 2^-`shift` J per count. + pub shift: u32, + } + + fn msr(device: &std::path::Path, register: u64) -> Option { + let mut raw = [0u8; 8]; + std::fs::File::open(device).ok()?.read_exact_at(&mut raw, register).ok()?; + Some(u64::from_le_bytes(raw)) + } + + /// Every counter domain's energy-status MSR, in domain order: Intel + /// family 6 (0x606 unit, 0x611 count), each die (`topology/die_id`, 0 + /// where absent) of each package; AMD family 0x17 and later or Hygon + /// (0xC0010299, 0xC001029B), each package. `None` when the CPU is + /// another, or any domain's unit does not read or reads zero. + pub fn packages() -> Option> { + let cpuinfo = std::fs::read_to_string("/proc/cpuinfo").ok()?; + let field = |key: &str| { + cpuinfo.lines().find_map(|l| l.split_once(':').filter(|(k, _)| k.trim() == key).map(|(_, v)| v.trim())) + }; + let family: u32 = field("cpu family")?.parse().ok()?; + let (unit, register, per_die) = match field("vendor_id")? { + "GenuineIntel" if family == 6 => (0x606, 0x611, true), + "AuthenticAMD" | "HygonGenuine" if family >= 0x17 => (0xC001_0299, 0xC001_029B, false), + _ => return None, + }; + let number = |p: PathBuf| std::fs::read_to_string(p).ok().and_then(|s| s.trim().parse::().ok()); + let mut first = std::collections::BTreeMap::<(u32, u32), u32>::new(); + for e in std::fs::read_dir("/sys/devices/system/cpu").ok()?.flatten() { + let name = e.file_name().to_string_lossy().into_owned(); + let Some(cpu) = name.strip_prefix("cpu").and_then(|n| n.parse::().ok()) else { continue }; + let Some(package) = number(e.path().join("topology/physical_package_id")) else { continue }; + let die = if per_die { number(e.path().join("topology/die_id")).unwrap_or(0) } else { 0 }; + let lowest = first.entry((package, die)).or_insert(cpu); + *lowest = (*lowest).min(cpu); + } + if first.is_empty() { + return None; + } + let dies = |package: u32| first.keys().filter(|(p, _)| *p == package).count(); + first + .iter() + .map(|(&(package, die), &cpu)| { + let device = PathBuf::from(format!("/dev/cpu/{cpu}/msr")); + let shift = ((msr(&device, unit)? >> 8) & 0x1F) as u32; + let name = + if dies(package) > 1 { format!("package-{package}-die-{die} msr") } else { format!("package-{package} msr") }; + (shift != 0).then_some(Package { name, device, register, shift }) + }) + .collect() + } + + /// One package's count in µJ: its 32-bit count, floored at its unit. + fn uj(p: &Package, raw: u64) -> u64 { + ((u128::from(raw & 0xFFFF_FFFF) * 1_000_000) >> p.shift) as u64 + } + + /// Every package's count in µJ, in order; `None` if one fails. + pub fn read(ps: &[Package]) -> Option> { + ps.iter().map(|p| msr(&p.device, p.register).map(|raw| uj(p, raw))).collect() + } + + /// The packages' energy between two reads, in µJ, each wrapping at + /// 2^32 counts. + pub fn energy(ps: &[Package], before: &[u64], after: &[u64]) -> u64 { + ps.iter() + .zip(before.iter().zip(after)) + .map(|(p, (&b, &a))| { + let range = (((1u128 << 32) * 1_000_000) >> p.shift) as u64; + if a >= b { a - b } else { range - b + a } + }) + .sum() + } + + /// The packages, as one line for the log. + pub fn listing(ps: &[Package]) -> String { + ps.iter() + .map(|p| format!("{} {:#x} on {} at 2^-{} J", p.name, p.register, p.device.display(), p.shift)) + .collect::>() + .join("; ") + } + } } use oracle::Verdict; @@ -382,13 +508,23 @@ fn the_rapl_status_agrees_with_the_oracle_and_gates_the_rung() { let status = rapl_status(); let watch = oracle::watch(Path::new(oracle::POWERCAP)); let (meter, why) = Meter::detect_explained(); + let msrs = oracle::msr::packages(); eprintln!( - "rapl hardware: {}; oracle {}; status={status:?} selected={:?} provenance={:?} detection={why:?}", + "rapl hardware: {}; msr: {}; oracle {}; status={status:?} access={:?} selected={:?} provenance={:?} \ + detection={why:?}", oracle::listing(Path::new(oracle::POWERCAP)), + msrs.as_deref().map_or("unreadable".to_string(), oracle::msr::listing), watch.summary(), + jcp_energy::rapl_access(), meter.rung(), meter.provenance() ); + // Where powercap lists no package and the oracle reads every package's + // MSR, the crate can read them too. + let no_zone = !oracle::zones(Path::new(oracle::POWERCAP)).iter().any(oracle::is_package); + if no_zone && msrs.is_some() { + assert_eq!(jcp_energy::rapl_access(), jcp_energy::RaplAccess::Readable, "the oracle read the MSRs"); + } assert_eq!(status == CounterStatus::Live, watch.verdict == Verdict::Live, "the status disagrees with the oracle"); match watch.verdict { Verdict::Unreadable | Verdict::NoPackage => assert_eq!(status, CounterStatus::Unavailable), @@ -419,6 +555,33 @@ fn a_cpu_burn_is_a_whole_hwshunt_reading_inside_the_package_only_bracket() { return; } let all = oracle::zones(Path::new(oracle::POWERCAP)); + if !all.iter().any(oracle::is_package) { + // No package zone: the rung reads the packages' MSRs, and so does the + // oracle's bracket. + let msrs = oracle::msr::packages().expect("the RAPL rung was selected with no package zone, so the MSRs read"); + let read = || oracle::msr::read(&msrs).expect("an oracle read of the MSRs"); + let wall = Instant::now(); + let o0 = read(); + let ((o1, cpu, o2), e) = meter.measure(|| { + let o1 = read(); + let cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)); + (o1, cpu, read()) + }); + let o3 = read(); + let (inner, outer) = (oracle::msr::energy(&msrs, &o1, &o2), oracle::msr::energy(&msrs, &o0, &o3)); + assert_eq!((e.provenance, e.coverage), (Provenance::HwShunt, Coverage::Whole), "a live RAPL span: {e:?}"); + let uj = e.measured_uj.expect("an HwShunt figure carries its measured figure"); + let line = format!("meter {uj} µJ; oracle MSR package-only inner {inner} outer {outer} µJ"); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + eprintln!( + "rapl burn: {cpu:?} of thread CPU in {:?}; energy {e:?}; {}; {line}", + wall.elapsed(), + oracle::msr::listing(&msrs) + ); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); + return; + } let set = oracle::bracket_set(&all); let wall = Instant::now(); let o0 = oracle::read_all(&set);