diff --git a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs index 0109f9776..2c9e36edf 100644 --- a/eoc/eoc-rs/crates/eoc-meter/src/lib.rs +++ b/eoc/eoc-rs/crates/eoc-meter/src/lib.rs @@ -30,11 +30,27 @@ //! //! - Linux RAPL is read through powercap (`/sys/class/powercap`) as the sum //! of every package zone — each top-level `intel-rapl:` zone whose `name` -//! reads `package-`, so every socket of a multi-socket server, and never -//! `psys` (the platform zone, whose energy includes the packages'), `dram` -//! or a sub-zone — each zone's wrap taken at its own `max_energy_range_uj`; -//! **only when every package counter reads and is live** ([`rapl_status`]; -//! the probe burns CPU on the calling thread). No external crate required. +//! reads `package-`, or `package-

-die-` (one die of a multi-die +//! Intel package, which the kernel lists as a zone of its own), so every +//! socket and every die, and never `psys` (the platform zone, whose energy +//! includes the packages'), `dram` or a sub-zone — each zone's wrap taken at +//! its own `max_energy_range_uj`; **only when every package counter reads +//! and is live** ([`rapl_status`]; the probe burns CPU on the calling +//! thread). No external crate required. Where powercap lists no package (a +//! kernel without the `intel_rapl_msr` driver, as Ubuntu without +//! `linux-modules-extra` ships on some bare-metal servers), the same +//! counters are read directly: each energy-status MSR (0x611 on Intel +//! family 6, `0xC001029B` on AMD Zen and Hygon) through the `msr` driver's +//! device file of its lowest-numbered CPU, `/dev/cpu//msr`, which needs +//! the `msr` module and `CAP_SYS_RAWIO`. On Intel the counter is per die, +//! on AMD and Hygon per package: the kernel treats the package counter as +//! package-scope only on AMD and Hygon, and as die-scope on a multi-die +//! Intel package (`arch/x86/events/rapl.c`), so a two-die Intel package is +//! two counters, both summed. The 32-bit count is converted at the counter's +//! own energy-status unit (bits 12:8 of the power-unit MSR, 0x606 or +//! `0xC0010299`), so it wraps at 2^32 counts; a power-unit MSR that reads +//! zero is no counter (some hypervisors return zeros). The same liveness +//! rule decides. //! - Apple Silicon IOReport Energy Model counters are read through Apple's //! IOReport C API (no sudo, no external binary) — **only when every CPU //! energy rail is live**, each die's on a multi-die part @@ -3714,9 +3730,14 @@ pub fn liveness_sampler_opens() -> usize { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CounterStatus { /// The counter could not be opened or read — not this platform, the - /// feature not built, no such device, no RAPL zone named `package-` - /// (a host that lists only `psys`), a locked-down sysfs (`energy_uj` is - /// root-only on most current kernels) — or it gave fewer than two usable + /// feature not built, no such device, no RAPL zone named `package-` or + /// `package-

-die-` (a host that lists only `psys`), a locked-down + /// sysfs (`energy_uj` is + /// root-only on most current kernels), or, where powercap lists no + /// package, no energy-status MSR that reads (the `msr` module not loaded, + /// or no `CAP_SYS_RAWIO` to open `/dev/cpu//msr`, as for an + /// unprivileged user; a CPU whose MSRs this crate does not know; a + /// power-unit MSR that reads zero) — or it gave fewer than two usable /// samples in five budgets. Never selected. Unavailable, /// The counter read, but did not keep advancing while the probe ran: it @@ -3760,12 +3781,15 @@ const VENDOR_COUNTER_LIVENESS_BUDGET: std::time::Duration = std::time::Duration: /// /// [`detect`] selects the RAPL rung only when this is [`CounterStatus::Live`]: /// every package counter — each top-level powercap zone whose `name` reads -/// `package-`, so each socket of a multi-socket server, and never `psys`, -/// `dram` or a sub-zone, wherever it sits — read, and each kept advancing +/// `package-` or `package-

-die-`, so each socket of a multi-socket +/// server and each die of a multi-die package, and never `psys`, `dram` or a +/// sub-zone, wherever it sits — read, and each kept advancing /// while the probing thread burned CPU, in at least five sample intervals, at /// least 80% of them, over at least 300 ms. The counter's figure sums them /// all, so one that holds, or is republished in lumps (a virtualised powercap -/// tree can do either), declines the rung. Where there is no package zone, or +/// tree can do either), declines the rung. Where powercap lists no package +/// zone, the packages are each package's energy-status MSR instead (see the +/// crate docs), under the same rule. Where there is no package either way, or /// a package counter is unreadable, it is [`CounterStatus::Unavailable`] /// without burning anything. Always `Unavailable` off Linux. The first call on /// a host whose counters read burns about 300 ms of the calling thread's CPU @@ -3808,9 +3832,9 @@ pub fn nvml_status() -> CounterStatus { } } -// Compiled for tests on every platform, so its zone listing, sampler, counter -// and gate run under test on a Mac too, over fake powercap directories; -// selected only on Linux. +// Compiled for tests on every platform, so its zone listing, MSR fallback, +// sampler, counter and gate run under test on a Mac too, over fake powercap +// directories and fake MSR device files; selected only on Linux. #[cfg(any(target_os = "linux", test))] #[cfg_attr(not(target_os = "linux"), allow(dead_code))] mod linux_rapl { @@ -3823,23 +3847,88 @@ mod linux_rapl { /// Where the kernel lists its powercap zones, RAPL's among them. pub(crate) const POWERCAP_ROOT: &str = "/sys/class/powercap"; - /// One RAPL package zone: a top-level `intel-rapl:` zone whose - /// `name` reads `package-`. + /// One RAPL package: a top-level `intel-rapl:` powercap zone whose + /// `name` reads `package-`, or, on a kernel without the powercap + /// driver, one energy-status MSR domain ([`Msr`]): a die on Intel, a + /// package on AMD and Hygon. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct Package { - /// `` in `intel-rapl:`. + /// `` in `intel-rapl:`; for an MSR domain, its + /// package's `physical_package_id` (its die is [`Msr::die`]). pub(crate) index: u32, - /// The zone's `max_energy_range_uj`, the value its `energy_uj` wraps - /// at, when that reads as a number above zero. + /// The value the package's microjoule figure wraps at: the zone's + /// `max_energy_range_uj` when that reads as a number above zero; for + /// an MSR package, 2^32 counts at its unit ([`Msr::range_uj`]). pub(crate) max_energy_range_uj: Option, + /// How to read an MSR package; `None` for a powercap zone. + pub(crate) msr: Option, } - /// Whether a top-level zone's `name` file names a package: `package-`, - /// `` one or more digits (the file ends in a newline). Not `psys` (the - /// platform zone, whose energy includes the packages'), `dram`, `core` - /// or `uncore`. + /// An energy-status MSR domain read directly, where the kernel lists no + /// powercap zone: on Intel one die of a package, on AMD and Hygon one + /// package ([`MsrScope`]). + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) struct Msr { + /// The domain's lowest-numbered CPU, whose `msr` device file it is + /// read through. + pub(crate) cpu: u32, + /// The domain's `die_id` within its package, where its package has + /// more than one die-scope domain (Intel, `package-

-die-` as the + /// kernel names such a zone); `None` where the domain is the whole + /// package. + pub(crate) die: Option, + /// The energy-status MSR: [`MSR_PKG_ENERGY_STATUS`] or + /// [`MSR_AMD_PKG_ENERGY_STATUS`]. + pub(crate) register: u64, + /// The energy-status unit, 2^-`unit_shift` J per count: bits 12:8 of + /// the power-unit MSR. Never zero. + pub(crate) unit_shift: u32, + } + + /// Intel's power-unit MSR (family 6). + pub(crate) const MSR_RAPL_POWER_UNIT: u64 = 0x606; + /// Intel's package energy-status MSR (family 6). + pub(crate) const MSR_PKG_ENERGY_STATUS: u64 = 0x611; + /// AMD Zen's (and Hygon's) power-unit MSR. + pub(crate) const MSR_AMD_RAPL_POWER_UNIT: u64 = 0xC001_0299; + /// AMD Zen's (and Hygon's) package energy-status MSR. + pub(crate) const MSR_AMD_PKG_ENERGY_STATUS: u64 = 0xC001_029B; + + impl Msr { + /// The package's energy, in microjoules, at a raw read of its + /// energy-status MSR: the low 32 bits are the count, rounded down. + pub(crate) fn uj(self, raw: u64) -> u64 { + ((u128::from(raw & 0xFFFF_FFFF) * 1_000_000) >> self.unit_shift) as u64 + } + + /// The microjoule figure's wrap: 2^32 counts. + pub(crate) fn range_uj(self) -> u64 { + (((1u128 << 32) * 1_000_000) >> self.unit_shift) as u64 + } + + /// The `msr` device file of the package's first CPU, under the + /// filesystem root `root`. + fn device(self, root: &Path) -> PathBuf { + root.join(format!("dev/cpu/{}/msr", self.cpu)) + } + } + + /// Whether a top-level zone's `name` file names a package counter: + /// `package-

`, or `package-

-die-` (one die of a multi-die Intel + /// package: the kernel's powercap driver lists one zone per die, named so, + /// where the RAPL MSRs are die-scope and a package has more than one die, + /// `intel_rapl_common.c`), each number one or more digits (the file ends + /// in a newline). Not `psys` (the platform zone, whose energy includes the + /// packages'), `dram`, `core`, `uncore`, or anything looser. pub(crate) fn is_package_name(name: &str) -> bool { - name.trim().strip_prefix("package-").is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit())) + let digits = |s: &str| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()); + let Some(rest) = name.trim().strip_prefix("package-") else { + return false; + }; + match rest.split_once("-die-") { + None => digits(rest), + Some((package, die)) => digits(package) && digits(die), + } } /// The index of a top-level RAPL zone directory, `intel-rapl:`. @@ -3884,6 +3973,7 @@ mod linux_rapl { is_package_name(&name).then(|| Package { index, max_energy_range_uj: read_u64(&dir.join("max_energy_range_uj")).filter(|&m| m > 0), + msr: None, }) }) .collect(); @@ -3891,14 +3981,163 @@ mod linux_rapl { packages } - /// Every package's `energy_uj` under `root`, in `packages` order. `None` - /// when there is no package, or when any one cannot be read: a figure that - /// left a package out would miss its energy while claiming the whole CPU. + /// Every package's energy in microjoules under `root`, in `packages` + /// order: a zone's `energy_uj` under the powercap directory `root`, or an + /// MSR package's count at its unit, read through its device file under the + /// filesystem root `root`. `None` when there is no package, or when any + /// one cannot be read: a figure that left a package out would miss its + /// energy while claiming the whole CPU. pub(crate) fn read_packages(root: &Path, packages: &[Package]) -> Option> { if packages.is_empty() { return None; } - packages.iter().map(|p| read_u64(&zone_dir(root, p.index).join("energy_uj"))).collect() + packages.iter().map(|p| read_package(root, p)).collect() + } + + /// One package's energy in microjoules under `root`, or `None` when it + /// cannot be read. + fn read_package(root: &Path, p: &Package) -> Option { + match p.msr { + None => read_u64(&zone_dir(root, p.index).join("energy_uj")), + Some(msr) => read_msr(&msr.device(root), msr.register).map(|raw| msr.uj(raw)), + } + } + + /// The 64-bit MSR `register` read through the `msr` driver's device file + /// `device`, where the register number is the file offset. `None` when the + /// file cannot be opened (no `msr` module, no `CAP_SYS_RAWIO`, no read + /// access) or the register cannot be read (the CPU does not implement it). + #[cfg(unix)] + fn read_msr(device: &Path, register: u64) -> Option { + use std::os::unix::fs::FileExt; + let file = std::fs::File::open(device).ok()?; + let mut raw = [0u8; 8]; + file.read_exact_at(&mut raw, register).ok()?; + Some(u64::from_le_bytes(raw)) + } + + #[cfg(not(unix))] + fn read_msr(_device: &Path, _register: u64) -> Option { + None + } + + /// The CPU's vendor and family from `/proc/cpuinfo` text: the first + /// CPU's `vendor_id` and `cpu family`. + pub(crate) fn cpu_vendor_family(cpuinfo: &str) -> Option<(&str, u32)> { + let field = |key: &str| { + cpuinfo.lines().find_map(|line| { + let (k, v) = line.split_once(':')?; + (k.trim() == key).then(|| v.trim()) + }) + }; + Some((field("vendor_id")?, field("cpu family")?.parse().ok()?)) + } + + /// What one energy-status MSR counts. The kernel's own RAPL drivers set + /// the rule: its perf PMU treats the package counter as package-scope only + /// on AMD and Hygon ("AMD/HYGON platforms have a per-PKG package energy + /// counter", `rapl_pkg_pmu_is_pkg_scope()` in `arch/x86/events/rapl.c`), + /// and on a multi-die Intel package (Cascade Lake-AP) "its RAPL MSRs are + /// die-scope", which its powercap driver lists as one zone per die, + /// `package-

-die-` (`intel_rapl_common.c`). + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(crate) enum MsrScope { + /// One counter per die of a package (Intel): a package of two dies + /// is two counters, each read on one of its own CPUs. + Die, + /// One counter per package (AMD Zen, Hygon), whatever its dies. + Package, + } + + /// The power-unit and energy-status MSRs for a CPU's vendor and family, + /// and what one counter covers; `None` where this crate knows none: Intel + /// family 6 (per die), AMD Zen (family 0x17 and later) and Hygon (per + /// package). + pub(crate) fn msr_registers(vendor: &str, family: u32) -> Option<(u64, u64, MsrScope)> { + match vendor { + "GenuineIntel" if family == 6 => Some((MSR_RAPL_POWER_UNIT, MSR_PKG_ENERGY_STATUS, MsrScope::Die)), + "AuthenticAMD" | "HygonGenuine" if family >= 0x17 => { + Some((MSR_AMD_RAPL_POWER_UNIT, MSR_AMD_PKG_ENERGY_STATUS, MsrScope::Package)) + } + _ => None, + } + } + + /// Each counter domain's lowest-numbered CPU, from + /// `/sys/devices/system/cpu/cpu/topology`, keyed by + /// (`physical_package_id`, die): the CPU's `die_id` where `scope` is + /// [`MsrScope::Die`] (0 where that file is absent, as on kernels before + /// it existed), and 0 for every CPU where it is [`MsrScope::Package`]. + fn first_cpu_per_domain(root: &Path, scope: MsrScope) -> std::collections::BTreeMap<(u32, u32), u32> { + let mut first = std::collections::BTreeMap::new(); + let Ok(entries) = std::fs::read_dir(root.join("sys/devices/system/cpu")) else { + return first; + }; + for entry in entries.flatten() { + let Some(cpu) = entry.file_name().to_str().and_then(|n| n.strip_prefix("cpu")?.parse::().ok()) else { + continue; + }; + let topology = entry.path().join("topology"); + let Some(package) = read_u64(&topology.join("physical_package_id")).and_then(|p| u32::try_from(p).ok()) else { + continue; + }; + let die = match scope { + MsrScope::Die => read_u64(&topology.join("die_id")).and_then(|d| u32::try_from(d).ok()).unwrap_or(0), + MsrScope::Package => 0, + }; + let lowest = first.entry((package, die)).or_insert(cpu); + *lowest = (*lowest).min(cpu); + } + first + } + + /// Every energy-status MSR domain under the filesystem root `root`, in + /// (package, die) order: on Intel each die of each package, on AMD and + /// Hygon each package ([`MsrScope`]), each read through the device file of + /// its lowest-numbered CPU (`/dev/cpu//msr`) at the unit its + /// power-unit MSR reads, and summed. Empty when the CPU is not one whose + /// MSRs this crate knows, when no CPU topology is listed, or when any + /// domain's power-unit MSR cannot be read or reads a zero unit — a figure + /// that left a die or a package out would miss its energy, and a zero unit + /// is a hypervisor's stand-in, not a counter. + pub(crate) fn msr_packages_under(root: &Path) -> Vec { + let Ok(cpuinfo) = std::fs::read_to_string(root.join("proc/cpuinfo")) else { + return Vec::new(); + }; + let Some((unit_register, register, scope)) = cpu_vendor_family(&cpuinfo).and_then(|(v, f)| msr_registers(v, f)) + else { + return Vec::new(); + }; + let domains = first_cpu_per_domain(root, scope); + let dies_of = |package: u32| domains.keys().filter(|(p, _)| *p == package).count(); + let mut packages = Vec::new(); + for (&(index, die), &cpu) in &domains { + let die = (dies_of(index) > 1).then_some(die); + let device = Msr { cpu, die, register, unit_shift: 0 }.device(root); + let Some(unit) = read_msr(&device, unit_register) else { + return Vec::new(); + }; + let unit_shift = ((unit >> 8) & 0x1F) as u32; + if unit_shift == 0 { + return Vec::new(); + } + let msr = Msr { cpu, die, register, unit_shift }; + packages.push(Package { index, max_energy_range_uj: Some(msr.range_uj()), msr: Some(msr) }); + } + packages + } + + /// This host's packages and the directory they are read under: the + /// powercap package zones under `powercap`, or, where it lists none, every + /// package's energy-status MSR under the filesystem root `fs_root`. With + /// no package either way, none, under `powercap`. + pub(crate) fn packages_from(powercap: &Path, fs_root: &Path) -> (PathBuf, Vec) { + let zones = packages_under(powercap); + if !zones.is_empty() { + return (powercap.to_path_buf(), zones); + } + let msrs = msr_packages_under(fs_root); + if msrs.is_empty() { (powercap.to_path_buf(), Vec::new()) } else { (fs_root.to_path_buf(), msrs) } } /// One zone's energy between two reads, modulo its own wrap. A counter @@ -3976,9 +4215,9 @@ mod linux_rapl { for ((p, &e0), &e1) in packages.iter().zip(last).zip(now) { let delta = zone_delta_uj(e0, e1, p.max_energy_range_uj).ok_or_else(|| { format!( - "RAPL zone intel-rapl:{} went down from {e0} to {e1} uJ with no readable \ + "RAPL {} went down from {e0} to {e1} uJ with no readable \ max_energy_range_uj; its energy between the two reads is unknown", - p.index + rail_name(p) ) })?; energy = energy.checked_add(delta).ok_or_else(|| "RAPL package energy overflows u64".to_string())?; @@ -3988,7 +4227,9 @@ mod linux_rapl { /// Reads RAPL's package energy through powercap: the sum over every /// package zone (`intel-rapl:` named `package-`), each zone's wrap - /// taken at its own `max_energy_range_uj`, as a running total. + /// taken at its own `max_energy_range_uj`, as a running total. Where + /// powercap lists no package, over every package's energy-status MSR + /// instead (see the crate docs), each wrapping at 2^32 counts at its unit. /// /// Constructed only on a host whose package counters the liveness probe /// found live, every one of them ([`crate::rapl_status`]); a live RAPL @@ -4011,22 +4252,25 @@ mod linux_rapl { /// The constructor behind [`Self::try_detect`], given this host's /// status: every status the gate does not admit declines. The counter - /// sums the package zones the host's probe watched. + /// sums the packages the host's probe watched, read where it read them. pub(crate) fn try_detect_given(status: CounterStatus) -> Result { - Self::admitted(status, Path::new(POWERCAP_ROOT), host_packages) + Self::admitted(status, host_packages) } - /// The counter over `packages` under the powercap directory `root`, - /// when `status` admits it; `packages` is asked for only then. - pub(crate) fn admitted(status: CounterStatus, root: &Path, packages: impl FnOnce() -> Vec) -> Result { + /// The counter over the packages `packages` gives, read under the + /// directory it gives with them, when `status` admits it; `packages` + /// is asked for only then. + pub(crate) fn admitted(status: CounterStatus, packages: impl FnOnce() -> (PathBuf, Vec)) -> Result { if rail_liveness::admits_vendor_counter(status) { - Ok(Self::over(root, packages())) + let (root, packages) = packages(); + Ok(Self::over(&root, packages)) } else { Err(Error::NoCounter) } } - /// The counter over `packages` under `root`, ungated. + /// The counter over `packages` under `root` (the powercap directory + /// for zones, the filesystem root for MSR packages), ungated. pub(crate) fn over(root: &Path, packages: Vec) -> Self { Self { root: root.to_path_buf(), packages, running: Mutex::new(Running::Unread) } } @@ -4044,7 +4288,12 @@ mod linux_rapl { } let mut running = self.running.lock().unwrap_or_else(std::sync::PoisonError::into_inner); let now = read_packages(&self.root, &self.packages).ok_or_else(|| { - Error::Io(format!("a RAPL package zone's energy_uj under {} could not be read", self.root.display())) + let what = if self.packages.iter().any(|p| p.msr.is_some()) { + "package's energy-status MSR (dev/cpu//msr)" + } else { + "package zone's energy_uj" + }; + Error::Io(format!("a RAPL {what} under {} could not be read", self.root.display())) })?; running.advance(&self.packages, &now) } @@ -4053,10 +4302,16 @@ mod linux_rapl { } } - /// The name a package's counter has in a liveness sample: its zone, - /// `intel-rapl:`. - pub(crate) fn rail_name(index: u32) -> String { - format!("intel-rapl:{index}") + /// The name a package's counter has in a liveness sample and in an error: + /// its zone, `intel-rapl:`; for an MSR domain, + /// `package- (MSR)`, or `package--die- (MSR)` for one + /// die of a multi-die package, so two dies of one package differ. + pub(crate) fn rail_name(p: &Package) -> String { + match p.msr { + None => format!("intel-rapl:{}", p.index), + Some(Msr { die: None, .. }) => format!("package-{} (MSR)", p.index), + Some(Msr { die: Some(die), .. }) => format!("package-{}-die-{die} (MSR)", p.index), + } } /// One liveness sample of the package counters `read` reads, in @@ -4069,7 +4324,7 @@ mod linux_rapl { if raw.len() != packages.len() { return None; } - Some(packages.iter().zip(raw).map(|(p, uj)| (rail_name(p.index), i64::try_from(uj).unwrap_or(i64::MAX))).collect()) + Some(packages.iter().zip(raw).map(|(p, uj)| (rail_name(p), i64::try_from(uj).unwrap_or(i64::MAX))).collect()) } /// The RAPL probe of `packages` on a given reader. With no package it is @@ -4098,44 +4353,52 @@ mod linux_rapl { ) } - /// What one probe found under a powercap directory: its package zones, + /// What one probe found: the packages (powercap zones, or energy-status + /// MSRs where powercap lists none), the directory they are read under, /// and whether every one of their counters is live. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct Host { + pub(crate) root: PathBuf, pub(crate) packages: Vec, pub(crate) status: CounterStatus, } - /// One probe of the package zones under `root`: list them - /// ([`packages_under`]), then probe all of their counters together - /// ([`status_with`]). - pub(crate) fn probe_root(root: &Path, tick: impl FnMut(), now: impl Fn() -> Duration) -> Host { - let packages = packages_under(root); - let status = status_with(&packages, || read_packages(root, &packages), tick, now); - Host { packages, status } + /// One probe of this host's packages: list them ([`packages_from`]: the + /// zones under `powercap`, or the MSRs under the filesystem root + /// `fs_root` where it lists none), then probe all of their counters + /// together ([`status_with`]). + pub(crate) fn probe_host(powercap: &Path, fs_root: &Path, tick: impl FnMut(), now: impl Fn() -> Duration) -> Host { + let (root, packages) = packages_from(powercap, fs_root); + let status = status_with(&packages, || read_packages(&root, &packages), tick, now); + Host { root, packages, status } } - /// This host's package zones and their liveness, probed once per process - /// and cached: the real powercap directory, the CPU-burning tick (a - /// package counter must advance while this thread burns CPU) and the - /// monotonic clock. Behind [`crate::rapl_status`] and the counter's - /// constructor, so the counter sums exactly the packages the probe found - /// live. + /// This host's packages and their liveness, probed once per process and + /// cached: the real powercap directory (or, where it lists no package, + /// the MSR device files under `/`), the CPU-burning tick (a package + /// counter must advance while this thread burns CPU) and the monotonic + /// clock. Behind [`crate::rapl_status`] and the counter's constructor, so + /// the counter sums exactly the packages the probe found live, read where + /// it read them. #[cfg(target_os = "linux")] pub(crate) fn host() -> &'static Host { static HOST: std::sync::OnceLock = std::sync::OnceLock::new(); - HOST.get_or_init(|| probe_root(Path::new(POWERCAP_ROOT), crate::clocks::cpu_burn_tick, crate::clocks::wall_now)) + HOST.get_or_init(|| { + probe_host(Path::new(POWERCAP_ROOT), Path::new("/"), crate::clocks::cpu_burn_tick, crate::clocks::wall_now) + }) } - /// The package zones the host's probe watched; none off Linux. - fn host_packages() -> Vec { + /// The packages the host's probe watched, and where they are read; none + /// off Linux. + fn host_packages() -> (PathBuf, Vec) { #[cfg(target_os = "linux")] { - host().packages.clone() + let host = host(); + (host.root.clone(), host.packages.clone()) } #[cfg(not(target_os = "linux"))] { - Vec::new() + (PathBuf::from(POWERCAP_ROOT), Vec::new()) } } @@ -4211,7 +4474,17 @@ mod linux_rapl { } fn pkg(index: u32, max: u64) -> Package { - Package { index, max_energy_range_uj: Some(max) } + Package { index, max_energy_range_uj: Some(max), msr: None } + } + + /// A filesystem root with no CPU topology and no MSR device file, so + /// the MSR fallback finds no package there. + const NO_MSR_ROOT: &str = "/no/such/root"; + + /// The host's probe over the powercap directory `root` alone: no MSR + /// package is ever found under [`NO_MSR_ROOT`]. + fn probe_root(root: &Path, tick: impl FnMut(), now: impl Fn() -> Duration) -> Host { + probe_host(root, Path::new(NO_MSR_ROOT), tick, now) } /// The probe over a fake directory, run as the host's probe runs @@ -4236,10 +4509,28 @@ mod linux_rapl { /// sub-zone, the control type and the MMIO interface are not. #[test] fn only_a_top_level_zone_named_package_n_is_a_package() { - for name in ["package-0", "package-1\n", "package-12"] { + for name in ["package-0", "package-1\n", "package-12", "package-0-die-0", "package-0-die-1\n", "package-3-die-12"] { assert!(is_package_name(name), "{name:?}"); } - for name in ["psys", "psys\n", "dram", "core", "uncore", "package-", "package-x", "package-0a", "Package-0", ""] { + for name in [ + "psys", + "psys\n", + "dram", + "core", + "uncore", + "package-", + "package-x", + "package-0a", + "Package-0", + "", + "package-0-die-", + "package--die-0", + "package-0-die-x", + "package-0-die-1-die-2", + "package-0-core-0", + "package-0-", + "package-0-die", + ] { assert!(!is_package_name(name), "{name:?}"); } assert_eq!(top_level_zone("intel-rapl:0"), Some(0)); @@ -4249,6 +4540,33 @@ mod linux_rapl { } } + /// A multi-die Intel package as the kernel lists it: one top-level zone + /// per die, `package-0-die-0` and `package-0-die-1`, each with its own + /// sub-zones, beside `psys`. Both dies are packages, both are summed + /// and both are watched; nothing else is. + #[test] + fn a_multi_die_package_sums_every_die_zone() { + let pc = Powercap::new("two-die"); + pc.zone("intel-rapl:0", "package-0-die-0", 1_000_000, Some(RANGE)) + .zone("intel-rapl:0:0", "dram", 90_000, Some(RANGE_B)) + .zone("intel-rapl:1", "package-0-die-1", 3_000_000, Some(RANGE_B)) + .zone("intel-rapl:1:0", "dram", 70_000, Some(RANGE_B)) + .zone("intel-rapl:2", "psys", 80_000_000, Some(RANGE)); + let packages = packages_under(pc.root()); + assert_eq!(packages, vec![pkg(0, RANGE), pkg(1, RANGE_B)], "both dies are listed"); + let counter = LinuxRaplCounter::over(pc.root(), packages.clone()); + let before = counter.read_microjoules().expect("both dies read"); + assert_eq!(before, 1_000_000 + 3_000_000); + for (dir, uj) in [("intel-rapl:0", 1_200_000), ("intel-rapl:1", 3_500_000), ("intel-rapl:0:0", 99_000), ("intel-rapl:2", 90_000_000)] { + pc.set(dir, uj); + } + assert_eq!(counter.read_microjoules().expect("both dies read") - before, 200_000 + 500_000, "each die's energy, nothing else"); + let held = probe_tree(&pc, |t| vec![("intel-rapl:0", 1_200_000 + 40_000 * t)]); + assert_eq!(held.status, CounterStatus::Frozen, "die 1 held its value: the sum would leave it out"); + let live = probe_tree(&pc, |t| vec![("intel-rapl:0", 50_000_000 + 40_000 * t), ("intel-rapl:1", 60_000_000 + 30_000 * t)]); + assert_eq!(live, Host { root: pc.root().to_path_buf(), packages, status: CounterStatus::Live }); + } + /// A two-socket server: the figure is both packages' energy and /// nothing else — not `psys` (which includes the packages, so it would /// count them twice), not a `dram` zone at the top level or under a @@ -4308,7 +4626,7 @@ mod linux_rapl { let pc = Powercap::new("psys-moving"); pc.zone("intel-rapl:0", "psys", 70_000_000, Some(RANGE)).zone("intel-rapl:1", "package-0", 2_000_000, Some(RANGE)); let host = probe_tree(&pc, |t| vec![("intel-rapl:0", 70_000_000 + 99_000 * t)]); - assert_eq!(host, Host { packages: vec![pkg(1, RANGE)], status: CounterStatus::Frozen }); + assert_eq!(host, Host { root: pc.root().to_path_buf(), packages: vec![pkg(1, RANGE)], status: CounterStatus::Frozen }); } /// Every package must keep advancing, because the figure sums them @@ -4318,7 +4636,10 @@ mod linux_rapl { fn every_package_counter_must_be_live() { let both = two_socket_server("live-both"); let host = probe_tree(&both, |t| vec![("intel-rapl:0", 1_000_000 + 40_000 * t), ("intel-rapl:1", 5_000_000 + 35_000 * t)]); - assert_eq!(host, Host { packages: vec![pkg(0, RANGE), pkg(1, RANGE_B)], status: CounterStatus::Live }); + assert_eq!( + host, + Host { root: both.root().to_path_buf(), packages: vec![pkg(0, RANGE), pkg(1, RANGE_B)], status: CounterStatus::Live } + ); let one = two_socket_server("live-one"); let host = probe_tree(&one, |t| vec![("intel-rapl:0", 1_000_000 + 40_000 * t), ("intel-rapl:2", 80_000_000 + 99_000 * t)]); assert_eq!(host.status, CounterStatus::Frozen, "package 1 held its value"); @@ -4358,7 +4679,10 @@ mod linux_rapl { let pc = Powercap::new("no-range"); pc.zone("intel-rapl:0", "package-0", 10_000, None).zone("intel-rapl:1", "package-1", 20_000, Some(0)); let packages = packages_under(pc.root()); - assert_eq!(packages, vec![Package { index: 0, max_energy_range_uj: None }, Package { index: 1, max_energy_range_uj: None }]); + assert_eq!( + packages, + vec![Package { index: 0, max_energy_range_uj: None, msr: None }, Package { index: 1, max_energy_range_uj: None, msr: None }] + ); let counter = LinuxRaplCounter::over(pc.root(), packages); let r0 = counter.read_microjoules().expect("read"); pc.set("intel-rapl:0", 15_000); @@ -4394,12 +4718,12 @@ mod linux_rapl { pc.zone("intel-rapl:0", "psys", 70_000_000, Some(RANGE)); let ticks = Cell::new(0u32); let host = probe_root(pc.root(), || ticks.set(ticks.get() + 1), || Duration::from_millis(25) * ticks.get()); - assert_eq!(host, Host { packages: Vec::new(), status: CounterStatus::Unavailable }); + assert_eq!(host, Host { root: pc.root().to_path_buf(), packages: Vec::new(), status: CounterStatus::Unavailable }); assert_eq!(ticks.get(), 0, "burned CPU probing a host with no package"); let asked = Cell::new(false); - let counter = LinuxRaplCounter::admitted(host.status, pc.root(), || { + let counter = LinuxRaplCounter::admitted(host.status, || { asked.set(true); - host.packages.clone() + (host.root.clone(), host.packages.clone()) }); assert!(matches!(counter, Err(Error::NoCounter))); assert!(!asked.get(), "listed the packages of a declined counter"); @@ -4407,7 +4731,10 @@ mod linux_rapl { let empty = Powercap::new("empty"); assert_eq!(probe_root(empty.root(), || panic!("ticked"), || Duration::ZERO).status, CounterStatus::Unavailable); let missing = Path::new("/no/such/powercap"); - assert_eq!(probe_root(missing, || panic!("ticked"), || Duration::ZERO), Host { packages: Vec::new(), status: CounterStatus::Unavailable }); + assert_eq!( + probe_root(missing, || panic!("ticked"), || Duration::ZERO), + Host { root: missing.to_path_buf(), packages: Vec::new(), status: CounterStatus::Unavailable } + ); } /// The RAPL probe on scripted reads of `packages`, with a clock that @@ -4469,6 +4796,276 @@ mod linux_rapl { ); } } + + /// The MSR path, over fake device files: a regular file whose bytes at + /// a register's offset are that register is read exactly as the `msr` + /// driver's device file is. + #[cfg(unix)] + mod msr { + use super::*; + use std::os::unix::fs::FileExt; + + const INTEL: &str = "processor\t: 0\nvendor_id\t: GenuineIntel\ncpu family\t: 6\nmodel\t\t: 85\n"; + const AMD: &str = "processor\t: 0\nvendor_id\t: AuthenticAMD\ncpu family\t: 26\nmodel\t\t: 112\n"; + /// A power-unit MSR whose energy-status unit (bits 12:8) is + /// 2^-14 J, the common Intel value (0xA0E03). + const UNIT_14: u64 = 0xA_0E03; + /// 2^-16 J, as on AMD Zen. + const UNIT_16: u64 = 0x10 << 8; + + /// A fake filesystem root for one test, removed when dropped: its + /// `/proc/cpuinfo`, each CPU's `physical_package_id`, and the CPUs' + /// `dev/cpu//msr` files. + struct MsrRoot(PathBuf); + + impl MsrRoot { + /// `package_of_cpu[n]` is CPU `n`'s package. + fn new(tag: &str, cpuinfo: &str, package_of_cpu: &[u32]) -> Self { + let topology: Vec<(u32, Option)> = package_of_cpu.iter().map(|&p| (p, None)).collect(); + Self::with_dies(tag, cpuinfo, &topology) + } + + /// `topology[n]` is CPU `n`'s package and, when given, its + /// `die_id` (no `die_id` file otherwise, as on kernels before + /// it existed). + fn with_dies(tag: &str, cpuinfo: &str, topology: &[(u32, Option)]) -> Self { + let root = std::env::temp_dir().join(format!("eoc-meter-msr-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("proc")).expect("create proc"); + std::fs::write(root.join("proc/cpuinfo"), cpuinfo).expect("write cpuinfo"); + for (cpu, (package, die)) in topology.iter().enumerate() { + let dir = root.join(format!("sys/devices/system/cpu/cpu{cpu}/topology")); + std::fs::create_dir_all(&dir).expect("create a CPU's topology"); + std::fs::write(dir.join("physical_package_id"), format!("{package}\n")).expect("write its package"); + if let Some(die) = die { + std::fs::write(dir.join("die_id"), format!("{die}\n")).expect("write its die"); + } + } + Self(root) + } + + /// Set CPU `cpu`'s MSR `register` to `value`. + fn set(&self, cpu: u32, register: u64, value: u64) -> &Self { + let dir = self.0.join(format!("dev/cpu/{cpu}")); + std::fs::create_dir_all(&dir).expect("create dev/cpu/"); + let file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(dir.join("msr")) + .expect("open the fake msr file"); + file.write_all_at(&value.to_le_bytes(), register).expect("write the register"); + self + } + + fn root(&self) -> &Path { + &self.0 + } + } + + impl Drop for MsrRoot { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A two-socket Intel host with no powercap driver, CPUs 0 and 1 on + /// package 0 and CPUs 2 and 3 on package 1: each package read on + /// its first CPU, at its unit, and summed. + #[test] + fn a_two_socket_host_without_powercap_reads_each_packages_msr() { + let host = MsrRoot::new("two-socket", INTEL, &[0, 0, 1, 1]); + host.set(0, MSR_RAPL_POWER_UNIT, UNIT_14).set(0, MSR_PKG_ENERGY_STATUS, 1_000); + host.set(2, MSR_RAPL_POWER_UNIT, UNIT_14).set(2, MSR_PKG_ENERGY_STATUS, 2_000); + let packages = msr_packages_under(host.root()); + let cpus: Vec<_> = packages.iter().map(|p| (p.index, p.msr.map(|m| m.cpu))).collect(); + assert_eq!(cpus, vec![(0, Some(0)), (1, Some(2))], "package 1 is read on its first CPU, 2"); + assert!(packages.iter().all(|p| p.msr.is_some_and(|m| (m.register, m.unit_shift) == (MSR_PKG_ENERGY_STATUS, 14)))); + assert!(packages.iter().all(|p| p.max_energy_range_uj == Some(262_144_000_000)), "2^32 counts of 2^-14 J"); + assert_eq!( + sample(&packages, || read_packages(host.root(), &packages)).map(|s| s.into_iter().map(|(n, _)| n).collect::>()), + Some(vec!["package-0 (MSR)".to_string(), "package-1 (MSR)".to_string()]), + "each package is its own rail" + ); + + let counter = LinuxRaplCounter::over(host.root(), packages); + // 1,000 and 2,000 counts of 61.035 µJ, each rounded down. + let r0 = counter.read_microjoules().expect("both packages read"); + assert_eq!(r0, 61_035 + 122_070); + host.set(0, MSR_PKG_ENERGY_STATUS, 3_000).set(2, MSR_PKG_ENERGY_STATUS, 2_100); + assert_eq!(counter.read_microjoules().expect("both packages read"), 183_105 + 128_173); + } + + /// The 32-bit count wraps at 2^32 counts, so the microjoule figure + /// wraps at that range, losing at most a microjoule; bits above the + /// count are not energy. + #[test] + fn the_count_wraps_at_2_to_the_32_counts() { + let host = MsrRoot::new("wrap", INTEL, &[0]); + host.set(0, MSR_RAPL_POWER_UNIT, UNIT_14).set(0, MSR_PKG_ENERGY_STATUS, 0xFFFF_FFF0); + let counter = LinuxRaplCounter::over(host.root(), msr_packages_under(host.root())); + let before = counter.read_microjoules().expect("read"); + host.set(0, MSR_PKG_ENERGY_STATUS, 0xABCD_0000_0000_0010); + // 32 counts of 61.035 µJ is 1,953.125 µJ. + assert_eq!(counter.read_microjoules().expect("read across the wrap") - before, 1_953); + } + + /// AMD Zen reads its own power-unit and energy-status MSRs. + #[test] + fn amd_zen_reads_its_own_msrs() { + let host = MsrRoot::new("amd", AMD, &[0, 0]); + host.set(0, MSR_AMD_RAPL_POWER_UNIT, UNIT_16).set(0, MSR_AMD_PKG_ENERGY_STATUS, 65_536); + let packages = msr_packages_under(host.root()); + assert_eq!(packages.len(), 1); + assert_eq!(packages[0].msr, Some(Msr { cpu: 0, die: None, register: MSR_AMD_PKG_ENERGY_STATUS, unit_shift: 16 })); + let counter = LinuxRaplCounter::over(host.root(), packages); + assert_eq!(counter.read_microjoules().expect("read"), 1_000_000, "65,536 counts of 2^-16 J is one joule"); + assert_eq!( + msr_registers("HygonGenuine", 0x18), + Some((MSR_AMD_RAPL_POWER_UNIT, MSR_AMD_PKG_ENERGY_STATUS, MsrScope::Package)) + ); + assert_eq!(msr_registers("GenuineIntel", 6), Some((MSR_RAPL_POWER_UNIT, MSR_PKG_ENERGY_STATUS, MsrScope::Die))); + assert_eq!(msr_registers("AuthenticAMD", 0x16), None, "pre-Zen AMD has no such MSR"); + } + + /// A multi-die Intel package (CPUs 0–1 on die 0, 2–3 on die 1, all + /// on package 0): Intel's RAPL MSRs are die-scope there, so it is + /// two counters, each read on its own die's first CPU, both summed, + /// with rails that tell the two dies apart. + #[test] + fn an_intel_package_of_two_dies_is_two_counters() { + let host = MsrRoot::with_dies("two-dies", INTEL, &[(0, Some(0)), (0, Some(0)), (0, Some(1)), (0, Some(1))]); + host.set(0, MSR_RAPL_POWER_UNIT, UNIT_14).set(0, MSR_PKG_ENERGY_STATUS, 1_000); + host.set(2, MSR_RAPL_POWER_UNIT, UNIT_14).set(2, MSR_PKG_ENERGY_STATUS, 2_000); + let packages = msr_packages_under(host.root()); + let domains: Vec<_> = packages.iter().map(|p| (p.index, p.msr.map(|m| (m.die, m.cpu)))).collect(); + assert_eq!(domains, vec![(0, Some((Some(0), 0))), (0, Some((Some(1), 2)))], "die 1 is read on its first CPU, 2"); + assert_eq!( + sample(&packages, || read_packages(host.root(), &packages)), + Some(vec![("package-0-die-0 (MSR)".to_string(), 61_035), ("package-0-die-1 (MSR)".to_string(), 122_070)]), + "each die is its own rail" + ); + let counter = LinuxRaplCounter::over(host.root(), packages.clone()); + let r0 = counter.read_microjoules().expect("both dies read"); + assert_eq!(r0, 61_035 + 122_070, "both dies summed"); + host.set(2, MSR_PKG_ENERGY_STATUS, 3_000); + assert_eq!(counter.read_microjoules().expect("both dies read") - r0, 183_105 - 122_070, "die 1's energy counts"); + + // A die whose MSR cannot be read leaves no figure for the package. + std::fs::remove_file(host.root().join("dev/cpu/2/msr")).expect("remove die 1's device"); + assert!(msr_packages_under(host.root()).is_empty(), "a die left out"); + } + + /// An AMD package whose CPUs report different `die_id`s (its CCDs) is + /// still one counter: AMD's package energy MSR is package-scope. + #[test] + fn an_amd_package_is_one_counter_whatever_its_dies() { + let host = MsrRoot::with_dies("amd-dies", AMD, &[(0, Some(0)), (0, Some(1)), (0, Some(2)), (0, Some(3))]); + host.set(0, MSR_AMD_RAPL_POWER_UNIT, UNIT_16).set(0, MSR_AMD_PKG_ENERGY_STATUS, 65_536); + let packages = msr_packages_under(host.root()); + assert_eq!(packages.len(), 1, "{packages:?}"); + assert_eq!(packages[0].msr.map(|m| (m.die, m.cpu)), Some((None, 0))); + assert_eq!(rail_name(&packages[0]), "package-0 (MSR)"); + } + + /// An Intel host with no `die_id` files is read per package, as + /// before the files existed; so is one whose every CPU is on die 0. + #[test] + fn an_intel_host_without_die_ids_is_read_per_package() { + for (tag, topology) in [ + ("no-die-id", vec![(0, None), (0, None), (1, None), (1, None)]), + ("die-0-only", vec![(0, Some(0)), (0, Some(0)), (1, Some(0)), (1, Some(0))]), + ] { + let host = MsrRoot::with_dies(tag, INTEL, &topology); + host.set(0, MSR_RAPL_POWER_UNIT, UNIT_14).set(0, MSR_PKG_ENERGY_STATUS, 1); + host.set(2, MSR_RAPL_POWER_UNIT, UNIT_14).set(2, MSR_PKG_ENERGY_STATUS, 1); + let packages = msr_packages_under(host.root()); + let domains: Vec<_> = packages.iter().map(|p| (p.index, p.msr.map(|m| (m.die, m.cpu)))).collect(); + assert_eq!(domains, vec![(0, Some((None, 0))), (1, Some((None, 2)))], "{tag}"); + assert_eq!(packages.iter().map(rail_name).collect::>(), ["package-0 (MSR)", "package-1 (MSR)"], "{tag}"); + } + } + + /// No MSR counter where the power-unit MSR reads a zero unit (a + /// hypervisor's stand-in, seen under KVM), where the CPU is not one + /// whose MSRs are known, or where a package's device file is + /// missing (no `msr` module, or no `CAP_SYS_RAWIO` to open it). + #[test] + fn a_zero_unit_an_unknown_cpu_or_a_missing_device_is_no_counter() { + let zero = MsrRoot::new("zero-unit", INTEL, &[0]); + zero.set(0, MSR_RAPL_POWER_UNIT, 0).set(0, MSR_PKG_ENERGY_STATUS, 0); + assert!(msr_packages_under(zero.root()).is_empty(), "a zero unit"); + + let arm = MsrRoot::new("arm", "processor\t: 0\nCPU implementer\t: 0x41\n", &[0]); + assert!(msr_packages_under(arm.root()).is_empty(), "no vendor_id"); + + let netburst = MsrRoot::new("family-15", "vendor_id\t: GenuineIntel\ncpu family\t: 15\n", &[0]); + netburst.set(0, MSR_RAPL_POWER_UNIT, UNIT_14); + assert!(msr_packages_under(netburst.root()).is_empty(), "not family 6"); + + let missing = MsrRoot::new("missing-device", INTEL, &[0, 1]); + missing.set(0, MSR_RAPL_POWER_UNIT, UNIT_14); + assert!(msr_packages_under(missing.root()).is_empty(), "package 1 has no device file"); + assert!(msr_packages_under(Path::new(NO_MSR_ROOT)).is_empty()); + } + + /// Powercap zones are preferred; the MSRs are read only where + /// powercap lists no package, and are then read under the + /// filesystem root. + #[test] + fn powercap_is_preferred_and_the_msrs_are_the_fallback() { + let host = MsrRoot::new("fallback", INTEL, &[0]); + host.set(0, MSR_RAPL_POWER_UNIT, UNIT_14).set(0, MSR_PKG_ENERGY_STATUS, 5); + let zones = two_socket_server("msr-fallback-zones"); + let (root, preferred) = packages_from(zones.root(), host.root()); + assert_eq!((root.as_path(), preferred), (zones.root(), vec![pkg(0, RANGE), pkg(1, RANGE_B)])); + + let psys_only = Powercap::new("msr-fallback-psys"); + psys_only.zone("intel-rapl:0", "psys", 70_000_000, Some(RANGE)); + let (root, fallback) = packages_from(psys_only.root(), host.root()); + assert_eq!(root.as_path(), host.root(), "an MSR package is read under the filesystem root"); + assert_eq!(fallback.len(), 1); + assert!(fallback[0].msr.is_some(), "powercap listed no package, so the MSR is read"); + + let (root, none) = packages_from(psys_only.root(), Path::new(NO_MSR_ROOT)); + assert_eq!((root.as_path(), none), (psys_only.root(), Vec::new()), "no package either way"); + } + + /// The production probe over MSR packages: live when every + /// package's count keeps advancing, frozen when one holds; the + /// counter it admits reads them where the probe did. + #[test] + fn the_production_probe_judges_msr_packages_by_the_same_rule() { + let empty = Powercap::new("msr-probe-empty"); + let run = |tag: &str, moving: &'static [u32]| { + let host = MsrRoot::new(tag, INTEL, &[0, 1]); + for cpu in [0, 1] { + host.set(cpu, MSR_RAPL_POWER_UNIT, UNIT_14).set(cpu, MSR_PKG_ENERGY_STATUS, 1_000_000); + } + let ticks = Cell::new(0u64); + let probed = probe_host( + empty.root(), + host.root(), + || { + ticks.set(ticks.get() + 1); + // 45 W over a 25 ms tick, in 61.035 µJ counts. + for &cpu in moving { + host.set(cpu, MSR_PKG_ENERGY_STATUS, 1_000_000 + 18_432 * ticks.get()); + } + }, + || Duration::from_millis(25) * ticks.get() as u32, + ); + (host, probed) + }; + let (host, live) = run("probe-both", &[0, 1]); + assert_eq!((live.root.as_path(), live.status, live.packages.len()), (host.root(), CounterStatus::Live, 2)); + let counter = LinuxRaplCounter::admitted(live.status, || (live.root.clone(), live.packages.clone())) + .expect("a live host admits the counter"); + assert!(counter.read_microjoules().expect("the counter reads the MSRs") > 0); + let (_host, frozen) = run("probe-one", &[0]); + assert_eq!(frozen.status, CounterStatus::Frozen, "package 1 held its count"); + } + } } } diff --git a/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs b/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs index 5fb841e47..2d48c49f7 100644 --- a/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs +++ b/eoc/eoc-rs/crates/eoc-meter/tests/rapl_hardware.rs @@ -5,13 +5,21 @@ //! runs in this process, and the tests take one lock: each burns CPU and //! reads machine-wide counters. The oracle reads `/sys/class/powercap` //! itself — every zone's `name`, `energy_uj` and `max_energy_range_uj` — and -//! applies the sustained-advance rule to the top-level `package-` zones -//! on its own. None of these tests passes without asserting: where the +//! applies the sustained-advance rule to the top-level `package-` (and +//! `package-

-die-`) zones on its own. None of these tests passes without asserting: where the //! counters cannot be read (no powercap; a CI container, whose //! `/sys/devices/virtual/powercap` Docker masks), the status must say so and //! agree with the oracle; where they are live, both must say live, and the //! meter's figure over a CPU burn must sit inside the oracle's package-only //! bracket and clearly below package+psys and package+core. +//! +//! Where powercap lists no package (a kernel without the powercap RAPL +//! driver), the meter reads each package's energy-status MSR, and so does the +//! oracle, on its own: the CPU's vendor and family, each package's first CPU, +//! the registers at their offsets in `/dev/cpu//msr`. Run as a user that +//! cannot open those files (no `CAP_SYS_RAWIO`), both must say unavailable; +//! as root, both live, with the meter's figure inside the oracle's MSR +//! package bracket. #![cfg(target_os = "linux")] @@ -116,9 +124,16 @@ mod oracle { z.dir.strip_prefix("intel-rapl:").is_some_and(digits) } - /// A package zone: top-level, named `package-`. + /// A package zone: top-level, named `package-`, or `package-

-die-` + /// (the kernel's zone for one die of a multi-die Intel package). pub fn is_package(z: &Zone) -> bool { - top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + let named = |n: &str| { + n.strip_prefix("package-").is_some_and(|rest| match rest.split_once("-die-") { + None => digits(rest), + Some((p, d)) => digits(p) && digits(d), + }) + }; + top_level(z) && z.name.as_deref().is_some_and(named) } /// The platform zone. @@ -188,23 +203,127 @@ mod oracle { } } - /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU - /// between samples (capped at 500 ms of wall time), until every package - /// has advanced in at least five intervals, at least 80% of those - /// observed, over at least 300 ms; or one second has passed with at least - /// five intervals; or five seconds have. + /// Where powercap lists no package, the oracle's own read of each + /// energy-status MSR: the CPU's vendor and family from `/proc/cpuinfo`, + /// each counter's lowest-numbered CPU from its `physical_package_id` and, + /// on Intel, its `die_id` (the kernel's RAPL MSRs are die-scope on Intel + /// and package-scope on AMD and Hygon, `arch/x86/events/rapl.c`), and the + /// registers read at their offsets in `/dev/cpu//msr` (the `msr` module + /// and `CAP_SYS_RAWIO`). + pub mod msr { + use std::os::unix::fs::FileExt; + + /// One energy-status MSR: a package's, or one die's on Intel. + #[derive(Debug, Clone, Copy)] + pub struct Package { + pub id: u32, + /// The die within the package (0 on AMD and Hygon, and where the + /// kernel lists no `die_id`). + pub die: u32, + pub cpu: u32, + pub register: u64, + /// 2^-`shift` J per count. + pub shift: u32, + } + + fn register(cpu: u32, at: u64) -> Option { + let file = std::fs::File::open(format!("/dev/cpu/{cpu}/msr")).ok()?; + let mut raw = [0u8; 8]; + file.read_exact_at(&mut raw, at).ok()?; + Some(u64::from_le_bytes(raw)) + } + + /// This host's counters, each read on its first CPU: per die on + /// Intel, per package on AMD and Hygon. `None` when the CPU's MSRs are + /// not Intel family 6's or AMD Zen's/Hygon's, or any counter's + /// power-unit MSR cannot be read or reads a zero unit. + pub fn packages() -> Option> { + let cpuinfo = std::fs::read_to_string("/proc/cpuinfo").ok()?; + let field = |key: &str| { + cpuinfo.lines().find_map(|l| l.split_once(':').filter(|(k, _)| k.trim() == key).map(|(_, v)| v.trim().to_string())) + }; + let family: u32 = field("cpu family")?.parse().ok()?; + let (unit_at, energy_at, per_die) = match field("vendor_id")?.as_str() { + "GenuineIntel" if family == 6 => (0x606, 0x611, true), + "AuthenticAMD" | "HygonGenuine" if family >= 0x17 => (0xC001_0299, 0xC001_029B, false), + _ => return None, + }; + let number = |p: std::path::PathBuf| std::fs::read_to_string(p).ok().and_then(|s| s.trim().parse::().ok()); + let mut first = std::collections::BTreeMap::<(u32, u32), u32>::new(); + for e in std::fs::read_dir("/sys/devices/system/cpu").ok()?.flatten() { + let name = e.file_name().to_string_lossy().into_owned(); + let Some(cpu) = name.strip_prefix("cpu").and_then(|n| n.parse::().ok()) else { continue }; + let Some(id) = number(e.path().join("topology/physical_package_id")) else { continue }; + let die = if per_die { number(e.path().join("topology/die_id")).unwrap_or(0) } else { 0 }; + let lowest = first.entry((id, die)).or_insert(cpu); + *lowest = (*lowest).min(cpu); + } + let mut packages = Vec::new(); + for ((id, die), cpu) in first { + let shift = ((register(cpu, unit_at)? >> 8) & 0x1F) as u32; + if shift == 0 { + return None; + } + packages.push(Package { id, die, cpu, register: energy_at, shift }); + } + (!packages.is_empty()).then_some(packages) + } + + /// A count's energy, µJ, rounded down: the low 32 bits at 2^-`shift` J. + pub fn uj(p: &Package, raw: u64) -> u64 { + (((raw & 0xFFFF_FFFF) as u128 * 1_000_000) >> p.shift) as u64 + } + + /// The value a package's microjoule figure wraps at: 2^32 counts. + pub fn range(p: &Package) -> u64 { + (((1u128 << 32) * 1_000_000) >> p.shift) as u64 + } + + /// Every package's energy, µJ, in order; `None` if any read fails. + pub fn read(packages: &[Package]) -> Option> { + packages.iter().map(|p| register(p.cpu, p.register).map(|raw| uj(p, raw))).collect() + } + + /// The packages' summed energy between two reads, each modulo 2^32 + /// counts. + pub fn energy(packages: &[Package], before: &[u64], after: &[u64]) -> u64 { + packages + .iter() + .zip(before.iter().zip(after)) + .map(|(p, (&b, &a))| super::delta(b, a, Some(range(p))).expect("an MSR count fell past its range")) + .sum() + } + } + + /// Sample every package's `energy_uj` (or, where powercap lists no + /// package, every package's energy-status MSR), burning 25 ms of this + /// thread's CPU between samples (capped at 500 ms of wall time), until + /// every package has advanced in at least five intervals, at least 80% of + /// those observed, over at least 300 ms; or one second has passed with at + /// least five intervals; or five seconds have. pub fn watch(root: &Path) -> Watch { let all = zones(root); let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); - let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); - let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; if packages.is_empty() { + if let Some(msrs) = msr::packages() { + let names = msrs.iter().map(|p| format!("package-{}-die-{} (MSR, cpu {})", p.id, p.die, p.cpu)).collect(); + return watch_reads(names, || msr::read(&msrs)); + } let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); - return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + let verdict = if readable { Verdict::NoPackage } else { Verdict::Unreadable }; + return Watch { verdict, packages: Vec::new(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; } + let names = packages.iter().map(|z| z.dir.clone()).collect(); + watch_reads(names, || read(root, &packages)) + } + + /// The sustained-advance run of [`watch`] over the packages `read` reads, + /// in `names` order. + fn watch_reads(names: Vec, read: impl Fn() -> Option>) -> Watch { + let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; let started = Instant::now(); - let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; - let mut advances = vec![0u32; packages.len()]; + let Some(mut prev) = read() else { return none(Verdict::Unreadable) }; + let mut advances = vec![0u32; names.len()]; let mut intervals = 0u32; let live = |adv: &[u32], n: u32, w: Duration| { w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) @@ -215,7 +334,7 @@ mod oracle { break; } burn(Duration::from_millis(25), Duration::from_millis(500)); - if let Some(now) = read(root, &packages) { + if let Some(now) = read() { intervals += 1; for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { *a += u32::from(n > p); @@ -412,6 +531,27 @@ fn a_cpu_burn_reads_inside_the_package_only_bracket() { return; }; let all = oracle::zones(Path::new(oracle::POWERCAP)); + if !all.iter().any(oracle::is_package) { + // No powercap package, so the live counter is over the packages' + // energy-status MSRs: bracket it with the oracle's own MSR reads. + let msrs = oracle::msr::packages().expect("the counter reads the MSRs, so the oracle's own read of them must too"); + let read = || oracle::msr::read(&msrs).expect("an oracle read of the MSRs"); + let wall = Instant::now(); + let o0 = read(); + let m0 = counter.read_microjoules().expect("a live counter reads"); + let o1 = read(); + let cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)); + let o2 = read(); + let m1 = counter.read_microjoules().expect("a live counter reads"); + let o3 = read(); + let (inner, outer, uj) = (oracle::msr::energy(&msrs, &o1, &o2), oracle::msr::energy(&msrs, &o0, &o3), m1 - m0); + let line = format!("meter {uj} µJ; oracle MSR package-only inner {inner} outer {outer} µJ ({} packages)", msrs.len()); + eprintln!("rapl burn: {cpu:?} of thread CPU in {:?}; {line}", wall.elapsed()); + assert!(inner > 0, "the packages did not advance over the burn: {line}"); + assert!(inner <= uj && uj <= outer, "the figure is outside the package-only bracket: {line}"); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); + return; + } let wall = Instant::now(); let mut cpu = Duration::ZERO; let (b, m0, m1) = oracle::around(