diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/Cargo.toml b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/Cargo.toml index fbca0214b..803cd9a78 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/Cargo.toml +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/Cargo.toml @@ -26,7 +26,7 @@ serde = ["dep:serde", "dep:serde_json"] # Per-platform energy backends. Off by default so the foundation is OS-neutral; # downstream crates / binaries enable what their target supports. -rapl = [] # Intel/AMD x86 — Linux sysfs powercap (HwShunt while its counter is live) +rapl = [] # Intel/AMD x86 — Linux sysfs powercap, or the energy-status MSRs where powercap lists no package (HwShunt while live) nvml = ["dep:nvml-wrapper", "dep:tracing"] # NVIDIA discrete — NVML cumulative-energy counter (HwShunt while live), power-draw integral (ModelBased) otherwise ioreport = [] # Apple Silicon M-series — real IOReport energy counter (HwShunt while every CPU rail is live), load-avg estimator fallback; and the per-process GPU counter (GPU rail × this process's GPU-time share) ina3221 = [] # NVIDIA Jetson (Orin / Thor) — sysfs i2c shunt reader diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs index 5b7ce966c..8b6591bf9 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/rapl.rs @@ -18,6 +18,29 @@ //! itself updates at ~1 kHz; below this readings are quantization //! noise). //! +//! **Without the powercap driver.** Some kernels ship without `intel_rapl_msr` +//! (Ubuntu without `linux-modules-extra`, as on bare-metal hosts rented by the +//! node), so no package zone is listed while the counters themselves are +//! there. Where powercap lists no package, the counter reads the same counters +//! directly: each RAPL domain's energy-status MSR (`MSR_PKG_ENERGY_STATUS`, +//! 0x611, on Intel family 6; `0xC001029B` on AMD Zen and Hygon), through the +//! `msr` driver's device file of the domain's first CPU (`/dev/cpu//msr`, +//! which needs the `msr` module, `CAP_SYS_RAWIO` and read access to the file). +//! A domain is what the kernel scopes these MSRs to: on AMD and Hygon the +//! package, whose energy counter is per package; on Intel a die of a package +//! (`topology/die_id` within `topology/physical_package_id`), since its RAPL +//! MSRs are die-scope on multi-die parts — on a single-die part, the package. +//! The 32-bit count is converted at the domain's own energy-status unit (bits +//! 12:8 of the power-unit MSR, 0x606 or `0xC0010299`), so it wraps at 2^32 +//! counts in microjoules, and its resolution is one count (about 61 μJ at the +//! common unit of 2^-14 J). A power-unit MSR that reads zero has no counter: +//! some hypervisors let a guest read these MSRs and return zeros. The same +//! liveness rule then decides whether the counter reads. +//! +//! **Multi-die packages.** The kernel names a multi-die Intel package's powercap +//! zones `package-

-die-`, one top-level zone per die; each is a package +//! zone here, and the figure sums them all. +//! //! **Liveness.** A vendor energy counter is a hardware reading only while it is //! demonstrably live for the domain the figure claims (JouleClaw SPEC §1.3). //! A readable `energy_uj` is not yet a live one: a hypervisor can pass the file @@ -56,24 +79,78 @@ pub const POWERCAP_ROOT: &str = "/sys/class/powercap"; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] const PROBE_TICK: Duration = Duration::from_millis(25); -/// One RAPL package zone: a top-level `intel-rapl:` zone whose `name` -/// reads `package-`. +/// One RAPL package: a top-level `intel-rapl:` powercap zone whose +/// `name` reads `package-`, or, on a kernel without the powercap driver, +/// the package's energy-status MSR ([`Msr`]). #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct Package { - /// The zone's directory, `/intel-rapl:`. + /// The zone's directory, `/intel-rapl:`; for an MSR + /// package, the device file it is read through, `/dev/cpu//msr`. dir: PathBuf, - /// `` in `intel-rapl:`. + /// `` in `intel-rapl:`; for an MSR package, its + /// `physical_package_id`. index: u32, - /// The zone's `max_energy_range_uj`, the value its `energy_uj` wraps at, - /// when that reads as a number above zero. + /// For an MSR package on Intel, its die within the package (`die_id`): the + /// kernel scopes Intel's RAPL MSRs per die. 0 for a powercap zone, an AMD + /// or Hygon package, and a single-die part. + die: u32, + /// The value the package's microjoule figure wraps at: the zone's + /// `max_energy_range_uj` when that reads as a number above zero; for an MSR + /// package, 2^32 counts at its unit ([`Msr::range_uj`]). max_energy_range_uj: Option, + /// How to read an MSR package; `None` for a powercap zone. + msr: Option, } -/// Whether a top-level zone's `name` file names a package: `package-`, `` -/// one or more digits (the file ends in a newline). Not `psys`, `dram`, `core` -/// or `uncore`. +/// A package read from its energy-status MSR, where the kernel lists no +/// powercap zone for it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Msr { + /// The energy-status MSR: [`MSR_PKG_ENERGY_STATUS`] or + /// [`MSR_AMD_PKG_ENERGY_STATUS`]. + register: u64, + /// The energy-status unit, 2^-`unit_shift` J per count: bits 12:8 of the + /// power-unit MSR. Never zero. + unit_shift: u32, +} + +/// Intel's power-unit MSR (family 6). +pub(crate) const MSR_RAPL_POWER_UNIT: u64 = 0x606; +/// Intel's package energy-status MSR (family 6). +pub(crate) const MSR_PKG_ENERGY_STATUS: u64 = 0x611; +/// AMD Zen's (and Hygon's) power-unit MSR. +pub(crate) const MSR_AMD_RAPL_POWER_UNIT: u64 = 0xC001_0299; +/// AMD Zen's (and Hygon's) package energy-status MSR. +pub(crate) const MSR_AMD_PKG_ENERGY_STATUS: u64 = 0xC001_029B; + +impl Msr { + /// The package's energy, in microjoules, at a raw read of its + /// energy-status MSR: the low 32 bits are the count. + fn uj(self, raw: u64) -> u64 { + ((u128::from(raw & 0xFFFF_FFFF) * 1_000_000) >> self.unit_shift) as u64 + } + + /// The microjoule figure's wrap: 2^32 counts. + fn range_uj(self) -> u64 { + (((1u128 << 32) * 1_000_000) >> self.unit_shift) as u64 + } + + /// One count, in whole microjoules rounded up: the counter's resolution. + fn count_uj(self) -> u64 { + 1_000_000u64.div_ceil(1u64 << self.unit_shift).max(1) + } +} + +/// Whether a top-level zone's `name` file names a package: `package-`, or +/// `package--die-` — the kernel's name for each die's zone of a multi-die +/// Intel package — `` and `` one or more digits (the file ends in a +/// newline). Not `psys`, `dram`, `core` or `uncore`. pub(crate) fn is_package_name(name: &str) -> bool { - name.trim().strip_prefix("package-").is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit())) + let digits = |n: &str| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit()); + name.trim().strip_prefix("package-").is_some_and(|n| match n.split_once("-die-") { + Some((package, die)) => digits(package) && digits(die), + None => digits(n), + }) } /// The index of a top-level RAPL zone directory, `intel-rapl:`. `None` @@ -94,7 +171,8 @@ fn read_u64(path: &Path) -> Option { } /// The package zones under the powercap directory `root`, in index order: every -/// top-level zone whose `name` reads `package-`, with the range its counter +/// top-level zone whose `name` reads `package-` (or `package--die-`, +/// [`is_package_name`]), with the range its counter /// wraps at. Everything else is left out, zone 0 included if it is not a /// package. Empty when no zone is a package, or `root` cannot be listed. pub(crate) fn packages_under(root: &Path) -> Vec { @@ -111,6 +189,8 @@ pub(crate) fn packages_under(root: &Path) -> Vec { max_energy_range_uj: read_u64(&dir.join("max_energy_range_uj")).filter(|&m| m > 0), dir, index, + die: 0, + msr: None, }) }) .collect(); @@ -118,14 +198,146 @@ pub(crate) fn packages_under(root: &Path) -> Vec { packages } -/// Every package's `energy_uj`, in order. `None` when there is no package, or -/// when any one cannot be read: a figure that left a package out would miss its -/// energy while claiming the whole CPU. +/// Every package's energy in microjoules (its `energy_uj`, or its MSR's count +/// at its unit), in order. `None` when there is no package, or when any one +/// cannot be read: a figure that left a package out would miss its energy +/// while claiming the whole CPU. pub(crate) fn read_packages(packages: &[Package]) -> Option> { if packages.is_empty() { return None; } - packages.iter().map(|p| read_u64(&p.dir.join("energy_uj"))).collect() + packages.iter().map(read_package).collect() +} + +/// One package's energy in microjoules, or `None` when it cannot be read. +fn read_package(p: &Package) -> Option { + match p.msr { + None => read_u64(&p.dir.join("energy_uj")), + Some(msr) => read_msr(&p.dir, msr.register).map(|raw| msr.uj(raw)), + } +} + +/// The 64-bit MSR `register` read through the `msr` driver's device file +/// `device`, where the register number is the file offset. `None` when the file +/// cannot be opened (no `msr` module, no `CAP_SYS_RAWIO`, no read access) or +/// the register cannot be read (the CPU does not implement it). +#[cfg(unix)] +fn read_msr(device: &Path, register: u64) -> Option { + use std::os::unix::fs::FileExt; + let file = std::fs::File::open(device).ok()?; + let mut raw = [0u8; 8]; + file.read_exact_at(&mut raw, register).ok()?; + Some(u64::from_le_bytes(raw)) +} + +#[cfg(not(unix))] +fn read_msr(_device: &Path, _register: u64) -> Option { + None +} + +/// The CPU's vendor and family from `/proc/cpuinfo` text: the first CPU's +/// `vendor_id` and `cpu family`. +fn cpu_vendor_family(cpuinfo: &str) -> Option<(&str, u32)> { + let field = |key: &str| { + cpuinfo.lines().find_map(|line| { + let (k, v) = line.split_once(':')?; + (k.trim() == key).then(|| v.trim()) + }) + }; + Some((field("vendor_id")?, field("cpu family")?.parse().ok()?)) +} + +/// The power-unit and energy-status MSRs for a CPU's vendor and family, or +/// `None` where this crate knows none: Intel family 6, AMD Zen (family 0x17 +/// and later) and Hygon. +fn msr_registers(vendor: &str, family: u32) -> Option<(u64, u64)> { + match vendor { + "GenuineIntel" if family == 6 => Some((MSR_RAPL_POWER_UNIT, MSR_PKG_ENERGY_STATUS)), + "AuthenticAMD" | "HygonGenuine" if family >= 0x17 => { + Some((MSR_AMD_RAPL_POWER_UNIT, MSR_AMD_PKG_ENERGY_STATUS)) + } + _ => None, + } +} + +/// Each RAPL domain's lowest-numbered CPU, keyed by (package, die), from +/// `/sys/devices/system/cpu/cpu/topology/`: its `physical_package_id` +/// and, where `per_die` (Intel), its `die_id` (0 where the file is absent). Linux +/// scopes Intel's RAPL MSRs per die and AMD's and Hygon's per package (its perf +/// RAPL driver: "AMD/HYGON platforms have a per-PKG package energy counter"; +/// "CLX-AP is multi-die and its RAPL MSRs are die-scope"), so on AMD and Hygon +/// the die is always 0. +fn first_cpu_per_domain(root: &Path, per_die: bool) -> std::collections::BTreeMap<(u32, u32), u32> { + let mut first = std::collections::BTreeMap::new(); + let Ok(entries) = std::fs::read_dir(root.join("sys/devices/system/cpu")) else { + return first; + }; + for entry in entries.flatten() { + let Some(cpu) = entry.file_name().to_str().and_then(|n| n.strip_prefix("cpu")?.parse::().ok()) else { + continue; + }; + let topology = entry.path().join("topology"); + let Some(package) = read_u64(&topology.join("physical_package_id")).and_then(|p| u32::try_from(p).ok()) else { + continue; + }; + let die = if per_die { + read_u64(&topology.join("die_id")).and_then(|d| u32::try_from(d).ok()).unwrap_or(0) + } else { + 0 + }; + let lowest = first.entry((package, die)).or_insert(cpu); + *lowest = (*lowest).min(cpu); + } + first +} + +/// Every RAPL domain's energy-status MSR under the filesystem root `root`, in +/// (package, die) order: each package on AMD and Hygon, each die of each +/// package on Intel ([`first_cpu_per_domain`]), read through the device file of +/// the domain's first CPU (`/dev/cpu//msr`), at the unit its +/// power-unit MSR reads. Empty when the CPU is not one whose MSRs this crate +/// knows, when no CPU topology is listed, or when any domain's power-unit MSR +/// cannot be read or reads a zero unit — a figure that left a domain out would +/// miss its energy, and a zero unit is a hypervisor's stand-in, not a counter. +pub(crate) fn msr_packages_under(root: &Path) -> Vec { + let Ok(cpuinfo) = std::fs::read_to_string(root.join("proc/cpuinfo")) else { + return Vec::new(); + }; + let Some((vendor, family)) = cpu_vendor_family(&cpuinfo) else { + return Vec::new(); + }; + let Some((unit_register, register)) = msr_registers(vendor, family) else { + return Vec::new(); + }; + let per_die = vendor == "GenuineIntel"; + let mut packages = Vec::new(); + for ((index, die), cpu) in first_cpu_per_domain(root, per_die) { + let dir = root.join(format!("dev/cpu/{cpu}/msr")); + let Some(unit) = read_msr(&dir, unit_register) else { + return Vec::new(); + }; + let unit_shift = ((unit >> 8) & 0x1F) as u32; + if unit_shift == 0 { + return Vec::new(); + } + let msr = Msr { register, unit_shift }; + packages.push(Package { dir, index, die, max_energy_range_uj: Some(msr.range_uj()), msr: Some(msr) }); + } + packages +} + +/// This host's packages: the powercap package zones under `powercap`, or, +/// where it lists none, every package's energy-status MSR under the filesystem +/// root `root`. +fn packages_from(powercap: &Path, root: &Path) -> Vec { + let zones = packages_under(powercap); + if zones.is_empty() { msr_packages_under(root) } else { zones } +} + +/// This host's packages: its powercap package zones, or its energy-status +/// MSRs where the kernel lists none. Off Linux, none. +fn host_packages() -> Vec { + if cfg!(target_os = "linux") { packages_from(Path::new(POWERCAP_ROOT), Path::new("/")) } else { Vec::new() } } /// One zone's energy between two reads, modulo its own wrap. A counter that went @@ -232,10 +444,18 @@ impl RaplCounter { /// and the composite meter falls through instead of selecting a counter /// whose every read errors, or reads zero, or leaves a socket out. /// + /// Where powercap lists no package zone, the counter is over every + /// package's energy-status MSR instead (see the module docs), under the + /// same rules: every package readable and live. Without the `msr` module or + /// `CAP_SYS_RAWIO` no MSR reads, and the counter is + /// [`CounterLiveness::Unavailable`]. + /// /// The first call for a package set in a process runs the liveness probe on /// the calling thread; later calls reuse the answer ([`rapl_liveness`]). pub fn new() -> Self { - Self::with_root(POWERCAP_ROOT) + let packages = host_packages(); + let liveness = package_liveness(&packages); + Self::given(packages, liveness) } /// Build over the package zones under an explicit powercap directory (a @@ -284,7 +504,7 @@ fn admits(liveness: CounterLiveness) -> bool { /// package zone is [`CounterLiveness::Unavailable`] with no probe. Off Linux it /// is [`CounterLiveness::Unavailable`], with no probe. pub fn rapl_liveness() -> CounterLiveness { - package_liveness(&packages_under(Path::new(POWERCAP_ROOT))) + package_liveness(&host_packages()) } /// The liveness of `packages`, every one of them, cached per package set. @@ -362,8 +582,10 @@ impl EnergyCounter for RaplCounter { } fn resolution_uj(&self) -> u64 { - // RAPL MSR native quantum. - 1 + // powercap reports microjoules; a package read from its MSR resolves + // one count at its unit (about 61 μJ at 2^-14 J), and the figure is + // only as fine as its coarsest package. + self.packages.iter().filter_map(|p| p.msr).map(Msr::count_uj).max().unwrap_or(1) } fn min_window_ns(&self) -> u64 { @@ -550,10 +772,13 @@ mod tests { /// sub-zone, the control type and the MMIO interface are not. #[test] fn only_a_top_level_zone_named_package_n_is_a_package() { - for name in ["package-0", "package-1\n", "package-12"] { + for name in ["package-0", "package-1\n", "package-12", "package-0-die-0", "package-1-die-1\n"] { assert!(is_package_name(name), "{name:?}"); } - for name in ["psys", "psys\n", "dram", "core", "uncore", "package-", "package-x", "package-0a", "Package-0", ""] { + for name in [ + "psys", "psys\n", "dram", "core", "uncore", "package-", "package-x", "package-0a", "Package-0", "", + "package-0-die-", "package--die-0", "package-0-die-x", "package-0-core-0", "package-0-die-0-die-1", + ] { assert!(!is_package_name(name), "{name:?}"); } assert_eq!(top_level_zone("intel-rapl:0"), Some(0)); @@ -563,6 +788,20 @@ mod tests { } } + /// A multi-die Intel package as the kernel lists it: one top-level zone per + /// die, named `package-

-die-`. Each is a package zone, and the figure + /// sums them. + #[test] + fn a_multi_die_package_sums_its_die_zones() { + let pc = Powercap::new("two-die-zones"); + pc.zone("intel-rapl:0", "package-0-die-0", 1_000_000, Some(RANGE)) + .zone("intel-rapl:1", "package-0-die-1", 2_000_000, Some(RANGE)) + .zone("intel-rapl:2", "psys", 80_000_000, Some(RANGE)); + let packages = packages_under(pc.root()); + assert_eq!(listed(&packages), vec![(0, Some(RANGE)), (1, Some(RANGE))]); + assert_eq!(uj(&live_counter(&pc)), 3_000_000, "both dies, not psys"); + } + /// A two-socket server: the figure is both packages' energy and nothing else /// — not `psys` (which includes the packages, so would count them twice), /// not a `dram` zone at the top level or under a package, not a core @@ -801,4 +1040,225 @@ mod tests { assert_eq!(liveness, CounterLiveness::Frozen, "package 1 held its value"); assert!(took >= LIVENESS_BUDGET, "a frozen verdict took only {took:?}"); } + + /// The MSR path, over fake device files: a regular file whose bytes at a + /// register's offset are that register is read exactly as the `msr` + /// driver's device file is. + #[cfg(unix)] + mod msr { + use super::*; + use std::os::unix::fs::FileExt; + + const INTEL: &str = "processor\t: 0\nvendor_id\t: GenuineIntel\ncpu family\t: 6\nmodel\t\t: 85\n"; + const AMD: &str = "processor\t: 0\nvendor_id\t: AuthenticAMD\ncpu family\t: 26\nmodel\t\t: 112\n"; + /// A power-unit MSR whose energy-status unit (bits 12:8) is 2^-14 J, + /// the common Intel value (0xA0E03). + const UNIT_14: u64 = 0xA_0E03; + /// 2^-16 J, as on AMD Zen. + const UNIT_16: u64 = 0x10 << 8; + + /// A fake filesystem root for one test, removed when dropped: its + /// `/proc/cpuinfo`, each CPU's `physical_package_id`, and the CPUs' + /// `dev/cpu//msr` files. + struct MsrRoot(PathBuf); + + impl MsrRoot { + /// `package_of_cpu[n]` is CPU `n`'s package; no CPU lists a die. + fn new(tag: &str, cpuinfo: &str, package_of_cpu: &[u32]) -> Self { + let root = std::env::temp_dir().join(format!("jouleclaw-msr-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("proc")).expect("create proc"); + std::fs::write(root.join("proc/cpuinfo"), cpuinfo).expect("write cpuinfo"); + for (cpu, package) in package_of_cpu.iter().enumerate() { + let topology = root.join(format!("sys/devices/system/cpu/cpu{cpu}/topology")); + std::fs::create_dir_all(&topology).expect("create a CPU's topology"); + std::fs::write(topology.join("physical_package_id"), format!("{package}\n")).expect("write its package"); + } + Self(root) + } + + /// `place_of_cpu[n]` is CPU `n`'s (package, die), each CPU listing + /// its `die_id`. + fn with_dies(tag: &str, cpuinfo: &str, place_of_cpu: &[(u32, u32)]) -> Self { + let packages: Vec = place_of_cpu.iter().map(|&(package, _)| package).collect(); + let host = Self::new(tag, cpuinfo, &packages); + for (cpu, (_, die)) in place_of_cpu.iter().enumerate() { + let topology = host.0.join(format!("sys/devices/system/cpu/cpu{cpu}/topology")); + std::fs::write(topology.join("die_id"), format!("{die}\n")).expect("write its die"); + } + host + } + + /// Set CPU `cpu`'s MSR `register` to `value`. + fn set(&self, cpu: u32, register: u64, value: u64) -> &Self { + let dir = self.0.join(format!("dev/cpu/{cpu}")); + std::fs::create_dir_all(&dir).expect("create dev/cpu/"); + let file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(dir.join("msr")) + .expect("open the fake msr file"); + file.write_all_at(&value.to_le_bytes(), register).expect("write the register"); + self + } + + fn root(&self) -> &Path { + &self.0 + } + } + + impl Drop for MsrRoot { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// A two-socket Intel host (CPUs 0–1 on package 0, 2–3 on package 1) + /// with no powercap driver: each package read on its first CPU, at its + /// unit, and summed. + #[test] + fn a_two_socket_host_without_powercap_reads_each_packages_msr() { + let host = MsrRoot::new("two-socket", INTEL, &[0, 0, 1, 1]); + host.set(0, MSR_RAPL_POWER_UNIT, UNIT_14).set(0, MSR_PKG_ENERGY_STATUS, 1_000); + host.set(2, MSR_RAPL_POWER_UNIT, UNIT_14).set(2, MSR_PKG_ENERGY_STATUS, 2_000); + let packages = msr_packages_under(host.root()); + let devices: Vec<_> = packages.iter().map(|p| (p.index, p.dir.clone())).collect(); + assert_eq!( + devices, + vec![(0, host.root().join("dev/cpu/0/msr")), (1, host.root().join("dev/cpu/2/msr"))], + "package 1 is read on its first CPU, 2" + ); + assert!(packages.iter().all(|p| p.msr == Some(Msr { register: MSR_PKG_ENERGY_STATUS, unit_shift: 14 }))); + assert!(packages.iter().all(|p| p.max_energy_range_uj == Some(262_144_000_000)), "2^32 counts of 2^-14 J"); + + let c = RaplCounter::given(packages, CounterLiveness::Live); + // 1,000 and 2,000 counts of 61.035 μJ, each floored. + assert_eq!(uj(&c), 61_035 + 122_070); + host.set(0, MSR_PKG_ENERGY_STATUS, 3_000).set(2, MSR_PKG_ENERGY_STATUS, 2_100); + assert_eq!(uj(&c), 183_105 + 128_173); + assert_eq!(c.resolution_uj(), 62, "one count, rounded up"); + assert_eq!((c.provenance(), c.domain()), (Provenance::HwShunt, EnergyDomain::CpuPkg)); + } + + /// A multi-die Intel package (CPUs 0–1 on die 0, 2–3 on die 1): the + /// kernel scopes Intel's RAPL MSRs per die, so each die is a domain, + /// read on its own first CPU, and the figure sums both. Reading the + /// package's first CPU alone would count one die. + #[test] + fn each_die_of_a_multi_die_intel_package_is_read_and_summed() { + let host = MsrRoot::with_dies("two-die", INTEL, &[(0, 0), (0, 0), (0, 1), (0, 1)]); + host.set(0, MSR_RAPL_POWER_UNIT, UNIT_14).set(0, MSR_PKG_ENERGY_STATUS, 1_000); + host.set(2, MSR_RAPL_POWER_UNIT, UNIT_14).set(2, MSR_PKG_ENERGY_STATUS, 2_000); + let packages = msr_packages_under(host.root()); + let domains: Vec<_> = packages.iter().map(|p| (p.index, p.die, p.dir.clone())).collect(); + assert_eq!( + domains, + vec![(0, 0, host.root().join("dev/cpu/0/msr")), (0, 1, host.root().join("dev/cpu/2/msr"))], + "die 1 is read on its first CPU, 2" + ); + let c = RaplCounter::given(packages, CounterLiveness::Live); + assert_eq!(uj(&c), 61_035 + 122_070, "both dies' counts"); + } + + /// AMD's package energy counter is per package, so CPUs whose `die_id` + /// differs within one package (the dies of a Zen part) are one domain, + /// read once. + #[test] + fn an_amd_package_is_one_domain_whatever_its_dies() { + let host = MsrRoot::with_dies("amd-dies", AMD, &[(0, 0), (0, 1), (0, 2), (0, 3)]); + host.set(0, MSR_AMD_RAPL_POWER_UNIT, UNIT_16).set(0, MSR_AMD_PKG_ENERGY_STATUS, 65_536); + let packages = msr_packages_under(host.root()); + assert_eq!(packages.len(), 1, "{packages:?}"); + assert_eq!((packages[0].index, packages[0].die), (0, 0)); + assert_eq!(uj(&RaplCounter::given(packages, CounterLiveness::Live)), 1_000_000); + } + + /// The 32-bit count wraps at 2^32 counts, so the microjoule figure wraps + /// at that range, losing at most a microjoule; bits above the count are + /// not energy. + #[test] + fn the_count_wraps_at_2_to_the_32_counts() { + let host = MsrRoot::new("wrap", INTEL, &[0]); + host.set(0, MSR_RAPL_POWER_UNIT, UNIT_14).set(0, MSR_PKG_ENERGY_STATUS, 0xFFFF_FFF0); + let c = RaplCounter::given(msr_packages_under(host.root()), CounterLiveness::Live); + let before = uj(&c); + host.set(0, MSR_PKG_ENERGY_STATUS, 0xABCD_0000_0000_0010); + // 32 counts of 61.035 μJ is 1,953.125 μJ. + assert_eq!(uj(&c) - before, 1_953); + } + + /// AMD Zen reads its own power-unit and energy-status MSRs. + #[test] + fn amd_zen_reads_its_own_msrs() { + let host = MsrRoot::new("amd", AMD, &[0, 0]); + host.set(0, MSR_AMD_RAPL_POWER_UNIT, UNIT_16).set(0, MSR_AMD_PKG_ENERGY_STATUS, 65_536); + let packages = msr_packages_under(host.root()); + assert_eq!(packages.len(), 1); + assert_eq!(packages[0].msr, Some(Msr { register: MSR_AMD_PKG_ENERGY_STATUS, unit_shift: 16 })); + let c = RaplCounter::given(packages, CounterLiveness::Live); + assert_eq!(uj(&c), 1_000_000, "65,536 counts of 2^-16 J is one joule"); + assert_eq!(c.resolution_uj(), 16); + } + + /// No MSR counter where the power-unit MSR reads a zero unit (a + /// hypervisor's stand-in, seen under KVM), where the CPU is not one whose + /// MSRs are known, or where a package's device file is missing. + #[test] + fn a_zero_unit_an_unknown_cpu_or_a_missing_device_is_no_counter() { + let zero = MsrRoot::new("zero-unit", INTEL, &[0]); + zero.set(0, MSR_RAPL_POWER_UNIT, 0).set(0, MSR_PKG_ENERGY_STATUS, 0); + assert!(msr_packages_under(zero.root()).is_empty(), "a zero unit"); + + let arm = MsrRoot::new("arm", "processor\t: 0\nCPU implementer\t: 0x41\n", &[0]); + assert!(msr_packages_under(arm.root()).is_empty(), "no vendor_id"); + + let netburst = MsrRoot::new("family-15", "vendor_id\t: GenuineIntel\ncpu family\t: 15\n", &[0]); + netburst.set(0, MSR_RAPL_POWER_UNIT, UNIT_14); + assert!(msr_packages_under(netburst.root()).is_empty(), "not family 6"); + + let missing = MsrRoot::new("missing-device", INTEL, &[0, 1]); + missing.set(0, MSR_RAPL_POWER_UNIT, UNIT_14); + assert!(msr_packages_under(missing.root()).is_empty(), "package 1 has no device file"); + } + + /// Powercap zones are preferred; the MSRs are read only where powercap + /// lists no package. + #[test] + fn powercap_is_preferred_and_the_msrs_are_the_fallback() { + let host = MsrRoot::new("fallback", INTEL, &[0]); + host.set(0, MSR_RAPL_POWER_UNIT, UNIT_14).set(0, MSR_PKG_ENERGY_STATUS, 5); + let zones = two_socket_server("fallback-zones"); + let preferred = packages_from(zones.root(), host.root()); + assert_eq!(preferred.len(), 2); + assert!(preferred.iter().all(|p| p.msr.is_none()), "powercap's zones were listed"); + + let empty = Powercap::new("fallback-empty"); + let fallback = packages_from(empty.root(), host.root()); + assert_eq!(fallback.len(), 1); + assert!(fallback[0].msr.is_some(), "powercap listed no package, so the MSR is read"); + } + + /// The production probe over MSR packages: live when every package's + /// count keeps advancing, frozen when one holds. + #[test] + fn the_production_probe_judges_msr_packages_by_the_same_rule() { + let run = |tag: &str, moving: &[u32]| { + let host = MsrRoot::new(tag, INTEL, &[0, 1]); + for cpu in [0, 1] { + host.set(cpu, MSR_RAPL_POWER_UNIT, UNIT_14).set(cpu, MSR_PKG_ENERGY_STATUS, 1_000_000); + } + let packages = msr_packages_under(&host.0); + let mut count = 1_000_000u64; + probe_packages_stepping(&packages, || { + count += 18_432; // 45 W over a 25 ms tick, in 61.035 μJ counts + for &cpu in moving { + host.set(cpu, MSR_PKG_ENERGY_STATUS, count); + } + }) + }; + assert_eq!(run("probe-both", &[0, 1]), CounterLiveness::Live); + assert_eq!(run("probe-one", &[0]), CounterLiveness::Frozen, "package 1 held its count"); + } + } } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs index b063a5dda..4867f6088 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/rapl_hardware.rs @@ -1,11 +1,15 @@ //! Linux RAPL against the hardware (feature `rapl`): this host's powercap -//! counters, read by the meter and by an oracle the tests declare here. +//! counters, or its energy-status MSRs where powercap lists no package, read by +//! the meter and by an oracle the tests declare here. //! //! In a test binary of its own, so the RAPL probe runs in this process, and //! the tests take one lock: each burns CPU and reads machine-wide counters. //! The oracle reads `/sys/class/powercap` itself — every zone's `name`, //! `energy_uj` and `max_energy_range_uj` — and applies the sustained-advance -//! rule to the top-level `package-` zones on its own. None of these tests +//! rule to the top-level package zones (`package-`, or `package--die-` +//! on a multi-die package) on its own. Where powercap lists no package it reads +//! the RAPL MSRs itself (`oracle::msr`), each domain on its first CPU: per +//! package on AMD and Hygon, per die on Intel. None of these tests //! passes without asserting: where the counters cannot be read (no powercap; //! a CI container, whose `/sys/devices/virtual/powercap` Docker masks), the //! liveness must say so and agree with the oracle; where they are live, both @@ -118,9 +122,14 @@ mod oracle { z.dir.strip_prefix("intel-rapl:").is_some_and(digits) } - /// A package zone: top-level, named `package-`. + /// A package zone: top-level, named `package-`, or `package--die-` + /// (one zone per die of a multi-die Intel package). pub fn is_package(z: &Zone) -> bool { - top_level(z) && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(digits) + top_level(z) + && z.name.as_deref().and_then(|n| n.strip_prefix("package-")).is_some_and(|n| match n.split_once("-die-") { + Some((package, die)) => digits(package) && digits(die), + None => digits(n), + }) } /// The platform zone. @@ -190,23 +199,52 @@ mod oracle { } } - /// Sample every package's `energy_uj`, burning 25 ms of this thread's CPU - /// between samples (capped at 500 ms of wall time), until every package - /// has advanced in at least five intervals, at least 80% of those + /// Sample every package's `energy_uj` under `root`, burning 25 ms of this + /// thread's CPU between samples (capped at 500 ms of wall time), until every + /// package has advanced in at least five intervals, at least 80% of those /// observed, over at least 300 ms; or one second has passed with at least /// five intervals; or five seconds have. pub fn watch(root: &Path) -> Watch { let all = zones(root); let packages: Vec<&Zone> = all.iter().filter(|z| is_package(z)).collect(); let names: Vec = packages.iter().map(|z| z.dir.clone()).collect(); - let none = |verdict| Watch { verdict, packages: names.clone(), intervals: 0, advances: Vec::new(), window: Duration::ZERO }; if packages.is_empty() { let readable = all.iter().any(|z| top_level(z) && z.energy_uj.is_some()); - return none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }); + return Watch::none(if readable { Verdict::NoPackage } else { Verdict::Unreadable }, names); } + watch_with(names, || read(root, &packages)) + } + + /// [`watch`] over this host: its powercap package zones, or, where powercap + /// lists none, its RAPL MSRs ([`msr::domains`]), sampled the same way. + pub fn watch_host() -> Watch { + let root = Path::new(POWERCAP); + if zones(root).iter().any(is_package) { + return watch(root); + } + match msr::domains() { + Some(domains) => { + let names = domains.iter().map(|d| d.name.clone()).collect(); + watch_with(names, || msr::read_all(&domains)) + } + None => { + let w = watch(root); + Watch::none(w.verdict, w.packages) + } + } + } + + impl Watch { + fn none(verdict: Verdict, packages: Vec) -> Watch { + Watch { verdict, packages, intervals: 0, advances: Vec::new(), window: Duration::ZERO } + } + } + + /// The sustained-advance run over the counters `read` returns, one per name. + fn watch_with(names: Vec, mut read: impl FnMut() -> Option>) -> Watch { let started = Instant::now(); - let Some(mut prev) = read(root, &packages) else { return none(Verdict::Unreadable) }; - let mut advances = vec![0u32; packages.len()]; + let Some(mut prev) = read() else { return Watch::none(Verdict::Unreadable, names) }; + let mut advances = vec![0u32; names.len()]; let mut intervals = 0u32; let live = |adv: &[u32], n: u32, w: Duration| { w >= Duration::from_millis(300) && adv.iter().all(|&a| a >= 5 && u64::from(a) * 5 >= u64::from(n) * 4) @@ -217,7 +255,7 @@ mod oracle { break; } burn(Duration::from_millis(25), Duration::from_millis(500)); - if let Some(now) = read(root, &packages) { + if let Some(now) = read() { intervals += 1; for ((a, n), p) in advances.iter_mut().zip(&now).zip(&prev) { *a += u32::from(n > p); @@ -230,6 +268,96 @@ mod oracle { Watch { verdict, packages: names, intervals, advances, window } } + /// The oracle's own reading of the RAPL energy-status MSRs, for a host + /// whose powercap lists no package: each domain's count through the `msr` + /// driver's device file of its first CPU. A domain is a package on AMD and + /// Hygon (a per-package counter) and a die of a package on Intel (the + /// kernel's die-scope RAPL MSRs). + pub mod msr { + use super::number; + use std::os::unix::fs::FileExt; + use std::path::{Path, PathBuf}; + + /// One RAPL domain's energy-status MSR. + pub struct Domain { + pub name: String, + device: PathBuf, + register: u64, + shift: u32, + } + + fn read(device: &Path, register: u64) -> Option { + let file = std::fs::File::open(device).ok()?; + let mut raw = [0u8; 8]; + file.read_exact_at(&mut raw, register).ok()?; + Some(u64::from_le_bytes(raw)) + } + + /// Every domain whose power-unit and energy-status MSRs read with a + /// non-zero unit; `None` when the CPU's MSRs are not known here, no + /// topology is listed, or any domain cannot be read. + pub fn domains() -> Option> { + let cpuinfo = std::fs::read_to_string("/proc/cpuinfo").ok()?; + let field = |key: &str| { + cpuinfo.lines().find_map(|l| l.split_once(':').filter(|(k, _)| k.trim() == key).map(|(_, v)| v.trim())) + }; + let family: u32 = field("cpu family")?.parse().ok()?; + let (unit, energy, per_die) = match field("vendor_id")? { + "GenuineIntel" if family == 6 => (0x606, 0x611, true), + "AuthenticAMD" | "HygonGenuine" if family >= 0x17 => (0xC001_0299, 0xC001_029B, false), + _ => return None, + }; + let mut first = std::collections::BTreeMap::<(u64, u64), u32>::new(); + for entry in std::fs::read_dir("/sys/devices/system/cpu").ok()?.flatten() { + let Some(cpu) = entry.file_name().to_str().and_then(|n| n.strip_prefix("cpu")?.parse::().ok()) else { + continue; + }; + let topology = entry.path().join("topology"); + let Some(package) = number(topology.join("physical_package_id")) else { continue }; + let die = if per_die { number(topology.join("die_id")).unwrap_or(0) } else { 0 }; + let lowest = first.entry((package, die)).or_insert(cpu); + *lowest = (*lowest).min(cpu); + } + if first.is_empty() { + return None; + } + first + .into_iter() + .map(|((package, die), cpu)| { + let device = PathBuf::from(format!("/dev/cpu/{cpu}/msr")); + let shift = ((read(&device, unit)? >> 8) & 0x1F) as u32; + if shift == 0 { + return None; + } + read(&device, energy)?; + Some(Domain { name: format!("msr package-{package}-die-{die} (cpu {cpu})"), device, register: energy, shift }) + }) + .collect() + } + + /// One domain's count, in µJ. + fn uj(d: &Domain) -> Option { + read(&d.device, d.register).map(|raw| ((u128::from(raw & 0xFFFF_FFFF) * 1_000_000) >> d.shift) as u64) + } + + /// Every domain's count, in µJ; `None` if one fails. + pub fn read_all(domains: &[Domain]) -> Option> { + domains.iter().map(uj).collect() + } + + /// The domains' summed energy between two reads, each modulo its 2^32 counts. + pub fn energy(domains: &[Domain], before: &[u64], after: &[u64]) -> u64 { + domains + .iter() + .zip(before.iter().zip(after)) + .map(|(d, (&b, &a))| { + let range = (((1u128 << 32) * 1_000_000) >> d.shift) as u64; + super::delta(b, a, Some(range)).expect("an MSR count above its range") + }) + .sum() + } + } + /// The zones under `root`, as one line for the log. pub fn listing(root: &Path) -> String { zones(root) @@ -379,7 +507,7 @@ use oracle::Verdict; fn the_rapl_liveness_agrees_with_the_oracle_and_gates_the_counter() { let _one = serial(); let liveness = rapl_liveness(); - let watch = oracle::watch(Path::new(oracle::POWERCAP)); + let watch = oracle::watch_host(); let counter = RaplCounter::new(); let composite = meter::detect_meter(); eprintln!( @@ -412,13 +540,37 @@ fn a_cpu_burn_reads_inside_the_package_only_bracket() { let _one = serial(); let counter = RaplCounter::new(); if !counter.is_available() { - let watch = oracle::watch(Path::new(oracle::POWERCAP)); + let watch = oracle::watch_host(); eprintln!("rapl burn: counter {:?}; oracle {}", counter.liveness(), watch.summary()); assert_ne!(watch.verdict, Verdict::Live, "the oracle saw live packages, yet the counter is not available"); assert!(counter.read().is_err()); return; } let all = oracle::zones(Path::new(oracle::POWERCAP)); + if !all.iter().any(oracle::is_package) { + // No powercap package: the counter reads the MSRs, and so does the oracle, + // outer and inner around the meter's reads. + let domains = oracle::msr::domains().expect("the counter reads the MSRs, so the oracle can"); + let read = || oracle::msr::read_all(&domains).expect("an oracle MSR read"); + let o0 = read(); + let r0 = counter.read().expect("a live counter reads"); + let o1 = read(); + let cpu = oracle::burn(Duration::from_millis(300), Duration::from_secs(10)); + let o2 = read(); + let r1 = counter.read().expect("a live counter reads"); + let o3 = read(); + for r in [&r0, &r1] { + assert_eq!((r.domain, r.provenance), (EnergyDomain::CpuPkg, Provenance::HwShunt), "{r:?}"); + } + let (uj, inner, outer) = + (r1.uj - r0.uj, oracle::msr::energy(&domains, &o1, &o2), oracle::msr::energy(&domains, &o0, &o3)); + let names: Vec<&str> = domains.iter().map(|d| d.name.as_str()).collect(); + eprintln!("rapl burn (MSR {names:?}): {cpu:?} of thread CPU; meter {uj} µJ; oracle inner {inner} outer {outer} µJ"); + assert!(inner > 0, "the domains did not advance over the burn"); + assert!(inner <= uj && uj <= outer, "the figure {uj} µJ is outside the oracle's MSR bracket [{inner}, {outer}]"); + assert!(cpu >= Duration::from_millis(300), "the burn got only {cpu:?} of CPU"); + return; + } let mut cpu = Duration::ZERO; let (b, r0, r1) = oracle::around( oracle::bracket_set(&all),