diff --git a/joule-code/joule-code-rs/crates/joule-code-remediate/src/lib.rs b/joule-code/joule-code-rs/crates/joule-code-remediate/src/lib.rs index e4cb1d709..aafd8c22e 100644 --- a/joule-code/joule-code-rs/crates/joule-code-remediate/src/lib.rs +++ b/joule-code/joule-code-rs/crates/joule-code-remediate/src/lib.rs @@ -99,11 +99,36 @@ struct EnergyTally { attr_uj: u64, windows: u64, attributed_windows: u64, + /// The probes whose readings the attributed windows apportioned, each + /// window's own (`Attribution::source`): what the meter source names. + read_by: BTreeSet<&'static str>, } impl EnergyTally { fn add(&mut self, a: &Attribution) { self.add_window(a.soc_cpu_uj, a.attributed_uj, a.attributed); + if a.attributed { + self.read_by.insert(a.source); + } + } + + /// How the figure was obtained, for [`RunReport::meter_source`]: an + /// attribution names the probes its windows' readings came from, as the + /// meter reported them; an estimate names the meter's hardware sources, + /// `meter_sources` ([`PowerMeter::sources`]), or says it has none. + fn source(&self, meter_sources: &[&str]) -> String { + if self.attributed() { + let read_by: Vec<&str> = self.read_by.iter().copied().collect(); + format!("attributed(ModelBased over {} HwShunt)", read_by.join("+")) + } else if meter_sources.is_empty() { + "estimated(Unaccounted): wall-clock × TDP, no live energy counter on this host".to_string() + } else { + format!( + "estimated(Unaccounted): not every window attributed; machine-wide figure over the \ + metered windows ({})", + meter_sources.join("+") + ) + } } /// The L3 window: its figure, an estimate, never attributed, with the @@ -1056,17 +1081,7 @@ impl Harness { }; let (soc_cpu_uj, attributed_uj, attributed) = (tally.soc_uj, tally.attr_uj, tally.attributed()); let wall = wall_start.elapsed(); - let base_source = if attributed { - "attributed(ModelBased over IOReport HwShunt)".to_string() - } else if meter_sources.is_empty() { - "estimated(Unaccounted): wall-clock × TDP, no live energy counter on this host".to_string() - } else { - format!( - "estimated(Unaccounted): not every window attributed; machine-wide figure over the \ - metered windows ({})", - meter_sources.join("+") - ) - }; + let base_source = tally.source(&meter_sources); let meter_source = match &l3 { Some(l3) => format!("{base_source}; {}", l3.describe()), None => base_source, @@ -1808,6 +1823,12 @@ fn answers() { assert_eq!(answer(), 42); } /// One metered cargo window, as `run_attributed` reports it. fn window(soc_cpu_uj: u64, attributed_uj: u64, attributed: bool) -> Attribution { + let source = if attributed { "ioreport-energy-model" } else { "estimator" }; + window_read_by(soc_cpu_uj, attributed_uj, attributed, source) + } + + /// [`window`], its figure read by the probe named `source`. + fn window_read_by(soc_cpu_uj: u64, attributed_uj: u64, attributed: bool, source: &'static str) -> Attribution { Attribution { soc_cpu_uj, attributed_uj, @@ -1818,9 +1839,32 @@ fn answers() { assert_eq!(answer(), 42); } attributed, // cargo runs its work in descendants: never a per-child reading. child_energy_uj: None, + source, } } + /// An attributed run's meter source names the probes its windows were read + /// by, as the meter reported them — never a counter it did not read. + #[test] + fn an_attributed_run_names_the_probes_that_read_it() { + let mut t = EnergyTally::default(); + t.add(&window_read_by(10_000_000, 2_500_000, true, "rapl-powercap")); + t.add(&window_read_by(30_000_000, 7_500_000, true, "rapl-powercap")); + assert!(t.attributed()); + let source = t.source(&["rapl-powercap", "nvml-total-energy"]); + assert_eq!(source, "attributed(ModelBased over rapl-powercap HwShunt)"); + assert!(!source.contains("IOReport"), "{source}"); + + t.add(&window_read_by(10_000_000, 2_500_000, true, "composite")); + assert_eq!(t.source(&[]), "attributed(ModelBased over composite+rapl-powercap HwShunt)"); + + // An estimate names the meter's sources, not any window's. + let mut est = EnergyTally::default(); + est.add(&window(20_000_000, 0, false)); + assert!(est.source(&["ioreport-energy-model"]).starts_with("estimated(Unaccounted)")); + assert!(est.source(&[]).contains("no live energy counter")); + } + /// A run whose windows were estimated is sealed as an estimate, and never /// as an apportioned zero. This is every macOS 27.0.1 host checked, where /// the meter has no live CPU counter. main sealed 0 J `ModelBased` there, diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-compliance/src/lib.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-compliance/src/lib.rs index 502172733..653e9950c 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-compliance/src/lib.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-compliance/src/lib.rs @@ -17,16 +17,21 @@ //! Every passport links a [`passport::MeasurementMethod`] to a //! [`jouleclaw_energy::Provenance`] tag — measurement honesty travels //! all the way from the platform counter into the compliance report. -//! RAPL and NVML flow through as `Provenance::HwShunt`. So does Apple -//! Silicon IOReport observed live (`MeasurementMethod::IoReportLive`): -//! a vendor energy counter's reading is a real reading, and what -//! decides its grade is provenance — was the counter live for the -//! domain the figure claims — not accuracy. A bare -//! `MeasurementMethod::IoReport` records no such observation; where -//! the CPU energy rail is frozen (macOS 27.0.1 for an unprivileged -//! reader) the figure came from the estimator, so it flows through as -//! `Provenance::Estimator`. A passport may say its inference energy -//! was measured only when its method is graded `HwShunt`. +//! RAPL flows through as `Provenance::HwShunt`. So do NVML's +//! cumulative-energy counter read while judged live +//! (`MeasurementMethod::NvmlLive`) and Apple Silicon IOReport observed +//! live (`MeasurementMethod::IoReportLive`): a vendor energy counter's +//! reading is a real reading, and what decides its grade is provenance +//! — was the counter live for the domain the figure claims — not +//! accuracy. A bare `MeasurementMethod::Nvml` records no such +//! observation; NVML's figure is then its sampled power draw integrated +//! over time, a vendor power model, so it flows through as +//! `Provenance::ModelBased`. A bare `MeasurementMethod::IoReport` +//! records none either; where the CPU energy rail is frozen (macOS +//! 27.0.1 for an unprivileged reader) the figure came from the +//! estimator, so it flows through as `Provenance::Estimator`. A +//! passport may say its inference energy was measured only when its +//! method is graded `HwShunt`. #![forbid(unsafe_code)] #![warn(missing_docs)] diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-compliance/src/passport.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-compliance/src/passport.rs index 62b5d929c..67f59db84 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-compliance/src/passport.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-compliance/src/passport.rs @@ -127,10 +127,26 @@ pub enum MeasurementMethod { /// counter is a hardware reading, graded `HwShunt` like RAPL and NVML: /// its trust gap is provenance, which liveness answers, not accuracy. IoReportLive, - /// NVIDIA Management Library. + /// NVIDIA Management Library, with no record that its cumulative-energy + /// counter was read while judged live. NVML gives a hardware reading only + /// from that counter (`nvmlDeviceGetTotalEnergyConsumption`), and only + /// where `jouleclaw_energy`'s liveness probe judged it live; elsewhere the + /// figure is the driver's sampled power draw integrated over wall time — + /// power × time, a vendor power model — graded `ModelBased`. Use + /// [`Self::NvmlLive`] for a figure read from the live counter. Nvml, - /// Combined GPU (NVML) + CPU (RAPL) measurement. + /// NVIDIA Management Library's cumulative-energy counter, read while it was + /// judged live (`jouleclaw_energy::CounterLiveness::Live`): a hardware + /// reading, graded `HwShunt`. + NvmlLive, + /// Combined GPU (NVML) + CPU (RAPL) measurement, with no record that the + /// NVML part came from its live cumulative-energy counter. A sum is graded + /// by its weakest part, so this is `ModelBased`, as [`Self::Nvml`] is. Use + /// [`Self::NvmlLiveRapl`] where the NVML part was read from the live counter. NvmlRapl, + /// Combined GPU (NVML's cumulative-energy counter, read while judged live) + + /// CPU (RAPL) measurement: both parts hardware readings, graded `HwShunt`. + NvmlLiveRapl, /// Derived from billing data or hardware specs — not measured. Estimated, } @@ -140,12 +156,27 @@ impl MeasurementMethod { /// Pure-Rust translation from the JouleClaw energy contract. pub fn provenance(&self) -> Provenance { match self { - Self::Rapl | Self::Nvml | Self::NvmlRapl | Self::IoReportLive => Provenance::HwShunt, + Self::Rapl | Self::NvmlLive | Self::NvmlLiveRapl | Self::IoReportLive => Provenance::HwShunt, + Self::Nvml | Self::NvmlRapl => Provenance::ModelBased, Self::IoReport => Provenance::Estimator, Self::Estimated => Provenance::Estimator, } } + /// The method for a figure NVML produced, from the grade of the readings + /// it was taken from (`jouleclaw_energy::EnergyReading::provenance`): + /// [`Self::NvmlLive`] only for readings of the live cumulative-energy + /// counter (`HwShunt`), [`Self::Nvml`] for the power-draw integral + /// (`ModelBased`), and [`Self::Estimated`] for an estimate. The method's + /// grade is the readings' grade, never better. + pub fn nvml_from_reading(provenance: Provenance) -> Self { + match provenance { + Provenance::HwShunt => Self::NvmlLive, + Provenance::ModelBased => Self::Nvml, + Provenance::Estimator => Self::Estimated, + } + } + /// Whether a figure obtained this way is a hardware reading from a live /// counter — the only grade under which inference energy may be called /// measured. @@ -161,7 +192,9 @@ impl fmt::Display for MeasurementMethod { Self::IoReport => write!(f, "IOReport"), Self::IoReportLive => write!(f, "IOReport (live)"), Self::Nvml => write!(f, "NVML"), + Self::NvmlLive => write!(f, "NVML (live)"), Self::NvmlRapl => write!(f, "NVML+RAPL"), + Self::NvmlLiveRapl => write!(f, "NVML (live)+RAPL"), Self::Estimated => write!(f, "Estimated"), } } @@ -520,7 +553,7 @@ impl PassportBuilder { /// energy measured (`inference_energy_measured`) when the energy /// profile's method is not graded as a live hardware reading /// ([`MeasurementMethod::is_live_hardware_reading`]): a bare - /// `IoReport`, or `Estimated`. + /// `IoReport` or `Nvml`, `NvmlRapl`, or `Estimated`. pub fn build(self) -> Result { let passport = AiServicePassport { passport_version: self.passport_version, @@ -639,7 +672,7 @@ mod tests { EnergyProfile { joules_per_input_token: 0.001, joules_per_output_token: 0.003, - measurement_method: MeasurementMethod::NvmlRapl, + measurement_method: MeasurementMethod::NvmlLiveRapl, sample_size: 10_000, benchmark_date: Utc::now(), idle_power_watts: 75.0, @@ -736,10 +769,19 @@ mod tests { #[test] fn measurement_method_maps_to_provenance() { assert_eq!(MeasurementMethod::Rapl.provenance(), Provenance::HwShunt); - assert_eq!(MeasurementMethod::Nvml.provenance(), Provenance::HwShunt); + // NVML is a hardware reading only from its cumulative-energy counter + // read while judged live. Without that record the figure may be the + // sampled power draw integrated over time — power × time, a vendor + // power model — and a sum is no better than its weakest part. + assert_eq!(MeasurementMethod::NvmlLive.provenance(), Provenance::HwShunt); + assert_eq!( + MeasurementMethod::NvmlLiveRapl.provenance(), + Provenance::HwShunt + ); + assert_eq!(MeasurementMethod::Nvml.provenance(), Provenance::ModelBased); assert_eq!( MeasurementMethod::NvmlRapl.provenance(), - Provenance::HwShunt + Provenance::ModelBased ); // IOReport read while its CPU rail was live is a hardware reading, the // grade of RAPL and NVML. A bare IoReport records no liveness: where the @@ -758,6 +800,23 @@ mod tests { ); } + /// A figure NVML produced is graded by the readings it was taken from: the + /// live cumulative-energy counter (`HwShunt`) is `NvmlLive`, the power-draw + /// integral (`ModelBased`) is `Nvml`, and the passport carries the readings' + /// own grade, never a better one. + #[test] + fn nvml_method_follows_the_reading_it_came_from() { + for (read, method) in [ + (Provenance::HwShunt, MeasurementMethod::NvmlLive), + (Provenance::ModelBased, MeasurementMethod::Nvml), + (Provenance::Estimator, MeasurementMethod::Estimated), + ] { + let m = MeasurementMethod::nvml_from_reading(read); + assert_eq!(m, method, "a {read} NVML reading"); + assert_eq!(m.provenance(), read, "a {read} NVML reading graded {} in the passport", m.provenance()); + } + } + #[test] fn build_passport_succeeds_with_all_fields() { let p = build_sample("svc-1", 0.35, EnergyLabel::A); @@ -785,14 +844,21 @@ mod tests { /// A passport may call its inference energy measured only when its method is /// graded as a live hardware reading. A bare IOReport records no liveness, so - /// paired with "measured" it is refused, as an estimate is; IOReport observed - /// live, RAPL and NVML are accepted. Unclaimed, every method builds. + /// paired with "measured" it is refused, as an estimate is; a bare NVML (or + /// NVML+RAPL) may be the power-draw integral, a model, and is refused too. + /// IOReport observed live, RAPL and NVML's live counter are accepted. + /// Unclaimed, every method builds. #[test] fn a_measured_claim_needs_a_live_hardware_reading() { - for method in [MeasurementMethod::IoReport, MeasurementMethod::Estimated] { + for (method, grade) in [ + (MeasurementMethod::IoReport, Provenance::Estimator), + (MeasurementMethod::Estimated, Provenance::Estimator), + (MeasurementMethod::Nvml, Provenance::ModelBased), + (MeasurementMethod::NvmlRapl, Provenance::ModelBased), + ] { match build_with(method, true) { Err(PassportError::MeasuredWithoutHardwareReading { method: m, provenance }) => { - assert_eq!((m, provenance), (method, Provenance::Estimator)); + assert_eq!((m, provenance), (method, grade)); } other => panic!("{method} paired with a measured claim was not refused: {other:?}"), } @@ -801,8 +867,8 @@ mod tests { for method in [ MeasurementMethod::IoReportLive, MeasurementMethod::Rapl, - MeasurementMethod::Nvml, - MeasurementMethod::NvmlRapl, + MeasurementMethod::NvmlLive, + MeasurementMethod::NvmlLiveRapl, ] { let p = build_with(method, true).unwrap_or_else(|e| panic!("{method} refused: {e}")); assert!(p.compliance.eu_ai_act.inference_energy_measured); @@ -850,7 +916,9 @@ mod tests { assert_eq!(format!("{}", MeasurementMethod::IoReport), "IOReport"); assert_eq!(format!("{}", MeasurementMethod::IoReportLive), "IOReport (live)"); assert_eq!(format!("{}", MeasurementMethod::Nvml), "NVML"); + assert_eq!(format!("{}", MeasurementMethod::NvmlLive), "NVML (live)"); assert_eq!(format!("{}", MeasurementMethod::NvmlRapl), "NVML+RAPL"); + assert_eq!(format!("{}", MeasurementMethod::NvmlLiveRapl), "NVML (live)+RAPL"); assert_eq!(format!("{}", MeasurementMethod::Estimated), "Estimated"); } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/span.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/span.rs index 9998d45bb..968b3c634 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/span.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/src/span.rs @@ -22,6 +22,17 @@ //! The CPU half reads every thread of this process, so a span counts whatever //! else this process runs at the same time; the GPU half every GPU client of //! this process. +//! +//! **The order of the readings.** A GPU snapshot can wait for the rail to +//! republish (up to [`crate::process_gpu::GPU_IDLE_WINDOW`] on a quiet GPU, +//! [`crate::process_gpu::RAIL_EDGE_WAIT`] on a busy one), so the snapshots are +//! the outermost readings at both ends, the CPU reading the innermost, and the +//! host meter's reading between them: at the start the GPU snapshot, then the +//! host meter, then the CPU; at the end the reverse. The CPU window then holds +//! neither the snapshots' CPU time nor the host meter's reads, and the +//! fallback's window is the work's own (and the CPU reads), never the +//! snapshots' waits — a time-based estimate over those would bill a span of a +//! few microseconds of work for up to two seconds of the meter's waiting. use std::time::Instant; @@ -120,6 +131,43 @@ pub struct SpanStart { at: Instant, } +/// The readings taken at a span's end ([`ProcessSpanMeter::finish`]). +struct SpanEnd { + cpu_nj: Option, + gpu: Result, + fallback: Result, +} + +/// The three readings a span takes at each end. A seam: the meter takes them +/// from its counters, and the tests from a clock of their own, so the order +/// [`read_start`] and [`read_end`] take them in is tested without hardware. +trait SpanReads { + /// This process's CPU energy, nJ, where its counter is live. + fn cpu_nj(&self) -> Option; + /// The GPU rail(s) and clients. May wait for the rail to republish. + fn gpu_snapshot(&self) -> Result; + /// The host meter's cumulative reading. + fn fallback_read(&self) -> Result; +} + +/// A span's starting readings, outermost first: the GPU snapshot, then the host +/// meter, then the CPU counter (see the module docs). +fn read_start(r: &impl SpanReads) -> SpanStart { + let gpu = r.gpu_snapshot(); + let fallback = r.fallback_read(); + let cpu_nj = r.cpu_nj(); + SpanStart { cpu_nj, gpu, fallback, at: Instant::now() } +} + +/// A span's ending readings, innermost first: the CPU counter, then the host +/// meter, then the GPU snapshot — [`read_start`]'s order reversed. +fn read_end(r: &impl SpanReads) -> SpanEnd { + let cpu_nj = r.cpu_nj(); + let fallback = r.fallback_read(); + let gpu = r.gpu_snapshot(); + SpanEnd { cpu_nj, gpu, fallback } +} + impl SpanStart { /// When the span started. pub fn at(&self) -> Instant { @@ -232,24 +280,22 @@ impl ProcessSpanMeter { self.fallback.active_name() } - /// Take the span's starting readings. The GPU snapshot comes first and the - /// CPU reading last (and the reverse in [`Self::finish`]), so the CPU - /// time the snapshots spend — a wait for the GPU rail to republish, and - /// IOReport samples — stays outside the CPU window. + /// Take the span's starting readings: the GPU snapshot first, then the host + /// meter, and the CPU reading last (the reverse in [`Self::finish`]). So + /// the CPU time the snapshots spend — a wait for the GPU rail to + /// republish, and IOReport samples — and the host meter's read stay + /// outside the CPU window, and the snapshots' waits outside the host + /// meter's window. pub fn start(&self) -> SpanStart { - let fallback = self.fallback.read().map_err(|e| EnergyError::Platform(e.to_string())); - let gpu = self.gpu.snapshot(); - SpanStart { cpu_nj: self.cpu_nj(), gpu, fallback, at: Instant::now() } + read_start(self) } /// End the span begun at `start`: its energy, or why there is none. pub fn finish(&self, start: SpanStart, uses: SpanUses) -> Result { - let cpu_end = self.cpu_nj(); - let gpu_end = self.gpu.snapshot(); - let fallback_end = self.fallback.read().map_err(|e| EnergyError::Platform(e.to_string())); - let cpu = self.cpu_window(start.cpu_nj, cpu_end); - let gpu = start.gpu.and_then(|s| gpu_end.and_then(|e| self.gpu.share(&s, &e))); - combine(cpu, gpu, uses, self.fallback.active_name(), (start.fallback, fallback_end)) + let end = read_end(self); + let cpu = self.cpu_window(start.cpu_nj, end.cpu_nj); + let gpu = start.gpu.and_then(|s| end.gpu.and_then(|e| self.gpu.share(&s, &e))); + combine(cpu, gpu, uses, self.fallback.active_name(), (start.fallback, end.fallback)) } /// Run `work`, which ran on what `uses` says, and meter it. @@ -259,17 +305,6 @@ impl ProcessSpanMeter { (out, self.finish(start, uses)) } - fn cpu_nj(&self) -> Option { - #[cfg(target_os = "macos")] - { - self.cpu.is_available().then(crate::process::energy_nj).flatten() - } - #[cfg(not(target_os = "macos"))] - { - None - } - } - fn cpu_window(&self, start: Option, end: Option) -> Result { #[cfg(target_os = "macos")] { @@ -286,10 +321,32 @@ impl ProcessSpanMeter { } } +impl SpanReads for ProcessSpanMeter { + fn cpu_nj(&self) -> Option { + #[cfg(target_os = "macos")] + { + self.cpu.is_available().then(crate::process::energy_nj).flatten() + } + #[cfg(not(target_os = "macos"))] + { + None + } + } + + fn gpu_snapshot(&self) -> Result { + self.gpu.snapshot() + } + + fn fallback_read(&self) -> Result { + self.fallback.read().map_err(|e| EnergyError::Platform(e.to_string())) + } +} + #[cfg(test)] mod tests { use super::*; use crate::EnergyDomain; + use std::cell::Cell; fn share(uj: u64) -> GpuShare { GpuShare { uj, self_ns: 1, rail_uj: Some(uj), all_ns: Some(1) } @@ -361,6 +418,75 @@ mod tests { assert!(none.to_string().contains("process CPU") && none.to_string().contains("`x`"), "{none}"); } + /// A host whose readings run on a clock of the test's own, in µs: the host + /// meter is a time-based estimate at 1 W (1 µJ a µs) and the CPU counter + /// counts 1 W of this process's CPU (1,000 nJ a µs). A GPU snapshot waits + /// `snapshot_wait_us` for the rail and gives no figure, and a host meter + /// read costs `read_us`. + struct Clocked { + now_us: Cell, + snapshot_wait_us: u64, + read_us: u64, + } + + impl Clocked { + fn pass(&self, us: u64) { + self.now_us.set(self.now_us.get() + us); + } + } + + impl SpanReads for Clocked { + fn cpu_nj(&self) -> Option { + Some(self.now_us.get() * 1_000) + } + fn gpu_snapshot(&self) -> Result { + self.pass(self.snapshot_wait_us); + Err(EnergyError::NotCurrent) + } + fn fallback_read(&self) -> Result { + let uj = self.now_us.get(); + self.pass(self.read_us); + Ok(EnergyReading { uj, timestamp_ns: 0, domain: EnergyDomain::SocTotal, provenance: Provenance::Estimator }) + } + } + + /// A fallback span's estimate scales with the work's own duration, never + /// with the GPU snapshots' waits: the host meter's window opens after the + /// starting snapshot and closes before the ending one. The CPU window stays + /// the innermost, holding neither the snapshots nor the host meter's reads. + /// (main read the host meter outside both snapshots, so an ~11 µs serve + /// was billed for up to two rail waits.) + #[test] + fn a_fallback_span_is_the_works_window_not_the_snapshots_waits() { + for snapshot_wait_us in [0, 100_000, 1_000_000] { + for work_us in [11, 5_000, 250_000] { + let host = Clocked { now_us: Cell::new(1_000), snapshot_wait_us, read_us: 3 }; + let start = read_start(&host); + host.pass(work_us); + let end = read_end(&host); + + let cpu = end.cpu_nj.unwrap() - start.cpu_nj.unwrap(); + assert_eq!(cpu, work_us * 1_000, "the CPU window is the work's alone (wait {snapshot_wait_us} µs)"); + + let e = combine( + Err(EnergyError::NotAdvanced), + Err(EnergyError::NotCurrent), + SpanUses::CpuGpu, + "est", + (start.fallback, end.fallback), + ) + .unwrap(); + assert!(matches!(e.source, SpanSource::Fallback { .. }), "{e:?}"); + // The work, plus at most the host meter's own read at one end. + assert!( + (work_us..=work_us + host.read_us).contains(&e.uj), + "{} µJ for {work_us} µs of work, snapshots waiting {snapshot_wait_us} µs each", + e.uj + ); + } + } + } + #[test] fn the_weaker_grade() { use Provenance::*; diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/process_cpu.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/process_cpu.rs index 75f0d5ce0..079016537 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/process_cpu.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-energy/tests/process_cpu.rs @@ -71,24 +71,46 @@ fn the_process_counter_meters_this_process_and_nothing_else() { assert_eq!((r.domain, r.provenance), (EnergyDomain::ProcessCpu, Provenance::HwShunt)); assert_eq!((counter.domain(), counter.provenance(), counter.name()), (EnergyDomain::ProcessCpu, Provenance::HwShunt, "process-cpu")); - // 2. 200 ms of one thread's CPU time reads 0.1 to 20 J. The floor is - // 0.1 J, not 0.2, because what 200 ms of CPU costs depends on the core - // class that ran it: 0.18-1.24 J on an M5 Max, and 0.171 J on a busy - // host (the oracle agreeing). The sleep below must read under 0.1 J, - // so a counter that reads wall time at a fixed power still fails one. + // 2. 200 ms of one thread's CPU time is a reading, and the reading is this + // process's own energy over the burn: no more than an oracle's window + // around it, and most of it. No joule figure is asserted — what 200 ms + // of CPU costs depends on the core class that ran it (0.099 J on an + // efficiency core at a load average of 114; 0.18-1.24 J otherwise on + // an M5 Max), so a fixed floor fails on the host's load, not on the + // meter. The oracle sleeps before each read: a thread that leaves its + // core has its energy folded into the process's count, so the oracle's + // window holds the whole burn. The counter's own reads may each miss + // up to a scheduling quantum of the reading thread, hence "most of". + // SAFETY: `getpid` has no preconditions. + let me = unsafe { libc::getpid() }; + std::thread::sleep(Duration::from_millis(2)); + let oracle_before = energy_nj_of(me); let t = Instant::now(); let ((), burned) = counter.measure(|| burn(Duration::from_millis(200))); let wall = t.elapsed(); + std::thread::sleep(Duration::from_millis(2)); + let oracle_nj = energy_nj_of(me) - oracle_before; let uj = burned.expect("a 200 ms burn is a reading"); - println!("process-cpu: 200 ms one-thread CPU burn = {:.3} J over {:.0} ms wall ({:.2} J per CPU-second)", uj as f64 / 1e6, wall.as_secs_f64() * 1e3, uj as f64 / 1e6 / 0.2); - assert!((100_000..=20_000_000).contains(&uj), "{uj} µJ for 200 ms of one thread's CPU"); + println!( + "process-cpu: 200 ms one-thread CPU burn = {:.3} J over {:.0} ms wall ({:.2} J per CPU-second); oracle window {:.3} J", + uj as f64 / 1e6, + wall.as_secs_f64() * 1e3, + uj as f64 / 1e6 / 0.2, + oracle_nj as f64 / 1e9 + ); + assert!(uj > 0, "a burn read as zero"); + assert!(uj * 1_000 <= oracle_nj + 1_000, "the burn ({uj} µJ) exceeds the oracle's window around it ({oracle_nj} nJ)"); + assert!(uj * 1_000 * 4 >= oracle_nj * 3, "the burn ({uj} µJ) is under three quarters of the oracle's window ({oracle_nj} nJ)"); - // 3. A 200 ms sleep reads under 0.1 J — or, if nothing of this process ran, - // is not a reading at all; never a zero. + // 3. A 200 ms sleep reads under a tenth of the burn — or, if nothing of + // this process ran, is not a reading at all; never a zero. A counter + // that reads wall time at a fixed power reads the sleep at 200 ms over + // the burn's wall time of the burn's figure — a sixth of it at the + // heaviest load seen here (1.26 s of wall for the burn) — and fails. let ((), slept) = counter.measure(|| std::thread::sleep(Duration::from_millis(200))); println!("process-cpu: 200 ms sleep = {slept:?} µJ"); match slept { - Ok(uj) => assert!(uj < 100_000, "{uj} µJ for a sleep"), + Ok(slept_uj) => assert!(slept_uj * 10 < uj, "{slept_uj} µJ for a sleep, against {uj} µJ for the burn"), Err(EnergyError::NotAdvanced) => {} Err(e) => panic!("a sleep window failed: {e}"), } @@ -136,7 +158,8 @@ fn the_process_counter_meters_this_process_and_nothing_else() { ); assert!(child_burned_nj >= 200_000_000, "the child did not burn: {child_burned_nj} nJ"); match parent { - Ok(uj) => assert!(uj < 100_000, "{uj} µJ in this process while only the child worked"), + // Under a tenth of what the child burned: this process only polled. + Ok(uj) => assert!(uj * 1_000 * 10 < child_burned_nj, "{uj} µJ in this process while only the child worked ({child_burned_nj} nJ)"), Err(EnergyError::NotAdvanced) => {} Err(e) => panic!("the parent window failed: {e}"), } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-fabric/src/lib.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-fabric/src/lib.rs index 08121a002..c700e9d30 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-fabric/src/lib.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-fabric/src/lib.rs @@ -53,6 +53,31 @@ pub use detect::detect_local; pub use place::{place, OpClass, OpDesc, Placement}; pub use reach::reach_remote; +/// The environment variable that names this machine's +/// calibration file, read and written. +pub const CALIBRATION_FILE_ENV: &str = "JOULECLAW_FABRIC_CALIBRATION"; + +/// This machine's calibration file: [`CALIBRATION_FILE_ENV`] +/// where it is set, else `$HOME/.jouleclaw/fabric-calibration.json`. +/// The one place the path is decided: +/// [`Fabric::apply_calibration_file`] reads it, and the +/// calibration writers (loader-gguf's `fabric_calibrate` and +/// `mpp_energy`) write it, so a file written under the variable +/// is the file the placer loads. +pub fn calibration_file_path() -> std::path::PathBuf { + calibration_file_path_from(std::env::var(CALIBRATION_FILE_ENV).ok(), std::env::var("HOME").ok()) +} + +/// [`calibration_file_path`] for given values of the variable +/// and `HOME`, so the rule is tested without touching the +/// process environment. +fn calibration_file_path_from(var: Option, home: Option) -> std::path::PathBuf { + match var { + Some(path) => path.into(), + None => format!("{}/.jouleclaw/fabric-calibration.json", home.unwrap_or_default()).into(), + } +} + // ───────────────────────────────────────────────────────────────────── // Members // ───────────────────────────────────────────────────────────────────── @@ -249,19 +274,15 @@ impl Fabric { } /// Overlay this machine's µJ figures from the calibration - /// file written by `fabric_calibrate` (loader-gguf). Path - /// from `JOULECLAW_FABRIC_CALIBRATION`, defaulting to - /// `~/.jouleclaw/fabric-calibration.json`. Calibrated - /// figures replace seeded entries — placement should run on - /// this machine's numbers, not a table written for another - /// one — at the grade each row declares (see + /// file written by `fabric_calibrate` (loader-gguf), at + /// [`calibration_file_path`]: `JOULECLAW_FABRIC_CALIBRATION`, + /// defaulting to `~/.jouleclaw/fabric-calibration.json`. + /// Calibrated figures replace seeded entries — placement + /// should run on this machine's numbers, not a table written + /// for another one — at the grade each row declares (see /// [`Self::apply_calibration`]). Returns the rows refused. pub fn apply_calibration_file(&mut self) -> Vec { - let path = std::env::var("JOULECLAW_FABRIC_CALIBRATION").unwrap_or_else(|_| { - let home = std::env::var("HOME").unwrap_or_default(); - format!("{home}/.jouleclaw/fabric-calibration.json") - }); - self.apply_calibration_path(std::path::Path::new(&path)) + self.apply_calibration_path(&calibration_file_path()) } /// [`Self::apply_calibration`] over the rows in the JSON @@ -481,6 +502,16 @@ mod tests { assert_eq!(serde_json::from_str::(&json).unwrap(), r); } + /// The calibration file is the variable's path where it is + /// set, whatever `HOME` is, and `~/.jouleclaw`'s otherwise. + #[test] + fn the_calibration_file_is_the_variables_where_it_is_set() { + let set = calibration_file_path_from(Some("/scratch/cal.json".into()), Some("/home/u".into())); + assert_eq!(set, std::path::PathBuf::from("/scratch/cal.json")); + let unset = calibration_file_path_from(None, Some("/home/u".into())); + assert_eq!(unset, std::path::PathBuf::from("/home/u/.jouleclaw/fabric-calibration.json")); + } + #[test] fn reachable_members_never_serve_without_detection() { let fabric = Fabric::discover(); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-harness-econ/examples/local_model_econ.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-harness-econ/examples/local_model_econ.rs index 11752f391..4c6fd2201 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-harness-econ/examples/local_model_econ.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-harness-econ/examples/local_model_econ.rs @@ -20,8 +20,8 @@ //! not move with anything else the host runs. Where it is not live, and off macOS, //! it is the host meter (`detect_meter`): the IOReport counter where its CPU rail //! is live, else a load-average × power-band ESTIMATOR (`Provenance::Estimator`). -//! Each dispatch prints which meter ran and whether its joules were measured or -//! estimated. Decode wall-time and token counts are exact measurements. A LOCAL +//! Each dispatch prints which meter ran and whether its joules were measured, +//! modeled or estimated. Decode wall-time and token counts are exact measurements. A LOCAL //! model is priced energy-only (no per-call API dollars). //! //! Run (defaults to a Nemotron-3-Nano-4B snapshot if present): diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-harness-econ/src/lib.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-harness-econ/src/lib.rs index 2c9dcee94..dcc3972c4 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-harness-econ/src/lib.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-harness-econ/src/lib.rs @@ -570,15 +570,18 @@ pub fn span_joules(span: Result) -> Result<(f64, Proven } } -/// How a report says a metered figure at `provenance` was obtained: `"measured"` for a -/// reading (a hardware shunt, a live vendor energy counter, or a vendor energy model), -/// `"estimated"` for an estimate. The local-model examples print each dispatch's joules -/// through it, beside the meter that ran, so an estimate — what -/// `jouleclaw_energy::meter::detect_meter` yields on Apple Silicon whose IOReport CPU -/// rail does not advance — is never called a measurement. +/// How a report says a metered figure at `provenance` was obtained, by its grade: +/// `"measured"` for a hardware reading (a shunt, or a live vendor energy counter), +/// `"modeled"` for a vendor energy model's figure (power × time, say), `"estimated"` +/// for an estimate. Only a hardware reading is called measured. The local-model +/// examples print each dispatch's joules through it, beside the meter that ran, so an +/// estimate — what `jouleclaw_energy::meter::detect_meter` yields on Apple Silicon +/// whose IOReport CPU rail does not advance — or a model's figure is never called a +/// measurement. pub fn joules_obtained(provenance: Provenance) -> &'static str { match provenance { - Provenance::HwShunt | Provenance::ModelBased => "measured", + Provenance::HwShunt => "measured", + Provenance::ModelBased => "modeled", Provenance::Estimator => "estimated", } } @@ -645,7 +648,6 @@ mod tests { assert!(why.contains("only part"), "{why}"); let why = span_joules(Err(EnergyError::GpuClientVanished { pid: 7, client: 1 })).unwrap_err(); assert!(why.contains("no figure"), "{why}"); - assert_eq!(joules_obtained(Provenance::ModelBased), "measured"); } /// A run that only ever hits a compiled skill is billed the skill's @@ -694,10 +696,12 @@ mod tests { assert_eq!(report.energy_provenance, Provenance::Estimator, "a skill's constant reported as a reading"); } + /// Only a hardware reading is called measured: a model's figure is "modeled", + /// an estimate "estimated". #[test] - fn an_estimate_is_never_reported_as_measured() { + fn an_estimate_or_a_model_is_never_reported_as_measured() { assert_eq!(joules_obtained(Provenance::Estimator), "estimated"); - assert_eq!(joules_obtained(Provenance::ModelBased), "measured"); + assert_eq!(joules_obtained(Provenance::ModelBased), "modeled"); assert_eq!(joules_obtained(Provenance::HwShunt), "measured"); } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-llm/src/backend.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-llm/src/backend.rs index 83d65561c..dde5ce2c1 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-llm/src/backend.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-llm/src/backend.rs @@ -5,6 +5,7 @@ //! with `"echo: "` — useful for tests, conformance vectors, and as a //! reference for the smallest possible compliant backend. +use jouleclaw_energy::Provenance; use serde::{Deserialize, Serialize}; /// Default "typical joules per call" for backends that do not override it. @@ -145,11 +146,11 @@ pub struct LlmResponse { pub input_tokens: u32, /// Token usage — output side. pub output_tokens: u32, - /// Energy reported by the backend, if it can measure or model the - /// upstream call's joule cost. `Some(j)` when the value came from a - /// real meter (`HwShunt` or `ModelBased`); `None` when the backend - /// has no meter — the tier then falls back to - /// [`LlmBackend::typical_joules_per_call`]. + /// The backend's own figure for the upstream call's joule cost, if it + /// has one, graded by [`LlmBackend::energy_provenance`]: a reading, a + /// calibrated model's figure, or an estimate, as the backend declares. + /// `None` when the backend has no figure for the call — the tier then + /// falls back to [`LlmBackend::typical_joules_per_call`], an estimate. pub energy_joules: Option, } @@ -197,9 +198,11 @@ pub enum LlmError { /// Backends MUST be `Send + Sync` so the tier (which is `Send`) can carry /// a reference behind a generic parameter. /// -/// Backends SHOULD report `energy_joules` honestly: `Some(j)` for real -/// measurements, `None` when no meter exists. Lying here breaks the -/// JouleClaw provenance contract. +/// Backends SHOULD report `energy_joules` honestly: `Some(j)` for the +/// backend's own figure for the call, `None` when it has none, and +/// declare how that figure was obtained through +/// [`energy_provenance`](Self::energy_provenance) — never better than it +/// was. Lying here breaks the JouleClaw provenance contract. pub trait LlmBackend: Send + Sync { /// Human-readable model name surfaced in answers and receipts. SHOULD /// be stable across calls for the same configuration. @@ -217,6 +220,17 @@ pub trait LlmBackend: Send + Sync { fn typical_joules_per_call(&self) -> f64 { DEFAULT_TYPICAL_JOULES } + + /// How the figures this backend reports in + /// [`LlmResponse::energy_joules`] were obtained (JCP SPEC §5): `HwShunt` + /// for a hardware reading over the call (a live energy counter), + /// `ModelBased` for a per-operation model calibrated on the host that + /// ran it, `Estimator` for anything else — a constant, a catalog cost, an + /// uncalibrated model. Defaults to `Estimator`: a figure is graded a + /// reading or a model only when the backend that produced it says so. + fn energy_provenance(&self) -> Provenance { + Provenance::Estimator + } } /// A deterministic, dependency-free backend that echoes the prompt back diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-llm/src/lib.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-llm/src/lib.rs index 3180d923a..c2d532f7a 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-llm/src/lib.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-llm/src/lib.rs @@ -36,11 +36,14 @@ //! ## Energy provenance //! //! Backends MUST set [`LlmResponse::energy_joules`] honestly: `Some(j)` -//! when the value came from a real upstream meter (HwShunt / -//! ModelBased) and `None` when no meter exists. On `None` the tier -//! falls back to [`LlmBackend::typical_joules_per_call`] — an -//! [`jouleclaw_energy::Provenance::Estimator`]-class number — and tags -//! the `Answer` accordingly via [`LlmCheapTier::report_provenance`]. +//! for the backend's own figure for the call and `None` when it has none, +//! and declare how its figures were obtained through +//! [`LlmBackend::energy_provenance`] — `HwShunt` for a reading over the +//! call, `ModelBased` for a model calibrated on the host, and by default +//! [`jouleclaw_energy::Provenance::Estimator`]: a number with nothing +//! declared behind it is an estimate. On `None` the tier falls back to +//! [`LlmBackend::typical_joules_per_call`] — an `Estimator`-class number. +//! [`LlmCheapTier::report_provenance`] gives the grade either way. #![forbid(unsafe_code)] diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-llm/src/tier.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-llm/src/tier.rs index 79d4a2d1b..7a567760d 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-llm/src/tier.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-llm/src/tier.rs @@ -134,13 +134,15 @@ impl LlmCheapTier { /// answer with given a backend response. Exposed so callers can pre- /// flight provenance decisions without dispatching. /// - /// * `Some(_)` energy from the backend → [`Provenance::ModelBased`] - /// (the backend has at least a vendor model — we cannot promise - /// HwShunt from this layer without inspecting the meter) - /// * `None` energy from the backend → [`Provenance::Estimator`] - pub fn report_provenance(response: &LlmResponse) -> Provenance { + /// * `Some(_)` energy from the backend → the grade the backend declares + /// for its figures ([`LlmBackend::energy_provenance`]), which is + /// [`Provenance::Estimator`] unless the backend says it read or + /// modelled the call: a bare number is not evidence of a meter + /// * `None` energy from the backend → [`Provenance::Estimator`] (the + /// backend's typical per-call constant) + pub fn report_provenance(&self, response: &LlmResponse) -> Provenance { match response.energy_joules { - Some(_) => Provenance::ModelBased, + Some(_) => self.backend.energy_provenance(), None => Provenance::Estimator, } } @@ -706,32 +708,71 @@ mod tests { assert!((est.confidence_floor - 1.0).abs() < f32::EPSILON); } - #[test] - fn report_provenance_model_based_when_energy_reported() { - let resp = LlmResponse { + fn response_with(energy_joules: Option) -> LlmResponse { + LlmResponse { text: "x".into(), finish_reason: FinishReason::Stop, input_tokens: 0, output_tokens: 0, - energy_joules: Some(0.1), - }; - assert_eq!( - LlmCheapTier::::report_provenance(&resp), - Provenance::ModelBased, - ); + energy_joules, + } } + /// A backend that reports a fixed per-call figure — no reading, no model — + /// and declares nothing about it, as a fixed-reply test backend does. + struct ConstantJoules; + impl LlmBackend for ConstantJoules { + fn model_name(&self) -> &str { + "constant" + } + fn complete(&self, _: &LlmRequest) -> Result { + Ok(response_with(Some(2.0))) + } + } + + /// A backend that declares its figures a grade. + struct Declares(Provenance); + impl LlmBackend for Declares { + fn model_name(&self) -> &str { + "declares" + } + fn complete(&self, _: &LlmRequest) -> Result { + Ok(response_with(Some(0.1))) + } + fn energy_provenance(&self) -> Provenance { + self.0 + } + } + + /// A figure is graded by how its backend says it was obtained. A bare + /// number with nothing declared behind it — a constant 2 J a call — is an + /// estimate, never `ModelBased`. + #[test] + fn a_reported_figure_takes_the_grade_its_backend_declares() { + let t = LlmCheapTier::new(ConstantJoules); + let resp = t.backend().complete(&LlmRequest::from_prompt("q", 8)).unwrap(); + assert_eq!(resp.energy_joules, Some(2.0)); + assert_eq!(t.report_provenance(&resp), Provenance::Estimator); + assert_eq!( + LlmCheapTier::new(EchoBackend::new()).report_provenance(&response_with(Some(0.1))), + Provenance::Estimator, + ); + for grade in [Provenance::HwShunt, Provenance::ModelBased, Provenance::Estimator] { + let t = LlmCheapTier::new(Declares(grade)); + assert_eq!(t.report_provenance(&response_with(Some(0.1))), grade); + } + } + + /// With no figure from the backend the tier bills its typical constant, + /// an estimate, whatever the backend declares for figures it reports. #[test] fn report_provenance_estimator_when_no_energy() { - let resp = LlmResponse { - text: "x".into(), - finish_reason: FinishReason::Stop, - input_tokens: 0, - output_tokens: 0, - energy_joules: None, - }; assert_eq!( - LlmCheapTier::::report_provenance(&resp), + LlmCheapTier::new(EchoBackend::new()).report_provenance(&response_with(None)), + Provenance::Estimator, + ); + assert_eq!( + LlmCheapTier::new(Declares(Provenance::HwShunt)).report_provenance(&response_with(None)), Provenance::Estimator, ); } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/fabric_calibrate.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/fabric_calibrate.rs index f9c4c5289..a9f170eb1 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/fabric_calibrate.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/fabric_calibrate.rs @@ -4,7 +4,9 @@ //! local member — NEON, AMX (Accelerate), Metal GPU //! (resident-weight MPS) — under an energy meter, and //! writes the measured µJ/GFLOP table the placer loads at -//! discovery (`~/.jouleclaw/fabric-calibration.json`). +//! discovery (`jouleclaw_fabric::calibration_file_path`: +//! `JOULECLAW_FABRIC_CALIBRATION` where it is set, else +//! `~/.jouleclaw/fabric-calibration.json`). //! //! Two meters, chosen as the live autotuner chooses //! (`fabric_dispatch::tune_shape_live`): @@ -112,9 +114,9 @@ fn main() { Shape { label: "prefill-batch (m=16)", m: 16, n: 4096, k: 2304 }, ]; - let home = std::env::var("HOME").unwrap_or_default(); - let dir = format!("{home}/.jouleclaw"); - let path = format!("{dir}/fabric-calibration.json"); + // The file the placer loads: the variable's where it is set. + let file = jouleclaw_fabric::calibration_file_path(); + let path = file.display().to_string(); let counter = AppleIoReportCounter::new(); let meter = if counter.is_available() { @@ -288,14 +290,16 @@ fn main() { } // Every row of a member this example measures is replaced: by // its row from this run, or by none. Other members' rows stay. - let existing: Vec = std::fs::read(&path) + let existing: Vec = std::fs::read(&file) .ok() .and_then(|b| serde_json::from_slice(&b).ok()) .unwrap_or_default(); let written: Vec = latest.keys().cloned().collect(); let rows = merge_calibration(existing, &FABRIC_CALIBRATE_MEMBERS, latest.into_values().collect()); - std::fs::create_dir_all(&dir).expect("mkdir"); - std::fs::write(&path, serde_json::to_vec_pretty(&rows).unwrap()).expect("write"); + if let Some(dir) = file.parent().filter(|d| !d.as_os_str().is_empty()) { + std::fs::create_dir_all(dir).expect("mkdir"); + } + std::fs::write(&file, serde_json::to_vec_pretty(&rows).unwrap()).expect("write"); println!("\nwrote {} to {path} — Fabric::discover() places them on these figures.", written.join(", ")); if !refused.is_empty() { for (member, why) in &refused { diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/mpp_energy.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/mpp_energy.rs index 657944639..2d4bbc458 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/mpp_energy.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf/examples/mpp_energy.rs @@ -34,9 +34,7 @@ fn main() { std::process::exit(1); } - let path = std::env::var("JOULECLAW_FABRIC_CALIBRATION").unwrap_or_else(|_| { - format!("{}/.jouleclaw/fabric-calibration.json", std::env::var("HOME").unwrap_or_default()) - }); + let path = jouleclaw_fabric::calibration_file_path().display().to_string(); let counter = jouleclaw_energy::apple_ioreport::AppleIoReportCounter::new(); if !counter.is_available() { return process_counters(&path); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-tokenpower-bench/src/lib.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-tokenpower-bench/src/lib.rs index 1fde40dfe..12ab1d48b 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-tokenpower-bench/src/lib.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-tokenpower-bench/src/lib.rs @@ -35,18 +35,28 @@ pub enum BenchError { // Energy provenance — matches jouleclaw-energy's worst-counter rule // ───────────────────────────────────────────────────────────────────── -/// Honest energy provenance tag. +/// Honest energy provenance tag: how a scenario's energy figure was +/// obtained (JCP SPEC §5). A figure MUST NOT claim a better grade than +/// that. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "PascalCase")] pub enum EnergyProvenance { - /// Hardware shunt — measured energy from an in-line current - /// sensor. The gold standard. + /// A hardware reading: an in-line shunt or coulomb counter, or a + /// vendor energy counter (Intel / AMD RAPL package counters, NVIDIA + /// NVML total energy, Apple IOReport energy rails) read while it was + /// observed live for the domain the figure claims. A counter that is + /// present but not live yields no figure of this grade. HwShunt, - /// Model-based — derived from vendor TDP + utilization counters - /// (NVIDIA NVML / Apple IOReport / AMD ROCm / Intel RAPL). + /// A power model: the device's own sampled power draw integrated + /// over time (NVML power draw, ROCm SMI average power), or wall time × + /// an active power calibrated on the host that ran the scenario (JCP + /// SPEC §5.1 rung 5). A model that was not calibrated on that host — + /// vendor TDP × utilization, a spec-sheet power — is not this grade: + /// it is an [`Estimator`](Self::Estimator). ModelBased, - /// Estimator — coarse heuristic (e.g. tokens × empirical-µJ - /// constant). Lowest fidelity. + /// An estimate: a constant per-token or per-call figure (tokens × an + /// empirical µJ constant), or an uncalibrated model such as vendor + /// TDP × utilization (JCP SPEC §5.1 rung 6). Lowest fidelity. Estimator, } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-verification-tier/src/llm.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-verification-tier/src/llm.rs index a2f19eac0..652c4bbc4 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-verification-tier/src/llm.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-verification-tier/src/llm.rs @@ -18,7 +18,8 @@ pub use jouleclaw_llm::{FinishReason, LlmBackend, LlmError, LlmRequest, LlmResponse}; /// A deterministic backend that returns a fixed reply regardless of -/// input, reporting a configurable per-call energy. Used to script +/// input, reporting a configurable per-call energy — a constant, graded +/// `Estimator` (the trait's default grade). Used to script /// agreement / disagreement scenarios in tests and to give downstream /// integration tests a backend that needs no inference stack. #[derive(Debug, Clone)] @@ -61,9 +62,10 @@ impl LlmBackend for StaticBackend { finish_reason: FinishReason::Stop, input_tokens: 0, output_tokens, - // Static backend models a known per-call cost — report it so - // the tier attributes real joules rather than the estimator - // fallback. + // The configured per-call figure, reported so the tier bills it + // rather than the typical-cost fallback. It is a constant — no + // reading or model behind it — so this backend keeps the default + // `Estimator` grade (`LlmBackend::energy_provenance`). energy_joules: Some(self.joules_per_call), }) } @@ -105,3 +107,21 @@ impl LlmBackend for FailingBackend { 2.0 } } + +#[cfg(test)] +mod tests { + use super::*; + use jouleclaw_energy::Provenance; + use jouleclaw_llm::LlmCheapTier; + + /// A fixed per-call figure is a constant, not a reading or a model: the + /// L3 tier grades it an estimate. + #[test] + fn a_static_backends_constant_is_graded_an_estimate() { + let b = StaticBackend::new("a", "x"); + assert_eq!(b.energy_provenance(), Provenance::Estimator); + let resp = b.complete(&LlmRequest::from_prompt("q", 8)).unwrap(); + assert_eq!(resp.energy_joules, Some(2.0)); + assert_eq!(LlmCheapTier::new(b).report_provenance(&resp), Provenance::Estimator); + } +} diff --git a/jouledesk/jouledesk-rs/crates/jouledesk-github/src/lib.rs b/jouledesk/jouledesk-rs/crates/jouledesk-github/src/lib.rs index 632cb73c0..71ec908d0 100644 --- a/jouledesk/jouledesk-rs/crates/jouledesk-github/src/lib.rs +++ b/jouledesk/jouledesk-rs/crates/jouledesk-github/src/lib.rs @@ -320,13 +320,16 @@ fn ledger_provenance(work: &Work) -> Option { }) } -/// How `work`'s energy figure was obtained, as a reply states it: a reading was -/// "measured via" its meter, an estimate "estimated by" it. `None` for work with no -/// figure, which a reply never shows. +/// How `work`'s energy figure was obtained, as a reply states it, by its grade: a +/// hardware reading (`HwShunt`) was "measured via" its meter, a model's figure +/// (`ModelBased`) "modeled by" it, an estimate "estimated by" it. Only a hardware +/// reading is called measured. `None` for work with no figure, which a reply never +/// shows. fn obtained(work: &Work) -> Option<&'static str> { match ledger_provenance(work)? { + EnergyProvenance::HwShunt => Some("measured via"), + EnergyProvenance::ModelBased => Some("modeled by"), EnergyProvenance::Estimator => Some("estimated by"), - EnergyProvenance::HwShunt | EnergyProvenance::ModelBased => Some("measured via"), } } @@ -743,11 +746,13 @@ pub(crate) mod tests { assert!(body.contains("measured via `apple-ioreport`, provenance `hw_shunt`"), "{body}"); let w = work(80_000, &Provenance::HwShunt.to_string(), "rapl"); assert_eq!(sealed(&w), ("HwShunt".to_string(), 80_000)); - // A vendor energy model's reading is sealed as one. + // A vendor energy model's figure is sealed as one, and posted as a model's + // figure: only a hardware reading is called measured. let w = work(80_000, &Provenance::ModelBased.to_string(), "vendor-model"); assert_eq!(sealed(&w), ("ModelBased".to_string(), 80_000)); let body = compose_github("dana", &w, "completed", true); - assert!(body.contains("measured via `vendor-model`, provenance `model_based`"), "{body}"); + assert!(body.contains("80.0 mJ (80000 µJ) — modeled by `vendor-model`, provenance `model_based`"), "{body}"); + assert!(!body.contains("measured"), "a model's figure is never called a measurement: {body}"); } #[test] @@ -839,8 +844,13 @@ pub(crate) mod tests { assert!(energy.contains(&format!("estimated by `{name}`, provenance `estimator`")), "{energy}"); assert!(!body.contains("measured"), "{body}"); } - Provenance::ModelBased | Provenance::HwShunt => { - assert_ne!(label, "Estimator"); + Provenance::ModelBased => { + assert_eq!(label, "ModelBased"); + assert!(energy.contains(&format!("modeled by `{name}`, provenance `model_based`")), "{energy}"); + assert!(!body.contains("measured"), "{body}"); + } + Provenance::HwShunt => { + assert_eq!(label, "HwShunt"); assert!(energy.contains(&format!("measured via `{name}`")), "{energy}"); } } diff --git a/jouledesk/jouledesk-rs/crates/jouledesk-github/src/main.rs b/jouledesk/jouledesk-rs/crates/jouledesk-github/src/main.rs index e37e79d8b..6dade703b 100644 --- a/jouledesk/jouledesk-rs/crates/jouledesk-github/src/main.rs +++ b/jouledesk/jouledesk-rs/crates/jouledesk-github/src/main.rs @@ -18,7 +18,7 @@ fn has(a: &[String], k: &str) -> bool { } /// A handled mention's energy for a log line: the sealed figure, or that there is none. -/// The reply body says whether the figure was measured or estimated, and by which meter. +/// The reply body says whether the figure was measured, modeled or estimated, and by which meter. fn energy(h: &Handled) -> String { if h.verified { format!("{} µJ", h.measured_uj) diff --git a/jouledesk/jouledesk-rs/crates/jouledesk-github/src/slack.rs b/jouledesk/jouledesk-rs/crates/jouledesk-github/src/slack.rs index 73ea51569..fca19e426 100644 --- a/jouledesk/jouledesk-rs/crates/jouledesk-github/src/slack.rs +++ b/jouledesk/jouledesk-rs/crates/jouledesk-github/src/slack.rs @@ -153,6 +153,10 @@ pub(crate) mod tests { let w = crate::tests::work(80_000, &Provenance::HwShunt.to_string(), "apple-ioreport"); let body = compose("U123", &w, "completed", true); assert!(body.contains("80.0 mJ measured via `apple-ioreport` (provenance `hw_shunt`)"), "{body}"); + let w = crate::tests::work(80_000, &Provenance::ModelBased.to_string(), "vendor-model"); + let body = compose("U123", &w, "completed", true); + assert!(body.contains("80.0 mJ modeled by `vendor-model` (provenance `model_based`)"), "{body}"); + assert!(!body.contains("measured"), "a model's figure is never called a measurement: {body}"); let w = crate::tests::work(0, super::super::UNMETERED, "apple-ioreport"); let body = compose("U123", &w, super::super::NOT_SEALED, false); assert!(body.contains("not metered") && body.contains("nothing sealed") && !body.contains("mJ"), "{body}"); diff --git a/joulehook/joulehook-rs/Cargo.lock b/joulehook/joulehook-rs/Cargo.lock index ea9836e82..eb2be9136 100644 --- a/joulehook/joulehook-rs/Cargo.lock +++ b/joulehook/joulehook-rs/Cargo.lock @@ -491,6 +491,8 @@ dependencies = [ "hex", "jcp-efficiency", "jcp-schema", + "mesh-openai", + "mesh-schema", "serde_json", ] @@ -529,6 +531,75 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mesh-core" +version = "0.1.0" +dependencies = [ + "ed25519-dalek", + "jcp-grant", + "jcp-schema", + "mesh-engine", + "mesh-meter", + "mesh-receipt", + "mesh-router", + "mesh-schema", + "thiserror", +] + +[[package]] +name = "mesh-engine" +version = "0.1.0" +dependencies = [ + "mesh-schema", +] + +[[package]] +name = "mesh-meter" +version = "0.1.0" +dependencies = [ + "mesh-schema", +] + +[[package]] +name = "mesh-openai" +version = "0.1.0" +dependencies = [ + "hex", + "jcp-schema", + "mesh-core", + "mesh-schema", + "serde_json", + "thiserror", +] + +[[package]] +name = "mesh-receipt" +version = "0.1.0" +dependencies = [ + "ed25519-dalek", + "jcp-schema", + "mesh-schema", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "mesh-router" +version = "0.1.0" +dependencies = [ + "mesh-schema", +] + +[[package]] +name = "mesh-schema" +version = "0.1.0" +dependencies = [ + "jcp-schema", + "serde", + "thiserror", +] + [[package]] name = "miniz_oxide" version = "0.8.9" diff --git a/joulehook/joulehook-rs/crates/joulehook-meter/Cargo.toml b/joulehook/joulehook-rs/crates/joulehook-meter/Cargo.toml index b27e1637d..436d01223 100644 --- a/joulehook/joulehook-rs/crates/joulehook-meter/Cargo.toml +++ b/joulehook/joulehook-rs/crates/joulehook-meter/Cargo.toml @@ -21,3 +21,9 @@ jcp-schema = { path = "../../../../jcp/jcp-rs/crates/jcp-schema" } ed25519-dalek = "2" serde_json = "1" hex = "0.4" + +[dev-dependencies] +# The label tests feed the bridge real Mesh output — Mesh's own wire labels and +# its OpenAI `usage.energy` block — not a hand-written copy of them. +mesh-schema = { path = "../../../../mesh/mesh-rs/crates/mesh-schema" } +mesh-openai = { path = "../../../../mesh/mesh-rs/crates/mesh-openai" } diff --git a/joulehook/joulehook-rs/crates/joulehook-meter/src/lib.rs b/joulehook/joulehook-rs/crates/joulehook-meter/src/lib.rs index 63d18a3d9..a56f7363a 100644 --- a/joulehook/joulehook-rs/crates/joulehook-meter/src/lib.rs +++ b/joulehook/joulehook-rs/crates/joulehook-meter/src/lib.rs @@ -70,12 +70,16 @@ pub fn bridge_receipt_default( ) } -/// Map Mesh's honest provenance label onto a [`jcp_schema::Energy`]. Unknown labels -/// fall to the estimator floor rather than overclaiming a measurement. +/// Map Mesh's honest provenance label onto a [`jcp_schema::Energy`]. Mesh writes +/// the family's wire labels (`HwShunt`, `ModelBased`, `Estimator` — its +/// `Provenance::jcr1_label`); the lower-case display spelling (`hw_shunt`, +/// `model_based`, `estimator`) that JouleClaw's energy crate prints is read as the +/// same grade. Any other label falls to the estimator floor rather than +/// overclaiming a measurement. fn energy_from(uj: u64, provenance: &str) -> Energy { match provenance { - "hw_shunt" => Energy::measured(uj), - "model_based" => Energy::modeled(uj), + "HwShunt" | "hw_shunt" => Energy::measured(uj), + "ModelBased" | "model_based" => Energy::modeled(uj), _ => Energy::estimator(uj), } } @@ -155,7 +159,7 @@ mod tests { "prompt_tokens": 8000, "completion_tokens": 900, "total_tokens": 8900, - "energy": { "uj": 2_700_000, "uj_per_accepted_token": 3000, "provenance": "model_based" } + "energy": { "uj": 2_700_000, "uj_per_accepted_token": 3000, "provenance": "ModelBased" } } }); let sealed = bridge_receipt( @@ -169,6 +173,54 @@ mod tests { assert_eq!(r.tokens_out, 900); assert_eq!(r.energy_uj, 2_700_000); assert_eq!(r.uj_per_accepted_token, 2_700_000 / 900); // 3000 µJ/accepted token + assert_eq!(r.provenance, "ModelBased"); + } + + /// Bridge a real Mesh response — `mesh-openai`'s chat completion, its + /// `usage.energy.provenance` written by Mesh's own wire label — and return + /// the provenance the sealed receipt carries. + fn bridged_grade_of_mesh_output(p: mesh_schema::Provenance) -> String { + let response = mesh_openai::chat_completion_response( + "chatcmpl-1", "m", 1_700_000_000, "ok", 12, 3, 66_900_000, Some(22_300_000), + p.jcr1_label(), + ); + let sealed = bridge_receipt_default(&response, "t", true, "ok", 0).expect("a usage.energy block seals"); + let pk = meter_steward_sk().verifying_key().to_bytes(); + jcp_efficiency::verify(&sealed, METER_STEWARD, &pk).expect("efficiency/1 verifies").provenance + } + + /// Mesh writes its grades as `HwShunt` / `ModelBased` / `Estimator`. Each must + /// reach the receipt as itself: a Mesh figure is neither under-graded to the + /// estimator floor nor raised. + #[test] + fn mesh_grades_reach_the_receipt_unchanged() { + use mesh_schema::Provenance as Mesh; + for p in [Mesh::HwShunt, Mesh::ModelBased, Mesh::Estimator] { + assert_eq!(bridged_grade_of_mesh_output(p), p.jcr1_label(), "Mesh's {p:?} figure"); + } + } + + /// The lower-case spelling (`hw_shunt` / `model_based` / `estimator`, the + /// JouleClaw energy crate's display form) is still read as the same grade, and + /// a label the family does not write falls to the estimator floor. + #[test] + fn lower_case_labels_and_unknown_labels() { + for (label, grade) in [ + ("hw_shunt", "HwShunt"), + ("model_based", "ModelBased"), + ("estimator", "Estimator"), + ("measured", "Estimator"), + ("hwshunt", "Estimator"), + ("", "Estimator"), + ] { + let response = mesh_openai::chat_completion_response( + "chatcmpl-1", "m", 1_700_000_000, "ok", 12, 3, 66_900_000, None, label, + ); + let sealed = bridge_receipt_default(&response, "t", true, "ok", 0).expect("seals"); + let pk = meter_steward_sk().verifying_key().to_bytes(); + let r = jcp_efficiency::verify(&sealed, METER_STEWARD, &pk).expect("verifies"); + assert_eq!(r.provenance, grade, "label {label:?}"); + } } #[test] diff --git a/joulehook/joulehook-rs/crates/joulehook-proxy/src/main.rs b/joulehook/joulehook-rs/crates/joulehook-proxy/src/main.rs index 3f8dbe14e..cc4536dc1 100644 --- a/joulehook/joulehook-rs/crates/joulehook-proxy/src/main.rs +++ b/joulehook/joulehook-rs/crates/joulehook-proxy/src/main.rs @@ -140,7 +140,7 @@ mod tests { "received_ceiling_uj": echoed, "usage": { "prompt_tokens": 100, "completion_tokens": 40, "total_tokens": 140, - "energy": { "uj": 120_000, "uj_per_accepted_token": 3000, "provenance": "model_based" } + "energy": { "uj": 120_000, "uj_per_accepted_token": 3000, "provenance": "ModelBased" } } }); let _ = req.respond(tiny_http::Response::from_string(resp.to_string())); diff --git a/mesh/mesh-rs/crates/mesh-meter/src/lib.rs b/mesh/mesh-rs/crates/mesh-meter/src/lib.rs index e51d43f13..6010221bf 100644 --- a/mesh/mesh-rs/crates/mesh-meter/src/lib.rs +++ b/mesh/mesh-rs/crates/mesh-meter/src/lib.rs @@ -828,9 +828,9 @@ mod process { /// advance over the span (a few microseconds of work, say), the span meter /// falls back to the family's host meter, whose grade it carries. Where /// that fallback is an estimate, the figure is this crate's - /// [`EstimatorMeter`] over the work's own window: the host meter's window - /// takes in the span meter's waits for the GPU rail at both ends, so its - /// estimate would bill the request for the meter's waiting. + /// [`EstimatorMeter`] over the work's own window, the estimator this crate + /// bills the rest of a partial span with, so an estimated request is + /// billed one way whichever path it took. pub struct ProcessMeter { span: ProcessSpanMeter, uses: SpanUses, @@ -902,9 +902,9 @@ mod process { Ok(s) if s.coverage == Coverage::Whole => match s.provenance { JpProvenance::HwShunt => Energy::measured(s.uj), JpProvenance::ModelBased => Energy::modeled(s.uj), - // Only the host meter's fallback is a whole estimate, over a - // window that takes in the span meter's own snapshot waits: - // this crate's estimate over the work's window instead. + // Only the host meter's fallback is a whole estimate: billed + // as this crate's estimate over the work's window, as the rest + // of a partial span is. JpProvenance::Estimator => Energy::estimator(rest_uj), }, Ok(s) => Energy::estimator(s.uj.saturating_add(rest_uj)), @@ -931,12 +931,11 @@ mod process { /// A span this process's counters could not read — a CPU counter that /// did not advance over a few microseconds of work — falls back to the - /// host meter, whose window opens before the span meter's first GPU - /// snapshot and closes after its last: a quarter of a second or more of - /// the meter's own waiting. Where that fallback is an estimate, the - /// request is billed this crate's estimate over the work's own window, - /// never the host estimator's over the meter's waits (9.48 J for 11 µs - /// of work, on an M5 Max). + /// host meter. Where that fallback is an estimate, the request is billed + /// this crate's estimate over the work's own window, never the host + /// estimator's figure (whose window, until the span meter read the host + /// meter inside its GPU snapshots, took in the meter's own waits: 9.48 J + /// for 11 µs of work, on an M5 Max). #[test] fn an_estimated_fallback_span_is_billed_the_works_own_window() { let fallback = Ok(SpanEnergy { diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/attribute.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/attribute.rs index 9c2ac90c0..e99787b4a 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/attribute.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/attribute.rs @@ -94,6 +94,13 @@ pub struct Attribution { /// the child did not reap is never covered and leaves no trace. Always /// `None` off macOS. pub child_energy_uj: Option, + /// Which probe produced the window's figure — the window's own + /// [`Measurement::source`](crate::probe::Measurement::source): a hardware + /// probe's name (`"ioreport-energy-model"`, `"rapl-powercap"`, ...), + /// `"composite"` where several probes read the window, or `"estimator"` + /// where the meter estimated it. Name the reading by this, never by the + /// platform. + pub source: &'static str, } #[cfg(all(target_os = "macos", target_arch = "aarch64"))] @@ -282,6 +289,7 @@ mod imp { // busy, so a zero reading here is a counter that held. attributed: attributed_ok && cpu_is_a_reading && soc_cpu_uj > 0, child_energy_uj, + source: measurement.source, }, )) } @@ -362,6 +370,7 @@ mod imp { wall: m.wall, attributed: false, child_energy_uj: None, + source: m.source, }, )) } @@ -464,6 +473,7 @@ mod tests { #[cfg(all(target_os = "macos", target_arch = "aarch64"))] assert_cpu_time_registered(&a); assert!(!a.attributed, "a window whose CPU counter held was attributed: {a:?}"); + assert_eq!(a.source, "estimator", "an estimated window is named as the estimator's: {a:?}"); } /// The positive path, which no host running macOS 27.0.1 can reach for @@ -486,6 +496,18 @@ mod tests { assert!(!a.attributed, "the unattributed fallback never attributes"); } + /// The window's figure names the probe that read it — the measurement's own + /// source — whichever probe that is, not the platform's usual one. + #[test] + fn an_attribution_names_the_probe_that_read_its_window() { + use crate::probe::tests::{meter_of, scripted}; + use sandbox_schema::DeviceClass::Cpu; + let mut meter = meter_of(vec![scripted("rapl-powercap", &[(Cpu, 1_000_000)], &[(Cpu, 3_000_000)])]); + let a = run_busy_child(&mut meter); + assert_eq!(a.soc_cpu_uj, 2_000_000, "the window's CPU reading"); + assert_eq!(a.source, "rapl-powercap", "{a:?}"); + } + /// A single-process child's own energy is read from the exited child /// before it is reaped: `sh -c 'awk …'` execs awk, so the child IS the /// work. Where the per-process counter is live the figure is there and