grade labels and fallback windows: every figure says how it was obtained #222
24 changed files with 728 additions and 173 deletions
|
|
@ -99,11 +99,36 @@ struct EnergyTally {
|
|||
attr_uj: u64,
|
||||
windows: u64,
|
||||
attributed_windows: u64,
|
||||
/// The probes whose readings the attributed windows apportioned, each
|
||||
/// window's own (`Attribution::source`): what the meter source names.
|
||||
read_by: BTreeSet<&'static str>,
|
||||
}
|
||||
|
||||
impl EnergyTally {
|
||||
fn add(&mut self, a: &Attribution) {
|
||||
self.add_window(a.soc_cpu_uj, a.attributed_uj, a.attributed);
|
||||
if a.attributed {
|
||||
self.read_by.insert(a.source);
|
||||
}
|
||||
}
|
||||
|
||||
/// How the figure was obtained, for [`RunReport::meter_source`]: an
|
||||
/// attribution names the probes its windows' readings came from, as the
|
||||
/// meter reported them; an estimate names the meter's hardware sources,
|
||||
/// `meter_sources` ([`PowerMeter::sources`]), or says it has none.
|
||||
fn source(&self, meter_sources: &[&str]) -> String {
|
||||
if self.attributed() {
|
||||
let read_by: Vec<&str> = self.read_by.iter().copied().collect();
|
||||
format!("attributed(ModelBased over {} HwShunt)", read_by.join("+"))
|
||||
} else if meter_sources.is_empty() {
|
||||
"estimated(Unaccounted): wall-clock × TDP, no live energy counter on this host".to_string()
|
||||
} else {
|
||||
format!(
|
||||
"estimated(Unaccounted): not every window attributed; machine-wide figure over the \
|
||||
metered windows ({})",
|
||||
meter_sources.join("+")
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// The L3 window: its figure, an estimate, never attributed, with the
|
||||
|
|
@ -1056,17 +1081,7 @@ impl Harness {
|
|||
};
|
||||
let (soc_cpu_uj, attributed_uj, attributed) = (tally.soc_uj, tally.attr_uj, tally.attributed());
|
||||
let wall = wall_start.elapsed();
|
||||
let base_source = if attributed {
|
||||
"attributed(ModelBased over IOReport HwShunt)".to_string()
|
||||
} else if meter_sources.is_empty() {
|
||||
"estimated(Unaccounted): wall-clock × TDP, no live energy counter on this host".to_string()
|
||||
} else {
|
||||
format!(
|
||||
"estimated(Unaccounted): not every window attributed; machine-wide figure over the \
|
||||
metered windows ({})",
|
||||
meter_sources.join("+")
|
||||
)
|
||||
};
|
||||
let base_source = tally.source(&meter_sources);
|
||||
let meter_source = match &l3 {
|
||||
Some(l3) => format!("{base_source}; {}", l3.describe()),
|
||||
None => base_source,
|
||||
|
|
@ -1808,6 +1823,12 @@ fn answers() { assert_eq!(answer(), 42); }
|
|||
|
||||
/// One metered cargo window, as `run_attributed` reports it.
|
||||
fn window(soc_cpu_uj: u64, attributed_uj: u64, attributed: bool) -> Attribution {
|
||||
let source = if attributed { "ioreport-energy-model" } else { "estimator" };
|
||||
window_read_by(soc_cpu_uj, attributed_uj, attributed, source)
|
||||
}
|
||||
|
||||
/// [`window`], its figure read by the probe named `source`.
|
||||
fn window_read_by(soc_cpu_uj: u64, attributed_uj: u64, attributed: bool, source: &'static str) -> Attribution {
|
||||
Attribution {
|
||||
soc_cpu_uj,
|
||||
attributed_uj,
|
||||
|
|
@ -1818,9 +1839,32 @@ fn answers() { assert_eq!(answer(), 42); }
|
|||
attributed,
|
||||
// cargo runs its work in descendants: never a per-child reading.
|
||||
child_energy_uj: None,
|
||||
source,
|
||||
}
|
||||
}
|
||||
|
||||
/// An attributed run's meter source names the probes its windows were read
|
||||
/// by, as the meter reported them — never a counter it did not read.
|
||||
#[test]
|
||||
fn an_attributed_run_names_the_probes_that_read_it() {
|
||||
let mut t = EnergyTally::default();
|
||||
t.add(&window_read_by(10_000_000, 2_500_000, true, "rapl-powercap"));
|
||||
t.add(&window_read_by(30_000_000, 7_500_000, true, "rapl-powercap"));
|
||||
assert!(t.attributed());
|
||||
let source = t.source(&["rapl-powercap", "nvml-total-energy"]);
|
||||
assert_eq!(source, "attributed(ModelBased over rapl-powercap HwShunt)");
|
||||
assert!(!source.contains("IOReport"), "{source}");
|
||||
|
||||
t.add(&window_read_by(10_000_000, 2_500_000, true, "composite"));
|
||||
assert_eq!(t.source(&[]), "attributed(ModelBased over composite+rapl-powercap HwShunt)");
|
||||
|
||||
// An estimate names the meter's sources, not any window's.
|
||||
let mut est = EnergyTally::default();
|
||||
est.add(&window(20_000_000, 0, false));
|
||||
assert!(est.source(&["ioreport-energy-model"]).starts_with("estimated(Unaccounted)"));
|
||||
assert!(est.source(&[]).contains("no live energy counter"));
|
||||
}
|
||||
|
||||
/// A run whose windows were estimated is sealed as an estimate, and never
|
||||
/// as an apportioned zero. This is every macOS 27.0.1 host checked, where
|
||||
/// the meter has no live CPU counter. main sealed 0 J `ModelBased` there,
|
||||
|
|
|
|||
|
|
@ -17,16 +17,21 @@
|
|||
//! Every passport links a [`passport::MeasurementMethod`] to a
|
||||
//! [`jouleclaw_energy::Provenance`] tag — measurement honesty travels
|
||||
//! all the way from the platform counter into the compliance report.
|
||||
//! RAPL and NVML flow through as `Provenance::HwShunt`. So does Apple
|
||||
//! Silicon IOReport observed live (`MeasurementMethod::IoReportLive`):
|
||||
//! a vendor energy counter's reading is a real reading, and what
|
||||
//! decides its grade is provenance — was the counter live for the
|
||||
//! domain the figure claims — not accuracy. A bare
|
||||
//! `MeasurementMethod::IoReport` records no such observation; where
|
||||
//! the CPU energy rail is frozen (macOS 27.0.1 for an unprivileged
|
||||
//! reader) the figure came from the estimator, so it flows through as
|
||||
//! `Provenance::Estimator`. A passport may say its inference energy
|
||||
//! was measured only when its method is graded `HwShunt`.
|
||||
//! RAPL flows through as `Provenance::HwShunt`. So do NVML's
|
||||
//! cumulative-energy counter read while judged live
|
||||
//! (`MeasurementMethod::NvmlLive`) and Apple Silicon IOReport observed
|
||||
//! live (`MeasurementMethod::IoReportLive`): a vendor energy counter's
|
||||
//! reading is a real reading, and what decides its grade is provenance
|
||||
//! — was the counter live for the domain the figure claims — not
|
||||
//! accuracy. A bare `MeasurementMethod::Nvml` records no such
|
||||
//! observation; NVML's figure is then its sampled power draw integrated
|
||||
//! over time, a vendor power model, so it flows through as
|
||||
//! `Provenance::ModelBased`. A bare `MeasurementMethod::IoReport`
|
||||
//! records none either; where the CPU energy rail is frozen (macOS
|
||||
//! 27.0.1 for an unprivileged reader) the figure came from the
|
||||
//! estimator, so it flows through as `Provenance::Estimator`. A
|
||||
//! passport may say its inference energy was measured only when its
|
||||
//! method is graded `HwShunt`.
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
#![warn(missing_docs)]
|
||||
|
|
|
|||
|
|
@ -127,10 +127,26 @@ pub enum MeasurementMethod {
|
|||
/// counter is a hardware reading, graded `HwShunt` like RAPL and NVML:
|
||||
/// its trust gap is provenance, which liveness answers, not accuracy.
|
||||
IoReportLive,
|
||||
/// NVIDIA Management Library.
|
||||
/// NVIDIA Management Library, with no record that its cumulative-energy
|
||||
/// counter was read while judged live. NVML gives a hardware reading only
|
||||
/// from that counter (`nvmlDeviceGetTotalEnergyConsumption`), and only
|
||||
/// where `jouleclaw_energy`'s liveness probe judged it live; elsewhere the
|
||||
/// figure is the driver's sampled power draw integrated over wall time —
|
||||
/// power × time, a vendor power model — graded `ModelBased`. Use
|
||||
/// [`Self::NvmlLive`] for a figure read from the live counter.
|
||||
Nvml,
|
||||
/// Combined GPU (NVML) + CPU (RAPL) measurement.
|
||||
/// NVIDIA Management Library's cumulative-energy counter, read while it was
|
||||
/// judged live (`jouleclaw_energy::CounterLiveness::Live`): a hardware
|
||||
/// reading, graded `HwShunt`.
|
||||
NvmlLive,
|
||||
/// Combined GPU (NVML) + CPU (RAPL) measurement, with no record that the
|
||||
/// NVML part came from its live cumulative-energy counter. A sum is graded
|
||||
/// by its weakest part, so this is `ModelBased`, as [`Self::Nvml`] is. Use
|
||||
/// [`Self::NvmlLiveRapl`] where the NVML part was read from the live counter.
|
||||
NvmlRapl,
|
||||
/// Combined GPU (NVML's cumulative-energy counter, read while judged live) +
|
||||
/// CPU (RAPL) measurement: both parts hardware readings, graded `HwShunt`.
|
||||
NvmlLiveRapl,
|
||||
/// Derived from billing data or hardware specs — not measured.
|
||||
Estimated,
|
||||
}
|
||||
|
|
@ -140,12 +156,27 @@ impl MeasurementMethod {
|
|||
/// Pure-Rust translation from the JouleClaw energy contract.
|
||||
pub fn provenance(&self) -> Provenance {
|
||||
match self {
|
||||
Self::Rapl | Self::Nvml | Self::NvmlRapl | Self::IoReportLive => Provenance::HwShunt,
|
||||
Self::Rapl | Self::NvmlLive | Self::NvmlLiveRapl | Self::IoReportLive => Provenance::HwShunt,
|
||||
Self::Nvml | Self::NvmlRapl => Provenance::ModelBased,
|
||||
Self::IoReport => Provenance::Estimator,
|
||||
Self::Estimated => Provenance::Estimator,
|
||||
}
|
||||
}
|
||||
|
||||
/// The method for a figure NVML produced, from the grade of the readings
|
||||
/// it was taken from (`jouleclaw_energy::EnergyReading::provenance`):
|
||||
/// [`Self::NvmlLive`] only for readings of the live cumulative-energy
|
||||
/// counter (`HwShunt`), [`Self::Nvml`] for the power-draw integral
|
||||
/// (`ModelBased`), and [`Self::Estimated`] for an estimate. The method's
|
||||
/// grade is the readings' grade, never better.
|
||||
pub fn nvml_from_reading(provenance: Provenance) -> Self {
|
||||
match provenance {
|
||||
Provenance::HwShunt => Self::NvmlLive,
|
||||
Provenance::ModelBased => Self::Nvml,
|
||||
Provenance::Estimator => Self::Estimated,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a figure obtained this way is a hardware reading from a live
|
||||
/// counter — the only grade under which inference energy may be called
|
||||
/// measured.
|
||||
|
|
@ -161,7 +192,9 @@ impl fmt::Display for MeasurementMethod {
|
|||
Self::IoReport => write!(f, "IOReport"),
|
||||
Self::IoReportLive => write!(f, "IOReport (live)"),
|
||||
Self::Nvml => write!(f, "NVML"),
|
||||
Self::NvmlLive => write!(f, "NVML (live)"),
|
||||
Self::NvmlRapl => write!(f, "NVML+RAPL"),
|
||||
Self::NvmlLiveRapl => write!(f, "NVML (live)+RAPL"),
|
||||
Self::Estimated => write!(f, "Estimated"),
|
||||
}
|
||||
}
|
||||
|
|
@ -520,7 +553,7 @@ impl PassportBuilder {
|
|||
/// energy measured (`inference_energy_measured`) when the energy
|
||||
/// profile's method is not graded as a live hardware reading
|
||||
/// ([`MeasurementMethod::is_live_hardware_reading`]): a bare
|
||||
/// `IoReport`, or `Estimated`.
|
||||
/// `IoReport` or `Nvml`, `NvmlRapl`, or `Estimated`.
|
||||
pub fn build(self) -> Result<AiServicePassport, PassportError> {
|
||||
let passport = AiServicePassport {
|
||||
passport_version: self.passport_version,
|
||||
|
|
@ -639,7 +672,7 @@ mod tests {
|
|||
EnergyProfile {
|
||||
joules_per_input_token: 0.001,
|
||||
joules_per_output_token: 0.003,
|
||||
measurement_method: MeasurementMethod::NvmlRapl,
|
||||
measurement_method: MeasurementMethod::NvmlLiveRapl,
|
||||
sample_size: 10_000,
|
||||
benchmark_date: Utc::now(),
|
||||
idle_power_watts: 75.0,
|
||||
|
|
@ -736,10 +769,19 @@ mod tests {
|
|||
#[test]
|
||||
fn measurement_method_maps_to_provenance() {
|
||||
assert_eq!(MeasurementMethod::Rapl.provenance(), Provenance::HwShunt);
|
||||
assert_eq!(MeasurementMethod::Nvml.provenance(), Provenance::HwShunt);
|
||||
// NVML is a hardware reading only from its cumulative-energy counter
|
||||
// read while judged live. Without that record the figure may be the
|
||||
// sampled power draw integrated over time — power × time, a vendor
|
||||
// power model — and a sum is no better than its weakest part.
|
||||
assert_eq!(MeasurementMethod::NvmlLive.provenance(), Provenance::HwShunt);
|
||||
assert_eq!(
|
||||
MeasurementMethod::NvmlLiveRapl.provenance(),
|
||||
Provenance::HwShunt
|
||||
);
|
||||
assert_eq!(MeasurementMethod::Nvml.provenance(), Provenance::ModelBased);
|
||||
assert_eq!(
|
||||
MeasurementMethod::NvmlRapl.provenance(),
|
||||
Provenance::HwShunt
|
||||
Provenance::ModelBased
|
||||
);
|
||||
// IOReport read while its CPU rail was live is a hardware reading, the
|
||||
// grade of RAPL and NVML. A bare IoReport records no liveness: where the
|
||||
|
|
@ -758,6 +800,23 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// A figure NVML produced is graded by the readings it was taken from: the
|
||||
/// live cumulative-energy counter (`HwShunt`) is `NvmlLive`, the power-draw
|
||||
/// integral (`ModelBased`) is `Nvml`, and the passport carries the readings'
|
||||
/// own grade, never a better one.
|
||||
#[test]
|
||||
fn nvml_method_follows_the_reading_it_came_from() {
|
||||
for (read, method) in [
|
||||
(Provenance::HwShunt, MeasurementMethod::NvmlLive),
|
||||
(Provenance::ModelBased, MeasurementMethod::Nvml),
|
||||
(Provenance::Estimator, MeasurementMethod::Estimated),
|
||||
] {
|
||||
let m = MeasurementMethod::nvml_from_reading(read);
|
||||
assert_eq!(m, method, "a {read} NVML reading");
|
||||
assert_eq!(m.provenance(), read, "a {read} NVML reading graded {} in the passport", m.provenance());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_passport_succeeds_with_all_fields() {
|
||||
let p = build_sample("svc-1", 0.35, EnergyLabel::A);
|
||||
|
|
@ -785,14 +844,21 @@ mod tests {
|
|||
|
||||
/// A passport may call its inference energy measured only when its method is
|
||||
/// graded as a live hardware reading. A bare IOReport records no liveness, so
|
||||
/// paired with "measured" it is refused, as an estimate is; IOReport observed
|
||||
/// live, RAPL and NVML are accepted. Unclaimed, every method builds.
|
||||
/// paired with "measured" it is refused, as an estimate is; a bare NVML (or
|
||||
/// NVML+RAPL) may be the power-draw integral, a model, and is refused too.
|
||||
/// IOReport observed live, RAPL and NVML's live counter are accepted.
|
||||
/// Unclaimed, every method builds.
|
||||
#[test]
|
||||
fn a_measured_claim_needs_a_live_hardware_reading() {
|
||||
for method in [MeasurementMethod::IoReport, MeasurementMethod::Estimated] {
|
||||
for (method, grade) in [
|
||||
(MeasurementMethod::IoReport, Provenance::Estimator),
|
||||
(MeasurementMethod::Estimated, Provenance::Estimator),
|
||||
(MeasurementMethod::Nvml, Provenance::ModelBased),
|
||||
(MeasurementMethod::NvmlRapl, Provenance::ModelBased),
|
||||
] {
|
||||
match build_with(method, true) {
|
||||
Err(PassportError::MeasuredWithoutHardwareReading { method: m, provenance }) => {
|
||||
assert_eq!((m, provenance), (method, Provenance::Estimator));
|
||||
assert_eq!((m, provenance), (method, grade));
|
||||
}
|
||||
other => panic!("{method} paired with a measured claim was not refused: {other:?}"),
|
||||
}
|
||||
|
|
@ -801,8 +867,8 @@ mod tests {
|
|||
for method in [
|
||||
MeasurementMethod::IoReportLive,
|
||||
MeasurementMethod::Rapl,
|
||||
MeasurementMethod::Nvml,
|
||||
MeasurementMethod::NvmlRapl,
|
||||
MeasurementMethod::NvmlLive,
|
||||
MeasurementMethod::NvmlLiveRapl,
|
||||
] {
|
||||
let p = build_with(method, true).unwrap_or_else(|e| panic!("{method} refused: {e}"));
|
||||
assert!(p.compliance.eu_ai_act.inference_energy_measured);
|
||||
|
|
@ -850,7 +916,9 @@ mod tests {
|
|||
assert_eq!(format!("{}", MeasurementMethod::IoReport), "IOReport");
|
||||
assert_eq!(format!("{}", MeasurementMethod::IoReportLive), "IOReport (live)");
|
||||
assert_eq!(format!("{}", MeasurementMethod::Nvml), "NVML");
|
||||
assert_eq!(format!("{}", MeasurementMethod::NvmlLive), "NVML (live)");
|
||||
assert_eq!(format!("{}", MeasurementMethod::NvmlRapl), "NVML+RAPL");
|
||||
assert_eq!(format!("{}", MeasurementMethod::NvmlLiveRapl), "NVML (live)+RAPL");
|
||||
assert_eq!(format!("{}", MeasurementMethod::Estimated), "Estimated");
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,17 @@
|
|||
//! The CPU half reads every thread of this process, so a span counts whatever
|
||||
//! else this process runs at the same time; the GPU half every GPU client of
|
||||
//! this process.
|
||||
//!
|
||||
//! **The order of the readings.** A GPU snapshot can wait for the rail to
|
||||
//! republish (up to [`crate::process_gpu::GPU_IDLE_WINDOW`] on a quiet GPU,
|
||||
//! [`crate::process_gpu::RAIL_EDGE_WAIT`] on a busy one), so the snapshots are
|
||||
//! the outermost readings at both ends, the CPU reading the innermost, and the
|
||||
//! host meter's reading between them: at the start the GPU snapshot, then the
|
||||
//! host meter, then the CPU; at the end the reverse. The CPU window then holds
|
||||
//! neither the snapshots' CPU time nor the host meter's reads, and the
|
||||
//! fallback's window is the work's own (and the CPU reads), never the
|
||||
//! snapshots' waits — a time-based estimate over those would bill a span of a
|
||||
//! few microseconds of work for up to two seconds of the meter's waiting.
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
|
|
@ -120,6 +131,43 @@ pub struct SpanStart {
|
|||
at: Instant,
|
||||
}
|
||||
|
||||
/// The readings taken at a span's end ([`ProcessSpanMeter::finish`]).
|
||||
struct SpanEnd {
|
||||
cpu_nj: Option<u64>,
|
||||
gpu: Result<GpuSnapshot, EnergyError>,
|
||||
fallback: Result<EnergyReading, EnergyError>,
|
||||
}
|
||||
|
||||
/// The three readings a span takes at each end. A seam: the meter takes them
|
||||
/// from its counters, and the tests from a clock of their own, so the order
|
||||
/// [`read_start`] and [`read_end`] take them in is tested without hardware.
|
||||
trait SpanReads {
|
||||
/// This process's CPU energy, nJ, where its counter is live.
|
||||
fn cpu_nj(&self) -> Option<u64>;
|
||||
/// The GPU rail(s) and clients. May wait for the rail to republish.
|
||||
fn gpu_snapshot(&self) -> Result<GpuSnapshot, EnergyError>;
|
||||
/// The host meter's cumulative reading.
|
||||
fn fallback_read(&self) -> Result<EnergyReading, EnergyError>;
|
||||
}
|
||||
|
||||
/// A span's starting readings, outermost first: the GPU snapshot, then the host
|
||||
/// meter, then the CPU counter (see the module docs).
|
||||
fn read_start(r: &impl SpanReads) -> SpanStart {
|
||||
let gpu = r.gpu_snapshot();
|
||||
let fallback = r.fallback_read();
|
||||
let cpu_nj = r.cpu_nj();
|
||||
SpanStart { cpu_nj, gpu, fallback, at: Instant::now() }
|
||||
}
|
||||
|
||||
/// A span's ending readings, innermost first: the CPU counter, then the host
|
||||
/// meter, then the GPU snapshot — [`read_start`]'s order reversed.
|
||||
fn read_end(r: &impl SpanReads) -> SpanEnd {
|
||||
let cpu_nj = r.cpu_nj();
|
||||
let fallback = r.fallback_read();
|
||||
let gpu = r.gpu_snapshot();
|
||||
SpanEnd { cpu_nj, gpu, fallback }
|
||||
}
|
||||
|
||||
impl SpanStart {
|
||||
/// When the span started.
|
||||
pub fn at(&self) -> Instant {
|
||||
|
|
@ -232,24 +280,22 @@ impl ProcessSpanMeter {
|
|||
self.fallback.active_name()
|
||||
}
|
||||
|
||||
/// Take the span's starting readings. The GPU snapshot comes first and the
|
||||
/// CPU reading last (and the reverse in [`Self::finish`]), so the CPU
|
||||
/// time the snapshots spend — a wait for the GPU rail to republish, and
|
||||
/// IOReport samples — stays outside the CPU window.
|
||||
/// Take the span's starting readings: the GPU snapshot first, then the host
|
||||
/// meter, and the CPU reading last (the reverse in [`Self::finish`]). So
|
||||
/// the CPU time the snapshots spend — a wait for the GPU rail to
|
||||
/// republish, and IOReport samples — and the host meter's read stay
|
||||
/// outside the CPU window, and the snapshots' waits outside the host
|
||||
/// meter's window.
|
||||
pub fn start(&self) -> SpanStart {
|
||||
let fallback = self.fallback.read().map_err(|e| EnergyError::Platform(e.to_string()));
|
||||
let gpu = self.gpu.snapshot();
|
||||
SpanStart { cpu_nj: self.cpu_nj(), gpu, fallback, at: Instant::now() }
|
||||
read_start(self)
|
||||
}
|
||||
|
||||
/// End the span begun at `start`: its energy, or why there is none.
|
||||
pub fn finish(&self, start: SpanStart, uses: SpanUses) -> Result<SpanEnergy, EnergyError> {
|
||||
let cpu_end = self.cpu_nj();
|
||||
let gpu_end = self.gpu.snapshot();
|
||||
let fallback_end = self.fallback.read().map_err(|e| EnergyError::Platform(e.to_string()));
|
||||
let cpu = self.cpu_window(start.cpu_nj, cpu_end);
|
||||
let gpu = start.gpu.and_then(|s| gpu_end.and_then(|e| self.gpu.share(&s, &e)));
|
||||
combine(cpu, gpu, uses, self.fallback.active_name(), (start.fallback, fallback_end))
|
||||
let end = read_end(self);
|
||||
let cpu = self.cpu_window(start.cpu_nj, end.cpu_nj);
|
||||
let gpu = start.gpu.and_then(|s| end.gpu.and_then(|e| self.gpu.share(&s, &e)));
|
||||
combine(cpu, gpu, uses, self.fallback.active_name(), (start.fallback, end.fallback))
|
||||
}
|
||||
|
||||
/// Run `work`, which ran on what `uses` says, and meter it.
|
||||
|
|
@ -259,17 +305,6 @@ impl ProcessSpanMeter {
|
|||
(out, self.finish(start, uses))
|
||||
}
|
||||
|
||||
fn cpu_nj(&self) -> Option<u64> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
self.cpu.is_available().then(crate::process::energy_nj).flatten()
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
{
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn cpu_window(&self, start: Option<u64>, end: Option<u64>) -> Result<u64, EnergyError> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
|
|
@ -286,10 +321,32 @@ impl ProcessSpanMeter {
|
|||
}
|
||||
}
|
||||
|
||||
impl SpanReads for ProcessSpanMeter {
|
||||
fn cpu_nj(&self) -> Option<u64> {
|
||||
#[cfg(target_os = "macos")]
|
||||
{
|
||||
self.cpu.is_available().then(crate::process::energy_nj).flatten()
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
{
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn gpu_snapshot(&self) -> Result<GpuSnapshot, EnergyError> {
|
||||
self.gpu.snapshot()
|
||||
}
|
||||
|
||||
fn fallback_read(&self) -> Result<EnergyReading, EnergyError> {
|
||||
self.fallback.read().map_err(|e| EnergyError::Platform(e.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::EnergyDomain;
|
||||
use std::cell::Cell;
|
||||
|
||||
fn share(uj: u64) -> GpuShare {
|
||||
GpuShare { uj, self_ns: 1, rail_uj: Some(uj), all_ns: Some(1) }
|
||||
|
|
@ -361,6 +418,75 @@ mod tests {
|
|||
assert!(none.to_string().contains("process CPU") && none.to_string().contains("`x`"), "{none}");
|
||||
}
|
||||
|
||||
/// A host whose readings run on a clock of the test's own, in µs: the host
|
||||
/// meter is a time-based estimate at 1 W (1 µJ a µs) and the CPU counter
|
||||
/// counts 1 W of this process's CPU (1,000 nJ a µs). A GPU snapshot waits
|
||||
/// `snapshot_wait_us` for the rail and gives no figure, and a host meter
|
||||
/// read costs `read_us`.
|
||||
struct Clocked {
|
||||
now_us: Cell<u64>,
|
||||
snapshot_wait_us: u64,
|
||||
read_us: u64,
|
||||
}
|
||||
|
||||
impl Clocked {
|
||||
fn pass(&self, us: u64) {
|
||||
self.now_us.set(self.now_us.get() + us);
|
||||
}
|
||||
}
|
||||
|
||||
impl SpanReads for Clocked {
|
||||
fn cpu_nj(&self) -> Option<u64> {
|
||||
Some(self.now_us.get() * 1_000)
|
||||
}
|
||||
fn gpu_snapshot(&self) -> Result<GpuSnapshot, EnergyError> {
|
||||
self.pass(self.snapshot_wait_us);
|
||||
Err(EnergyError::NotCurrent)
|
||||
}
|
||||
fn fallback_read(&self) -> Result<EnergyReading, EnergyError> {
|
||||
let uj = self.now_us.get();
|
||||
self.pass(self.read_us);
|
||||
Ok(EnergyReading { uj, timestamp_ns: 0, domain: EnergyDomain::SocTotal, provenance: Provenance::Estimator })
|
||||
}
|
||||
}
|
||||
|
||||
/// A fallback span's estimate scales with the work's own duration, never
|
||||
/// with the GPU snapshots' waits: the host meter's window opens after the
|
||||
/// starting snapshot and closes before the ending one. The CPU window stays
|
||||
/// the innermost, holding neither the snapshots nor the host meter's reads.
|
||||
/// (main read the host meter outside both snapshots, so an ~11 µs serve
|
||||
/// was billed for up to two rail waits.)
|
||||
#[test]
|
||||
fn a_fallback_span_is_the_works_window_not_the_snapshots_waits() {
|
||||
for snapshot_wait_us in [0, 100_000, 1_000_000] {
|
||||
for work_us in [11, 5_000, 250_000] {
|
||||
let host = Clocked { now_us: Cell::new(1_000), snapshot_wait_us, read_us: 3 };
|
||||
let start = read_start(&host);
|
||||
host.pass(work_us);
|
||||
let end = read_end(&host);
|
||||
|
||||
let cpu = end.cpu_nj.unwrap() - start.cpu_nj.unwrap();
|
||||
assert_eq!(cpu, work_us * 1_000, "the CPU window is the work's alone (wait {snapshot_wait_us} µs)");
|
||||
|
||||
let e = combine(
|
||||
Err(EnergyError::NotAdvanced),
|
||||
Err(EnergyError::NotCurrent),
|
||||
SpanUses::CpuGpu,
|
||||
"est",
|
||||
(start.fallback, end.fallback),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(e.source, SpanSource::Fallback { .. }), "{e:?}");
|
||||
// The work, plus at most the host meter's own read at one end.
|
||||
assert!(
|
||||
(work_us..=work_us + host.read_us).contains(&e.uj),
|
||||
"{} µJ for {work_us} µs of work, snapshots waiting {snapshot_wait_us} µs each",
|
||||
e.uj
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_weaker_grade() {
|
||||
use Provenance::*;
|
||||
|
|
|
|||
|
|
@ -71,24 +71,46 @@ fn the_process_counter_meters_this_process_and_nothing_else() {
|
|||
assert_eq!((r.domain, r.provenance), (EnergyDomain::ProcessCpu, Provenance::HwShunt));
|
||||
assert_eq!((counter.domain(), counter.provenance(), counter.name()), (EnergyDomain::ProcessCpu, Provenance::HwShunt, "process-cpu"));
|
||||
|
||||
// 2. 200 ms of one thread's CPU time reads 0.1 to 20 J. The floor is
|
||||
// 0.1 J, not 0.2, because what 200 ms of CPU costs depends on the core
|
||||
// class that ran it: 0.18-1.24 J on an M5 Max, and 0.171 J on a busy
|
||||
// host (the oracle agreeing). The sleep below must read under 0.1 J,
|
||||
// so a counter that reads wall time at a fixed power still fails one.
|
||||
// 2. 200 ms of one thread's CPU time is a reading, and the reading is this
|
||||
// process's own energy over the burn: no more than an oracle's window
|
||||
// around it, and most of it. No joule figure is asserted — what 200 ms
|
||||
// of CPU costs depends on the core class that ran it (0.099 J on an
|
||||
// efficiency core at a load average of 114; 0.18-1.24 J otherwise on
|
||||
// an M5 Max), so a fixed floor fails on the host's load, not on the
|
||||
// meter. The oracle sleeps before each read: a thread that leaves its
|
||||
// core has its energy folded into the process's count, so the oracle's
|
||||
// window holds the whole burn. The counter's own reads may each miss
|
||||
// up to a scheduling quantum of the reading thread, hence "most of".
|
||||
// SAFETY: `getpid` has no preconditions.
|
||||
let me = unsafe { libc::getpid() };
|
||||
std::thread::sleep(Duration::from_millis(2));
|
||||
let oracle_before = energy_nj_of(me);
|
||||
let t = Instant::now();
|
||||
let ((), burned) = counter.measure(|| burn(Duration::from_millis(200)));
|
||||
let wall = t.elapsed();
|
||||
std::thread::sleep(Duration::from_millis(2));
|
||||
let oracle_nj = energy_nj_of(me) - oracle_before;
|
||||
let uj = burned.expect("a 200 ms burn is a reading");
|
||||
println!("process-cpu: 200 ms one-thread CPU burn = {:.3} J over {:.0} ms wall ({:.2} J per CPU-second)", uj as f64 / 1e6, wall.as_secs_f64() * 1e3, uj as f64 / 1e6 / 0.2);
|
||||
assert!((100_000..=20_000_000).contains(&uj), "{uj} µJ for 200 ms of one thread's CPU");
|
||||
println!(
|
||||
"process-cpu: 200 ms one-thread CPU burn = {:.3} J over {:.0} ms wall ({:.2} J per CPU-second); oracle window {:.3} J",
|
||||
uj as f64 / 1e6,
|
||||
wall.as_secs_f64() * 1e3,
|
||||
uj as f64 / 1e6 / 0.2,
|
||||
oracle_nj as f64 / 1e9
|
||||
);
|
||||
assert!(uj > 0, "a burn read as zero");
|
||||
assert!(uj * 1_000 <= oracle_nj + 1_000, "the burn ({uj} µJ) exceeds the oracle's window around it ({oracle_nj} nJ)");
|
||||
assert!(uj * 1_000 * 4 >= oracle_nj * 3, "the burn ({uj} µJ) is under three quarters of the oracle's window ({oracle_nj} nJ)");
|
||||
|
||||
// 3. A 200 ms sleep reads under 0.1 J — or, if nothing of this process ran,
|
||||
// is not a reading at all; never a zero.
|
||||
// 3. A 200 ms sleep reads under a tenth of the burn — or, if nothing of
|
||||
// this process ran, is not a reading at all; never a zero. A counter
|
||||
// that reads wall time at a fixed power reads the sleep at 200 ms over
|
||||
// the burn's wall time of the burn's figure — a sixth of it at the
|
||||
// heaviest load seen here (1.26 s of wall for the burn) — and fails.
|
||||
let ((), slept) = counter.measure(|| std::thread::sleep(Duration::from_millis(200)));
|
||||
println!("process-cpu: 200 ms sleep = {slept:?} µJ");
|
||||
match slept {
|
||||
Ok(uj) => assert!(uj < 100_000, "{uj} µJ for a sleep"),
|
||||
Ok(slept_uj) => assert!(slept_uj * 10 < uj, "{slept_uj} µJ for a sleep, against {uj} µJ for the burn"),
|
||||
Err(EnergyError::NotAdvanced) => {}
|
||||
Err(e) => panic!("a sleep window failed: {e}"),
|
||||
}
|
||||
|
|
@ -136,7 +158,8 @@ fn the_process_counter_meters_this_process_and_nothing_else() {
|
|||
);
|
||||
assert!(child_burned_nj >= 200_000_000, "the child did not burn: {child_burned_nj} nJ");
|
||||
match parent {
|
||||
Ok(uj) => assert!(uj < 100_000, "{uj} µJ in this process while only the child worked"),
|
||||
// Under a tenth of what the child burned: this process only polled.
|
||||
Ok(uj) => assert!(uj * 1_000 * 10 < child_burned_nj, "{uj} µJ in this process while only the child worked ({child_burned_nj} nJ)"),
|
||||
Err(EnergyError::NotAdvanced) => {}
|
||||
Err(e) => panic!("the parent window failed: {e}"),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -53,6 +53,31 @@ pub use detect::detect_local;
|
|||
pub use place::{place, OpClass, OpDesc, Placement};
|
||||
pub use reach::reach_remote;
|
||||
|
||||
/// The environment variable that names this machine's
|
||||
/// calibration file, read and written.
|
||||
pub const CALIBRATION_FILE_ENV: &str = "JOULECLAW_FABRIC_CALIBRATION";
|
||||
|
||||
/// This machine's calibration file: [`CALIBRATION_FILE_ENV`]
|
||||
/// where it is set, else `$HOME/.jouleclaw/fabric-calibration.json`.
|
||||
/// The one place the path is decided:
|
||||
/// [`Fabric::apply_calibration_file`] reads it, and the
|
||||
/// calibration writers (loader-gguf's `fabric_calibrate` and
|
||||
/// `mpp_energy`) write it, so a file written under the variable
|
||||
/// is the file the placer loads.
|
||||
pub fn calibration_file_path() -> std::path::PathBuf {
|
||||
calibration_file_path_from(std::env::var(CALIBRATION_FILE_ENV).ok(), std::env::var("HOME").ok())
|
||||
}
|
||||
|
||||
/// [`calibration_file_path`] for given values of the variable
|
||||
/// and `HOME`, so the rule is tested without touching the
|
||||
/// process environment.
|
||||
fn calibration_file_path_from(var: Option<String>, home: Option<String>) -> std::path::PathBuf {
|
||||
match var {
|
||||
Some(path) => path.into(),
|
||||
None => format!("{}/.jouleclaw/fabric-calibration.json", home.unwrap_or_default()).into(),
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
// Members
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
|
|
@ -249,19 +274,15 @@ impl Fabric {
|
|||
}
|
||||
|
||||
/// Overlay this machine's µJ figures from the calibration
|
||||
/// file written by `fabric_calibrate` (loader-gguf). Path
|
||||
/// from `JOULECLAW_FABRIC_CALIBRATION`, defaulting to
|
||||
/// `~/.jouleclaw/fabric-calibration.json`. Calibrated
|
||||
/// figures replace seeded entries — placement should run on
|
||||
/// this machine's numbers, not a table written for another
|
||||
/// one — at the grade each row declares (see
|
||||
/// file written by `fabric_calibrate` (loader-gguf), at
|
||||
/// [`calibration_file_path`]: `JOULECLAW_FABRIC_CALIBRATION`,
|
||||
/// defaulting to `~/.jouleclaw/fabric-calibration.json`.
|
||||
/// Calibrated figures replace seeded entries — placement
|
||||
/// should run on this machine's numbers, not a table written
|
||||
/// for another one — at the grade each row declares (see
|
||||
/// [`Self::apply_calibration`]). Returns the rows refused.
|
||||
pub fn apply_calibration_file(&mut self) -> Vec<RejectedCalibration> {
|
||||
let path = std::env::var("JOULECLAW_FABRIC_CALIBRATION").unwrap_or_else(|_| {
|
||||
let home = std::env::var("HOME").unwrap_or_default();
|
||||
format!("{home}/.jouleclaw/fabric-calibration.json")
|
||||
});
|
||||
self.apply_calibration_path(std::path::Path::new(&path))
|
||||
self.apply_calibration_path(&calibration_file_path())
|
||||
}
|
||||
|
||||
/// [`Self::apply_calibration`] over the rows in the JSON
|
||||
|
|
@ -481,6 +502,16 @@ mod tests {
|
|||
assert_eq!(serde_json::from_str::<CalibrationEntry>(&json).unwrap(), r);
|
||||
}
|
||||
|
||||
/// The calibration file is the variable's path where it is
|
||||
/// set, whatever `HOME` is, and `~/.jouleclaw`'s otherwise.
|
||||
#[test]
|
||||
fn the_calibration_file_is_the_variables_where_it_is_set() {
|
||||
let set = calibration_file_path_from(Some("/scratch/cal.json".into()), Some("/home/u".into()));
|
||||
assert_eq!(set, std::path::PathBuf::from("/scratch/cal.json"));
|
||||
let unset = calibration_file_path_from(None, Some("/home/u".into()));
|
||||
assert_eq!(unset, std::path::PathBuf::from("/home/u/.jouleclaw/fabric-calibration.json"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reachable_members_never_serve_without_detection() {
|
||||
let fabric = Fabric::discover();
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@
|
|||
//! not move with anything else the host runs. Where it is not live, and off macOS,
|
||||
//! it is the host meter (`detect_meter`): the IOReport counter where its CPU rail
|
||||
//! is live, else a load-average × power-band ESTIMATOR (`Provenance::Estimator`).
|
||||
//! Each dispatch prints which meter ran and whether its joules were measured or
|
||||
//! estimated. Decode wall-time and token counts are exact measurements. A LOCAL
|
||||
//! Each dispatch prints which meter ran and whether its joules were measured,
|
||||
//! modeled or estimated. Decode wall-time and token counts are exact measurements. A LOCAL
|
||||
//! model is priced energy-only (no per-call API dollars).
|
||||
//!
|
||||
//! Run (defaults to a Nemotron-3-Nano-4B snapshot if present):
|
||||
|
|
|
|||
|
|
@ -570,15 +570,18 @@ pub fn span_joules(span: Result<SpanEnergy, EnergyError>) -> Result<(f64, Proven
|
|||
}
|
||||
}
|
||||
|
||||
/// How a report says a metered figure at `provenance` was obtained: `"measured"` for a
|
||||
/// reading (a hardware shunt, a live vendor energy counter, or a vendor energy model),
|
||||
/// `"estimated"` for an estimate. The local-model examples print each dispatch's joules
|
||||
/// through it, beside the meter that ran, so an estimate — what
|
||||
/// `jouleclaw_energy::meter::detect_meter` yields on Apple Silicon whose IOReport CPU
|
||||
/// rail does not advance — is never called a measurement.
|
||||
/// How a report says a metered figure at `provenance` was obtained, by its grade:
|
||||
/// `"measured"` for a hardware reading (a shunt, or a live vendor energy counter),
|
||||
/// `"modeled"` for a vendor energy model's figure (power × time, say), `"estimated"`
|
||||
/// for an estimate. Only a hardware reading is called measured. The local-model
|
||||
/// examples print each dispatch's joules through it, beside the meter that ran, so an
|
||||
/// estimate — what `jouleclaw_energy::meter::detect_meter` yields on Apple Silicon
|
||||
/// whose IOReport CPU rail does not advance — or a model's figure is never called a
|
||||
/// measurement.
|
||||
pub fn joules_obtained(provenance: Provenance) -> &'static str {
|
||||
match provenance {
|
||||
Provenance::HwShunt | Provenance::ModelBased => "measured",
|
||||
Provenance::HwShunt => "measured",
|
||||
Provenance::ModelBased => "modeled",
|
||||
Provenance::Estimator => "estimated",
|
||||
}
|
||||
}
|
||||
|
|
@ -645,7 +648,6 @@ mod tests {
|
|||
assert!(why.contains("only part"), "{why}");
|
||||
let why = span_joules(Err(EnergyError::GpuClientVanished { pid: 7, client: 1 })).unwrap_err();
|
||||
assert!(why.contains("no figure"), "{why}");
|
||||
assert_eq!(joules_obtained(Provenance::ModelBased), "measured");
|
||||
}
|
||||
|
||||
/// A run that only ever hits a compiled skill is billed the skill's
|
||||
|
|
@ -694,10 +696,12 @@ mod tests {
|
|||
assert_eq!(report.energy_provenance, Provenance::Estimator, "a skill's constant reported as a reading");
|
||||
}
|
||||
|
||||
/// Only a hardware reading is called measured: a model's figure is "modeled",
|
||||
/// an estimate "estimated".
|
||||
#[test]
|
||||
fn an_estimate_is_never_reported_as_measured() {
|
||||
fn an_estimate_or_a_model_is_never_reported_as_measured() {
|
||||
assert_eq!(joules_obtained(Provenance::Estimator), "estimated");
|
||||
assert_eq!(joules_obtained(Provenance::ModelBased), "measured");
|
||||
assert_eq!(joules_obtained(Provenance::ModelBased), "modeled");
|
||||
assert_eq!(joules_obtained(Provenance::HwShunt), "measured");
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
//! with `"echo: "` — useful for tests, conformance vectors, and as a
|
||||
//! reference for the smallest possible compliant backend.
|
||||
|
||||
use jouleclaw_energy::Provenance;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Default "typical joules per call" for backends that do not override it.
|
||||
|
|
@ -145,11 +146,11 @@ pub struct LlmResponse {
|
|||
pub input_tokens: u32,
|
||||
/// Token usage — output side.
|
||||
pub output_tokens: u32,
|
||||
/// Energy reported by the backend, if it can measure or model the
|
||||
/// upstream call's joule cost. `Some(j)` when the value came from a
|
||||
/// real meter (`HwShunt` or `ModelBased`); `None` when the backend
|
||||
/// has no meter — the tier then falls back to
|
||||
/// [`LlmBackend::typical_joules_per_call`].
|
||||
/// The backend's own figure for the upstream call's joule cost, if it
|
||||
/// has one, graded by [`LlmBackend::energy_provenance`]: a reading, a
|
||||
/// calibrated model's figure, or an estimate, as the backend declares.
|
||||
/// `None` when the backend has no figure for the call — the tier then
|
||||
/// falls back to [`LlmBackend::typical_joules_per_call`], an estimate.
|
||||
pub energy_joules: Option<f64>,
|
||||
}
|
||||
|
||||
|
|
@ -197,9 +198,11 @@ pub enum LlmError {
|
|||
/// Backends MUST be `Send + Sync` so the tier (which is `Send`) can carry
|
||||
/// a reference behind a generic parameter.
|
||||
///
|
||||
/// Backends SHOULD report `energy_joules` honestly: `Some(j)` for real
|
||||
/// measurements, `None` when no meter exists. Lying here breaks the
|
||||
/// JouleClaw provenance contract.
|
||||
/// Backends SHOULD report `energy_joules` honestly: `Some(j)` for the
|
||||
/// backend's own figure for the call, `None` when it has none, and
|
||||
/// declare how that figure was obtained through
|
||||
/// [`energy_provenance`](Self::energy_provenance) — never better than it
|
||||
/// was. Lying here breaks the JouleClaw provenance contract.
|
||||
pub trait LlmBackend: Send + Sync {
|
||||
/// Human-readable model name surfaced in answers and receipts. SHOULD
|
||||
/// be stable across calls for the same configuration.
|
||||
|
|
@ -217,6 +220,17 @@ pub trait LlmBackend: Send + Sync {
|
|||
fn typical_joules_per_call(&self) -> f64 {
|
||||
DEFAULT_TYPICAL_JOULES
|
||||
}
|
||||
|
||||
/// How the figures this backend reports in
|
||||
/// [`LlmResponse::energy_joules`] were obtained (JCP SPEC §5): `HwShunt`
|
||||
/// for a hardware reading over the call (a live energy counter),
|
||||
/// `ModelBased` for a per-operation model calibrated on the host that
|
||||
/// ran it, `Estimator` for anything else — a constant, a catalog cost, an
|
||||
/// uncalibrated model. Defaults to `Estimator`: a figure is graded a
|
||||
/// reading or a model only when the backend that produced it says so.
|
||||
fn energy_provenance(&self) -> Provenance {
|
||||
Provenance::Estimator
|
||||
}
|
||||
}
|
||||
|
||||
/// A deterministic, dependency-free backend that echoes the prompt back
|
||||
|
|
|
|||
|
|
@ -36,11 +36,14 @@
|
|||
//! ## Energy provenance
|
||||
//!
|
||||
//! Backends MUST set [`LlmResponse::energy_joules`] honestly: `Some(j)`
|
||||
//! when the value came from a real upstream meter (HwShunt /
|
||||
//! ModelBased) and `None` when no meter exists. On `None` the tier
|
||||
//! falls back to [`LlmBackend::typical_joules_per_call`] — an
|
||||
//! [`jouleclaw_energy::Provenance::Estimator`]-class number — and tags
|
||||
//! the `Answer` accordingly via [`LlmCheapTier::report_provenance`].
|
||||
//! for the backend's own figure for the call and `None` when it has none,
|
||||
//! and declare how its figures were obtained through
|
||||
//! [`LlmBackend::energy_provenance`] — `HwShunt` for a reading over the
|
||||
//! call, `ModelBased` for a model calibrated on the host, and by default
|
||||
//! [`jouleclaw_energy::Provenance::Estimator`]: a number with nothing
|
||||
//! declared behind it is an estimate. On `None` the tier falls back to
|
||||
//! [`LlmBackend::typical_joules_per_call`] — an `Estimator`-class number.
|
||||
//! [`LlmCheapTier::report_provenance`] gives the grade either way.
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
|
|
|
|||
|
|
@ -134,13 +134,15 @@ impl<B: LlmBackend> LlmCheapTier<B> {
|
|||
/// answer with given a backend response. Exposed so callers can pre-
|
||||
/// flight provenance decisions without dispatching.
|
||||
///
|
||||
/// * `Some(_)` energy from the backend → [`Provenance::ModelBased`]
|
||||
/// (the backend has at least a vendor model — we cannot promise
|
||||
/// HwShunt from this layer without inspecting the meter)
|
||||
/// * `None` energy from the backend → [`Provenance::Estimator`]
|
||||
pub fn report_provenance(response: &LlmResponse) -> Provenance {
|
||||
/// * `Some(_)` energy from the backend → the grade the backend declares
|
||||
/// for its figures ([`LlmBackend::energy_provenance`]), which is
|
||||
/// [`Provenance::Estimator`] unless the backend says it read or
|
||||
/// modelled the call: a bare number is not evidence of a meter
|
||||
/// * `None` energy from the backend → [`Provenance::Estimator`] (the
|
||||
/// backend's typical per-call constant)
|
||||
pub fn report_provenance(&self, response: &LlmResponse) -> Provenance {
|
||||
match response.energy_joules {
|
||||
Some(_) => Provenance::ModelBased,
|
||||
Some(_) => self.backend.energy_provenance(),
|
||||
None => Provenance::Estimator,
|
||||
}
|
||||
}
|
||||
|
|
@ -706,32 +708,71 @@ mod tests {
|
|||
assert!((est.confidence_floor - 1.0).abs() < f32::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_provenance_model_based_when_energy_reported() {
|
||||
let resp = LlmResponse {
|
||||
fn response_with(energy_joules: Option<f64>) -> LlmResponse {
|
||||
LlmResponse {
|
||||
text: "x".into(),
|
||||
finish_reason: FinishReason::Stop,
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
energy_joules: Some(0.1),
|
||||
};
|
||||
assert_eq!(
|
||||
LlmCheapTier::<EchoBackend>::report_provenance(&resp),
|
||||
Provenance::ModelBased,
|
||||
);
|
||||
energy_joules,
|
||||
}
|
||||
}
|
||||
|
||||
/// A backend that reports a fixed per-call figure — no reading, no model —
|
||||
/// and declares nothing about it, as a fixed-reply test backend does.
|
||||
struct ConstantJoules;
|
||||
impl LlmBackend for ConstantJoules {
|
||||
fn model_name(&self) -> &str {
|
||||
"constant"
|
||||
}
|
||||
fn complete(&self, _: &LlmRequest) -> Result<LlmResponse, LlmError> {
|
||||
Ok(response_with(Some(2.0)))
|
||||
}
|
||||
}
|
||||
|
||||
/// A backend that declares its figures a grade.
|
||||
struct Declares(Provenance);
|
||||
impl LlmBackend for Declares {
|
||||
fn model_name(&self) -> &str {
|
||||
"declares"
|
||||
}
|
||||
fn complete(&self, _: &LlmRequest) -> Result<LlmResponse, LlmError> {
|
||||
Ok(response_with(Some(0.1)))
|
||||
}
|
||||
fn energy_provenance(&self) -> Provenance {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
/// A figure is graded by how its backend says it was obtained. A bare
|
||||
/// number with nothing declared behind it — a constant 2 J a call — is an
|
||||
/// estimate, never `ModelBased`.
|
||||
#[test]
|
||||
fn a_reported_figure_takes_the_grade_its_backend_declares() {
|
||||
let t = LlmCheapTier::new(ConstantJoules);
|
||||
let resp = t.backend().complete(&LlmRequest::from_prompt("q", 8)).unwrap();
|
||||
assert_eq!(resp.energy_joules, Some(2.0));
|
||||
assert_eq!(t.report_provenance(&resp), Provenance::Estimator);
|
||||
assert_eq!(
|
||||
LlmCheapTier::new(EchoBackend::new()).report_provenance(&response_with(Some(0.1))),
|
||||
Provenance::Estimator,
|
||||
);
|
||||
for grade in [Provenance::HwShunt, Provenance::ModelBased, Provenance::Estimator] {
|
||||
let t = LlmCheapTier::new(Declares(grade));
|
||||
assert_eq!(t.report_provenance(&response_with(Some(0.1))), grade);
|
||||
}
|
||||
}
|
||||
|
||||
/// With no figure from the backend the tier bills its typical constant,
|
||||
/// an estimate, whatever the backend declares for figures it reports.
|
||||
#[test]
|
||||
fn report_provenance_estimator_when_no_energy() {
|
||||
let resp = LlmResponse {
|
||||
text: "x".into(),
|
||||
finish_reason: FinishReason::Stop,
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
energy_joules: None,
|
||||
};
|
||||
assert_eq!(
|
||||
LlmCheapTier::<EchoBackend>::report_provenance(&resp),
|
||||
LlmCheapTier::new(EchoBackend::new()).report_provenance(&response_with(None)),
|
||||
Provenance::Estimator,
|
||||
);
|
||||
assert_eq!(
|
||||
LlmCheapTier::new(Declares(Provenance::HwShunt)).report_provenance(&response_with(None)),
|
||||
Provenance::Estimator,
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@
|
|||
//! local member — NEON, AMX (Accelerate), Metal GPU
|
||||
//! (resident-weight MPS) — under an energy meter, and
|
||||
//! writes the measured µJ/GFLOP table the placer loads at
|
||||
//! discovery (`~/.jouleclaw/fabric-calibration.json`).
|
||||
//! discovery (`jouleclaw_fabric::calibration_file_path`:
|
||||
//! `JOULECLAW_FABRIC_CALIBRATION` where it is set, else
|
||||
//! `~/.jouleclaw/fabric-calibration.json`).
|
||||
//!
|
||||
//! Two meters, chosen as the live autotuner chooses
|
||||
//! (`fabric_dispatch::tune_shape_live`):
|
||||
|
|
@ -112,9 +114,9 @@ fn main() {
|
|||
Shape { label: "prefill-batch (m=16)", m: 16, n: 4096, k: 2304 },
|
||||
];
|
||||
|
||||
let home = std::env::var("HOME").unwrap_or_default();
|
||||
let dir = format!("{home}/.jouleclaw");
|
||||
let path = format!("{dir}/fabric-calibration.json");
|
||||
// The file the placer loads: the variable's where it is set.
|
||||
let file = jouleclaw_fabric::calibration_file_path();
|
||||
let path = file.display().to_string();
|
||||
|
||||
let counter = AppleIoReportCounter::new();
|
||||
let meter = if counter.is_available() {
|
||||
|
|
@ -288,14 +290,16 @@ fn main() {
|
|||
}
|
||||
// Every row of a member this example measures is replaced: by
|
||||
// its row from this run, or by none. Other members' rows stay.
|
||||
let existing: Vec<jouleclaw_fabric::CalibrationEntry> = std::fs::read(&path)
|
||||
let existing: Vec<jouleclaw_fabric::CalibrationEntry> = std::fs::read(&file)
|
||||
.ok()
|
||||
.and_then(|b| serde_json::from_slice(&b).ok())
|
||||
.unwrap_or_default();
|
||||
let written: Vec<String> = latest.keys().cloned().collect();
|
||||
let rows = merge_calibration(existing, &FABRIC_CALIBRATE_MEMBERS, latest.into_values().collect());
|
||||
std::fs::create_dir_all(&dir).expect("mkdir");
|
||||
std::fs::write(&path, serde_json::to_vec_pretty(&rows).unwrap()).expect("write");
|
||||
if let Some(dir) = file.parent().filter(|d| !d.as_os_str().is_empty()) {
|
||||
std::fs::create_dir_all(dir).expect("mkdir");
|
||||
}
|
||||
std::fs::write(&file, serde_json::to_vec_pretty(&rows).unwrap()).expect("write");
|
||||
println!("\nwrote {} to {path} — Fabric::discover() places them on these figures.", written.join(", "));
|
||||
if !refused.is_empty() {
|
||||
for (member, why) in &refused {
|
||||
|
|
|
|||
|
|
@ -34,9 +34,7 @@ fn main() {
|
|||
std::process::exit(1);
|
||||
}
|
||||
|
||||
let path = std::env::var("JOULECLAW_FABRIC_CALIBRATION").unwrap_or_else(|_| {
|
||||
format!("{}/.jouleclaw/fabric-calibration.json", std::env::var("HOME").unwrap_or_default())
|
||||
});
|
||||
let path = jouleclaw_fabric::calibration_file_path().display().to_string();
|
||||
let counter = jouleclaw_energy::apple_ioreport::AppleIoReportCounter::new();
|
||||
if !counter.is_available() {
|
||||
return process_counters(&path);
|
||||
|
|
|
|||
|
|
@ -35,18 +35,28 @@ pub enum BenchError {
|
|||
// Energy provenance — matches jouleclaw-energy's worst-counter rule
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Honest energy provenance tag.
|
||||
/// Honest energy provenance tag: how a scenario's energy figure was
|
||||
/// obtained (JCP SPEC §5). A figure MUST NOT claim a better grade than
|
||||
/// that.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "PascalCase")]
|
||||
pub enum EnergyProvenance {
|
||||
/// Hardware shunt — measured energy from an in-line current
|
||||
/// sensor. The gold standard.
|
||||
/// A hardware reading: an in-line shunt or coulomb counter, or a
|
||||
/// vendor energy counter (Intel / AMD RAPL package counters, NVIDIA
|
||||
/// NVML total energy, Apple IOReport energy rails) read while it was
|
||||
/// observed live for the domain the figure claims. A counter that is
|
||||
/// present but not live yields no figure of this grade.
|
||||
HwShunt,
|
||||
/// Model-based — derived from vendor TDP + utilization counters
|
||||
/// (NVIDIA NVML / Apple IOReport / AMD ROCm / Intel RAPL).
|
||||
/// A power model: the device's own sampled power draw integrated
|
||||
/// over time (NVML power draw, ROCm SMI average power), or wall time ×
|
||||
/// an active power calibrated on the host that ran the scenario (JCP
|
||||
/// SPEC §5.1 rung 5). A model that was not calibrated on that host —
|
||||
/// vendor TDP × utilization, a spec-sheet power — is not this grade:
|
||||
/// it is an [`Estimator`](Self::Estimator).
|
||||
ModelBased,
|
||||
/// Estimator — coarse heuristic (e.g. tokens × empirical-µJ
|
||||
/// constant). Lowest fidelity.
|
||||
/// An estimate: a constant per-token or per-call figure (tokens × an
|
||||
/// empirical µJ constant), or an uncalibrated model such as vendor
|
||||
/// TDP × utilization (JCP SPEC §5.1 rung 6). Lowest fidelity.
|
||||
Estimator,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@
|
|||
pub use jouleclaw_llm::{FinishReason, LlmBackend, LlmError, LlmRequest, LlmResponse};
|
||||
|
||||
/// A deterministic backend that returns a fixed reply regardless of
|
||||
/// input, reporting a configurable per-call energy. Used to script
|
||||
/// input, reporting a configurable per-call energy — a constant, graded
|
||||
/// `Estimator` (the trait's default grade). Used to script
|
||||
/// agreement / disagreement scenarios in tests and to give downstream
|
||||
/// integration tests a backend that needs no inference stack.
|
||||
#[derive(Debug, Clone)]
|
||||
|
|
@ -61,9 +62,10 @@ impl LlmBackend for StaticBackend {
|
|||
finish_reason: FinishReason::Stop,
|
||||
input_tokens: 0,
|
||||
output_tokens,
|
||||
// Static backend models a known per-call cost — report it so
|
||||
// the tier attributes real joules rather than the estimator
|
||||
// fallback.
|
||||
// The configured per-call figure, reported so the tier bills it
|
||||
// rather than the typical-cost fallback. It is a constant — no
|
||||
// reading or model behind it — so this backend keeps the default
|
||||
// `Estimator` grade (`LlmBackend::energy_provenance`).
|
||||
energy_joules: Some(self.joules_per_call),
|
||||
})
|
||||
}
|
||||
|
|
@ -105,3 +107,21 @@ impl LlmBackend for FailingBackend {
|
|||
2.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use jouleclaw_energy::Provenance;
|
||||
use jouleclaw_llm::LlmCheapTier;
|
||||
|
||||
/// A fixed per-call figure is a constant, not a reading or a model: the
|
||||
/// L3 tier grades it an estimate.
|
||||
#[test]
|
||||
fn a_static_backends_constant_is_graded_an_estimate() {
|
||||
let b = StaticBackend::new("a", "x");
|
||||
assert_eq!(b.energy_provenance(), Provenance::Estimator);
|
||||
let resp = b.complete(&LlmRequest::from_prompt("q", 8)).unwrap();
|
||||
assert_eq!(resp.energy_joules, Some(2.0));
|
||||
assert_eq!(LlmCheapTier::new(b).report_provenance(&resp), Provenance::Estimator);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -320,13 +320,16 @@ fn ledger_provenance(work: &Work) -> Option<EnergyProvenance> {
|
|||
})
|
||||
}
|
||||
|
||||
/// How `work`'s energy figure was obtained, as a reply states it: a reading was
|
||||
/// "measured via" its meter, an estimate "estimated by" it. `None` for work with no
|
||||
/// figure, which a reply never shows.
|
||||
/// How `work`'s energy figure was obtained, as a reply states it, by its grade: a
|
||||
/// hardware reading (`HwShunt`) was "measured via" its meter, a model's figure
|
||||
/// (`ModelBased`) "modeled by" it, an estimate "estimated by" it. Only a hardware
|
||||
/// reading is called measured. `None` for work with no figure, which a reply never
|
||||
/// shows.
|
||||
fn obtained(work: &Work) -> Option<&'static str> {
|
||||
match ledger_provenance(work)? {
|
||||
EnergyProvenance::HwShunt => Some("measured via"),
|
||||
EnergyProvenance::ModelBased => Some("modeled by"),
|
||||
EnergyProvenance::Estimator => Some("estimated by"),
|
||||
EnergyProvenance::HwShunt | EnergyProvenance::ModelBased => Some("measured via"),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -743,11 +746,13 @@ pub(crate) mod tests {
|
|||
assert!(body.contains("measured via `apple-ioreport`, provenance `hw_shunt`"), "{body}");
|
||||
let w = work(80_000, &Provenance::HwShunt.to_string(), "rapl");
|
||||
assert_eq!(sealed(&w), ("HwShunt".to_string(), 80_000));
|
||||
// A vendor energy model's reading is sealed as one.
|
||||
// A vendor energy model's figure is sealed as one, and posted as a model's
|
||||
// figure: only a hardware reading is called measured.
|
||||
let w = work(80_000, &Provenance::ModelBased.to_string(), "vendor-model");
|
||||
assert_eq!(sealed(&w), ("ModelBased".to_string(), 80_000));
|
||||
let body = compose_github("dana", &w, "completed", true);
|
||||
assert!(body.contains("measured via `vendor-model`, provenance `model_based`"), "{body}");
|
||||
assert!(body.contains("80.0 mJ (80000 µJ) — modeled by `vendor-model`, provenance `model_based`"), "{body}");
|
||||
assert!(!body.contains("measured"), "a model's figure is never called a measurement: {body}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -839,8 +844,13 @@ pub(crate) mod tests {
|
|||
assert!(energy.contains(&format!("estimated by `{name}`, provenance `estimator`")), "{energy}");
|
||||
assert!(!body.contains("measured"), "{body}");
|
||||
}
|
||||
Provenance::ModelBased | Provenance::HwShunt => {
|
||||
assert_ne!(label, "Estimator");
|
||||
Provenance::ModelBased => {
|
||||
assert_eq!(label, "ModelBased");
|
||||
assert!(energy.contains(&format!("modeled by `{name}`, provenance `model_based`")), "{energy}");
|
||||
assert!(!body.contains("measured"), "{body}");
|
||||
}
|
||||
Provenance::HwShunt => {
|
||||
assert_eq!(label, "HwShunt");
|
||||
assert!(energy.contains(&format!("measured via `{name}`")), "{energy}");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ fn has(a: &[String], k: &str) -> bool {
|
|||
}
|
||||
|
||||
/// A handled mention's energy for a log line: the sealed figure, or that there is none.
|
||||
/// The reply body says whether the figure was measured or estimated, and by which meter.
|
||||
/// The reply body says whether the figure was measured, modeled or estimated, and by which meter.
|
||||
fn energy(h: &Handled) -> String {
|
||||
if h.verified {
|
||||
format!("{} µJ", h.measured_uj)
|
||||
|
|
|
|||
|
|
@ -153,6 +153,10 @@ pub(crate) mod tests {
|
|||
let w = crate::tests::work(80_000, &Provenance::HwShunt.to_string(), "apple-ioreport");
|
||||
let body = compose("U123", &w, "completed", true);
|
||||
assert!(body.contains("80.0 mJ measured via `apple-ioreport` (provenance `hw_shunt`)"), "{body}");
|
||||
let w = crate::tests::work(80_000, &Provenance::ModelBased.to_string(), "vendor-model");
|
||||
let body = compose("U123", &w, "completed", true);
|
||||
assert!(body.contains("80.0 mJ modeled by `vendor-model` (provenance `model_based`)"), "{body}");
|
||||
assert!(!body.contains("measured"), "a model's figure is never called a measurement: {body}");
|
||||
let w = crate::tests::work(0, super::super::UNMETERED, "apple-ioreport");
|
||||
let body = compose("U123", &w, super::super::NOT_SEALED, false);
|
||||
assert!(body.contains("not metered") && body.contains("nothing sealed") && !body.contains("mJ"), "{body}");
|
||||
|
|
|
|||
71
joulehook/joulehook-rs/Cargo.lock
generated
71
joulehook/joulehook-rs/Cargo.lock
generated
|
|
@ -491,6 +491,8 @@ dependencies = [
|
|||
"hex",
|
||||
"jcp-efficiency",
|
||||
"jcp-schema",
|
||||
"mesh-openai",
|
||||
"mesh-schema",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
|
|
@ -529,6 +531,75 @@ version = "2.8.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "mesh-core"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"ed25519-dalek",
|
||||
"jcp-grant",
|
||||
"jcp-schema",
|
||||
"mesh-engine",
|
||||
"mesh-meter",
|
||||
"mesh-receipt",
|
||||
"mesh-router",
|
||||
"mesh-schema",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mesh-engine"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"mesh-schema",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mesh-meter"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"mesh-schema",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mesh-openai"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"hex",
|
||||
"jcp-schema",
|
||||
"mesh-core",
|
||||
"mesh-schema",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mesh-receipt"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"ed25519-dalek",
|
||||
"jcp-schema",
|
||||
"mesh-schema",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mesh-router"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"mesh-schema",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mesh-schema"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"jcp-schema",
|
||||
"serde",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
|
|
|
|||
|
|
@ -21,3 +21,9 @@ jcp-schema = { path = "../../../../jcp/jcp-rs/crates/jcp-schema" }
|
|||
ed25519-dalek = "2"
|
||||
serde_json = "1"
|
||||
hex = "0.4"
|
||||
|
||||
[dev-dependencies]
|
||||
# The label tests feed the bridge real Mesh output — Mesh's own wire labels and
|
||||
# its OpenAI `usage.energy` block — not a hand-written copy of them.
|
||||
mesh-schema = { path = "../../../../mesh/mesh-rs/crates/mesh-schema" }
|
||||
mesh-openai = { path = "../../../../mesh/mesh-rs/crates/mesh-openai" }
|
||||
|
|
|
|||
|
|
@ -70,12 +70,16 @@ pub fn bridge_receipt_default(
|
|||
)
|
||||
}
|
||||
|
||||
/// Map Mesh's honest provenance label onto a [`jcp_schema::Energy`]. Unknown labels
|
||||
/// fall to the estimator floor rather than overclaiming a measurement.
|
||||
/// Map Mesh's honest provenance label onto a [`jcp_schema::Energy`]. Mesh writes
|
||||
/// the family's wire labels (`HwShunt`, `ModelBased`, `Estimator` — its
|
||||
/// `Provenance::jcr1_label`); the lower-case display spelling (`hw_shunt`,
|
||||
/// `model_based`, `estimator`) that JouleClaw's energy crate prints is read as the
|
||||
/// same grade. Any other label falls to the estimator floor rather than
|
||||
/// overclaiming a measurement.
|
||||
fn energy_from(uj: u64, provenance: &str) -> Energy {
|
||||
match provenance {
|
||||
"hw_shunt" => Energy::measured(uj),
|
||||
"model_based" => Energy::modeled(uj),
|
||||
"HwShunt" | "hw_shunt" => Energy::measured(uj),
|
||||
"ModelBased" | "model_based" => Energy::modeled(uj),
|
||||
_ => Energy::estimator(uj),
|
||||
}
|
||||
}
|
||||
|
|
@ -155,7 +159,7 @@ mod tests {
|
|||
"prompt_tokens": 8000,
|
||||
"completion_tokens": 900,
|
||||
"total_tokens": 8900,
|
||||
"energy": { "uj": 2_700_000, "uj_per_accepted_token": 3000, "provenance": "model_based" }
|
||||
"energy": { "uj": 2_700_000, "uj_per_accepted_token": 3000, "provenance": "ModelBased" }
|
||||
}
|
||||
});
|
||||
let sealed = bridge_receipt(
|
||||
|
|
@ -169,6 +173,54 @@ mod tests {
|
|||
assert_eq!(r.tokens_out, 900);
|
||||
assert_eq!(r.energy_uj, 2_700_000);
|
||||
assert_eq!(r.uj_per_accepted_token, 2_700_000 / 900); // 3000 µJ/accepted token
|
||||
assert_eq!(r.provenance, "ModelBased");
|
||||
}
|
||||
|
||||
/// Bridge a real Mesh response — `mesh-openai`'s chat completion, its
|
||||
/// `usage.energy.provenance` written by Mesh's own wire label — and return
|
||||
/// the provenance the sealed receipt carries.
|
||||
fn bridged_grade_of_mesh_output(p: mesh_schema::Provenance) -> String {
|
||||
let response = mesh_openai::chat_completion_response(
|
||||
"chatcmpl-1", "m", 1_700_000_000, "ok", 12, 3, 66_900_000, Some(22_300_000),
|
||||
p.jcr1_label(),
|
||||
);
|
||||
let sealed = bridge_receipt_default(&response, "t", true, "ok", 0).expect("a usage.energy block seals");
|
||||
let pk = meter_steward_sk().verifying_key().to_bytes();
|
||||
jcp_efficiency::verify(&sealed, METER_STEWARD, &pk).expect("efficiency/1 verifies").provenance
|
||||
}
|
||||
|
||||
/// Mesh writes its grades as `HwShunt` / `ModelBased` / `Estimator`. Each must
|
||||
/// reach the receipt as itself: a Mesh figure is neither under-graded to the
|
||||
/// estimator floor nor raised.
|
||||
#[test]
|
||||
fn mesh_grades_reach_the_receipt_unchanged() {
|
||||
use mesh_schema::Provenance as Mesh;
|
||||
for p in [Mesh::HwShunt, Mesh::ModelBased, Mesh::Estimator] {
|
||||
assert_eq!(bridged_grade_of_mesh_output(p), p.jcr1_label(), "Mesh's {p:?} figure");
|
||||
}
|
||||
}
|
||||
|
||||
/// The lower-case spelling (`hw_shunt` / `model_based` / `estimator`, the
|
||||
/// JouleClaw energy crate's display form) is still read as the same grade, and
|
||||
/// a label the family does not write falls to the estimator floor.
|
||||
#[test]
|
||||
fn lower_case_labels_and_unknown_labels() {
|
||||
for (label, grade) in [
|
||||
("hw_shunt", "HwShunt"),
|
||||
("model_based", "ModelBased"),
|
||||
("estimator", "Estimator"),
|
||||
("measured", "Estimator"),
|
||||
("hwshunt", "Estimator"),
|
||||
("", "Estimator"),
|
||||
] {
|
||||
let response = mesh_openai::chat_completion_response(
|
||||
"chatcmpl-1", "m", 1_700_000_000, "ok", 12, 3, 66_900_000, None, label,
|
||||
);
|
||||
let sealed = bridge_receipt_default(&response, "t", true, "ok", 0).expect("seals");
|
||||
let pk = meter_steward_sk().verifying_key().to_bytes();
|
||||
let r = jcp_efficiency::verify(&sealed, METER_STEWARD, &pk).expect("verifies");
|
||||
assert_eq!(r.provenance, grade, "label {label:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ mod tests {
|
|||
"received_ceiling_uj": echoed,
|
||||
"usage": {
|
||||
"prompt_tokens": 100, "completion_tokens": 40, "total_tokens": 140,
|
||||
"energy": { "uj": 120_000, "uj_per_accepted_token": 3000, "provenance": "model_based" }
|
||||
"energy": { "uj": 120_000, "uj_per_accepted_token": 3000, "provenance": "ModelBased" }
|
||||
}
|
||||
});
|
||||
let _ = req.respond(tiny_http::Response::from_string(resp.to_string()));
|
||||
|
|
|
|||
|
|
@ -828,9 +828,9 @@ mod process {
|
|||
/// advance over the span (a few microseconds of work, say), the span meter
|
||||
/// falls back to the family's host meter, whose grade it carries. Where
|
||||
/// that fallback is an estimate, the figure is this crate's
|
||||
/// [`EstimatorMeter`] over the work's own window: the host meter's window
|
||||
/// takes in the span meter's waits for the GPU rail at both ends, so its
|
||||
/// estimate would bill the request for the meter's waiting.
|
||||
/// [`EstimatorMeter`] over the work's own window, the estimator this crate
|
||||
/// bills the rest of a partial span with, so an estimated request is
|
||||
/// billed one way whichever path it took.
|
||||
pub struct ProcessMeter {
|
||||
span: ProcessSpanMeter,
|
||||
uses: SpanUses,
|
||||
|
|
@ -902,9 +902,9 @@ mod process {
|
|||
Ok(s) if s.coverage == Coverage::Whole => match s.provenance {
|
||||
JpProvenance::HwShunt => Energy::measured(s.uj),
|
||||
JpProvenance::ModelBased => Energy::modeled(s.uj),
|
||||
// Only the host meter's fallback is a whole estimate, over a
|
||||
// window that takes in the span meter's own snapshot waits:
|
||||
// this crate's estimate over the work's window instead.
|
||||
// Only the host meter's fallback is a whole estimate: billed
|
||||
// as this crate's estimate over the work's window, as the rest
|
||||
// of a partial span is.
|
||||
JpProvenance::Estimator => Energy::estimator(rest_uj),
|
||||
},
|
||||
Ok(s) => Energy::estimator(s.uj.saturating_add(rest_uj)),
|
||||
|
|
@ -931,12 +931,11 @@ mod process {
|
|||
|
||||
/// A span this process's counters could not read — a CPU counter that
|
||||
/// did not advance over a few microseconds of work — falls back to the
|
||||
/// host meter, whose window opens before the span meter's first GPU
|
||||
/// snapshot and closes after its last: a quarter of a second or more of
|
||||
/// the meter's own waiting. Where that fallback is an estimate, the
|
||||
/// request is billed this crate's estimate over the work's own window,
|
||||
/// never the host estimator's over the meter's waits (9.48 J for 11 µs
|
||||
/// of work, on an M5 Max).
|
||||
/// host meter. Where that fallback is an estimate, the request is billed
|
||||
/// this crate's estimate over the work's own window, never the host
|
||||
/// estimator's figure (whose window, until the span meter read the host
|
||||
/// meter inside its GPU snapshots, took in the meter's own waits: 9.48 J
|
||||
/// for 11 µs of work, on an M5 Max).
|
||||
#[test]
|
||||
fn an_estimated_fallback_span_is_billed_the_works_own_window() {
|
||||
let fallback = Ok(SpanEnergy {
|
||||
|
|
|
|||
|
|
@ -94,6 +94,13 @@ pub struct Attribution {
|
|||
/// the child did not reap is never covered and leaves no trace. Always
|
||||
/// `None` off macOS.
|
||||
pub child_energy_uj: Option<u64>,
|
||||
/// Which probe produced the window's figure — the window's own
|
||||
/// [`Measurement::source`](crate::probe::Measurement::source): a hardware
|
||||
/// probe's name (`"ioreport-energy-model"`, `"rapl-powercap"`, ...),
|
||||
/// `"composite"` where several probes read the window, or `"estimator"`
|
||||
/// where the meter estimated it. Name the reading by this, never by the
|
||||
/// platform.
|
||||
pub source: &'static str,
|
||||
}
|
||||
|
||||
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
|
||||
|
|
@ -282,6 +289,7 @@ mod imp {
|
|||
// busy, so a zero reading here is a counter that held.
|
||||
attributed: attributed_ok && cpu_is_a_reading && soc_cpu_uj > 0,
|
||||
child_energy_uj,
|
||||
source: measurement.source,
|
||||
},
|
||||
))
|
||||
}
|
||||
|
|
@ -362,6 +370,7 @@ mod imp {
|
|||
wall: m.wall,
|
||||
attributed: false,
|
||||
child_energy_uj: None,
|
||||
source: m.source,
|
||||
},
|
||||
))
|
||||
}
|
||||
|
|
@ -464,6 +473,7 @@ mod tests {
|
|||
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
|
||||
assert_cpu_time_registered(&a);
|
||||
assert!(!a.attributed, "a window whose CPU counter held was attributed: {a:?}");
|
||||
assert_eq!(a.source, "estimator", "an estimated window is named as the estimator's: {a:?}");
|
||||
}
|
||||
|
||||
/// The positive path, which no host running macOS 27.0.1 can reach for
|
||||
|
|
@ -486,6 +496,18 @@ mod tests {
|
|||
assert!(!a.attributed, "the unattributed fallback never attributes");
|
||||
}
|
||||
|
||||
/// The window's figure names the probe that read it — the measurement's own
|
||||
/// source — whichever probe that is, not the platform's usual one.
|
||||
#[test]
|
||||
fn an_attribution_names_the_probe_that_read_its_window() {
|
||||
use crate::probe::tests::{meter_of, scripted};
|
||||
use sandbox_schema::DeviceClass::Cpu;
|
||||
let mut meter = meter_of(vec![scripted("rapl-powercap", &[(Cpu, 1_000_000)], &[(Cpu, 3_000_000)])]);
|
||||
let a = run_busy_child(&mut meter);
|
||||
assert_eq!(a.soc_cpu_uj, 2_000_000, "the window's CPU reading");
|
||||
assert_eq!(a.source, "rapl-powercap", "{a:?}");
|
||||
}
|
||||
|
||||
/// A single-process child's own energy is read from the exited child
|
||||
/// before it is reaped: `sh -c 'awk …'` execs awk, so the child IS the
|
||||
/// work. Where the per-process counter is live the figure is there and
|
||||
|
|
|
|||
Loading…
Reference in a new issue