diff --git a/jcp/jcp-rs/Cargo.lock b/jcp/jcp-rs/Cargo.lock index 7c0c0ffcb..8c5a1f0f6 100644 --- a/jcp/jcp-rs/Cargo.lock +++ b/jcp/jcp-rs/Cargo.lock @@ -630,6 +630,7 @@ dependencies = [ "jcp-runtime", "jcp-schema", "sandbox-meter", + "sandbox-schema", "serde_json", ] diff --git a/jcp/jcp-rs/crates/jcp-action-receipt/docs/AR-1-spec.md b/jcp/jcp-rs/crates/jcp-action-receipt/docs/AR-1-spec.md index 9c00c7cba..417616dc5 100644 --- a/jcp/jcp-rs/crates/jcp-action-receipt/docs/AR-1-spec.md +++ b/jcp/jcp-rs/crates/jcp-action-receipt/docs/AR-1-spec.md @@ -19,7 +19,7 @@ what an audit actually asks for, and **none of them binds to the others**: |---|---|---| | `jcp-receipt` | authority | grant chain, verdict, budget drawn | | `sandbox-proof` | environment | attestation, isolation tier, substrate | -| JCR-1 | work | input hash, measured joules, energy provenance | +| JCR-1 | work | input hash, joules, and how they were obtained (energy provenance) | | *(nothing)* | **effect** | what changed, and whether it can be undone | So no single artifact answers: *under what authority did this agent do that, @@ -73,7 +73,7 @@ ActionReceipt { // binding 3 — work work_receipt: ContentId input_hash: Hash - joules_uj: u64 + joules_uj: u64 // the work's cost, graded by energy_provenance energy_provenance: "HwShunt" | "ModelBased" | "Estimator" // binding 4 — effect (new; nothing else records this) @@ -114,6 +114,10 @@ them into a single undifferentiated number. A measured figure and an estimated one are not interchangeable, and a receipt that treats them as such is lying by arithmetic. +`joules_uj` is the work's cost however it was obtained. It is a hardware +reading only when `energy_provenance` is `HwShunt`, and a verifier or display +MUST NOT present it as a measurement otherwise. + ## 3. The three rules These MUST be enforced at verification, re-derived from the wire projection — @@ -201,8 +205,10 @@ not prove those artifacts are themselves sound — that is their own verifiers' job, and a complete audit runs all of them. **The energy cross-check is a consistency check, not a proof.** An attestation -asserts; joules are measured. Binding both means a forged attestation must also -maintain a plausible energy profile across an independently sampled counter, and -that the failure is visible rather than silent. An adversary with physical +asserts; joules are metered, and where `energy_provenance` is `HwShunt` they are +read off an independently sampled counter. Binding both means a forged +attestation must also maintain a plausible energy profile across that counter, +and that the failure is visible rather than silent. An `Estimator` figure offers +no such cross-check, which is why it is labelled. An adversary with physical access who can forge a quote can also influence measurement. Claiming more than this would be exactly the overreach the design exists to avoid. diff --git a/jcp/jcp-rs/crates/jcp-action-receipt/examples/end_to_end.rs b/jcp/jcp-rs/crates/jcp-action-receipt/examples/end_to_end.rs index 414b77a44..e42177598 100644 --- a/jcp/jcp-rs/crates/jcp-action-receipt/examples/end_to_end.rs +++ b/jcp/jcp-rs/crates/jcp-action-receipt/examples/end_to_end.rs @@ -15,8 +15,10 @@ //! is really refused; //! - the authorisation is a **real `decide` call** against a real budget, and //! the receipt records the verdict that call returned; -//! - the joules are **measured**, and the meter's provenance is carried through -//! verbatim. On a host with no counter it reports `Estimator` and this example +//! - the joules are **metered**, and the span's provenance is carried through +//! verbatim: `HwShunt` only when every record the meter returned is a +//! hardware reading. Where the host has no live counter — or the span's CPU +//! counter held — the meter returns its `Estimator` record, and this example //! says so rather than inventing `HwShunt`; //! - the tier is stamped from what is **actually running**. This is in-process //! with no isolation boundary crossed, so it is `S0`. Claiming `S5` without a @@ -136,13 +138,29 @@ fn main() -> Result<(), Box> { std::hint::black_box(acc); let joules_uj = measurement.total_uj(); - let provenance = if is_hw { "HwShunt" } else { "Estimator" }; + // The span's own provenance, the weakest of its records. A hardware meter + // still returns its estimator record for a span whose CPU counter held, + // so `is_hardware()` says nothing about this span. + let provenance = { + use sandbox_meter::Provenance as P; + let records = &measurement.devices; + if records.is_empty() || records.iter().any(|d| d.provenance == P::Estimator) { + "Estimator" + } else if records.iter().all(|d| d.provenance == P::HwShunt && d.measured_uj.is_some()) { + "HwShunt" + } else { + "ModelBased" + } + }; println!( "3. work metered — {joules_uj} µJ [{provenance}] over {:.2}s via {sources}", measurement.wall.as_secs_f64() ); - if !is_hw { - println!(" note: no hardware counter here; provenance is Estimator, not HwShunt"); + if provenance != "HwShunt" { + println!( + " note: this span was not a hardware reading; provenance is {provenance}, not HwShunt ({})", + if is_hw { "its CPU counter held" } else { "no live energy counter here" } + ); } // Build the REAL work receipt from the devices the meter actually reported. diff --git a/jcp/jcp-rs/crates/jcp-action-receipt/src/lib.rs b/jcp/jcp-rs/crates/jcp-action-receipt/src/lib.rs index 9a80a9004..7aa48a722 100644 --- a/jcp/jcp-rs/crates/jcp-action-receipt/src/lib.rs +++ b/jcp/jcp-rs/crates/jcp-action-receipt/src/lib.rs @@ -9,7 +9,7 @@ //! |---|---|---| //! | `jcp-receipt` | authority | grant chain, verdict, budget drawn | //! | `sandbox-proof` | environment | attestation, isolation tier, substrate | -//! | JCR-1 | work | input hash, measured joules, energy provenance | +//! | JCR-1 | work | input hash, joules, and how they were obtained (energy provenance) | //! | *(nothing)* | **effect** | what changed, and whether it can be undone | //! //! AR-1 binds the first three **by content-id** and supplies the fourth. It is @@ -201,7 +201,10 @@ pub struct ActionReceipt { pub work_receipt: ContentId, /// Hash of the input the work consumed. pub input_hash: String, - /// Measured cost in microjoules. + /// The work's energy cost in microjoules. Whether it is a hardware + /// reading, a model or an estimate is not this field's to say: it is + /// graded by [`energy_provenance`](Self::energy_provenance), and is a + /// reading only when that says `HwShunt`. pub joules_uj: u64, /// How that figure was obtained. Never collapsed into a single number: /// `HwShunt` and `Estimator` are not the same claim. diff --git a/jcp/jcp-rs/crates/jcp-gateway/Cargo.toml b/jcp/jcp-rs/crates/jcp-gateway/Cargo.toml index ce35e114d..43a595e91 100644 --- a/jcp/jcp-rs/crates/jcp-gateway/Cargo.toml +++ b/jcp/jcp-rs/crates/jcp-gateway/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jcp-gateway" -description = "The energy-native MCP gateway: a JCP enforcement point in front of an MCP server. Every tools/call is gated by a capability grant, the upstream call is metered on real hardware, the MEASURED joules are debited from the grant's budget, and the decision seals into a hash-chained receipt log. The chokepoint position the AI-gateway field occupies — denominated in joules, with a tamper-evident audit, not dollars and a span." +description = "The energy-native MCP gateway: a JCP enforcement point in front of an MCP server. Every tools/call is gated by a capability grant, the upstream call is metered (a hardware reading where the host exposes a live energy counter, an estimate where it does not, each sealed as what it is), the metered joules are debited from the grant's budget, and the decision seals into a hash-chained receipt log. The chokepoint position the AI-gateway field occupies — denominated in joules, with a tamper-evident audit, not dollars and a span." version.workspace = true edition.workspace = true license.workspace = true @@ -15,6 +15,9 @@ jcp-receipt = { path = "../jcp-receipt" } jcp-bridge = { path = "../jcp-bridge" } jcp-flow = { path = "../jcp-flow" } sandbox-meter = { path = "../../../../sandbox/sandbox-rs/crates/sandbox-meter" } +# The meter's device classes: a hardware span with no CPU record is a reading +# of part of the call, and is sealed as one. +sandbox-schema = { path = "../../../../sandbox/sandbox-rs/crates/sandbox-schema" } ed25519-dalek = { workspace = true } serde_json = { workspace = true } diff --git a/jcp/jcp-rs/crates/jcp-gateway/src/bin/jcp_gateway.rs b/jcp/jcp-rs/crates/jcp-gateway/src/bin/jcp_gateway.rs index 84b8c4438..84126ad08 100644 --- a/jcp/jcp-rs/crates/jcp-gateway/src/bin/jcp_gateway.rs +++ b/jcp/jcp-rs/crates/jcp-gateway/src/bin/jcp_gateway.rs @@ -17,6 +17,18 @@ use jcp_gateway::{Gateway, Relayed, Upstream}; use std::io::{BufRead, BufReader, Write}; use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; +/// The energy grade a verified receipt seals: its `energy_provenance`, plus +/// `energy_coverage` when that is `Partial`. `None` if it does not verify. +fn sealed_grade(envelope: &[u8], steward: &str, pubkey: &[u8; 32]) -> Option { + let core = jcp_receipt::jcr1::verify_cose_sign1(envelope, steward, pubkey).ok()?; + let v: serde_json::Value = jcp_schema::from_canonical_cbor(&core).ok()?; + let provenance = v.get("energy_provenance")?.as_str()?; + Some(match v.get("energy_coverage").and_then(serde_json::Value::as_str) { + Some(coverage) => format!("{provenance}, {coverage}"), + None => provenance.to_string(), + }) +} + /// The upstream MCP server as a child process speaking JSON-RPC over stdio. struct ChildServer { _child: Child, @@ -80,6 +92,8 @@ fn main() -> std::io::Result<()> { let grant_refs: Vec<(&str, u64)> = grants.iter().map(|(t, b)| (t.as_str(), *b)).collect(); let steward_sk = SigningKey::from_bytes(&[0x5e; 32]); + let steward_pk = steward_sk.verifying_key().to_bytes(); + let steward = format!("did:jcp:gateway:{server}"); let mut gw = Gateway::new(&server, &grant_refs, steward_sk); eprintln!( "jcp-gateway: server={server} tools={:?} metering={}", @@ -100,7 +114,14 @@ fn main() -> std::io::Result<()> { } match gw.relay(&line, &mut up)? { Relayed::Allowed { response, measured_uj, budget_remaining_uj } => { - eprintln!("jcp-gateway: allowed, {measured_uj} µJ measured, {budget_remaining_uj} µJ left"); + // How the joules were obtained, read back from the receipt the + // gateway just sealed, so this line cannot disagree with it. + let grade = gw + .receipts() + .last() + .and_then(|r| sealed_grade(r, &steward, &steward_pk)) + .unwrap_or_else(|| "unverified".into()); + eprintln!("jcp-gateway: allowed, {measured_uj} µJ [{grade}], {budget_remaining_uj} µJ left"); if !response.is_empty() { writeln!(stdout, "{response}")?; stdout.flush()?; diff --git a/jcp/jcp-rs/crates/jcp-gateway/src/lib.rs b/jcp/jcp-rs/crates/jcp-gateway/src/lib.rs index 36568a1fe..1492e5045 100644 --- a/jcp/jcp-rs/crates/jcp-gateway/src/lib.rs +++ b/jcp/jcp-rs/crates/jcp-gateway/src/lib.rs @@ -8,9 +8,12 @@ //! `(server, tool)`, so a grant for `read_file` cannot authorize `shell`. //! 2. The call is authorized *before* it runs ([`Runtime::authorize`]) — but //! nothing is debited yet, because the cost isn't known until it runs. -//! 3. The upstream call is **metered on real hardware** -//! ([`sandbox_meter::PowerMeter`]) — the measured joules, not an estimate. -//! 4. The measured cost is debited from the grant ([`Runtime::charge`]); when +//! 3. The upstream call is **metered** ([`sandbox_meter::PowerMeter`]): a +//! hardware reading where this host exposes a live energy counter, and the +//! meter's wall-clock estimate where it does not. The figure keeps the +//! provenance of the span it came from — `HwShunt` only for a reading, +//! `Estimator` for an estimate — in the ledger and in the receipt. +//! 4. The metered cost is debited from the grant ([`Runtime::charge`]); when //! the budget is exhausted the next call is denied. //! 5. Every decision — allow and deny — seals into a **hash-chained receipt //! log** ([`jcp_receipt::ReceiptLog`]): a tamper-evident, non-repudiable @@ -36,7 +39,8 @@ use jcp_receipt::{FlowSeal, ReceiptLog}; use jcp_runtime::Runtime; use jcp_schema::flow::{FlowCeiling, Readers}; use jcp_schema::{Capability, Caveat, ContentId, Did, Energy, FlowFacts, Grant, SignedGrant}; -use sandbox_meter::PowerMeter; +use sandbox_meter::{Measurement, PowerMeter}; +use sandbox_schema::DeviceClass; use serde_json::{json, Value}; use std::collections::{BTreeMap, BTreeSet}; @@ -53,6 +57,9 @@ pub enum Relayed { /// A non-`tools/call` message forwarded verbatim; here is the response. PassedThrough(String), /// A `tools/call` the gateway authorized, metered, and forwarded. + /// `measured_uj` is the figure debited from the grant: a hardware reading + /// or the meter's estimate. The sealed receipt's `energy_provenance`, and + /// the same label beside the figure in the response's `_meta`, say which. Allowed { response: String, measured_uj: u64, budget_remaining_uj: u64 }, /// A `tools/call` the gateway refused; the upstream was never contacted. /// `response` is a JSON-RPC error to hand back to the client. @@ -61,7 +68,7 @@ pub enum Relayed { Notified, } -/// The gateway: a JCP runtime, the tool grants, a hardware meter, and a +/// The gateway: a JCP runtime, the tool grants, an energy meter, and a /// hash-chained receipt log, in front of one MCP server. pub struct Gateway { server: String, @@ -168,7 +175,10 @@ impl Gateway { self.log.entries() } - /// Whether the active meter reads real hardware (vs. the estimator). + /// Whether the active meter has a hardware energy source (vs. the + /// estimator alone). This says what the meter can read, not what a call + /// did: a span whose CPU counter held is estimated even on a hardware + /// meter, and each call's receipt carries its own span's provenance. pub fn metering_is_hardware(&self) -> bool { self.meter.is_hardware() } @@ -225,7 +235,10 @@ impl Gateway { label: witness.label.clone(), witness_id: witness.id(), eval_uj: eval.total_uj(), - eval_prov: eval_provenance(&self.meter), + // The span's own provenance: over a scan this short the CPU + // counter can hold, and the span is then an estimate even on a + // hardware meter. + eval_prov: span_provenance(&eval), }; let seal = FlowSeal { facts: Some(facts.clone()), ceiling: ceiling_digest(&grant) }; // Keep the witness discoverable for a later audit of this verdict. @@ -243,7 +256,7 @@ impl Gateway { }); } - // Phase 2: run the upstream call, metered on real hardware. + // Phase 2: run the upstream call, metered. let (response, measurement) = { let mut response = String::new(); let (res, m) = self.meter.measure(|| up.call(line)); @@ -252,11 +265,17 @@ impl Gateway { } (response, m) }; - let measured_uj = measurement.total_uj(); + // The call's cost as what it is: a hardware reading, or the meter's + // estimate when this host exposes no live counter or the span's CPU + // counter held. Either way it is debited, so a runaway is enforced. + let cost = energy_of(&measurement); + let measured_uj = cost.uj(); - // Phase 3: debit the measured joules (and the tainted sub-budget, if the + // Phase 3: debit the metered joules (and the tainted sub-budget, if the // call was tainted); seal the accounting decision as jcp/3 when tainted. - let decision = self.runtime.charge_flow(&grant, Energy::measured(measured_uj), &facts); + // The receipt seals `cost`'s own provenance: an estimate never enters + // the ledger, or the evidence, as a reading. + let decision = self.runtime.charge_flow(&grant, cost, &facts); self.log.append_flow(&frame, &grant, &decision, &seal); // Phase 4: the result becomes a labelled source. Its frame is Trust::Data @@ -278,7 +297,7 @@ impl Gateway { // receives evidence for its own request rather than having to trust // that the gateway logged something somewhere. let response = match self.log.entries().last() { - Some(r) => attach_receipt(&response, r, measured_uj, decision.budget_remaining_uj), + Some(r) => attach_receipt(&response, r, &cost, decision.budget_remaining_uj), None => response, }; @@ -345,7 +364,17 @@ fn jcp_meta(params: &Value) -> Option<&Value> { /// be presented, disputed, or forwarded, and that is most of what a receipt is /// for. Returns the response unchanged if it is not a JSON-RPC result object — /// this must never mangle an upstream payload it does not understand. -fn attach_receipt(response: &str, receipt: &[u8], measured_uj: u64, remaining_uj: u64) -> String { +/// +/// The plaintext figure is graded beside itself. `measured_uj` keeps its name +/// for the clients already reading it, but it is the call's COST, which is a +/// hardware reading only where the span was one; on a host with no live energy +/// counter it is the meter's estimate. So `energy_provenance` (`HwShunt`, +/// `ModelBased` or `Estimator`) rides next to it, with `energy_coverage: +/// "Partial"` when the figure reads only part of the call, the same two labels +/// the sealed receipt carries. A client that reads only the plaintext therefore +/// never takes an estimate for a reading, and never has to decode the receipt +/// to find out which it got. +fn attach_receipt(response: &str, receipt: &[u8], cost: &Energy, remaining_uj: u64) -> String { let Ok(mut v) = serde_json::from_str::(response) else { return response.to_string(); }; @@ -357,14 +386,16 @@ fn attach_receipt(response: &str, receipt: &[u8], measured_uj: u64, remaining_uj .or_insert_with(|| json!({})) .as_object_mut(); let Some(meta) = meta else { return response.to_string() }; - meta.insert( - JCP_META_NS.to_string(), - json!({ - "receipt_b64": b64(receipt), - "measured_uj": measured_uj, - "budget_remaining_uj": remaining_uj, - }), - ); + let mut jcp = json!({ + "receipt_b64": b64(receipt), + "measured_uj": cost.uj(), + "energy_provenance": cost.provenance.jcr1_label(), + "budget_remaining_uj": remaining_uj, + }); + if !cost.is_whole() { + jcp["energy_coverage"] = json!(cost.coverage.label()); + } + meta.insert(JCP_META_NS.to_string(), jcp); v.to_string() } @@ -426,12 +457,40 @@ fn declared_witness_ids(params: &Value) -> BTreeSet { .unwrap_or_default() } -/// The provenance label to seal for the monitor's own metered evaluation. -fn eval_provenance(meter: &PowerMeter) -> jcp_schema::Provenance { - if meter.is_hardware() { +/// The provenance of one metered span: the weakest of its records, because a +/// sum is only as good as its weakest part. The meter returns its estimator +/// record when the host exposes no live counter, and also when the span's CPU +/// counter held — even on a hardware meter. So a span is labelled by its own +/// records, never by [`PowerMeter::is_hardware`]. +fn span_provenance(m: &Measurement) -> jcp_schema::Provenance { + use sandbox_meter::Provenance as Grade; + if m.devices.is_empty() || m.devices.iter().any(|d| d.provenance == Grade::Estimator) { + jcp_schema::Provenance::Estimator + } else if m + .devices + .iter() + .all(|d| d.provenance == Grade::HwShunt && d.measured_uj.is_some()) + { jcp_schema::Provenance::HwShunt } else { - jcp_schema::Provenance::Estimator + jcp_schema::Provenance::ModelBased + } +} + +/// The energy a metered span enters the ledger and the receipt as, built with +/// jcp-schema's own constructors: an estimate as an estimate, a reading as a +/// reading. A hardware reading with no CPU record — a GPU counter alone, as on +/// a host that exposes NVML but not RAPL — reads only part of the call, so it +/// is sealed `Partial` instead of claiming the whole call's energy. +fn energy_of(m: &Measurement) -> Energy { + let uj = m.total_uj(); + match span_provenance(m) { + jcp_schema::Provenance::HwShunt if m.devices.iter().any(|d| d.class == DeviceClass::Cpu) => { + Energy::measured(uj) + } + jcp_schema::Provenance::HwShunt => Energy::measured_partial(uj), + jcp_schema::Provenance::ModelBased => Energy::modeled(uj), + jcp_schema::Provenance::Estimator => Energy::estimator(uj), } } @@ -509,10 +568,48 @@ mod tests { let b64_returned = meta["receipt_b64"].as_str().expect("receipt_b64"); assert_eq!(b64_returned, b64(g.receipts().last().unwrap())); + // The plaintext figure carries its grade, and it is the grade the + // receipt sealed: a client reading `measured_uj` without decoding the + // receipt is told whether it is a reading or an estimate. + let steward_pub = SigningKey::from_bytes(&[7u8; 32]).verifying_key().to_bytes(); + let sealed = jcr1_payload(g.receipts().last().unwrap(), "did:jcp:gateway:files", &steward_pub); + let grade = meta["energy_provenance"].as_str().expect("energy_provenance beside measured_uj"); + assert_eq!(Some(grade), sealed.get("energy_provenance").and_then(Value::as_str), "{meta} vs {sealed}"); + assert_eq!(meta.get("energy_coverage"), sealed.get("energy_coverage"), "{meta} vs {sealed}"); + eprintln!("gateway _meta: hardware={} {meta}", g.metering_is_hardware()); + if !g.metering_is_hardware() { + assert_eq!(grade, "Estimator", "an estimate labelled as something else: {meta}"); + } + // And the upstream's own payload survived untouched beside it. assert_eq!(v["result"]["ok"].as_bool(), Some(true)); } + /// `measured_uj` keeps its name for compatibility but is the call's cost, + /// so its grade rides beside it in plaintext for every shape the cost + /// takes: `energy_provenance` always, `energy_coverage` only when the + /// figure covers part of the call (whole is the default and is omitted, + /// as in the sealed receipt). + #[test] + fn the_plaintext_figure_carries_its_grade() { + let ok = r#"{"jsonrpc":"2.0","id":1,"result":{"ok":true}}"#; + let ptr = format!("/result/_meta/{}", JCP_META_NS.replace('/', "~1")); + for (cost, provenance, coverage) in [ + (Energy::estimator(60_000), "Estimator", None), + (Energy::measured(1_300), "HwShunt", None), + (Energy::measured_partial(3_000), "HwShunt", Some("Partial")), + (Energy::modeled(700), "ModelBased", None), + ] { + let v: Value = serde_json::from_str(&attach_receipt(ok, b"r", &cost, 9)).unwrap(); + let meta = v.pointer(&ptr).expect("the jcp _meta block"); + assert_eq!(meta["measured_uj"].as_u64(), Some(cost.uj()), "{meta}"); + assert_eq!(meta["energy_provenance"].as_str(), Some(provenance), "{meta}"); + assert_eq!(meta.get("energy_coverage").and_then(Value::as_str), coverage, "{meta}"); + assert_eq!(meta["budget_remaining_uj"].as_u64(), Some(9)); + assert_eq!(v["result"]["ok"].as_bool(), Some(true), "the upstream payload is untouched"); + } + } + /// A refusal the caller cannot see is an enforcement nobody can audit, so a /// denial carries its sealed record too. #[test] @@ -554,10 +651,10 @@ mod tests { #[test] fn a_response_the_gateway_cannot_parse_is_returned_untouched() { let junk = "not json at all"; - assert_eq!(attach_receipt(junk, b"r", 1, 2), junk); + assert_eq!(attach_receipt(junk, b"r", &Energy::estimator(1), 2), junk); // Valid JSON, but no `result` object to extend. let err = r#"{"jsonrpc":"2.0","id":1,"error":{"code":-1,"message":"x"}}"#; - assert_eq!(attach_receipt(err, b"r", 1, 2), err); + assert_eq!(attach_receipt(err, b"r", &Energy::estimator(1), 2), err); } /// Base64 is RFC 4648 with padding — pinned against known vectors, because a @@ -573,6 +670,9 @@ mod tests { assert_eq!(b64(b"foobar"), "Zm9vYmFy"); } + /// A call that ran costs joules and is debited on every host: read off a + /// live counter, or estimated where there is none. An estimate is still + /// debited, so the budget binds on a host that exposes no reading. #[test] fn granted_tool_is_allowed_metered_and_debited() { let mut g = gw(); @@ -581,10 +681,8 @@ mod tests { match r { Relayed::Allowed { measured_uj, budget_remaining_uj, .. } => { assert_eq!(up.calls, 1, "upstream was contacted"); - if g.metering_is_hardware() { - assert!(measured_uj > 0, "a real span costs joules"); - assert!(budget_remaining_uj < 50_000_000, "budget was debited"); - } + assert!(measured_uj > 0, "a real span costs joules, measured or estimated"); + assert_eq!(budget_remaining_uj, 50_000_000 - measured_uj, "budget was debited by the cost"); } other => panic!("expected Allowed, got {other:?}"), } @@ -616,7 +714,8 @@ mod tests { #[test] fn budget_exhaustion_denies_further_calls_and_the_log_verifies() { - // A tiny budget so a couple of measured calls exhaust it. + // A tiny budget so a couple of metered calls exhaust it — on every + // host: an estimate is debited like a reading. let mut g = Gateway::new("files", &[("read_file", 500)], SigningKey::from_bytes(&[7u8; 32])); let mut up = MockServer { calls: 0 }; let steward_pub = SigningKey::from_bytes(&[7u8; 32]).verifying_key().to_bytes(); @@ -633,11 +732,9 @@ mod tests { _ => unreachable!(), } } - if g.metering_is_hardware() { - assert!(allowed >= 1, "at least one call fit the budget"); - assert!(denied >= 1, "the budget eventually exhausts"); - assert_eq!(up.calls, allowed, "denied calls never reached upstream"); - } + assert!(allowed >= 1, "at least one call fit the budget"); + assert!(denied >= 1, "the budget eventually exhausts"); + assert_eq!(up.calls, allowed, "denied calls never reached upstream"); // The full decision log — allows and denies — is hash-chain intact. assert_eq!( jcp_receipt::verify_log(g.receipts(), "did:jcp:gateway:files", &steward_pub), @@ -645,6 +742,90 @@ mod tests { ); } + // ── a call's energy is sealed as what it is ────────────────────────────── + + use sandbox_meter::{DeviceEnergy, Provenance as Grade}; + use std::time::Duration; + + fn rec(class: DeviceClass, measured_uj: Option, estimated_uj: u64, provenance: Grade) -> DeviceEnergy { + DeviceEnergy { class, measured_uj, estimated_uj, provenance } + } + + fn span(devices: Vec) -> Measurement { + Measurement { devices, wall: Duration::from_millis(3), source: "test" } + } + + /// Every shape a metered span takes, and the energy it enters the ledger + /// and the receipt as. The first is what every macOS 27.0.1 host checked + /// now gets; main sealed it as `Energy::measured`, an estimate presented + /// as a reading. + #[test] + fn a_metered_span_enters_the_ledger_as_what_it_is() { + use jcp_schema::{Coverage, Provenance}; + // The meter's estimator record — no live counter, or a CPU counter + // that held across the span: wall × TDP, debited as an estimate. + let est = span(vec![rec(DeviceClass::Cpu, None, 60_000, Grade::Estimator)]); + assert_eq!(energy_of(&est), Energy::estimator(60_000)); + assert_eq!(span_provenance(&est), Provenance::Estimator); + + // A live CPU counter with the NPU and GPU beside it: a whole-call reading. + let live = span(vec![ + rec(DeviceClass::Cpu, Some(1_234), 1_234, Grade::HwShunt), + rec(DeviceClass::Npu, Some(0), 0, Grade::HwShunt), + rec(DeviceClass::Gpu, Some(66), 66, Grade::HwShunt), + ]); + assert_eq!(energy_of(&live), Energy::measured(1_300)); + assert_eq!(span_provenance(&live), Provenance::HwShunt); + + // A GPU counter alone (NVML without RAPL): a reading of part of the call. + let gpu = span(vec![rec(DeviceClass::Gpu, Some(3_000), 3_000, Grade::HwShunt)]); + assert_eq!(energy_of(&gpu), Energy::measured_partial(3_000)); + assert_eq!(energy_of(&gpu).coverage, Coverage::Partial); + + // A sum is only as good as its weakest record. + let mixed = span(vec![ + rec(DeviceClass::Cpu, Some(10), 10, Grade::HwShunt), + rec(DeviceClass::Gpu, None, 500, Grade::Estimator), + ]); + assert_eq!(energy_of(&mixed), Energy::estimator(510)); + let modeled = span(vec![rec(DeviceClass::Cpu, None, 700, Grade::ModelBased)]); + assert_eq!(energy_of(&modeled), Energy::modeled(700)); + assert_eq!(span_provenance(&modeled), Provenance::ModelBased); + + for m in [&est, &live, &gpu, &mixed, &modeled] { + let e = energy_of(m); + e.check_provenance().unwrap(); + assert_eq!(e.uj(), m.total_uj(), "the debit is the span's own figure"); + } + } + + /// End to end on this host: the receipt a call gets back says how its + /// joules were obtained, as the meter's span said it. Where the meter can + /// only estimate — every macOS 27.0.1 host checked, whose CPU energy rail + /// is frozen — the sealed provenance is `Estimator`, never `HwShunt`. + #[test] + fn a_calls_receipt_says_how_its_joules_were_obtained() { + let steward_pub = SigningKey::from_bytes(&[7u8; 32]).verifying_key().to_bytes(); + let mut g = gw(); + let mut up = MockServer { calls: 0 }; + let Relayed::Allowed { measured_uj, .. } = g.relay(&call("read_file", 1), &mut up).unwrap() else { + panic!("expected Allowed") + }; + let payload = jcr1_payload(g.receipts().last().unwrap(), "did:jcp:gateway:files", &steward_pub); + let sealed = payload.get("energy_provenance").and_then(Value::as_str).expect("energy_provenance"); + eprintln!("gateway call: hardware={} sealed={sealed} cost_uj={measured_uj}", g.metering_is_hardware()); + assert_eq!(payload.get("cost_uj").and_then(Value::as_u64), Some(measured_uj)); + assert!(measured_uj > 0, "a call that ran costs joules, measured or estimated"); + if g.metering_is_hardware() { + // A hardware meter reads the span, or estimates it if its CPU counter held. + assert!(matches!(sealed, "HwShunt" | "Estimator"), "{payload}"); + } else { + // No hardware source: every span is the estimator's, and says so. + assert_eq!(sealed, "Estimator", "an estimate sealed as a reading: {payload}"); + assert!(payload.get("energy_coverage").is_none(), "{payload}"); + } + } + // ── the end-to-end attack the whole feature exists to stop ─────────────── use jcp_schema::flow::{Integrity, Label, Origins}; @@ -723,6 +904,13 @@ mod tests { assert_eq!(payload.get("jcp_receipt").and_then(Value::as_str), Some("jcp/3")); let flow = payload.get("flow").expect("a flow block"); assert_eq!(flow.get("verdict").and_then(Value::as_str), Some("above")); + // The monitor's own energy is sealed with its own span's provenance. + let flow_prov = flow.get("flow_energy_provenance").and_then(Value::as_str); + if g.metering_is_hardware() { + assert!(matches!(flow_prov, Some("HwShunt" | "Estimator")), "{flow}"); + } else { + assert_eq!(flow_prov, Some("Estimator"), "an estimated scan sealed as a reading: {flow}"); + } // And the sealed witness is disclosable and re-verifies offline. let witness_hex = flow.get("witness").and_then(Value::as_str).unwrap(); diff --git a/joule-code/joule-code-rs/crates/joule-code-remediate/Cargo.toml b/joule-code/joule-code-rs/crates/joule-code-remediate/Cargo.toml index 9c1e4832d..c2e15276f 100644 --- a/joule-code/joule-code-rs/crates/joule-code-remediate/Cargo.toml +++ b/joule-code/joule-code-rs/crates/joule-code-remediate/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "joule-code-remediate" -description = "The reference verified-fix harness: scan a Rust workspace, remediate at the cheapest sufficient cascade tier (L1 lawful codemods first), gate every fix behind a verification pipeline whose stages ARE the contract obligations, and emit a JouleContract receipt with hardware-measured joules sealed by a sandbox meter attestation. A fix that fails its gate is reverted, never shipped." +description = "The reference verified-fix harness: scan a Rust workspace, remediate at the cheapest sufficient cascade tier (L1 lawful codemods first), gate every fix behind a verification pipeline whose stages ARE the contract obligations, and emit a JouleContract receipt whose joules are graded by how they were obtained: a hardware reading apportioned to the run and anchored to a sealed sandbox meter envelope only where the host exposes a live CPU energy counter, an estimate sealed as one everywhere else. A fix that fails its gate is reverted, never shipped." version.workspace = true edition.workspace = true license.workspace = true @@ -26,7 +26,10 @@ thiserror = { workspace = true } # fix's formal obligations through the joulecontract→proof vericoding bridge over # a real `z3`, and a refuted/unproved obligation reverts the fix like any other # failed gate. Decision-procedure ladder only — a formal gate must never fall -# through to a model stand-in (see `examples/formal_fix.rs`). +# through to a model stand-in (see `examples/formal_fix.rs`). Without `z3` on +# PATH an SMT obligation is Unproved and the gate fails closed; the formal-tier +# tests assert that branch on a solverless host, and `JOULE_FIX_EXPECT_Z3=1` +# makes a missing `z3` fail them instead (set it wherever z3 is installed). joulecontract-proof = { path = "../../../../joulecontract/joulecontract-rs/crates/joulecontract-proof" } proof-cascade = { path = "../../../../proof/proof-rs/crates/proof-cascade" } proof-solvers = { path = "../../../../proof/proof-rs/crates/proof-solvers" } diff --git a/joule-code/joule-code-rs/crates/joule-code-remediate/src/bin/joule_fix.rs b/joule-code/joule-code-rs/crates/joule-code-remediate/src/bin/joule_fix.rs index a3104a87e..35285fac0 100644 --- a/joule-code/joule-code-rs/crates/joule-code-remediate/src/bin/joule_fix.rs +++ b/joule-code/joule-code-rs/crates/joule-code-remediate/src/bin/joule_fix.rs @@ -26,20 +26,37 @@ fn main() { for g in &report.gates { println!(" [{}] {} — {}", if g.passed { "✓" } else { "✗" }, g.gate, g.detail); } - println!( - "\nenergy: {:.3} J attributed of {:.3} J machine-wide SoC-CPU over {:.2}s ({})", - report.attributed_uj as f64 / 1e6, - report.soc_cpu_uj as f64 / 1e6, - report.wall.as_secs_f64(), - report.meter_source, - ); + if report.attributed { + println!( + "\nenergy: {:.3} J attributed of {:.3} J machine-wide SoC-CPU over {:.2}s ({})", + report.attributed_uj as f64 / 1e6, + report.soc_cpu_uj as f64 / 1e6, + report.wall.as_secs_f64(), + report.meter_source, + ); + } else { + println!( + "\nenergy: {:.3} J estimated, not attributed, over {:.2}s ({})", + report.total_uj() as f64 / 1e6, + report.wall.as_secs_f64(), + report.meter_source, + ); + } if let Some(uj) = report.uj_per_verified_fix() { - println!("energy per verified fix: {:.3} J (attributed)", uj as f64 / 1e6); + println!( + "energy per verified fix: {:.3} J ({})", + uj as f64 / 1e6, + if report.attributed { "attributed" } else { "estimated" } + ); } if let Some(f) = &report.formal { println!( - "\nformal proof: {} proved, {} refuted, {} unproved — {:.3} J (real z3, ModelBased)", - f.proved, f.refuted, f.unproved, f.proof_uj as f64 / 1e6, + "\nformal proof: {} proved, {} refuted, {} unproved — {:.3} J ({}, ModelBased)", + f.proved, + f.refuted, + f.unproved, + f.proof_uj as f64 / 1e6, + if proof_solvers::z3_available() { "real z3" } else { "no z3 on PATH: SMT goals left undecided" }, ); for e in &f.entries { print!(" [{}] {}", if e.verdict == "proved" { "✓" } else { "✗" }, e.obligation); diff --git a/joule-code/joule-code-rs/crates/joule-code-remediate/src/lib.rs b/joule-code/joule-code-rs/crates/joule-code-remediate/src/lib.rs index bf2ac1365..3189912be 100644 --- a/joule-code/joule-code-rs/crates/joule-code-remediate/src/lib.rs +++ b/joule-code/joule-code-rs/crates/joule-code-remediate/src/lib.rs @@ -6,16 +6,18 @@ //! JouleContract whose `pre`/`post`/`inv` clauses are the pipeline stages //! (baseline capture, build, no-new-test-failures, lint-delta, frame //! confinement, diff guard), discharges each one with the stage's actual -//! outcome, and seals the result with hardware-measured energy. Three -//! consequences, by construction: +//! outcome, and seals the result with its energy, graded by how that energy +//! was obtained. Three consequences, by construction: //! //! 1. **A fix that fails its gate is reverted and never ships** — but the //! refusal still produces a receipt: an undischarged obligation on the //! record beats a silently dropped fix. -//! 2. **The energy figure is measured, not estimated**, whenever the host has -//! a hardware counter ([`sandbox_meter::PowerMeter`]); the meter -//! attestation inside the JouleContract receipt is a sealed sandbox JCR-1 -//! envelope — one standard's receipt is the other's evidence. +//! 2. **The energy figure rests on a hardware reading** whenever the host +//! exposes a live CPU energy counter ([`sandbox_meter::PowerMeter`]), and +//! is then anchored to a sealed sandbox JCR-1 meter envelope — one +//! standard's receipt is the other's evidence. Where it does not, the +//! figure is an estimate and is sealed as one (JCR-1 `Estimator`, +//! JouleContract `Unaccounted`), never as a reading. //! 3. **The receipt grades honestly**: receipt-local validation, JR-2 at //! most, generator and verifier self-declared and labeled as such. Higher //! rigor is a deployment's verifier-of-record work, exactly per the @@ -36,7 +38,7 @@ use joulecontract_schema::{ Ceiling, ContentAddress, Contract, EnergyClass, GovernanceClass, ObligationKind, Profile, Rigor, }; -use sandbox_meter::{run_attributed, PowerMeter}; +use sandbox_meter::{run_attributed, Attribution, PowerMeter}; use serde::{Deserialize, Serialize}; // The formally-verified fix tier. `DischargeStatus`/`Formalization`/ // `ContractDischarge` come from the proof bridge; the bridge's own @@ -57,12 +59,87 @@ use std::time::{Duration, Instant}; /// run unmetered. struct MeterAcc { meter: PowerMeter, - /// Σ machine-wide SoC-CPU energy over the metered windows (the upper bound). + /// This run's metered windows, reset at the start of each run. + tally: EnergyTally, +} + +/// The energy of one run's metered cargo windows, and the figure and grades +/// the receipts carry for it. +/// +/// A window is attributed when a hardware CPU reading that advanced was +/// apportioned to the child's process tree (`ModelBased`). Otherwise its +/// figure is not a reading of this workload. It is the meter's wall × TDP +/// estimate where the host exposes no live CPU counter (every macOS 27.0.1 +/// host checked), or where the counter held; or it is a machine-wide figure +/// the meter could not apportion. The run is `ModelBased` only when every +/// window was attributed, because a sum is only as good as its weakest part. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct EnergyTally { + /// Σ machine-wide SoC-CPU figure over the windows (the upper bound). soc_uj: u64, - /// Σ energy attributed to this run's process trees (ModelBased apportion). + /// Σ figure apportioned to this run's process trees. attr_uj: u64, - /// True once at least one window rested on real per-process accounting. - attributed: bool, + windows: u64, + attributed_windows: u64, +} + +impl EnergyTally { + fn add(&mut self, a: &Attribution) { + self.soc_uj = self.soc_uj.saturating_add(a.soc_cpu_uj); + self.attr_uj = self.attr_uj.saturating_add(a.attributed_uj); + self.windows += 1; + if a.attributed { + self.attributed_windows += 1; + } + } + + /// True when every metered window rested on real per-process accounting + /// of a hardware CPU reading. + fn attributed(&self) -> bool { + self.windows > 0 && self.attributed_windows == self.windows + } + + /// The figure the receipts carry. When attributed, it is the apportioned + /// reading. Otherwise it is the unapportioned total over the windows, as + /// an estimate. An apportioned estimate is no better than an estimate, + /// and the apportioning share is 0 whenever the machine's tick counters + /// (which update in bursts up to ~1 s apart) missed a window, which would + /// put 0 J on the receipt for work that ran. + fn figure_uj(&self) -> u64 { + if self.attributed() { + self.attr_uj + } else { + self.soc_uj + } + } + + /// The sandbox JCR-1 meter record for the figure. It is `ModelBased` for + /// an attribution, and otherwise the `Estimator` grade with no measured + /// figure. + fn meter_record(&self) -> sandbox_meter::DeviceEnergy { + sandbox_meter::DeviceEnergy { + class: sandbox_schema::DeviceClass::Cpu, + measured_uj: None, + estimated_uj: self.figure_uj(), + provenance: if self.attributed() { + sandbox_meter::Provenance::ModelBased + } else { + sandbox_meter::Provenance::Estimator + }, + } + } + + /// The JouleContract energy class for the figure. It is `ModelBased`, + /// anchored to the sealed meter receipt, for an attribution. Otherwise it + /// is `Unaccounted`, JouleContract's grade for energy estimated where the + /// substrate exposes no reading. + fn energy_class(&self) -> EnergyClass { + if self.attributed() { + EnergyClass::ModelBased + } else { + EnergyClass::Unaccounted + } + } } #[derive(Debug, thiserror::Error)] @@ -75,6 +152,10 @@ pub enum HarnessError { Spawn(String), #[error("contract error: {0}")] Contract(String), + /// Every gate passed but the fix could not be committed, so nothing + /// shipped. The message says where the verified change was left. + #[error("the fix passed every gate but could not be committed: {0}")] + Ship(String), } /// One pipeline stage's outcome — the discharge evidence for one obligation. @@ -116,7 +197,8 @@ pub struct FormalEntry { pub closed_by: Option, /// The counter-model witness (when refuted by a decision procedure). pub counter_model: Option, - /// Measured joules for this obligation's proof attempts. + /// Joules for this obligation's proof attempts, as the proof cascade + /// accounted them (`ModelBased`, in-process). pub uj: u64, } @@ -126,8 +208,8 @@ pub struct FormalOutcome { pub proved: usize, pub refuted: usize, pub unproved: usize, - /// Total measured joules across all proof attempts (`ModelBased`, in-process - /// — distinct from the cargo-attributed gate energy). + /// Total joules across all proof attempts (`ModelBased`, in-process — + /// distinct from the cargo-attributed gate energy). pub proof_uj: u64, /// True iff at least one formalization was supplied and **every** one /// proved. This is the gate: any refutation or unproven goal fails it, and @@ -149,14 +231,17 @@ pub struct RunReport { pub shipped: bool, /// The branch carrying the fix, when shipped. pub branch: Option, - /// Energy attributed to this run's process trees (`ModelBased` - /// apportionment of a real SoC measurement), microjoules — the receipt's - /// figure, and the honest per-workload cost. + /// Energy apportioned to this run's process trees, microjoules. When + /// `attributed`, it is a `ModelBased` apportionment of a real SoC reading: + /// the receipts' figure, and the honest per-workload cost. Otherwise it is + /// at best an estimate, and is context only. pub attributed_uj: u64, - /// Machine-wide SoC-CPU energy over the metered windows, microjoules — - /// the upper bound, for context. + /// Machine-wide SoC-CPU figure over the metered windows, microjoules — the + /// upper bound. When not `attributed`, this is the receipts' figure, as an + /// estimate. pub soc_cpu_uj: u64, - /// True when the figure rests on real per-process accounting. + /// True when every metered window rested on real per-process accounting + /// of a hardware CPU reading. False means the receipts carry an estimate. pub attributed: bool, pub wall: Duration, pub meter_source: String, @@ -176,16 +261,24 @@ pub struct RunReport { } impl RunReport { - /// Microjoules for the whole verified run (measure → fix → gate): the - /// energy ATTRIBUTED to this run's process trees, not the machine-wide - /// total. `soc_cpu_uj` carries the machine-wide upper bound. + /// Microjoules for the whole verified run (measure → fix → gate), the + /// figure the receipts carry. When `attributed`, it is the energy + /// ATTRIBUTED to this run's process trees, not the machine-wide total. + /// Otherwise it is the machine-wide figure over the metered windows, which + /// is then an estimate and is sealed as one (`Estimator`, `Unaccounted`). + /// It is never an apportioned zero for work that ran. pub fn total_uj(&self) -> u64 { - self.attributed_uj + if self.attributed { + self.attributed_uj + } else { + self.soc_cpu_uj + } } /// The headline metric nobody else publishes: microjoules per verified, - /// shipped fix. `None` when nothing shipped (energy was still spent and - /// is still on the receipt — refusals aren't free, and saying so is the + /// shipped fix. It is attributed when `attributed` and an estimate + /// otherwise. `None` when nothing shipped (energy was still spent and is + /// still on the receipt — refusals aren't free, and saying so is the /// point). pub fn uj_per_verified_fix(&self) -> Option { if !self.shipped { @@ -282,6 +375,17 @@ pub struct Harness { /// Per-process energy attribution, accumulated across cargo subprocesses. /// `RefCell` because `run()` takes `&self` and feeds it call by call. meter: RefCell, + /// Test builds only: the target directory every `cargo` run is handed + /// (`--target-dir`), the workspace's own `target/`, so a test's scratch + /// crate builds where it does with no `CARGO_TARGET_DIR`, whatever the + /// test process inherited. Every scratch crate is a package named + /// `scratch` at 0.1.0, and cargo names its artifacts without the path, so + /// under one exported `CARGO_TARGET_DIR` the concurrent tests' crates + /// built over each other's artifacts and fingerprints. Not in production + /// builds: `joule-fix` hands cargo no such flag, and the user's + /// environment, `CARGO_TARGET_DIR` included, decides where it builds. + #[cfg(test)] + cargo_target_dir: PathBuf, } impl Harness { @@ -298,6 +402,8 @@ impl Harness { if !inside { return Err(HarnessError::NotGit(workspace)); } + #[cfg(test)] + let cargo_target_dir = workspace.join("target"); Ok(Harness { workspace, max_diff_lines: 400, @@ -314,10 +420,10 @@ impl Harness { }), meter: RefCell::new(MeterAcc { meter: PowerMeter::detect(), - soc_uj: 0, - attr_uj: 0, - attributed: false, + tally: EnergyTally::default(), }), + #[cfg(test)] + cargo_target_dir, }) } @@ -331,13 +437,12 @@ impl Harness { // The real compute is `cargo`; meter and attribute it. Other programs // (git, rustfmt) are negligible harness overhead and run unmetered. if program == "cargo" { + let args = self.cargo_args(args); let mut acc = self.meter.borrow_mut(); - let MeterAcc { meter, soc_uj, attr_uj, attributed } = &mut *acc; - let (ok, text, a) = run_attributed(meter, program, args, &self.workspace) + let MeterAcc { meter, tally } = &mut *acc; + let (ok, text, a) = run_attributed(meter, program, &args, &self.workspace) .map_err(|e| HarnessError::Spawn(format!("{program}: {e}")))?; - *soc_uj = soc_uj.saturating_add(a.soc_cpu_uj); - *attr_uj = attr_uj.saturating_add(a.attributed_uj); - *attributed |= a.attributed; + tally.add(&a); return Ok((ok, text)); } let out = Command::new(program) @@ -350,6 +455,19 @@ impl Harness { Ok((out.status.success(), text)) } + /// The arguments a `cargo` run is given: `args`, as passed. In test + /// builds only, `--target-dir /target` follows the subcommand + /// (every cargo run here has one: `clippy`, `test`, `build`), and the + /// flag outranks an inherited `CARGO_TARGET_DIR`. + fn cargo_args<'a>(&'a self, args: &[&'a str]) -> Vec<&'a str> { + #[cfg(test)] + if let Some((sub, rest)) = args.split_first() { + let dir = self.cargo_target_dir.to_str().expect("a test workspace path is UTF-8"); + return [*sub, "--target-dir", dir].into_iter().chain(rest.iter().copied()).collect(); + } + args.to_vec() + } + fn git_head(&self) -> Result { let (ok, out) = self.run("git", &["rev-parse", "HEAD"])?; if !ok { @@ -422,12 +540,12 @@ impl Harness { let prior_rev = self.git_head()?; let prior = ContentId::of_bytes(prior_rev.as_bytes()); - // Reset the per-run attribution accumulator; cargo calls feed it. - { - let mut acc = self.meter.borrow_mut(); - acc.soc_uj = 0; - acc.attr_uj = 0; - } + // Reset the per-run tally, all of it; this run's cargo calls feed it. + // An earlier run's windows are not this run's energy: without the + // reset, a second run on this Harness would seal the first run's + // cargo figure into its own receipts (on every host, estimated or + // not), and an attributed earlier window could grade this one. + self.meter.borrow_mut().tally = EnergyTally::default(); let wall_start = Instant::now(); // ---- Stage 0: baseline (the `pre` obligation's evidence). @@ -582,11 +700,18 @@ impl Harness { // The rung above build+test, and the one the market has not reached. let formal = formal_spec.as_ref().map(|s| self.discharge_formal(s)); if let Some(f) = &formal { + // The detail rides into a failed gate's receipt entry, so it names + // the solver only when there was one to decide the goals. + let solver = if proof_solvers::z3_available() { + "real z3" + } else { + "no z3 on PATH: SMT goals left undecided" + }; gates.push(GateOutcome { gate: "formal-proof".into(), passed: f.all_proved, detail: format!( - "{} proved, {} refuted, {} unproved ({:.3} J, real z3)", + "{} proved, {} refuted, {} unproved ({:.3} J, {solver})", f.proved, f.refuted, f.unproved, @@ -601,14 +726,37 @@ impl Harness { let branch = if shipped { let name = format!("joule-fix/{}", &prior_rev[..12.min(prior_rev.len())]); - let _ = self.run("git", &["checkout", "-b", &name]); + // Shipped means committed. Every step is checked: an unchecked + // `checkout -b` that fails (the branch exists) would let the + // commit land on whatever branch is current, and an unchecked + // `commit` that fails (no git identity, a rejecting hook) would + // report a fix as shipped while it sits uncommitted in the tree. + let (ok, out) = self.run("git", &["checkout", "-b", &name])?; + if !ok { + return Err(HarnessError::Ship(format!( + "git checkout -b {name}: {} (the verified change is left uncommitted in the work tree)", + out.trim() + ))); + } // -u: stage only tracked modifications — the harness owns the // tracked diff; untracked artifacts (target/) are not its fix. - let _ = self.run("git", &["add", "-u", "--", "."]); - let _ = self.run( + let (ok, out) = self.run("git", &["add", "-u", "--", "."])?; + if !ok { + return Err(HarnessError::Ship(format!( + "git add on {name}: {} (the verified change is left uncommitted on {name})", + out.trim() + ))); + } + let (ok, out) = self.run( "git", &["commit", "-m", &format!("joule-fix: {} lint findings remediated, all gates green", findings_before.saturating_sub(findings_after))], - ); + )?; + if !ok { + return Err(HarnessError::Ship(format!( + "git commit on {name}: {} (the verified change is left staged on {name})", + out.trim() + ))); + } // A gate-verified stochastic fix is a lesson worth keeping: // compile it into a deterministic skill so the next recurrence // never pays the model's joules again. @@ -632,33 +780,36 @@ impl Harness { None }; - // ---- Energy: read the per-process attribution accumulated across the - // run's cargo subprocesses. The receipt carries the ATTRIBUTED figure - // (ModelBased — an apportionment of a real SoC measurement), not the - // machine-wide total; the SoC total rides along as honest context. - let (soc_cpu_uj, attributed_uj, attributed) = { + // ---- Energy: the per-window accounting accumulated across the run's + // cargo subprocesses. When every window was attributed, the receipts + // carry the ATTRIBUTED figure (ModelBased — an apportionment of a real + // SoC reading), not the machine-wide total. Otherwise they carry an + // estimate and grade it as one — never a figure signed as a reading. + let (tally, meter_sources) = { let acc = self.meter.borrow(); - (acc.soc_uj, acc.attr_uj, acc.attributed && acc.meter.is_hardware()) + (acc.tally, acc.meter.sources()) }; + let (soc_cpu_uj, attributed_uj, attributed) = (tally.soc_uj, tally.attr_uj, tally.attributed()); let wall = wall_start.elapsed(); let meter_source = if attributed { "attributed(ModelBased over IOReport HwShunt)".to_string() + } else if meter_sources.is_empty() { + "estimated(Unaccounted): wall-clock × TDP, no live energy counter on this host".to_string() } else { - "estimator".to_string() + format!( + "estimated(Unaccounted): not every window attributed; machine-wide figure over the \ + metered windows ({})", + meter_sources.join("+") + ) }; - // The sandbox meter receipt records the attributed figure as a - // ModelBased CPU reading; the JouleContract receipt anchors to it. - let device = sandbox_meter::DeviceEnergy { - class: sandbox_schema::DeviceClass::Cpu, - measured_uj: None, - estimated_uj: attributed_uj, - provenance: sandbox_meter::Provenance::ModelBased, - }; + // The sandbox meter receipt seals the receipts' figure with its grade: + // ModelBased for an attribution, the Estimator grade for an estimate. + // The JouleContract receipt anchors to it only when attributed. let meter_receipt = sandbox_meter::Receipt::build( sandbox_schema::ContentId::of_bytes(prior_rev.as_bytes()), sandbox_schema::Tier::S0, - vec![device], + vec![tally.meter_record()], 1, [3u8; 8], ) @@ -670,8 +821,7 @@ impl Harness { let receipt = self.receipt( &contract, &op, - attributed_uj, - attributed, + &tally, &meter_envelope, &gates, findings_before, @@ -1072,8 +1222,7 @@ impl Harness { &self, contract: &Contract, op: &CodingOp, - attributed_uj: u64, - attributed: bool, + energy: &EnergyTally, meter_envelope: &[u8], gates: &[GateOutcome], before: usize, @@ -1136,18 +1285,15 @@ impl Harness { }) .collect(); - // The figure is an apportionment of a real SoC measurement: ModelBased, - // never Metered (a process is not a shunt). A ModelBased receipt is - // anchored to the sealed sandbox meter receipt that carries the - // measurement the apportionment rests on; without hardware it is - // honestly Unaccounted. - let provenance = if attributed { - EnergyClass::ModelBased - } else { - EnergyClass::Unaccounted - }; + // An attributed figure is an apportionment of a real SoC reading: + // ModelBased, never Metered (a process is not a shunt), and anchored + // to the sealed sandbox meter receipt that carries it. Unless every + // window was attributed, the figure is an estimate, and Unaccounted + // (energy estimated where the substrate exposes no reading) is the + // honest class. + let provenance = energy.energy_class(); let energy_anchor = - attributed.then(|| ContentAddress::of_bytes(meter_envelope)); + energy.attributed().then(|| ContentAddress::of_bytes(meter_envelope)); // No MeterBinding: that attests a Metered (direct-shunt) figure, which // this is not. The energy_anchor is the ModelBased evidence path. let meter: Option = None; @@ -1169,7 +1315,7 @@ impl Harness { }, discharge, execution: vec![tier_addr], - joules_spent: attributed_uj as f64 / 1e6, + joules_spent: energy.figure_uj() as f64 / 1e6, meter, energy_anchor, joules_allowed: contract.ceiling.enpi_joules, @@ -1222,24 +1368,77 @@ fn answers() { assert_eq!(answer(), 42); } "#, ) .unwrap(); - sh(&dir, "git init -q && git add -A && git -c user.email=t@t -c user.name=t commit -qm init"); + // A repo-local identity: the harness's own ship commit must work in + // any environment, including a CI container with no global git + // identity (where it used to fail silently and leave the tree dirty). + sh(&dir, "git init -q && git config user.email t@t && git config user.name t && git add -A && git commit -qm init"); dir } + fn git_out(dir: &std::path::Path, args: &[&str]) -> String { + let o = Command::new("git").args(args).current_dir(dir).output().unwrap(); + assert!(o.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&o.stderr)); + String::from_utf8_lossy(&o.stdout).trim().to_string() + } + #[test] fn verified_fix_end_to_end_ships_with_receipt() { let dir = scratch_crate("ship"); + let before = git_out(&dir, &["rev-parse", "HEAD"]); let mut h = Harness::new(&dir).unwrap(); h.skill_store = dir.join("skills.json"); + // A test's cargo runs build in the scratch crate's own `target/`: + // `--target-dir` follows the subcommand and outranks an inherited + // `CARGO_TARGET_DIR`, under which every scratch crate (each named + // `scratch`) would share one target directory with the other tests'. + let own_target = dir.join("target"); + assert_eq!( + h.cargo_args(&["test", "--workspace"]), + ["test", "--target-dir", own_target.to_str().unwrap(), "--workspace"] + ); let report = h.run_once().unwrap(); + assert!(own_target.join("debug").is_dir(), "the scratch crate did not build in its own target directory"); assert!(report.findings_before > 0, "scratch crate has findings"); assert!(report.shipped, "gates: {:?}", report.gates); assert!(report.gates.iter().all(|g| g.passed)); assert_eq!(report.findings_after, Some(0)); - assert!(report.branch.is_some()); - assert!(report.total_uj() > 0); - assert!(report.uj_per_verified_fix().is_some()); + // Shipped means committed: the fix is a commit on the reported branch, + // on top of where the run started, and nothing is left uncommitted. + let branch = report.branch.clone().expect("a shipped fix names its branch"); + assert_eq!(git_out(&dir, &["rev-parse", "--abbrev-ref", "HEAD"]), branch); + assert_eq!(git_out(&dir, &["rev-parse", "HEAD~1"]), before, "one commit on top of the start"); + assert!(git_out(&dir, &["status", "--porcelain", "--untracked-files=no"]).is_empty(), "shipped, yet the tree is dirty"); + // Work ran, so the figure is never zero — attributed or estimated. + assert!(report.total_uj() > 0, "{} µJ ({})", report.total_uj(), report.meter_source); + assert!(report.uj_per_verified_fix().is_some_and(|uj| uj > 0)); + + // Both receipts grade the figure by how it was obtained. + eprintln!( + "joule-fix run: attributed={} total_uj={} soc_cpu_uj={} attributed_uj={} ({})", + report.attributed, report.total_uj(), report.soc_cpu_uj, report.attributed_uj, report.meter_source + ); + assert_eq!(report.receipt.joules_spent, report.total_uj() as f64 / 1e6); + let sealed = sandbox_meter::jcr1::open(&report.meter_envelope).expect("meter envelope").payload; + let seals = |grade: &str| sealed.windows(grade.len()).any(|w| w == grade.as_bytes()); + if report.attributed { + assert_eq!(report.receipt.provenance, EnergyClass::ModelBased); + assert!(seals("ModelBased") && !seals("Estimator"), "meter envelope grade"); + } else { + assert_eq!(report.receipt.provenance, EnergyClass::Unaccounted); + assert!(report.receipt.energy_anchor.is_none()); + assert!( + seals("Estimator") && !seals("ModelBased"), + "an estimate sealed in the meter envelope as ModelBased" + ); + } + // Where the meter cannot read CPU energy at all, nothing is attributed. + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + if sandbox_meter::ioreport_status() != sandbox_meter::IoReportStatus::Live { + assert!(!report.attributed, "attributed with no live CPU rail ({:?})", sandbox_meter::ioreport_status()); + } + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + assert!(!report.attributed, "the unattributed fallback never attributes"); // The receipt validates locally and tells the truth about itself: // self-verified ⇒ independence false ⇒ JR-1 ceiling, stated. @@ -1250,6 +1449,145 @@ fn answers() { assert_eq!(answer(), 42); } let _ = std::fs::remove_dir_all(&dir); } + /// A fix that passes every gate but cannot be committed did not ship. + /// The harness used to discard `git commit`'s result and report + /// `shipped: true` with a branch name while the fix sat uncommitted. + /// That is what a CI container with no git identity produced. + #[test] + fn a_fix_whose_commit_is_rejected_is_never_reported_shipped() { + let dir = scratch_crate("commit-rejected"); + sh(&dir, "printf '#!/bin/sh\\nexit 1\\n' > .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit"); + let before = git_out(&dir, &["rev-parse", "HEAD"]); + let mut h = Harness::new(&dir).unwrap(); + h.skill_store = dir.join("skills.json"); + match h.run_once() { + Err(HarnessError::Ship(msg)) => assert!(msg.contains("git commit"), "{msg}"), + other => panic!("a rejected commit must not ship: {:?}", other.map(|r| (r.shipped, r.branch))), + } + assert_eq!(git_out(&dir, &["rev-parse", "HEAD"]), before, "nothing was committed"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// When the fix's branch already exists, the harness must not commit the + /// fix onto whatever branch happens to be checked out. + #[test] + fn a_fix_never_lands_on_the_current_branch_when_its_own_branch_exists() { + let dir = scratch_crate("branch-exists"); + let before = git_out(&dir, &["rev-parse", "HEAD"]); + let current = git_out(&dir, &["rev-parse", "--abbrev-ref", "HEAD"]); + git_out(&dir, &["branch", &format!("joule-fix/{}", &before[..12])]); + let mut h = Harness::new(&dir).unwrap(); + h.skill_store = dir.join("skills.json"); + match h.run_once() { + Err(HarnessError::Ship(msg)) => assert!(msg.contains("checkout -b"), "{msg}"), + other => panic!("an existing branch must stop the ship: {:?}", other.map(|r| (r.shipped, r.branch))), + } + assert_eq!(git_out(&dir, &["rev-parse", "--abbrev-ref", "HEAD"]), current); + assert_eq!(git_out(&dir, &["rev-parse", "HEAD"]), before, "the fix was committed onto {current}"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// One metered cargo window, as `run_attributed` reports it. + fn window(soc_cpu_uj: u64, attributed_uj: u64, attributed: bool) -> Attribution { + Attribution { + soc_cpu_uj, + attributed_uj, + proc_cpu_secs: 1.0, + busy_cpu_secs: if attributed_uj > 0 { 4.0 } else { 0.0 }, + share: if attributed_uj > 0 { 0.25 } else { 0.0 }, + wall: Duration::from_secs(1), + attributed, + } + } + + /// A run whose windows were estimated is sealed as an estimate, and never + /// as an apportioned zero. This is every macOS 27.0.1 host checked, where + /// the meter has no live CPU counter. main sealed 0 J `ModelBased` there, + /// and the core fix alone still sealed the apportioned estimate + /// `ModelBased` in the meter envelope, and 0 J whenever the tick counters + /// missed every window. + #[test] + fn an_unattributed_run_is_sealed_as_an_estimate_never_an_apportioned_zero() { + let mut t = EnergyTally::default(); + // The machine tick counters missed this window: share 0. + t.add(&window(20_000_000, 0, false)); + t.add(&window(40_000_000, 10_000_000, false)); + assert!(!t.attributed()); + assert_eq!(t.figure_uj(), 60_000_000, "the unapportioned estimate"); + assert_eq!(t.energy_class(), EnergyClass::Unaccounted); + let r = t.meter_record(); + assert_eq!(r.provenance, sandbox_meter::Provenance::Estimator); + assert_eq!((r.measured_uj, r.estimated_uj), (None, 60_000_000)); + r.check_provenance().unwrap(); + + // Even when every window's share was lost, the run is not free. + let mut missed = EnergyTally::default(); + missed.add(&window(20_000_000, 0, false)); + assert_eq!(missed.figure_uj(), 20_000_000); + } + + /// Each run on a Harness seals its own cargo windows only. The tally a + /// run leaves behind (here with a stale window added on top, so the + /// assertion does not depend on run 1's figure) never reaches the next + /// run's figure, grade or receipts. + #[test] + fn a_second_run_seals_only_its_own_cargo_windows() { + const STALE_UJ: u64 = 1_000_000_000_000_000; // 1 GJ: no scratch run comes near it + let dir = scratch_crate("rerun"); + let mut h = Harness::new(&dir).unwrap(); + h.skill_store = dir.join("skills.json"); + let first = h.run_once().unwrap(); + assert!(first.shipped, "gates: {:?}", first.gates); + { + // Run 1's windows are still in the tally when run 1 returns. + let acc = h.meter.borrow(); + assert!(acc.tally.windows > 0, "run 1 metered its cargo calls"); + assert_eq!(acc.tally.soc_uj, first.soc_cpu_uj); + } + h.meter.borrow_mut().tally.add(&window(STALE_UJ, STALE_UJ, true)); + + let second = h.run_once().unwrap(); + eprintln!( + "joule-fix rerun: first soc_cpu_uj={} second soc_cpu_uj={} attributed={} ({})", + first.soc_cpu_uj, second.soc_cpu_uj, second.attributed, second.meter_source + ); + assert!(second.soc_cpu_uj < STALE_UJ, "run 2 sealed an earlier run's windows: {} µJ", second.soc_cpu_uj); + assert!(second.attributed_uj < STALE_UJ, "{}", second.attributed_uj); + assert!(second.total_uj() > 0, "run 2's own cargo calls ran"); + assert!(second.receipt.joules_spent < STALE_UJ as f64 / 1e6); + assert_eq!(second.receipt.joules_spent, second.total_uj() as f64 / 1e6); + let acc = h.meter.borrow(); + assert_eq!(acc.tally.soc_uj, second.soc_cpu_uj, "the tally holds run 2's windows alone"); + assert_eq!(acc.tally.attributed(), second.attributed); + drop(acc); + let _ = std::fs::remove_dir_all(&dir); + } + + /// `ModelBased` only when every window was attributed: one estimated + /// window makes the sum an estimate. main labelled the run attributed if + /// any window was, and never reset that between runs. + #[test] + fn a_run_is_attributed_only_when_every_window_was() { + let mut all = EnergyTally::default(); + all.add(&window(10_000_000, 2_500_000, true)); + all.add(&window(30_000_000, 7_500_000, true)); + assert!(all.attributed()); + assert_eq!(all.figure_uj(), 10_000_000, "the apportioned reading"); + assert_eq!(all.energy_class(), EnergyClass::ModelBased); + assert_eq!(all.meter_record().provenance, sandbox_meter::Provenance::ModelBased); + all.meter_record().check_provenance().unwrap(); + + let mut mixed = all; + mixed.add(&window(20_000_000, 0, false)); + assert!(!mixed.attributed()); + assert_eq!(mixed.figure_uj(), 60_000_000, "the unapportioned estimate"); + assert_eq!(mixed.energy_class(), EnergyClass::Unaccounted); + assert_eq!(mixed.meter_record().provenance, sandbox_meter::Provenance::Estimator); + + // Nothing metered is not an attribution either. + assert!(!EnergyTally::default().attributed()); + } + #[test] fn l3_fix_passes_the_same_gates() { // A finding clippy --fix can NOT auto-fix (approx_constant): L1 @@ -1469,52 +1807,103 @@ fn pins_source_text() { std::fs::write(dir.join(".joule-fix/formal.json"), spec.to_string()).unwrap(); } + /// Whether the formal-tier tests run against a real `z3`. + /// + /// They follow this repo's z3 convention (proof-solvers, joulecontract- + /// proof, and the standards-build workflow's "Install z3" step): a suite + /// passes with or without a solver, and asserts something either way. + /// With `z3` the SMT rung decides the obligation and its verdict is + /// asserted. Without it the obligation is Unproved, never a fabricated + /// proof, and the formal gate fails closed, so the fix is reverted — that + /// is asserted instead. Neither branch returns without asserting, and each + /// logs which one ran. + /// + /// Where a solver is supposed to be installed (a CI job that installs z3, + /// a dev box), set `JOULE_FIX_EXPECT_Z3=1`: a missing `z3` then FAILS + /// these tests rather than quietly taking the solverless branch. + fn z3_for_formal_test(test: &str) -> bool { + let present = proof_solvers::z3_available(); + let expected = std::env::var("JOULE_FIX_EXPECT_Z3").as_deref() == Ok("1"); + assert!(present || !expected, "{test}: JOULE_FIX_EXPECT_Z3=1, but there is no working `z3` on PATH"); + eprintln!( + "{test}: z3 {}", + if present { "present, asserting its verdict" } else { "absent, asserting the formal gate fails closed" } + ); + present + } + + /// Without a solver nothing decides the obligation: it is Unproved (not + /// proved, and not refuted either: there is no counter-model without a + /// solver), the formal gate fails and says why, and the fix is reverted + /// although every other gate passed. + fn assert_the_formal_gate_failed_closed(dir: &std::path::Path, report: &RunReport) { + let formal = report.formal.as_ref().expect("formal gate ran"); + assert_eq!((formal.proved, formal.refuted, formal.unproved), (0, 0, 1), "{:?}", formal.entries); + assert!(!formal.all_proved); + let g = report.gates.iter().find(|g| g.gate == "formal-proof").unwrap(); + assert!(!g.passed); + assert!(g.detail.contains("no z3 on PATH") && !g.detail.contains("real z3"), "{}", g.detail); + assert!( + report.gates.iter().filter(|g| g.gate != "formal-proof").all(|g| g.passed), + "only the undecided proof blocks this fix: {:?}", + report.gates + ); + assert!(!report.shipped, "an undecided formal obligation must block shipping"); + let src = std::fs::read_to_string(dir.join("src/main.rs")).unwrap(); + assert!(src.contains("return 42;"), "the operational fix was reverted"); + assert!(dir.join(".joule-fix/formal-discharge.json").exists(), "the discharge evidence is written either way"); + } + #[test] fn a_proven_formal_obligation_lets_the_fix_ship() { - if !proof_solvers::z3_available() { - return; // no solver: the formal gate can't run — covered below - } + let z3 = z3_for_formal_test("a_proven_formal_obligation_lets_the_fix_ship"); let dir = scratch_crate("formal-ok"); write_formal_spec(&dir, "(- debits credits)"); // the correct postcondition let mut h = Harness::new(&dir).unwrap(); h.skill_store = dir.join("skills.json"); let report = h.run_once().unwrap(); - let formal = report.formal.as_ref().expect("formal gate ran"); - assert!(formal.all_proved, "entries: {:?}", formal.entries); - assert_eq!((formal.proved, formal.refuted, formal.unproved), (1, 0, 0)); - assert!(formal.proof_uj > 0, "real z3 burned measured joules"); - let g = report.gates.iter().find(|g| g.gate == "formal-proof").unwrap(); - assert!(g.passed); - assert!(report.shipped, "all gates incl. formal green: {:?}", report.gates); - // the discharge evidence was written for an auditor - assert!(dir.join(".joule-fix/formal-discharge.json").exists()); + if z3 { + let formal = report.formal.as_ref().expect("formal gate ran"); + assert!(formal.all_proved, "entries: {:?}", formal.entries); + assert_eq!((formal.proved, formal.refuted, formal.unproved), (1, 0, 0)); + assert!(formal.proof_uj > 0, "the proof attempt was accounted joules"); + let g = report.gates.iter().find(|g| g.gate == "formal-proof").unwrap(); + assert!(g.passed); + assert!(report.shipped, "all gates incl. formal green: {:?}", report.gates); + // the discharge evidence was written for an auditor + assert!(dir.join(".joule-fix/formal-discharge.json").exists()); + } else { + assert_the_formal_gate_failed_closed(&dir, &report); + } let _ = std::fs::remove_dir_all(&dir); } #[test] fn a_refuted_formal_obligation_reverts_the_fix() { - if !proof_solvers::z3_available() { - return; - } + let z3 = z3_for_formal_test("a_refuted_formal_obligation_reverts_the_fix"); let dir = scratch_crate("formal-refute"); write_formal_spec(&dir, "(+ debits credits)"); // the BUG: z3 finds a counter-model let mut h = Harness::new(&dir).unwrap(); h.skill_store = dir.join("skills.json"); let report = h.run_once().unwrap(); - let formal = report.formal.as_ref().expect("formal gate ran"); - assert_eq!(formal.refuted, 1, "z3 refuted the false postcondition"); - assert!(!formal.all_proved); - // the witness rode through from the cascade - assert!(formal.entries.iter().any(|e| e.verdict == "refuted")); - let g = report.gates.iter().find(|g| g.gate == "formal-proof").unwrap(); - assert!(!g.passed); - // a refuted formal obligation reverts the fix exactly like a failed test - assert!(!report.shipped, "a refuted proof must block shipping"); - let src = std::fs::read_to_string(dir.join("src/main.rs")).unwrap(); - assert!(src.contains("return 42;"), "the operational fix was reverted"); + if z3 { + let formal = report.formal.as_ref().expect("formal gate ran"); + assert_eq!(formal.refuted, 1, "z3 refuted the false postcondition"); + assert!(!formal.all_proved); + // the witness rode through from the cascade + assert!(formal.entries.iter().any(|e| e.verdict == "refuted")); + let g = report.gates.iter().find(|g| g.gate == "formal-proof").unwrap(); + assert!(!g.passed); + // a refuted formal obligation reverts the fix exactly like a failed test + assert!(!report.shipped, "a refuted proof must block shipping"); + let src = std::fs::read_to_string(dir.join("src/main.rs")).unwrap(); + assert!(src.contains("return 42;"), "the operational fix was reverted"); + } else { + assert_the_formal_gate_failed_closed(&dir, &report); + } let _ = std::fs::remove_dir_all(&dir); } diff --git a/joule-ui/README.md b/joule-ui/README.md index 86961f1d3..54f33fc23 100644 --- a/joule-ui/README.md +++ b/joule-ui/README.md @@ -28,7 +28,8 @@ renderer is product surface, owned by the consumer. an agent (the protocol draws no distinction; both are traced). `run_interaction` authorizes it against a capability *before it runs* (a denied action's handler never executes), meters it in joules, - charges the measured cost, and an `InteractionLog` seals it as the next + charges its cost (a reading or an estimate, and which), and an + `InteractionLog` seals it as the next link of a **hash-chained** receipt. The actor is bound into the signed core, so an interaction cannot be re-attributed without breaking the chain. A joule-ui session becomes a tamper-evident ledger of *who fired diff --git a/joule-ui/SPEC.md b/joule-ui/SPEC.md index 3580ea03a..6030a4404 100644 --- a/joule-ui/SPEC.md +++ b/joule-ui/SPEC.md @@ -62,17 +62,44 @@ provides. ## §5 Energy and provenance Every interaction carries a two-part energy figure in integer **microjoules**: -`measured_uj` (present only from a hardware interface) and `estimated_uj` -(always present), tagged with a **Provenance** — `HwShunt` (a hardware energy -reading), `ModelBased` (a calibrated apportionment), or `Estimator` (a coarse -constant). This is the family's shared energy model; a figure is reported with -the honesty of how it was obtained, never promoted. +`measured_uj` (present only for a hardware reading) and `estimated_uj` (always +present), tagged with a **Provenance** — `HwShunt` (a hardware energy reading), +`ModelBased` (a calibrated apportionment), or `Estimator` (a coarse estimate). +This is the family's shared energy model; a figure is reported with the honesty +of how it was obtained, never promoted. Measurement itself is a **recognized edge**, not part of this standard: a -`Meter` is any source of a cumulative microjoule counter, and the sibling -**Sandbox** standard's `PowerMeter` (real RAPL/IOReport, and CPU-time -apportionment) is the reference one. joule-ui defines what an interaction -records; the host wires the meter it trusts. +`Meter` is any source of a cumulative microjoule counter, and it declares the +provenance of its readings. A meter's figures are `Estimator` unless the meter +declares otherwise: a meter opts in to `ModelBased` for a calibrated model, and +to `HwShunt` only for readings of a live hardware counter (below). A meter +MUST NOT declare `HwShunt` for figures that are not such readings — a +deterministic stub, a model, a replayed counter. (The conformance pack's +fixture meter, §8, is the one exception: it declares `HwShunt` only so the +vectors exercise the reading path of the signed core, and like the pack's +published steward key it attests nothing.) The sibling +**Sandbox** standard's `PowerMeter` is the reference meter: it reads vendor +energy counters (RAPL, NVML, Apple IOReport) while they are live, apportions a +reading to a process tree by CPU time (`ModelBased`), and falls back to a +wall-clock estimate, labelled `Estimator`, where no counter is live. joule-ui +defines what an interaction records; the host wires the meter it trusts. + +**Vendor energy counters.** A reading from a vendor energy counter (RAPL, +NVML, Apple IOReport, ...) MAY be graded as a hardware reading only while the +counter is demonstrably live for the domain the figure claims: the +implementation MUST have observed the counter covering that domain advance +under load (for Apple IOReport: the CPU energy rail itself advancing in at +least 5 sample intervals, in at least 4 of every 5, over at least 300 ms while +the probing thread burns CPU). A counter that is present but not live — frozen, +or republished in lumps — MUST NOT produce a hardware-graded figure; the +implementation falls back to its estimator or reports the work unmetered, and +SHOULD record why. A figure MUST NOT claim coverage of a domain its counter +does not read; where the format has a coverage field, such a figure is +`Partial`. joule-ui's energy record has no coverage field, so a figure from a +counter that does not read the domain the work ran on — a GPU counter alone, +for a handler that ran on the CPU — MUST NOT be graded `HwShunt` for the +interaction: the reference adapters seal such a span as the meter's +`Estimator` figure for its wall time. ## §6 The interaction lifecycle @@ -84,11 +111,22 @@ records; the host wires the meter it trusts. two-phase `(authorize, charge)` gate, and the sibling **JCP** standard's runtime — gating on a signed, joule-budgeted grant — is the reference one. 2. **Meter the handler.** The handler runs once; the meter is sampled before - and after; the difference is the interaction's energy. With no hardware - meter the figure is honestly `Estimator`. -3. **Charge the measured cost.** The measured joules are debited from the - capability's remaining budget — so the budget that stops a runaway UI is - energy *actually spent*, settled after the fact, not a token estimate. + and after; the difference is the interaction's energy, graded with the + meter's declared provenance (§5). When the meter gives no reading — either + sample is absent — the handler's wall-clock duration is charged at the + meter's declared estimator power (20 W unless the meter declares its own), + rounded up to whole microjoules, and the figure is `Estimator`. A handler + that ran MUST NOT be recorded at 0 µJ for want of a reading: work that ran + is not free, and a figure of zero would let a budget never bind on a host + without a counter. A denied action's handler never ran, and its record + carries 0 µJ `Estimator`. +3. **Charge the cost, with its provenance.** The interaction's figure — a + reading or an estimate — is debited from the capability's remaining budget, + and the authorizer is handed the figure's provenance with it. So the budget + binds on every host, settled after the fact against the energy the work + cost as nearly as the host can know it; and an authorizer that keeps its own + record of the debit MUST record it with that provenance, never an estimate + as a reading. The result is an **Interaction**: the surface's content address (what the user was looking at), the action id, the content addresses of the inputs and the @@ -135,7 +173,8 @@ edge, not required.) the exhaustive error list (empty for a valid tree). - **C2 — interaction metering.** Given a surface, action, actor, inputs, a stub meter, and a handler, reproduce the Interaction: verdict, energy, content - addresses, and the sealed actor. + addresses, and the sealed actor. The stub meter declares `HwShunt` as a + fixture grade, so the recorded energy carries `measured_uj` (§5). - **C3 — authorization.** A denied action produces a deny Interaction whose outputs are the genesis sentinel and whose handler did not run; the actor is still named on the denial. @@ -145,11 +184,13 @@ edge, not required.) ## §9 Recognized edges joule-ui is self-contained at its core and composes outward at named edges: -the **Sandbox** standard supplies the hardware `Meter`; **JCP** supplies the -budgeted-grant `Authorizer`; **JouleContract** / in-toto+DSSE supply an optional -cross-family receipt envelope; and any transport (MCP, REST, SSE, gRPC, Web -MoQ) carries the validated surface and the interaction stream. None are required -to use the protocol; each is the reference answer where a host needs that edge. +the **Sandbox** standard supplies the reference `Meter` (a hardware reading +while a vendor counter is live, its labelled estimate otherwise); **JCP** +supplies the budgeted-grant `Authorizer`; **JouleContract** / in-toto+DSSE +supply an optional cross-family receipt envelope; and any transport (MCP, REST, +SSE, gRPC, Web MoQ) carries the validated surface and the interaction stream. +None are required to use the protocol; each is the reference answer where a +host needs that edge. ## §10 Non-goals diff --git a/joule-ui/conformance/README.md b/joule-ui/conformance/README.md index e80b6a0c4..87fcd4401 100644 --- a/joule-ui/conformance/README.md +++ b/joule-ui/conformance/README.md @@ -27,7 +27,11 @@ them. There is no renderer vector (the renderer is the host's), no transport vector (the consumer's — MCP/SSE/Web MoQ), and no widget-library vector (product surface). The metering and capability mechanisms are **recognized edges** (SPEC §9): C2/C3/C4 use a deterministic stub meter and a stub budget so the vectors are -reproducible, while a deployment wires the Sandbox `PowerMeter` (real hardware) -and a JCP grant. What conformance guarantees is the load-bearing claim — that an +reproducible, while a deployment wires the Sandbox `PowerMeter` (a hardware +reading while a vendor counter is live, its labelled estimate otherwise) and a +JCP grant. The stub meter declares `HwShunt` explicitly, so C2 and C4 exercise +the reading path of the signed core (`measured_uj` present); like the steward +key, that grade is a fixture and attests nothing. Outside the pack, a meter that +declares no grade is `Estimator` (SPEC §5). What conformance guarantees is the load-bearing claim — that an interaction is validated, authorized-before-running, metered, and sealed into a tamper-evident chain, identically across implementations. diff --git a/joule-ui/joule-ui-rs/Cargo.lock b/joule-ui/joule-ui-rs/Cargo.lock index 4d2e7e1ef..2e5dad160 100644 --- a/joule-ui/joule-ui-rs/Cargo.lock +++ b/joule-ui/joule-ui-rs/Cargo.lock @@ -341,6 +341,7 @@ dependencies = [ "jcp-schema", "joule-ui-core", "sandbox-meter", + "sandbox-schema", "serde", "serde_json", "thiserror", @@ -371,6 +372,7 @@ dependencies = [ "joule-ui-interaction", "joule-ui-render", "sandbox-meter", + "sandbox-schema", "serde", "serde_json", ] diff --git a/joule-ui/joule-ui-rs/crates/joule-ui-conformance/src/lib.rs b/joule-ui/joule-ui-rs/crates/joule-ui-conformance/src/lib.rs index 04dddb654..52cf3946b 100644 --- a/joule-ui/joule-ui-rs/crates/joule-ui-conformance/src/lib.rs +++ b/joule-ui/joule-ui-rs/crates/joule-ui-conformance/src/lib.rs @@ -19,7 +19,8 @@ use joule_ui_core::{ WidgetSchema, validate, }; use joule_ui_interaction::{ - Authorizer, ContentId, Interaction, InteractionLog, Meter, SealedInteraction, run_interaction, + Authorizer, ContentId, Energy, Interaction, InteractionLog, Meter, Provenance, SealedInteraction, + run_interaction, }; use std::cell::Cell; @@ -30,6 +31,13 @@ pub const STEWARD_SEED: [u8; 32] = [0x17; 32]; /// A deterministic meter: each sample advances by `step` µJ, so a before/after /// difference is fixed. +/// +/// It declares `HwShunt` explicitly. The published C2 and C4 vectors were +/// sealed with this fixture standing in for a hardware counter, so they carry +/// `measured_uj` and exercise the reading path of the signed core (SPEC §7.1); +/// keeping that grade keeps every vector byte-identical. It is a fixture grade, +/// like the published steward key: it attests nothing, and outside the pack a +/// stub meter that declares nothing is `Estimator` (SPEC §5). pub struct StepMeter { counter: Cell, step: u64, @@ -45,6 +53,9 @@ impl Meter for StepMeter { self.counter.set(v + self.step); Some(v) } + fn provenance(&self) -> Provenance { + Provenance::HwShunt + } } /// A joule budget — the JCP edge in miniature, deterministic. @@ -59,8 +70,8 @@ impl Authorizer for Budget { Err("budget-exceeded".into()) } } - fn charge(&mut self, _action: &str, measured_uj: u64) -> u64 { - self.remaining = self.remaining.saturating_sub(measured_uj); + fn charge(&mut self, _action: &str, cost: &Energy) -> u64 { + self.remaining = self.remaining.saturating_sub(cost.uj()); self.remaining } fn grant_id(&self) -> Option { @@ -194,7 +205,7 @@ mod tests { let it = c2_interaction(); assert!(it.verdict.is_allow()); assert_eq!(it.energy.uj(), 4200); - assert_eq!(it.energy.provenance, joule_ui_interaction::Provenance::HwShunt); + assert_eq!(it.energy.provenance, Provenance::HwShunt); assert_ne!(it.outputs, ContentId::genesis()); } diff --git a/joule-ui/joule-ui-rs/crates/joule-ui-interaction/Cargo.toml b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/Cargo.toml index a2a6003dd..bee1be358 100644 --- a/joule-ui/joule-ui-rs/crates/joule-ui-interaction/Cargo.toml +++ b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/Cargo.toml @@ -17,10 +17,14 @@ ed25519-dalek = { workspace = true } [dev-dependencies] joule-ui-core = { path = "../joule-ui-core" } -# The Sandbox standard's real hardware meter — the reference Meter edge -# (SPEC §5). A dev-dependency keeps the core decoupled while proving the edge -# composes against real RAPL/IOReport. +# The Sandbox standard's meter — the reference Meter edge (SPEC §5): a hardware +# reading where the host exposes a live energy counter (RAPL, NVML, or Apple +# IOReport while its CPU rail is live), its labelled estimate otherwise. A +# dev-dependency keeps the core decoupled while proving the edge composes. sandbox-meter = { path = "../../../../sandbox/sandbox-rs/crates/sandbox-meter" } +# Device classes for the scripted spans `tests/span_meter.rs` replays through +# the examples' meter adapter. +sandbox-schema = { path = "../../../../sandbox/sandbox-rs/crates/sandbox-schema" } # The JCP standard's runtime + grant — the reference Authorizer edge (SPEC §6). # Dev-deps: the core stays decoupled; the example proves a real signed, # joule-budgeted grant gates a joule-ui interaction. diff --git a/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/jcp_metered.rs b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/jcp_metered.rs index 898dacde4..372eb265e 100644 --- a/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/jcp_metered.rs +++ b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/jcp_metered.rs @@ -1,27 +1,37 @@ //! The full "metered, attested UI" with **both recognized edges real at once**: //! a joule-ui interaction gated by an actual signed **JCP grant** (the //! Authorizer edge, SPEC §6) and metered by the actual **Sandbox PowerMeter** -//! (the Meter edge, SPEC §5). Real hardware joules are debited from a real -//! capability's budget until the grant runs out, and then the **real JCP -//! runtime** — not a stub — denies the next interaction. Each one seals into -//! the chain. +//! (the Meter edge, SPEC §5). Each interaction's figure is debited from the +//! grant's budget, and once the budget is spent the **real JCP runtime** — not +//! a stub — denies the next interaction. Each one seals into the chain. +//! +//! Each interaction is metered as one `PowerMeter` span and sealed with that +//! span's own figure and provenance (the `SpanMeter` in `support/`, the +//! adapter `joule-ui-serve` uses), and the output says which it is. With a +//! live hardware counter whose CPU rail advanced, it is `HwShunt`: the +//! counters' machine-wide delta over the interaction. With none (every Apple +//! Silicon host checked on macOS 27.0.1) it is the meter's `Estimator` figure, +//! wall-clock × TDP for the interaction's span — never 0 µJ for work that ran. +//! +//! The grant is debited with the figure AND its provenance: `Authorizer::charge` +//! is handed the interaction's sealed figure with its grade, and the authorizer +//! here hands the JCP runtime an estimate as an estimate, never wrapped as a +//! measurement. JCP debits a budget by the reading if there is one, else by the +//! estimate, so estimated interactions spend the grant too. //! //! ```bash //! cargo run -p joule-ui-interaction --example jcp_metered --release //! ``` -use joule_ui_interaction::{Authorizer, ContentId, InteractionLog, Meter, run_interaction, verify_chain}; +use joule_ui_interaction::{ + Authorizer, ContentId, Energy, InteractionLog, Provenance, run_interaction, verify_chain, +}; use sandbox_meter::PowerMeter; -use std::cell::RefCell; use std::time::{Duration, Instant}; -/// The Sandbox `PowerMeter` as a joule-ui `Meter` (SPEC §5 edge). -struct SandboxMeter(RefCell); -impl Meter for SandboxMeter { - fn sample_uj(&self) -> Option { - self.0.borrow_mut().sample_uj() - } -} +#[path = "support/span_meter.rs"] +mod span_meter; +use span_meter::SpanMeter; /// The JCP runtime as a joule-ui `Authorizer` (SPEC §6 edge): a real signed, /// joule-budgeted grant, enforced by the real two-phase JCP runtime. @@ -52,16 +62,35 @@ impl Authorizer for JcpAuthorizer { jcp_schema::Verdict::Deny(r) => Err(r.label().to_string()), } } - fn charge(&mut self, _action: &str, measured_uj: u64) -> u64 { - self.runtime - .charge(&self.grant, jcp_schema::Energy::measured(measured_uj)) - .budget_remaining_uj + fn charge(&mut self, _action: &str, cost: &Energy) -> u64 { + // The figure's own provenance, as the interaction is sealed with it: a + // measurement is charged as one, an estimate as an estimate. + let uj = cost.uj(); + let cost = match cost.provenance { + Provenance::HwShunt => jcp_schema::Energy::measured(uj), + Provenance::ModelBased => jcp_schema::Energy::modeled(uj), + Provenance::Estimator => jcp_schema::Energy::estimator(uj), + }; + self.runtime.charge(&self.grant, cost).budget_remaining_uj } fn grant_id(&self) -> Option { Some(self.grant.id().to_hex()) } } +/// What an interaction's figure is, read from its own record: a 0 for work +/// that ran is no figure, and only a hardware record is called a reading. +fn figure(e: &Energy) -> String { + match (e.uj(), e.provenance) { + (0, p) => format!("no figure (0 µJ [{}]: the meter gave no reading for this work)", p.label()), + (uj, Provenance::HwShunt) => format!("{uj} µJ [HwShunt]"), + (uj, p) => format!("{uj} µJ [{}, an estimate]", p.label()), + } +} + +/// The grant's budget, as minted below. +const BUDGET_UJ: u64 = 15_000_000; + fn main() { // A real signed JCP grant for the `apply` action, with a joule budget that // a couple of real interactions will exhaust. @@ -74,7 +103,7 @@ fn main() { jcp_schema::ContentId::of_bytes(b"apply"), ["apply"], ), - budget_uj: 15_000_000, // 15 J + budget_uj: BUDGET_UJ, // 15 J // Joules-only grant: no money ceiling, no caveats. An empty caveat list is // omitted from the canonical encoding, so this stays byte-identical to the // grant this example minted before JCP gained dual ceilings. @@ -86,20 +115,38 @@ fn main() { nonce: [0; 16], }; let sg = jcp_grant::issue(&issuer, grant.clone()).expect("issue grant"); - let mut auth = JcpAuthorizer { runtime: jcp_runtime::Runtime::new(), sg, grant, caller: did }; let pm = PowerMeter::detect(); println!( - "edges: JCP grant (15 J budget, real signed) + {} meter", - if pm.is_hardware() { "Sandbox hardware" } else { "estimator" } + "edges: JCP grant (15 J budget, real signed) + {}", + if pm.is_hardware() { + format!("Sandbox hardware meter ({})", pm.sources().join(", ")) + } else { + let why = match sandbox_meter::ioreport_status() { + // Apple Silicon: say why the SoC counters were declined. + sandbox_meter::IoReportStatus::Unavailable => String::new(), + status => format!(" ({status})"), + }; + format!( + "no hardware probe{why} — each interaction below is charged and sealed with the \ + estimator's wall-clock × TDP figure for its own span" + ) + } ); - let meter = SandboxMeter(RefCell::new(pm)); + let meter = SpanMeter::new(pm); + let mut auth = JcpAuthorizer { + runtime: jcp_runtime::Runtime::new(), + sg, + grant, + caller: did, + }; let surface = ContentId::of_bytes(b"
"); let mut log = InteractionLog::new(ed25519_dalek::SigningKey::from_bytes(&[8u8; 32])); - // Fire interactions doing ~150ms of real work each until the real grant's - // real joules run out and the real JCP runtime denies the next one. + // Fire interactions doing ~150ms of real work each. Each one's figure is + // debited from the grant; if the figures spend it, the real JCP runtime + // denies the next one. for n in 1..=6u64 { let it = run_interaction(&surface, "apply", "did:key:z6MkDemoUser", &n.to_le_bytes(), &meter, Some(&mut auth), || { let t = Instant::now(); @@ -111,21 +158,49 @@ fn main() { } std::hint::black_box(x).to_le_bytes().to_vec() }); + let charged = if it.verdict.is_allow() { figure(&it.energy) } else { "not run".into() }; println!( - " apply #{n}: {} · {} µJ [{}] · grant {} µJ left", + " apply #{n}: {} · {charged} · grant {} µJ left", it.verdict.tag(), - it.energy.uj(), - it.energy.provenance.label(), it.budget_remaining_uj.map(|b| b.to_string()).unwrap_or_else(|| "—".into()), ); log.record(it); } let verified = verify_chain(log.entries()).expect("chain verifies"); - let allowed = log.entries().iter().filter(|e| e.interaction.verdict.is_allow()).count(); + let allowed: Vec<&Energy> = log + .entries() + .iter() + .filter(|e| e.interaction.verdict.is_allow()) + .map(|e| &e.interaction.energy) + .collect(); + let denied = verified - allowed.len(); + let sealed: u64 = allowed.iter().map(|e| e.uj()).sum(); + // What the grant was actually debited: the runtime never debits past the + // budget, so the last figure may have been only partly charged. + let left = log + .entries() + .iter() + .rev() + .filter(|e| e.interaction.verdict.is_allow()) + .find_map(|e| e.interaction.budget_remaining_uj) + .unwrap_or(BUDGET_UJ); + let debited = BUDGET_UJ - left; + let unmetered = allowed.iter().filter(|e| e.uj() == 0).count(); + // Label the figures by the sealed records, not by the meter. + let what = if unmetered > 0 { + format!("{unmetered} had no figure, 0 µJ for work that ran") + } else if allowed.iter().all(|e| e.provenance == Provenance::HwShunt) { + format!("{sealed} µJ sealed, machine-wide hardware counter deltas (HwShunt)") + } else if allowed.iter().all(|e| e.provenance == Provenance::Estimator) { + format!("{sealed} µJ sealed, estimated (wall-clock × TDP over each interaction's span), not measured") + } else { + format!("{sealed} µJ sealed, partly estimated, not measured") + }; + let what = format!("{what}; {debited} of the {BUDGET_UJ} µJ budget debited"); println!( - "\n{verified} interactions sealed: {allowed} allowed (real joules, debited from a real grant), \ - {} denied by the real JCP runtime when the budget ran out.", - verified - allowed + "\n{verified} interactions sealed: {} allowed; {what}; {denied} denied by the real JCP runtime{}.", + allowed.len(), + if denied > 0 { " once the budget was spent" } else { ", as the budget was never spent" } ); } diff --git a/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/metered_surface.rs b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/metered_surface.rs index 464865222..4ce0f4a35 100644 --- a/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/metered_surface.rs +++ b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/metered_surface.rs @@ -13,13 +13,14 @@ use joule_ui_core::{ AllowedChildren, PropSchema, PropType, PropValue, Registry, Widget, WidgetSchema, validate, }; use joule_ui_interaction::{ - Authorizer, ContentId, InteractionLog, Meter, run_interaction, verify_chain, + Authorizer, ContentId, Energy, InteractionLog, Meter, run_interaction, verify_chain, }; use std::cell::Cell; /// A wall-clock-free deterministic meter standing in for the Sandbox standard's -/// real `PowerMeter` (which reads RAPL/IOReport). The point of the example is -/// the *protocol*; the hardware figure is the Sandbox edge. +/// `PowerMeter`. The point of the example is the *protocol*. It declares no +/// hardware grade, so its figures are sealed as `Estimator`: a hardware +/// figure is the Sandbox edge's to supply, and only from a live counter. struct DemoMeter(Cell); impl Meter for DemoMeter { fn sample_uj(&self) -> Option { @@ -41,8 +42,8 @@ impl Authorizer for Budget { Err("budget-exceeded".into()) } } - fn charge(&mut self, _action: &str, measured_uj: u64) -> u64 { - self.remaining = self.remaining.saturating_sub(measured_uj); + fn charge(&mut self, _action: &str, cost: &Energy) -> u64 { + self.remaining = self.remaining.saturating_sub(cost.uj()); self.remaining } fn grant_id(&self) -> Option { diff --git a/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/real_meter.rs b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/real_meter.rs index a02a1b437..79767cac6 100644 --- a/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/real_meter.rs +++ b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/real_meter.rs @@ -1,26 +1,38 @@ //! A joule-ui interaction metered by the **real Sandbox `PowerMeter`** — the -//! reference Meter edge (SPEC §5), reading actual hardware energy (Apple -//! Silicon IOReport / Intel-AMD RAPL) rather than a stub. Proves the -//! recognized-edge design composes end to end: a fired Action's joules are a -//! hardware measurement, sealed into the chain. +//! reference Meter edge (SPEC §5) — rather than a stub. Proves the +//! recognized-edge design composes end to end: a fired Action's energy +//! record is sealed into the chain. +//! +//! What that record is depends on the host, and the output says which. Each +//! interaction is metered as one `PowerMeter` span and sealed with that +//! span's own figure and provenance (the `SpanMeter` in `support/`, the +//! adapter `joule-ui-serve` uses). Where the meter has a live hardware counter +//! (Intel/AMD RAPL; Apple Silicon IOReport only when its CPU rail is live) and +//! the span's CPU counter advanced, the figure is `HwShunt`: the counters' +//! machine-wide delta over the interaction. Otherwise (every Apple Silicon +//! host checked on macOS 27.0.1) it is the meter's `Estimator` figure, +//! wall-clock × TDP for the interaction's own span: an estimate, labelled as +//! one, and never 0 µJ for work that ran. //! //! ```bash //! cargo run -p joule-ui-interaction --example real_meter --release //! ``` -use joule_ui_interaction::{ContentId, InteractionLog, Meter, run_interaction, verify_chain}; +use joule_ui_interaction::{ContentId, Energy, InteractionLog, Provenance, run_interaction, verify_chain}; use sandbox_meter::PowerMeter; -use std::cell::RefCell; use std::time::{Duration, Instant}; -/// Adapts the Sandbox `PowerMeter` (whose cumulative `sample_uj` needs `&mut`) -/// to joule-ui's `Meter` (which samples through `&self`) via interior -/// mutability. This tiny shim is the whole "Sandbox is the reference Meter -/// edge" claim, made real. -struct SandboxMeter(RefCell); -impl Meter for SandboxMeter { - fn sample_uj(&self) -> Option { - self.0.borrow_mut().sample_uj() +#[path = "support/span_meter.rs"] +mod span_meter; +use span_meter::SpanMeter; + +/// What an interaction's figure is, read from its own record: a 0 for work +/// that ran is no figure, and only a hardware record is called a reading. +fn figure(e: &Energy) -> String { + match (e.uj(), e.provenance) { + (0, p) => format!("no figure (0 µJ [{}]: the meter gave no reading for this work)", p.label()), + (uj, Provenance::HwShunt) => format!("{uj} µJ [HwShunt]"), + (uj, p) => format!("{uj} µJ [{}, an estimate]", p.label()), } } @@ -28,18 +40,29 @@ fn main() { let pm = PowerMeter::detect(); let hardware = pm.is_hardware(); println!( - "meter: {} ({})", - if hardware { "Sandbox PowerMeter — hardware" } else { "no hardware probe — estimator" }, - pm.sources().join(", ") + "meter: {}", + if hardware { + format!("Sandbox PowerMeter — hardware ({})", pm.sources().join(", ")) + } else { + let why = match sandbox_meter::ioreport_status() { + // Apple Silicon: say why the SoC counters were declined. + sandbox_meter::IoReportStatus::Unavailable => String::new(), + status => format!(" ({status})"), + }; + format!( + "no hardware probe{why} — each interaction below is sealed with the estimator's \ + wall-clock × TDP figure for its own span" + ) + } ); - let meter = SandboxMeter(RefCell::new(pm)); + let meter = SpanMeter::new(pm); let surface = ContentId::of_bytes(b"
"); let signing = ed25519_dalek::SigningKey::from_bytes(&[5u8; 32]); let mut log = InteractionLog::new(signing); // Three real interactions: each handler does ~150ms of genuine CPU work - // (the "computation behind the UI"), metered on real hardware. + // (the "computation behind the UI"), metered by whatever the meter can read. for n in 1..=3u64 { let it = run_interaction(&surface, "render", "did:key:z6MkDemoUser", &n.to_le_bytes(), &meter, None, || { let t = Instant::now(); @@ -51,19 +74,26 @@ fn main() { } std::hint::black_box(x).to_le_bytes().to_vec() }); - println!( - " render #{n}: {} · {} µJ [{}]", - it.verdict.tag(), - it.energy.uj(), - it.energy.provenance.label(), - ); + println!(" render #{n}: {} · {}", it.verdict.tag(), figure(&it.energy)); log.record(it); } let verified = verify_chain(log.entries()).expect("chain verifies"); - println!( - "\nsealed chain: {verified} links verified, {} µJ total{}", - log.total_uj(), - if hardware { " — measured on real hardware (HwShunt)" } else { "" }, - ); + // Label the total by the sealed records, not by the meter: "measured" + // only when every record is a hardware reading that moved. + let records: Vec<&Energy> = log.entries().iter().map(|e| &e.interaction.energy).collect(); + let unmetered = records.iter().filter(|e| e.uj() == 0).count(); + let label = if unmetered > 0 { + format!( + " — {unmetered} of {} interaction(s) have no figure (0 µJ for work that ran): not a measurement", + records.len() + ) + } else if records.iter().all(|e| e.provenance == Provenance::HwShunt) { + " — hardware counter deltas (HwShunt), machine-wide over each interaction".to_string() + } else if records.iter().all(|e| e.provenance == Provenance::Estimator) { + " — estimated (wall-clock × TDP over each interaction's span), not measured".to_string() + } else { + " — partly estimated, not measured".to_string() + }; + println!("\nsealed chain: {verified} links verified, {} µJ total{label}", log.total_uj()); } diff --git a/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/support/span_meter.rs b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/support/span_meter.rs new file mode 100644 index 000000000..61c2dcaed --- /dev/null +++ b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/examples/support/span_meter.rs @@ -0,0 +1,115 @@ +//! The Sandbox `PowerMeter` as a joule-ui `Meter` (SPEC §5 edge) over managed +//! spans, shared by the `real_meter` and `jcp_metered` examples. It is the +//! adapter `joule-ui-serve` seals its served interactions with. +//! +//! Every `sample_uj` closes the span the previous sample opened, adds that +//! span's figure to a running counter, and opens the next. So the difference +//! of two consecutive samples is exactly one span's figure, and `provenance` +//! is that span's own, which `run_interaction` asks for right after its second +//! sample. A span is `HwShunt` only when it was a hardware reading of the CPU +//! the handler ran on, whose CPU counter advanced. Otherwise it is the meter's +//! `Estimator` figure (wall-clock × TDP, never 0 for work that ran): on a host +//! with no live energy counter — every macOS 27.0.1 host checked, whose CPU +//! energy rail is frozen — when the span's CPU counter held while another +//! class moved, and when the span has no CPU reading at all (an NVML-only +//! host reads the GPU alone: see [`interaction_figure`]). +//! +//! The examples' old adapter sampled `PowerMeter::sample_uj`, a sum of raw +//! counters under the trait's default `HwShunt`. Where no hardware probe +//! existed that was `None`, and each interaction that ran was sealed at +//! `0 µJ [Estimator]`. +//! +//! `tests/span_meter.rs` includes this file too, and drives the adapter over +//! scripted spans. + +use joule_ui_interaction::{Meter, Provenance}; +use sandbox_meter::probe::Span; +use sandbox_meter::{Measurement, PowerMeter}; +use std::cell::RefCell; +use std::time::Duration; + +/// A meter that measures managed spans: the Sandbox `PowerMeter` does, and a +/// test can script one. +pub trait Spans { + type Open; + fn open(&mut self) -> Self::Open; + fn close(&mut self, span: Self::Open) -> Measurement; + /// The meter's estimate for `wall` of work, with no reading. + fn estimate(&self, wall: Duration) -> Measurement; +} + +impl Spans for PowerMeter { + type Open = Span; + fn open(&mut self) -> Span { + self.start() + } + fn close(&mut self, span: Span) -> Measurement { + span.stop(self) + } + fn estimate(&self, wall: Duration) -> Measurement { + PowerMeter::estimate(self, wall) + } +} + +/// `RefCell` because sampling mutates the meter while the `Meter` trait +/// samples through `&self`. Each example drives one interaction at a time, so +/// a span is exclusive to one interaction. +pub struct SpanMeter(RefCell>); + +struct State { + meter: M, + open: Option, + counter_uj: u64, + last: Provenance, +} + +impl SpanMeter { + pub fn new(meter: M) -> Self { + SpanMeter(RefCell::new(State { meter, open: None, counter_uj: 0, last: Provenance::Estimator })) + } +} + +impl Meter for SpanMeter { + fn sample_uj(&self) -> Option { + let mut s = self.0.borrow_mut(); + if let Some(span) = s.open.take() { + let m = s.meter.close(span); + let m = interaction_figure(m, |wall| s.meter.estimate(wall)); + s.counter_uj = s.counter_uj.saturating_add(m.total_uj()); + s.last = span_provenance(&m); + } + let next = s.meter.open(); + s.open = Some(next); + Some(s.counter_uj) + } + + /// The provenance of the span the last sample closed. + fn provenance(&self) -> Provenance { + self.0.borrow().last + } +} + +/// The figure an interaction is sealed with, from its span. joule-ui seals +/// one figure per interaction, with no field to say that figure covers part +/// of the work, so a span with no hardware reading of the CPU is not a +/// reading of the interaction: an NVML-only probe set reads the GPU alone, +/// and its figure would be sealed `HwShunt` for work that ran on the CPU. Such +/// a span is sealed as the meter's estimate for its wall time (`Estimator`, +/// never 0 for work that ran); a span that reads the CPU passes through, +/// with any GPU record beside it. +fn interaction_figure(m: Measurement, estimate: impl FnOnce(Duration) -> Measurement) -> Measurement { + if m.reads_cpu() { m } else { estimate(m.wall) } +} + +/// A span's provenance: the weakest of its records, because a sum is only as +/// good as its weakest part. +fn span_provenance(m: &Measurement) -> Provenance { + use sandbox_meter::Provenance as Grade; + if m.devices.is_empty() || m.devices.iter().any(|d| d.provenance == Grade::Estimator) { + Provenance::Estimator + } else if m.devices.iter().all(|d| d.provenance == Grade::HwShunt && d.measured_uj.is_some()) { + Provenance::HwShunt + } else { + Provenance::ModelBased + } +} diff --git a/joule-ui/joule-ui-rs/crates/joule-ui-interaction/src/lib.rs b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/src/lib.rs index 6d1240088..5c6ac642c 100644 --- a/joule-ui/joule-ui-rs/crates/joule-ui-interaction/src/lib.rs +++ b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/src/lib.rs @@ -14,7 +14,8 @@ //! The two heavy mechanisms — measuring energy and gating on a budget — are //! **recognized edges**, not hard dependencies. A [`Meter`] is any source of a //! cumulative microjoule counter (the sibling Sandbox standard's `PowerMeter` -//! is one, and reads real hardware); an [`Authorizer`] is any two-phase +//! is one: a hardware reading where the host exposes a live energy counter, +//! an estimate where it does not); an [`Authorizer`] is any two-phase //! capability check (the sibling JCP standard's runtime is one, gating on a //! signed joule budget). joule-ui supplies the receipt, the chain, and the //! lifecycle; the host wires whichever edge it trusts. @@ -23,10 +24,11 @@ //! //! [`run_interaction`] is the whole protocol in one function: hash the inputs, //! **authorize before running** (a denied action never executes its handler), -//! meter the handler, **charge the measured cost**, hash the outputs, and -//! return an [`Interaction`]. An [`InteractionLog`] then seals it as the next -//! link in a chain — allows and denies alike, because a denial excised from -//! the record is exactly the audit a malicious host would want to drop. +//! meter the handler, **charge its cost with the cost's provenance**, hash the +//! outputs, and return an [`Interaction`]. An [`InteractionLog`] then seals it +//! as the next link in a chain — allows and denies alike, because a denial +//! excised from the record is exactly the audit a malicious host would want to +//! drop. use serde::{Deserialize, Serialize}; @@ -87,28 +89,48 @@ impl Energy { } } +/// The power, in milliwatts, the estimator of last resort charges a handler's +/// wall-clock time at when its meter gives no reading (SPEC §6): 20 W. +/// Deliberately coarse, and a meter that knows its host's power envelope +/// declares its own through [`Meter::estimator_milliwatts`]. The figure it +/// produces is always graded `Estimator`. +pub const DEFAULT_ESTIMATOR_MILLIWATTS: u64 = 20_000; + /// A cumulative microjoule counter — the metering edge. The Sandbox standard's -/// `PowerMeter` is one implementation (real RAPL/IOReport). `sample_uj` returns +/// `PowerMeter` is one implementation: a hardware reading where the host +/// exposes a live energy counter, its estimator otherwise. `sample_uj` returns /// the running counter; the lifecycle takes a before/after difference. `None` -/// means no reading, and the interaction falls to `Estimator`. +/// means no reading, and the lifecycle estimates the handler's energy from its +/// wall-clock time at [`estimator_milliwatts`](Meter::estimator_milliwatts), +/// graded `Estimator` (SPEC §6). /// /// A meter MUST also declare the **provenance** of its readings, so the receipt /// is honest about *how the number was obtained*. This is load-bearing: a /// deterministic stub or a calibrated model returns a number too, and stamping /// it `HwShunt` (a hardware measurement) is exactly the dishonesty this standard /// exists to forbid — a figure is reported with the honesty of how it was -/// obtained, never promoted. Real hardware samplers keep the `HwShunt` default; -/// a model declares `ModelBased`; a coarse constant declares `Estimator`. +/// obtained, never promoted. So the grade is `Estimator` unless a meter says +/// otherwise: a model declares `ModelBased`, and a meter declares `HwShunt` +/// only for a reading of a hardware counter it has observed to be live for the +/// work it covers (SPEC §5). A counter that is present but frozen, or +/// republished in lumps, is not live, and its figures are not `HwShunt`. pub trait Meter { fn sample_uj(&self) -> Option; - /// How honest this meter's readings are. Defaults to `HwShunt` — implement - /// only on a real hardware interface; override for a model or estimator. + /// How honest this meter's readings are. Defaults to `Estimator`: a meter + /// opts in to a stronger grade, and only for readings that earn it. fn provenance(&self) -> Provenance { - Provenance::HwShunt + Provenance::Estimator + } + /// The power the lifecycle charges a handler's wall-clock time at when + /// this meter gives no reading, in milliwatts. Defaults to + /// [`DEFAULT_ESTIMATOR_MILLIWATTS`]. + fn estimator_milliwatts(&self) -> u64 { + DEFAULT_ESTIMATOR_MILLIWATTS } } -/// A meter with no interface — always `None` (interactions fall to `Estimator`). +/// A meter with no interface — always `None`, so every interaction that runs +/// is estimated from its wall-clock time and graded `Estimator`. pub struct NoMeter; impl Meter for NoMeter { fn sample_uj(&self) -> Option { @@ -119,17 +141,30 @@ impl Meter for NoMeter { } } +/// The estimator of last resort: `wall` at `milliwatts`, in microjoules, +/// rounded up and never 0 — this is only asked for work that ran, and work +/// that ran is not free. (1 mW for 1 ns is 1e-6 µJ, so µJ = mW × ns / 1e6.) +pub fn estimate_uj(wall: std::time::Duration, milliwatts: u64) -> u64 { + let nuj = wall.as_nanos().saturating_mul(u128::from(milliwatts)); + u64::try_from(nuj.div_ceil(1_000_000)).unwrap_or(u64::MAX).max(1) +} + /// The capability edge: two-phase, because an interaction's true cost is /// unknown until its handler runs. [`authorize`](Authorizer::authorize) gates -/// the action *before* it executes (on any remaining budget); [`charge`] debits -/// the measured cost *after*. The JCP standard's runtime implements exactly -/// this shape over a signed joule-budgeted grant. +/// the action *before* it executes (on any remaining budget); +/// [`charge`](Authorizer::charge) debits its cost *after*. The JCP standard's +/// runtime implements exactly this shape over a signed joule-budgeted grant. pub trait Authorizer { /// May `action` fire? `Ok(budget_remaining_uj)` to allow, `Err(reason)` to /// deny. Called before the handler runs and debits nothing. fn authorize(&mut self, action: &str) -> Result; - /// Debit the measured cost after the handler ran; returns budget left. - fn charge(&mut self, action: &str, measured_uj: u64) -> u64; + /// Debit the interaction's cost after the handler ran; returns budget + /// left. `cost` is the figure the interaction is sealed with, and carries + /// its provenance: a hardware reading, a model, or an estimate. Debit + /// [`cost.uj()`](Energy::uj) whichever it is, so the budget binds on a + /// host with no reading; an authorizer that seals its own record of the + /// debit seals it with `cost.provenance`, never as a reading it was not. + fn charge(&mut self, action: &str, cost: &Energy) -> u64; /// The grant identity this authorizer enforces, for the receipt. fn grant_id(&self) -> Option { None @@ -185,8 +220,14 @@ pub struct Interaction { /// Run one interaction end to end (SPEC §6). Hashes the inputs, authorizes /// before running (a denied action's handler never executes), meters the -/// handler, charges the measured cost, hashes the outputs. The handler returns -/// the output bytes (a serialized widget tree or `UiDelta`). +/// handler, charges its cost with the cost's provenance, hashes the outputs. +/// The handler returns the output bytes (a serialized widget tree or +/// `UiDelta`). +/// +/// A handler that ran is never costed at 0 µJ for want of a reading: when the +/// meter gives none, its wall-clock time is estimated at the meter's +/// [`estimator_milliwatts`](Meter::estimator_milliwatts) and graded +/// `Estimator`, so a budget binds on every host. pub fn run_interaction( surface: &ContentId, action: &str, @@ -222,10 +263,13 @@ where // Phase 2: meter the handler. The reading is tagged with the meter's // DECLARED provenance — a number from a model or a stub is never promoted - // to `HwShunt`. `measured_uj` is carried only for a hardware shunt; a - // model/estimator figure rides in `estimated_uj` (SPEC §5). + // to `HwShunt`, and a meter that declares nothing is `Estimator`. + // `measured_uj` is carried only for a hardware shunt; a model/estimator + // figure rides in `estimated_uj` (SPEC §5). let before = meter.sample_uj(); + let started = std::time::Instant::now(); let output = handler(); + let wall = started.elapsed(); let after = meter.sample_uj(); let energy = match (before, after) { (Some(a), Some(b)) => { @@ -244,11 +288,15 @@ where }, } } - _ => Energy::estimator(0), + // No reading for work that ran: estimate it from the handler's own + // wall-clock time rather than seal 0 µJ, which would cost it nothing + // and let a budget never bind on a host without a counter (SPEC §6). + _ => Energy::estimator(estimate_uj(wall, meter.estimator_milliwatts())), }; - // Phase 3: charge the measured cost. - let budget_remaining_uj = authorizer.map(|a| a.charge(action, energy.uj())); + // Phase 3: charge the cost, with its provenance — an estimate is debited + // as an estimate, never handed on as a reading. + let budget_remaining_uj = authorizer.map(|a| a.charge(action, &energy)); Interaction { surface: surface.clone(), @@ -409,7 +457,8 @@ mod tests { use super::*; /// A test meter whose counter advances by a fixed step each sample, so a - /// before/after difference is deterministic. + /// before/after difference is deterministic. It declares no grade, so its + /// figures are the trait default's: `Estimator`. struct StepMeter(std::cell::Cell, u64); impl Meter for StepMeter { fn sample_uj(&self) -> Option { @@ -419,11 +468,32 @@ mod tests { } } + /// The same counter, opted in to the hardware grade — standing in for a + /// meter reading a live hardware counter, to exercise the `HwShunt` path. + struct HwStepMeter(StepMeter); + impl Meter for HwStepMeter { + fn sample_uj(&self) -> Option { + self.0.sample_uj() + } + fn provenance(&self) -> Provenance { + Provenance::HwShunt + } + } + + fn hw_step(step: u64) -> HwStepMeter { + HwStepMeter(StepMeter(std::cell::Cell::new(0), step)) + } + /// A budget authorizer — the JCP edge in miniature: allow while budget - /// remains, deny when exhausted, debit the measured cost. + /// remains, deny when exhausted, debit the cost. It records every figure + /// it was charged, so a test can see what it was handed. struct Budget { remaining: u64, id: String, + charged: Vec, + } + fn budget(remaining: u64, id: &str) -> Budget { + Budget { remaining, id: id.into(), charged: Vec::new() } } impl Authorizer for Budget { fn authorize(&mut self, _action: &str) -> Result { @@ -433,8 +503,9 @@ mod tests { Err("budget-exceeded".into()) } } - fn charge(&mut self, _action: &str, measured_uj: u64) -> u64 { - self.remaining = self.remaining.saturating_sub(measured_uj); + fn charge(&mut self, _action: &str, cost: &Energy) -> u64 { + self.charged.push(*cost); + self.remaining = self.remaining.saturating_sub(cost.uj()); self.remaining } fn grant_id(&self) -> Option { @@ -442,13 +513,23 @@ mod tests { } } + /// Work a handler does for at least `ms` of wall time. + fn busy(ms: u64) -> Vec { + let t = std::time::Instant::now(); + let mut x = 0u64; + while t.elapsed() < std::time::Duration::from_millis(ms) { + x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + } + std::hint::black_box(x).to_le_bytes().to_vec() + } + fn surface() -> ContentId { ContentId::of_bytes(b"
") } #[test] fn an_interaction_is_metered_and_sealed() { - let meter = StepMeter(std::cell::Cell::new(0), 1500); + let meter = hw_step(1500); let it = run_interaction( &surface(), "apply", @@ -459,8 +540,7 @@ mod tests { || br#"{"op":"replace","path":"/0","widget":{"name":"Image"}}"#.to_vec(), ); assert!(it.verdict.is_allow()); - assert_eq!(it.energy.provenance, Provenance::HwShunt); - assert_eq!(it.energy.uj(), 1500); // 1500 → 3000 delta + assert_eq!(it.energy, Energy::measured(1500)); // a declared hardware reading, 0 → 1500 assert_ne!(it.outputs, ContentId::genesis()); let mut log = InteractionLog::new(SigningKey::from_bytes(&[3u8; 32])); @@ -472,7 +552,7 @@ mod tests { #[test] fn a_denied_action_never_runs_its_handler_and_is_still_sealed() { let meter = NoMeter; - let mut budget = Budget { remaining: 0, id: "grant-xyz".into() }; + let mut budget = budget(0, "grant-xyz"); let mut ran = false; let it = run_interaction( &surface(), @@ -490,6 +570,9 @@ mod tests { assert_eq!(it.verdict, Verdict::Deny("budget-exceeded".into())); assert_eq!(it.outputs, ContentId::genesis()); assert_eq!(it.grant_id.as_deref(), Some("grant-xyz")); + // Nothing ran, so nothing is costed and nothing is charged. + assert_eq!(it.energy, Energy::estimator(0)); + assert!(budget.charged.is_empty(), "a denial debits nothing"); // The denial still seals — and chains. let mut log = InteractionLog::new(SigningKey::from_bytes(&[3u8; 32])); @@ -553,11 +636,11 @@ mod tests { #[test] fn budget_is_debited_across_interactions() { - let mut budget = Budget { remaining: 5000, id: "g".into() }; + let mut budget = budget(5000, "g"); let mut log = InteractionLog::new(SigningKey::from_bytes(&[9u8; 32])); // Three interactions of 1500 µJ each: third leaves 500. for _ in 0..3 { - let meter = StepMeter(std::cell::Cell::new(0), 1500); + let meter = hw_step(1500); let it = run_interaction(&surface(), "apply", "did:key:z6MkTestActor", b"x", &meter, Some(&mut budget), || b"y".to_vec()); log.record(it); } @@ -571,4 +654,115 @@ mod tests { log.record(it); assert_eq!(verify_chain(log.entries()), Ok(4)); } + + // ── a figure is graded by what it is, and work that ran is never free ── + + /// A meter that declares nothing is an estimator: a stub's number is not a + /// hardware reading until the meter says it is one. + #[test] + fn a_meter_is_an_estimator_unless_it_opts_in() { + struct Silent; + impl Meter for Silent { + fn sample_uj(&self) -> Option { + Some(0) + } + } + assert_eq!(Silent.provenance(), Provenance::Estimator); + assert_eq!(NoMeter.provenance(), Provenance::Estimator); + assert_eq!(Silent.estimator_milliwatts(), DEFAULT_ESTIMATOR_MILLIWATTS); + + let meter = StepMeter(std::cell::Cell::new(0), 1500); + let it = run_interaction(&surface(), "apply", "did:key:z6MkTestActor", b"x", &meter, None, || b"y".to_vec()); + assert_eq!(it.energy, Energy::estimator(1500), "an undeclared counter's delta is an estimate"); + assert_eq!(it.energy.measured_uj, None); + + let it = run_interaction(&surface(), "apply", "did:key:z6MkTestActor", b"x", &hw_step(1500), None, || b"y".to_vec()); + assert_eq!(it.energy, Energy::measured(1500), "a meter that opts in is sealed as a reading"); + } + + /// With no reading, the handler's wall-clock time is estimated at the + /// meter's declared power — never 0 µJ for work that ran — and graded + /// `Estimator`. + #[test] + fn work_that_ran_without_a_reading_is_estimated_never_zero() { + let it = run_interaction(&surface(), "apply", "did:key:z6MkTestActor", b"x", &NoMeter, None, || busy(20)); + assert!(it.verdict.is_allow()); + assert_eq!((it.energy.provenance, it.energy.measured_uj), (Provenance::Estimator, None)); + // At least 20 ms at 20 W = 400_000 µJ. + assert!(it.energy.estimated_uj >= 400_000, "{:?}", it.energy); + + // A meter that knows its power envelope declares it, and the estimate + // scales with it: 20 ms at 1 W is at least 20_000 µJ, and well under + // what the same work costs at 20 W. + struct OneWatt; + impl Meter for OneWatt { + fn sample_uj(&self) -> Option { + None + } + fn estimator_milliwatts(&self) -> u64 { + 1_000 + } + } + let low = run_interaction(&surface(), "apply", "did:key:z6MkTestActor", b"x", &OneWatt, None, || busy(20)); + assert_eq!(low.energy.provenance, Provenance::Estimator); + assert!((20_000..400_000).contains(&low.energy.estimated_uj), "{:?}", low.energy); + + // Even an instant handler is not free. + let instant = run_interaction(&surface(), "apply", "did:key:z6MkTestActor", b"x", &NoMeter, None, || b"y".to_vec()); + assert!(instant.energy.uj() >= 1, "{:?}", instant.energy); + } + + #[test] + fn the_estimate_is_wall_time_at_the_declared_power_rounded_up() { + use std::time::Duration; + assert_eq!(estimate_uj(Duration::from_millis(1), 20_000), 20_000); // 1 ms at 20 W = 20 mJ + assert_eq!(estimate_uj(Duration::from_secs(2), 1_500), 3_000_000); // 2 s at 1.5 W = 3 J + assert_eq!(estimate_uj(Duration::from_nanos(1), 20_000), 1); // 0.02 µJ rounds up + assert_eq!(estimate_uj(Duration::ZERO, 20_000), 1); // work that ran is never free + assert_eq!(estimate_uj(Duration::from_secs(u64::MAX), u64::MAX), u64::MAX); // saturates + } + + /// On a host with no reading, a budget still binds: each interaction that + /// ran spends its estimate. Sealed at 0 µJ, as before, the budget never + /// moved and a runaway UI was never stopped. + #[test] + fn a_budget_binds_on_a_host_with_no_reading() { + let mut budget = budget(1_000_000, "g"); // 1 J: 50 ms of work at 20 W + let mut log = InteractionLog::new(SigningKey::from_bytes(&[9u8; 32])); + let mut ran = 0; + for _ in 0..10 { + let it = run_interaction(&surface(), "apply", "did:key:z6MkTestActor", b"x", &NoMeter, Some(&mut budget), || { + ran += 1; + busy(20) + }); + let allowed = it.verdict.is_allow(); + log.record(it); + if !allowed { + break; + } + } + assert!(ran <= 3, "a 1 J budget stopped ~20 ms interactions at 20 W within three; {ran} ran"); + assert_eq!(budget.remaining, 0); + assert!(!log.entries().last().unwrap().interaction.verdict.is_allow(), "the budget denied the next one"); + assert!(budget.charged.iter().all(|e| e.provenance == Provenance::Estimator), "{:?}", budget.charged); + assert_eq!(verify_chain(log.entries()), Ok(log.entries().len())); + } + + /// The authorizer is handed the figure the interaction is sealed with, and + /// its provenance: a reading as a reading, an estimate as an estimate. + #[test] + fn charge_is_handed_the_sealed_figure_and_its_provenance() { + let mut budget = budget(u64::MAX, "g"); + let hw = hw_step(1500); + let est = StepMeter(std::cell::Cell::new(0), 700); + let mut sealed = Vec::new(); + for meter in [&hw as &dyn Meter, &est, &NoMeter] { + let it = run_interaction(&surface(), "apply", "did:key:z6MkTestActor", b"x", meter, Some(&mut budget), || busy(1)); + sealed.push(it.energy); + } + assert_eq!(budget.charged, sealed, "charge sees exactly what is sealed"); + let grades: Vec = budget.charged.iter().map(|e| e.provenance).collect(); + assert_eq!(grades, [Provenance::HwShunt, Provenance::Estimator, Provenance::Estimator]); + assert_eq!((budget.charged[0].measured_uj, budget.charged[1].measured_uj), (Some(1500), None)); + } } diff --git a/joule-ui/joule-ui-rs/crates/joule-ui-interaction/tests/span_meter.rs b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/tests/span_meter.rs new file mode 100644 index 000000000..ec2ac32b5 --- /dev/null +++ b/joule-ui/joule-ui-rs/crates/joule-ui-interaction/tests/span_meter.rs @@ -0,0 +1,86 @@ +//! The examples' `SpanMeter` (`examples/support/span_meter.rs`), driven over +//! scripted spans through the real `run_interaction`: each interaction is +//! sealed with its own span's figure and grade, and a span with no reading of +//! the CPU is never sealed as a hardware reading of the interaction. + +#[path = "../examples/support/span_meter.rs"] +#[allow(dead_code)] +mod span_meter; + +use joule_ui_interaction::{ContentId, InteractionLog, Provenance, run_interaction, verify_chain}; +use sandbox_meter::{DeviceEnergy, Measurement, Provenance as Grade}; +use sandbox_schema::DeviceClass; +use span_meter::{SpanMeter, Spans}; +use std::collections::VecDeque; +use std::time::Duration; + +/// A meter whose spans replay a script, one per close, and whose estimate is +/// 20 W, the Sandbox estimator's default. +struct Script(VecDeque); + +impl Spans for Script { + type Open = (); + fn open(&mut self) {} + fn close(&mut self, _span: ()) -> Measurement { + self.0.pop_front().expect("script exhausted") + } + fn estimate(&self, wall: Duration) -> Measurement { + let uj = (wall.as_secs_f64() * 20e6) as u64; + Measurement { devices: vec![record(DeviceClass::Cpu, Grade::Estimator, uj)], wall, source: "estimator" } + } +} + +fn record(class: DeviceClass, provenance: Grade, uj: u64) -> DeviceEnergy { + DeviceEnergy { class, measured_uj: (provenance == Grade::HwShunt).then_some(uj), estimated_uj: uj, provenance } +} + +/// An 8 ms span of these records. +fn span_of(devices: Vec, source: &'static str) -> Measurement { + Measurement { devices, wall: Duration::from_millis(8), source } +} + +/// Seal `n` interactions over the scripted spans — the first interaction's +/// own span, then for each later one the idle gap before it and its own span +/// — through the real `run_interaction`, returning the log. +fn seal(script: Vec, n: u8) -> InteractionLog { + assert_eq!(script.len(), 2 * usize::from(n) - 1, "setup: one span for the first, two for each later one"); + let meter = SpanMeter::new(Script(script.into())); + let surface = ContentId::of_bytes(b"
"); + let mut log = InteractionLog::new(ed25519_dalek::SigningKey::from_bytes(&[9u8; 32])); + for i in 1..=n { + log.record(run_interaction(&surface, "apply", "did:key:z6MkTestActor", &[i], &meter, None, || vec![i])); + } + log +} + +/// A GPU-only span — NVML's reading, no CPU record, as on a Linux host whose +/// only live counter is the GPU's — is not a reading of the interaction, +/// which ran on the CPU. It is sealed as the meter's estimate for the span's +/// 8 ms (160 000 µJ at 20 W), `Estimator`: not the GPU's 9 000 µJ as +/// `HwShunt`, and not 0. The span that reads the CPU beside the GPU (RAPL and +/// NVML) is a reading, sealed as one; and one that reads only the CPU is too. +#[test] +fn a_gpu_only_span_is_sealed_as_an_estimate_never_as_a_reading() { + let gpu_only = span_of(vec![record(DeviceClass::Gpu, Grade::HwShunt, 9_000)], "nvml-total-energy"); + assert!(gpu_only.fully_measured() && !gpu_only.reads_cpu(), "setup: a GPU reading with no CPU record"); + let gap = || span_of(vec![record(DeviceClass::Cpu, Grade::Estimator, 999_999)], "estimator"); + let both = span_of( + vec![record(DeviceClass::Cpu, Grade::HwShunt, 1_500), record(DeviceClass::Gpu, Grade::HwShunt, 9_000)], + "composite", + ); + let cpu = span_of(vec![record(DeviceClass::Cpu, Grade::HwShunt, 2_000)], "rapl-powercap"); + let log = seal(vec![gpu_only, gap(), both, gap(), cpu], 3); + + assert_eq!(verify_chain(log.entries()), Ok(3)); + let sealed: Vec<_> = log.entries().iter().map(|e| e.interaction.energy).collect(); + let got: Vec<_> = sealed.iter().map(|e| (e.provenance, e.measured_uj, e.estimated_uj)).collect(); + assert_eq!( + got, + [ + (Provenance::Estimator, None, 160_000), + (Provenance::HwShunt, Some(10_500), 10_500), + (Provenance::HwShunt, Some(2_000), 2_000), + ], + "{sealed:?}" + ); +} diff --git a/joule-ui/joule-ui-rs/crates/joule-ui-mcp/src/lib.rs b/joule-ui/joule-ui-rs/crates/joule-ui-mcp/src/lib.rs index 6d325a101..9f82064d5 100644 --- a/joule-ui/joule-ui-rs/crates/joule-ui-mcp/src/lib.rs +++ b/joule-ui/joule-ui-rs/crates/joule-ui-mcp/src/lib.rs @@ -183,7 +183,7 @@ mod tests { Ok(1_000) } } - fn charge(&mut self, _action: &str, _measured_uj: u64) -> u64 { + fn charge(&mut self, _action: &str, _cost: &joule_ui_interaction::Energy) -> u64 { 1_000 } } @@ -202,7 +202,9 @@ mod tests { let it = meter_mcp_action(&action("submit"), &meter, None, || b"".to_vec()); assert!(matches!(it.verdict, Verdict::Allow)); assert_eq!(it.actor, "did:key:z6MkActor", "the actor is sealed in"); - assert_eq!(it.energy.uj(), 1_500, "the action carries its measured cost"); + assert_eq!(it.energy.uj(), 1_500, "the action carries its metered cost"); + // A stub meter declares no hardware grade, so its figure is an estimate. + assert_eq!(it.energy.provenance, joule_ui_interaction::Provenance::Estimator); // the component became the surface assert_eq!(it.surface, action("submit").surface()); } diff --git a/joule-ui/joule-ui-rs/crates/joule-ui-serve/Cargo.toml b/joule-ui/joule-ui-rs/crates/joule-ui-serve/Cargo.toml index 482f636ba..2601f3b5d 100644 --- a/joule-ui/joule-ui-rs/crates/joule-ui-serve/Cargo.toml +++ b/joule-ui/joule-ui-rs/crates/joule-ui-serve/Cargo.toml @@ -18,11 +18,19 @@ ed25519-dalek = { workspace = true } # Render the host's own verifying key as a `did:key:z…` — the default actor # attributed to interactions fired against an unauthenticated served surface. bs58 = "0.5" -# The reference Meter edge (SPEC §5): the live demo meters on REAL hardware -# (Apple Silicon IOReport / RAPL), so every receipt's HwShunt is true and the -# joules vary with the work — not a stub mislabelled as a measurement. +# The reference Meter edge (SPEC §5): the live demo meters each interaction as +# a Sandbox PowerMeter span. A receipt is HwShunt only when its span was a +# hardware reading of the CPU (a live RAPL or IOReport counter, with any live +# NVML GPU reading beside it), and Estimator otherwise, with the meter's +# wall-clock estimate as its figure: a GPU reading alone does not cover work +# that ran on the CPU. The joules vary with the work either way, and no +# estimate is sealed as a measurement. sandbox-meter = { path = "../../../../sandbox/sandbox-rs/crates/sandbox-meter" } +[dev-dependencies] +# Device classes for the scripted spans the binary's tests replay. +sandbox-schema = { path = "../../../../sandbox/sandbox-rs/crates/sandbox-schema" } + [[bin]] name = "joule-ui-serve" path = "src/bin/joule_ui_serve.rs" diff --git a/joule-ui/joule-ui-rs/crates/joule-ui-serve/src/bin/joule_ui_serve.rs b/joule-ui/joule-ui-rs/crates/joule-ui-serve/src/bin/joule_ui_serve.rs index cb1bca2a4..a056b4208 100644 --- a/joule-ui/joule-ui-rs/crates/joule-ui-serve/src/bin/joule_ui_serve.rs +++ b/joule-ui/joule-ui-rs/crates/joule-ui-serve/src/bin/joule_ui_serve.rs @@ -15,9 +15,10 @@ //! position always lands, each landing a real sealed, metered interaction. use joule_ui_core::Widget; -use joule_ui_interaction::Meter; +use joule_ui_interaction::{Meter, Provenance}; use joule_ui_serve::{Host, parse_request}; -use sandbox_meter::PowerMeter; +use sandbox_meter::probe::Span; +use sandbox_meter::{Measurement, PowerMeter}; use std::cell::RefCell; use std::io::{Read, Write}; use std::net::{TcpListener, TcpStream}; @@ -26,20 +27,122 @@ use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; -/// The **real** Sandbox `PowerMeter` as a joule-ui `Meter` (SPEC §5 edge): it -/// reads actual hardware energy (Apple Silicon IOReport / Intel-AMD RAPL), so -/// the receipt's `HwShunt` is true and the joules are a measurement, not a -/// constant. `RefCell` because `sample_uj` mutates the probe and the `Meter` -/// trait samples through `&self`; the host mutex serialises access across the -/// worker pool, so the metered window is exclusive — required for an honest -/// before/after delta. Where no hardware probe exists, `sample_uj` returns -/// `None` and the interaction is honestly `Estimator`, never a faked figure. -struct HardwareMeter(RefCell); -impl Meter for HardwareMeter { +/// A meter that measures managed spans: the Sandbox `PowerMeter` does, and a +/// test can script one. +trait Spans { + type Open; + fn open(&mut self) -> Self::Open; + fn close(&mut self, span: Self::Open) -> Measurement; + /// The meter's estimate for `wall` of work, with no reading. + fn estimate(&self, wall: Duration) -> Measurement; +} + +impl Spans for PowerMeter { + type Open = Span; + fn open(&mut self) -> Span { + self.start() + } + fn close(&mut self, span: Span) -> Measurement { + span.stop(self) + } + fn estimate(&self, wall: Duration) -> Measurement { + PowerMeter::estimate(self, wall) + } +} + +/// The Sandbox `PowerMeter` as a joule-ui `Meter` (SPEC §5 edge), sealing each +/// interaction with the provenance of the span it was metered over. +/// +/// Every `sample_uj` closes the span the previous sample opened, adds that +/// span's figure to a running counter, and opens the next. So the difference +/// of two consecutive samples is exactly one span's figure, and `provenance` +/// is that span's own, which `run_interaction` asks for right after its second +/// sample. A span is `HwShunt` only when it was a hardware reading of the CPU +/// the handler ran on. It is the meter's `Estimator` figure (wall-clock × TDP, +/// never 0 for work that ran) when the host exposes no live energy counter — +/// every macOS 27.0.1 host checked, whose CPU energy rail is frozen — when the +/// span's CPU counter held while another class, a GPU still compositing, +/// moved, and when the span has no CPU reading at all: an NVML-only host reads +/// the GPU alone, and joule-ui seals one figure per interaction with no field +/// to say it covers part of the work ([`interaction_figure`]). The old sampler +/// summed raw counters under the trait's default `HwShunt`, so it could seal +/// such GPU energy as the energy of this CPU work, and it sealed 0 J wherever +/// no hardware probe existed. +/// +/// `RefCell` because sampling mutates the probe while the `Meter` trait samples +/// through `&self`. The host mutex serialises access across the worker pool, +/// so a span is exclusive to one interaction. +struct SpanMeter(RefCell>); + +struct SpanState { + meter: M, + open: Option, + counter_uj: u64, + last: Provenance, +} + +impl SpanMeter { + fn new(meter: M) -> Self { + SpanMeter(RefCell::new(SpanState { + meter, + open: None, + counter_uj: 0, + last: Provenance::Estimator, + })) + } +} + +impl Meter for SpanMeter { fn sample_uj(&self) -> Option { - self.0.borrow_mut().sample_uj() + let mut s = self.0.borrow_mut(); + if let Some(span) = s.open.take() { + let m = s.meter.close(span); + let m = interaction_figure(m, |wall| s.meter.estimate(wall)); + s.counter_uj = s.counter_uj.saturating_add(m.total_uj()); + s.last = span_provenance(&m); + } + let next = s.meter.open(); + s.open = Some(next); + Some(s.counter_uj) + } + + fn provenance(&self) -> Provenance { + self.0.borrow().last + } +} + +/// The meter the server seals interactions with. +fn served_meter(meter: PowerMeter) -> Box { + Box::new(SpanMeter::new(meter)) +} + +/// The figure an interaction is sealed with, from its span. joule-ui seals +/// one figure per interaction, with no field to say that figure covers part +/// of the work, so a span with no hardware reading of the CPU is not a +/// reading of the interaction: an NVML-only probe set reads the GPU alone, +/// and its figure would be sealed `HwShunt` for work that ran on the CPU. Such +/// a span is sealed as the meter's estimate for its wall time (`Estimator`, +/// never 0 for work that ran); a span that reads the CPU passes through, +/// with any GPU record beside it. +fn interaction_figure(m: Measurement, estimate: impl FnOnce(Duration) -> Measurement) -> Measurement { + if m.reads_cpu() { m } else { estimate(m.wall) } +} + +/// A span's provenance: the weakest of its records, because a sum is only as +/// good as its weakest part. +fn span_provenance(m: &Measurement) -> Provenance { + use sandbox_meter::Provenance as Grade; + if m.devices.is_empty() || m.devices.iter().any(|d| d.provenance == Grade::Estimator) { + Provenance::Estimator + } else if m + .devices + .iter() + .all(|d| d.provenance == Grade::HwShunt && d.measured_uj.is_some()) + { + Provenance::HwShunt + } else { + Provenance::ModelBased } - // provenance() defaults to HwShunt — correct, because this is real hardware. } /// The output the demo computes: a "processed" image whose intensity is the @@ -49,9 +152,10 @@ fn output(intensity: f64) -> Widget { Widget::leaf("Image").prop("intensity", intensity) } -/// The demo's "compute": a real, bounded CPU workload scaled by intensity, so -/// the metered joules genuinely reflect work done — more intensity, more -/// compute, more joules — rather than a flat number. ~8 ms per intensity step. +/// The demo's "compute": a real, bounded CPU workload scaled by intensity, +/// ~8 ms per intensity step. More intensity is more compute and more joules, +/// not a flat number, whether the joules are read off a live counter or +/// estimated from the span's wall time. fn process(intensity: f64) -> Widget { let budget = Duration::from_millis((intensity.clamp(1.0, 10.0) * 8.0) as u64); let t = Instant::now(); @@ -75,7 +179,7 @@ fn main() -> std::io::Result<()> { let host = Host::new( "joule-ui — metered surface", output(1.0), - Box::new(HardwareMeter(RefCell::new(PowerMeter::detect()))), + served_meter(PowerMeter::detect()), Box::new(|_action, inputs| { let intensity: f64 = serde_json::from_slice(inputs).unwrap_or(1.0); process(intensity) @@ -232,3 +336,135 @@ input[type=range]{width:100%}\ .jui-placeholder{display:grid;place-items:center;height:140px;background:#11151a;border:1px dashed #2a2f36;color:#9aa4af;font-size:1.2rem}\ .jui-note{margin-top:1.25rem;font-size:.85rem;color:#9aa4af;font-family:ui-monospace,monospace}\ a{color:#2d6cdf}"; + +#[cfg(test)] +mod tests { + use super::*; + use ed25519_dalek::SigningKey; + use joule_ui_interaction::verify_chain; + use sandbox_meter::{DeviceEnergy, Provenance as Grade}; + use sandbox_schema::DeviceClass; + use std::collections::VecDeque; + + /// A meter whose spans replay a script, one per close, and whose + /// estimate is 20 W, the Sandbox estimator's default. + struct Script(VecDeque); + impl Spans for Script { + type Open = (); + fn open(&mut self) {} + fn close(&mut self, _span: ()) -> Measurement { + self.0.pop_front().expect("script exhausted") + } + fn estimate(&self, wall: Duration) -> Measurement { + let uj = (wall.as_secs_f64() * 20e6) as u64; + Measurement { devices: vec![record(DeviceClass::Cpu, Grade::Estimator, uj)], wall, source: "estimator" } + } + } + + fn record(class: DeviceClass, provenance: Grade, uj: u64) -> DeviceEnergy { + DeviceEnergy { class, measured_uj: (provenance == Grade::HwShunt).then_some(uj), estimated_uj: uj, provenance } + } + + /// An 8 ms span of these records. + fn span_of(devices: Vec, source: &'static str) -> Measurement { + Measurement { devices, wall: Duration::from_millis(8), source } + } + + fn span(provenance: Grade, uj: u64) -> Measurement { + span_of(vec![record(DeviceClass::Cpu, provenance, uj)], "script") + } + + fn host(meter: Box) -> Host { + Host::new( + "t", + output(1.0), + meter, + Box::new(|_action, inputs| process(serde_json::from_slice(inputs).unwrap_or(1.0))), + SigningKey::from_bytes(&[12u8; 32]), + ) + } + + fn fire(h: &mut Host) -> serde_json::Value { + serde_json::from_str(&h.handle("POST", "/action", br#"{"action":"apply","value":3}"#).body).unwrap() + } + + /// Through the real host, each fired action is sealed with its own span's + /// provenance and figure: a hardware reading as `HwShunt`, the meter's + /// estimate as `Estimator` with that estimate — not `HwShunt`, and not 0. + #[test] + fn each_served_interaction_carries_its_own_spans_provenance() { + let script = Script(VecDeque::from([ + span(Grade::HwShunt, 1_500), // interaction 1: a hardware reading + span(Grade::Estimator, 999_999), // the idle gap between the two + span(Grade::Estimator, 160_000), // interaction 2: the estimator's figure + ])); + let mut h = host(Box::new(SpanMeter::new(script))); + let first = fire(&mut h); + assert_eq!((first["provenance"].as_str(), first["measured_uj"].as_u64()), (Some("HwShunt"), Some(1_500))); + let second = fire(&mut h); + assert_eq!((second["provenance"].as_str(), second["measured_uj"].as_u64()), (Some("Estimator"), Some(160_000))); + + let chain = h.entries(); + assert_eq!(verify_chain(chain), Ok(2)); + let (a, b) = (chain[0].interaction.energy, chain[1].interaction.energy); + assert_eq!((a.provenance, a.measured_uj, a.estimated_uj), (Provenance::HwShunt, Some(1_500), 1_500)); + assert_eq!((b.provenance, b.measured_uj, b.estimated_uj), (Provenance::Estimator, None, 160_000)); + } + + /// A GPU-only span — NVML's reading, no CPU record, as on a Linux host + /// whose only live counter is the GPU's — is not a reading of the + /// interaction, which ran on the CPU. It is sealed as the meter's estimate + /// for the span's 8 ms (160 000 µJ at 20 W), `Estimator`: not the GPU's + /// 9 000 µJ as `HwShunt`, and not 0. A span that reads the CPU beside the + /// GPU (RAPL and NVML) is a reading, and is sealed as one. + #[test] + fn a_gpu_only_span_is_sealed_as_an_estimate_never_as_a_reading() { + let gpu_only = span_of(vec![record(DeviceClass::Gpu, Grade::HwShunt, 9_000)], "nvml-total-energy"); + assert!(gpu_only.fully_measured() && !gpu_only.reads_cpu(), "setup: a GPU reading with no CPU record"); + let both = span_of( + vec![record(DeviceClass::Cpu, Grade::HwShunt, 1_500), record(DeviceClass::Gpu, Grade::HwShunt, 9_000)], + "composite", + ); + let script = Script(VecDeque::from([gpu_only, span(Grade::Estimator, 999_999), both])); + let mut h = host(Box::new(SpanMeter::new(script))); + + let first = fire(&mut h); + assert_eq!((first["provenance"].as_str(), first["measured_uj"].as_u64()), (Some("Estimator"), Some(160_000))); + let second = fire(&mut h); + assert_eq!((second["provenance"].as_str(), second["measured_uj"].as_u64()), (Some("HwShunt"), Some(10_500))); + + let chain = h.entries(); + assert_eq!(verify_chain(chain), Ok(2)); + let (a, b) = (chain[0].interaction.energy, chain[1].interaction.energy); + assert_eq!((a.provenance, a.measured_uj, a.estimated_uj), (Provenance::Estimator, None, 160_000)); + assert_eq!((b.provenance, b.measured_uj, b.estimated_uj), (Provenance::HwShunt, Some(10_500), 10_500)); + } + + /// The served meter on this host, end to end: an interaction that ran is + /// never sealed at 0 J, and is `HwShunt` only from a hardware meter. Every + /// macOS 27.0.1 host checked has no hardware source, so there each + /// interaction is sealed as the estimator's figure for its work. + #[test] + fn the_served_meter_never_seals_zero_or_an_estimate_as_a_reading() { + let pm = PowerMeter::detect(); + let hardware = pm.is_hardware(); + let mut h = host(served_meter(pm)); + for _ in 0..2 { + fire(&mut h); + } + for sealed in h.entries() { + let e = sealed.interaction.energy; + eprintln!("served interaction: hardware={hardware} {e:?}"); + assert!(e.uj() > 0, "~24 ms of work sealed at 0 J: {e:?}"); + match e.provenance { + Provenance::HwShunt => { + assert!(hardware, "a reading from a meter with no hardware source: {e:?}"); + assert_eq!(e.measured_uj, Some(e.estimated_uj)); + } + Provenance::Estimator => assert_eq!(e.measured_uj, None), + Provenance::ModelBased => panic!("this meter never apportions: {e:?}"), + } + } + assert_eq!(verify_chain(h.entries()), Ok(2)); + } +} diff --git a/joule-ui/joule-ui-rs/crates/joule-ui-serve/src/lib.rs b/joule-ui/joule-ui-rs/crates/joule-ui-serve/src/lib.rs index 2014e52af..becc55ae8 100644 --- a/joule-ui/joule-ui-rs/crates/joule-ui-serve/src/lib.rs +++ b/joule-ui/joule-ui-rs/crates/joule-ui-serve/src/lib.rs @@ -190,6 +190,8 @@ impl Host { "html": html, "verdict": it.verdict.tag(), "reason": match &it.verdict { joule_ui_interaction::Verdict::Deny(r) => Some(r.clone()), _ => None }, + // The figure, under the key the page already reads; it is a + // reading only when the grade beside it says `HwShunt`. "measured_uj": it.energy.uj(), "provenance": it.energy.provenance.label(), "budget_remaining_uj": it.budget_remaining_uj, @@ -217,6 +219,8 @@ mod tests { use joule_ui_core::{PropValue, Widget}; use joule_ui_interaction::verify_chain; + /// A deterministic stub. It declares no grade, so what it reads is served + /// as what it is: an estimate. struct StepMeter(Cell); impl Meter for StepMeter { fn sample_uj(&self) -> Option { @@ -261,8 +265,11 @@ mod tests { assert_eq!(r.status, 200); let j: Value = serde_json::from_str(&r.body).unwrap(); assert_eq!(j["verdict"], "allow"); + // `measured_uj` keeps its name for the page reading it, and is the + // figure whatever its grade; the grade rides beside it. A stub's + // figure is never served as a reading. assert_eq!(j["measured_uj"], 2500); - assert_eq!(j["provenance"], "HwShunt"); + assert_eq!(j["provenance"], "Estimator"); assert!(j["html"].as_str().unwrap().contains("intensity 7")); assert_eq!(j["seq"], 0); diff --git a/sandbox/README.md b/sandbox/README.md index 6cf1b9acc..bd97bf043 100644 --- a/sandbox/README.md +++ b/sandbox/README.md @@ -6,7 +6,7 @@ Sandbox is the **isolated-execution substrate** of the Transaction Science open Every other sandbox in the field isolates a **CPU process** — a container, a microVM, a Linux process pinned to a core. The accelerator, where present, is an opaque passthrough that nobody meters and nobody attests. But agentic and AI work does not run on a CPU. It runs across a **fabric** — CPU, GPU, NPU, TPU, photonic, quantum. So in this standard the unit of isolation is not a process; it is a **heterogeneous unit of work**, and the accelerator is a first-class citizen of the isolation boundary: a capability you grant, a resource you meter in joules, and a line item you attest. -A Sandbox run carries exactly four guarantees: **isolation** (the unit runs at the lowest tier of an eight-rung ladder, S0–S7, that satisfies its needs, and cannot reach past it), **capability** (every resource the unit touches — each device class included — was explicitly granted; ungranted access is denied), **metering** (the run emits a signed energy receipt with per-device-class joule attribution and an honest provenance tag on every reading — and the reference meter reads real hardware counters: Apple Silicon's energy model via IOReport, Intel/AMD RAPL via powercap, NVIDIA's total-energy counter via NVML, each tagged `HwShunt`; where no counter exists the reading degrades to a labeled estimator, never a fabricated measurement; and because a hardware counter reports the whole SoC, the meter can **attribute** it to a single process tree by CPU-time apportionment — tagged `ModelBased`, since the reading is a measurement but the split is a model), and **attestation** (a signed statement of *what image ran, at what tier, under what grants, on what hardware, for what cost*, with a deterministic-replay journal hash where one was produced). +A Sandbox run carries exactly four guarantees: **isolation** (the unit runs at the lowest tier of an eight-rung ladder, S0–S7, that satisfies its needs, and cannot reach past it), **capability** (every resource the unit touches — each device class included — was explicitly granted; ungranted access is denied), **metering** (the run emits a signed energy receipt with per-device-class joule attribution and an honest provenance tag on every reading — and the reference meter reads vendor energy counters: Apple Silicon's energy model via IOReport, Intel/AMD RAPL via powercap, NVIDIA's total-energy counter via NVML. A reading is tagged `HwShunt` only while its counter is demonstrably live: each vendor counter is used only if every rail of it that the meter sums into one figure keeps advancing while the meter's probe burns CPU — in at least 5 sample intervals, at least 4 of every 5, over at least 300 ms — every IOReport CPU energy rail (the rail itself, one per die on a multi-die part), every RAPL package and every NVML GPU; and a span across which the CPU counter held is never a reading. A counter that is absent, or present but not live — frozen, or republished in lumps, as the IOReport CPU rails are on macOS 27 — yields no hardware-graded figure: the span degrades to the labeled `Estimator` (wall-clock × TDP), never a fabricated measurement, and `ioreport_status()`, `rapl_status()` and `nvml_status()` say why a host is estimated. Because a live counter reports the whole SoC, the meter can **attribute** it to a single process tree by CPU-time apportionment — tagged `ModelBased`, since the reading is a measurement but the split is a model), and **attestation** (a signed statement of *what image ran, at what tier, under what grants, on what hardware, for what cost*, with a deterministic-replay journal hash where one was produced). The substrate is intentionally **ownerless**: no protocol-level token, permissive licensing on spec text and reference code, and conformance vectors so any implementer can verify canonicity without a trust relationship with the authors. A conforming host runs a unit, enforces its grants, meters its energy, and seals its attestation **without contacting any single steward's servers as a condition of operation**. Transaction Science is one steward: it publishes the spec, ships a reference implementation, and operates commercial services around the substrate — not its proprietor. @@ -30,7 +30,7 @@ Three consequences, normative in [`SPEC.md`](SPEC.md): ## Heterogeneity is normative, not an extension -The clause that distinguishes this standard from a process sandbox is in [`SPEC.md`](SPEC.md) §2.3 and §6.2: where a unit is granted an accelerator capability, a conforming host **MUST** attribute energy to that device class separately in the receipt, **MUST** record the device identity (class, model, driver) in the attestation, and at tiers S5 and above **MUST** report the measurement provenance honestly (`HwShunt` where the device exposes an energy interface — NVML, RAPL, ROCm SMI, Level Zero — `ModelBased` or `Estimator` otherwise). A host that runs accelerated work but reports a single undifferentiated CPU energy figure is **not conformant**. +The clause that distinguishes this standard from a process sandbox is in [`SPEC.md`](SPEC.md) §2.3 and §6.2: where a unit is granted an accelerator capability, a conforming host **MUST** attribute energy to that device class separately in the receipt, **MUST** record the device identity (class, model, driver) in the attestation, and at tiers S5 and above **MUST** report the measurement provenance honestly (`HwShunt` where the device exposes a live energy interface — NVML, RAPL, ROCm SMI, Level Zero — `ModelBased` or `Estimator` otherwise). A host that runs accelerated work but reports a single undifferentiated CPU energy figure is **not conformant**. ## Lineage diff --git a/sandbox/sandbox-rs/crates/sandbox-exec-microvm/examples/boot_microvm.rs b/sandbox/sandbox-rs/crates/sandbox-exec-microvm/examples/boot_microvm.rs index 19a6d0f12..84aff69e2 100644 --- a/sandbox/sandbox-rs/crates/sandbox-exec-microvm/examples/boot_microvm.rs +++ b/sandbox/sandbox-rs/crates/sandbox-exec-microvm/examples/boot_microvm.rs @@ -13,8 +13,10 @@ //! It spawns `firecracker --api-sock `, drives `PUT /boot-source`, //! `/drives/rootfs`, `/machine-config`, and the `InstanceStart` action through //! our unix-socket HTTP/1.1 client, lets the guest run briefly, and reads the -//! package energy counter before and after. A genuine `HwShunt` figure where -//! the MSR is readable; an honest `ModelBased` vCPU-seconds estimate otherwise. +//! package energy counters (every package's, and every die's) before and +//! after. A genuine `HwShunt` figure where the MSRs read and every package's +//! counter proved live; an honest `ModelBased` vCPU-seconds estimate +//! otherwise. use sandbox_exec_microvm::firecracker::{FirecrackerConfig, FirecrackerVmm, VmState, Vmm}; use sandbox_exec_microvm::power::PowerMeter; @@ -49,10 +51,17 @@ fn main() { } // The meter: real RAPL via MSR where available (root on Nitro metal), - // else None → an honest vCPU-seconds ModelBased estimate. + // every package's and every die's, once each has proved live; else None + // → an honest vCPU-seconds ModelBased estimate. #[cfg(target_os = "linux")] let meter: Option = - sandbox_exec_microvm::power::MsrRaplMeter::package0(); + match sandbox_exec_microvm::power::MsrRaplMeter::host() { + Ok(m) => Some(m), + Err(why) => { + println!("no MSR meter: {why}"); + None + } + }; #[cfg(not(target_os = "linux"))] let meter: Option = None; @@ -92,7 +101,7 @@ fn main() { let device = match (before, after) { (Some(a), Some(b)) => { let uj = b.saturating_sub(a); - println!("CPU energy (RAPL pkg-0, MSR): {:.3} J over the window", uj as f64 / 1e6); + println!("CPU energy (RAPL, every package, MSR): {:.3} J over the window", uj as f64 / 1e6); DeviceEnergy { class: DeviceClass::Cpu, measured_uj: Some(uj), diff --git a/sandbox/sandbox-rs/crates/sandbox-exec-microvm/src/lib.rs b/sandbox/sandbox-rs/crates/sandbox-exec-microvm/src/lib.rs index 86187620d..4a793ab39 100644 --- a/sandbox/sandbox-rs/crates/sandbox-exec-microvm/src/lib.rs +++ b/sandbox/sandbox-rs/crates/sandbox-exec-microvm/src/lib.rs @@ -92,8 +92,12 @@ impl MicroVmExecutor { fn cpu_energy(&self, before: Option, after: Option, vcpu_seconds: u64) -> DeviceEnergy { match (before, after) { - // S5 with a hardware interface: HwShunt measured (SPEC §6.3). - (Some(b), Some(a)) if a >= b => DeviceEnergy { + // S5 with a hardware interface: HwShunt measured (SPEC §6.3) - but + // only if the counter advanced. vCPU work ran, so a CPU counter that + // held across it is not a reading of that work (a frozen or + // lump-refreshed counter); it falls to the model estimate below, + // never a measured zero. (Same rule as sandbox-meter's `Span::stop`.) + (Some(b), Some(a)) if a > b => DeviceEnergy { class: DeviceClass::Cpu, measured_uj: Some(a - b), estimated_uj: a - b, @@ -270,6 +274,26 @@ mod tests { assert_eq!(cpu.estimated_uj, 4 * 12_000); } + /// vCPU work ran, so a CPU counter that held across it is not a reading + /// of that work: the figure is the model estimate, never a measured zero. + /// A counter that advanced is the reading. + #[test] + fn a_cpu_counter_that_held_across_a_run_is_not_a_measured_zero() { + let exec = MicroVmExecutor::new( + Box::new(MockVmm { state: VmState::Completed, vcpu_seconds: 4 }), + template(), + Box::new(NoMeter), + ); + let held = exec.cpu_energy(Some(5_000), Some(5_000), 4); + assert_eq!(held.provenance, Provenance::ModelBased, "a held counter sealed as {held:?}"); + assert_eq!(held.measured_uj, None); + assert_eq!(held.estimated_uj, 4 * 12_000); + + let advanced = exec.cpu_energy(Some(5_000), Some(8_000), 4); + assert_eq!(advanced.provenance, Provenance::HwShunt); + assert_eq!(advanced.measured_uj, Some(3_000)); + } + #[test] fn failed_vm_is_terminated_as_tamper() { // A VMM that reports a failed guest must surface Outcome::Terminated with diff --git a/sandbox/sandbox-rs/crates/sandbox-exec-microvm/src/power.rs b/sandbox/sandbox-rs/crates/sandbox-exec-microvm/src/power.rs index 5b2760737..8ae1d28e7 100644 --- a/sandbox/sandbox-rs/crates/sandbox-exec-microvm/src/power.rs +++ b/sandbox/sandbox-rs/crates/sandbox-exec-microvm/src/power.rs @@ -5,6 +5,26 @@ //! (SPEC §6.3). A [`PowerMeter`] abstracts that interface so the executor can //! prefer a hardware reading and fall back to an honest `ModelBased` estimate //! where none exists — tagging the provenance accordingly. +//! +//! **A CPU figure is every package's, and only a live counter's.** Both CPU +//! meters here follow the rule sandbox-meter's RAPL probe implements. The +//! figure sums every CPU package — every socket, and on a part whose RAPL +//! counter is per die, every die — and nothing that is not one: never one +//! socket's energy as the whole CPU's, never `psys` (whose count already +//! includes the packages') or `dram`. Every package or none: a package that +//! cannot be read leaves the meter unopened rather than out of the sum. Each +//! package's counter is wrap-corrected at its own range, and a counter whose +//! delta cannot be known gives no figure. And the meter reads nothing until +//! every package's counter has kept advancing under the sustained-advance +//! liveness rule (each one in at least five sample intervals, four in five of +//! those observed, over at least 300 ms, while the probing thread burned CPU): +//! a counter that held, or stepped only in a burst, is not a reading of a +//! run. A meter that did not open, or whose count broke, samples `None`, and +//! the executor estimates. + +use sandbox_meter::probe::{ProbeError, RaplPackages}; +use std::path::Path; +use std::sync::Mutex; /// A source of cumulative energy readings for one device, in microjoules. /// `sample()` returns the running counter; the executor takes a before/after @@ -15,86 +35,324 @@ pub trait PowerMeter: Send + Sync { fn sample_uj(&self) -> Option; } -/// The Linux RAPL / powercap CPU energy interface -/// (`/sys/class/powercap/intel-rapl:0/energy_uj`), or AMD's equivalent. Reads -/// the cumulative microjoule counter. Available only where the sysfs node -/// exists and is readable; elsewhere `sample_uj` returns `None`. +/// Linux RAPL CPU energy through the powercap interface +/// (`/sys/class/powercap/intel-rapl:/energy_uj`, Intel's or AMD's): the +/// sum of every top-level zone whose `name` is `package-`, each +/// wrap-corrected at its own `max_energy_range_uj`, as a cumulative counter +/// from when the meter opened. It is sandbox-meter's own RAPL probe +/// ([`RaplPackages`]): not `psys`, `dram`, a package's subzones or the +/// `intel-rapl-mmio` interface to the same packages; every package or none (a +/// package whose counter or range does not read, or a top-level zone whose +/// `name` does not, leaves the meter unopened); a package with no readable, +/// non-zero range never opens, and a fall from above a package's range breaks +/// the count for good, so that read and every later one sample `None`. It +/// opens only once every package counter has proved live. pub struct RaplMeter { - path: String, + /// The package counters, or why there are none. + packages: Result, String>, } impl RaplMeter { - /// The standard Intel RAPL package-0 energy node. - pub fn intel_package0() -> Self { - RaplMeter { - path: "/sys/class/powercap/intel-rapl:0/energy_uj".into(), - } + /// This host's package zones, read only when sandbox-meter's cached + /// [`rapl_status`](sandbox_meter::rapl_status) is `Live` (probed once per + /// process: about a second of CPU on the first caller's thread where + /// powercap reads). Otherwise a meter that samples `None` and says why + /// ([`unavailable`](Self::unavailable)); always so off Linux. + pub fn host() -> RaplMeter { + Self::opened(RaplPackages::host()) } - /// A RAPL/powercap node at an explicit sysfs path. - pub fn at(path: impl Into) -> Self { - RaplMeter { path: path.into() } + /// The package zones of the powercap tree at `root`, read only when every + /// one proves live under the same rule, with `tick` driving the load + /// between liveness samples. For a tree other than the host's sysfs: a + /// test's fake tree. + pub fn live_in(root: &Path, tick: impl FnMut()) -> RaplMeter { + Self::opened(RaplPackages::live_in(root, tick)) + } + + fn opened(packages: Result) -> RaplMeter { + RaplMeter { packages: packages.map(Mutex::new).map_err(|e| e.to_string()) } + } + + /// Why this meter reads nothing, if it did not open. + pub fn unavailable(&self) -> Option<&str> { + self.packages.as_ref().err().map(String::as_str) } } impl PowerMeter for RaplMeter { + /// The packages' cumulative energy since the meter opened; `None` if it + /// did not open, if a package did not read, and from the read that broke + /// the count on. fn sample_uj(&self) -> Option { - std::fs::read_to_string(&self.path) - .ok() - .and_then(|s| s.trim().parse::().ok()) + let packages = self.packages.as_ref().ok()?; + let mut packages = packages.lock().ok()?; + packages.cumulative_uj().ok() } } -/// RAPL package energy read **directly from the MSR** (`MSR_PKG_ENERGY_STATUS`, -/// `0x611`), scaled by the energy unit in `MSR_RAPL_POWER_UNIT` (`0x606`). This -/// is the same physical counter `RaplMeter` reads through powercap — but some -/// hosts expose the MSR while omitting the `intel_rapl` sysfs driver entirely. -/// AWS Nitro bare-metal is exactly this case: `/dev/cpu/N/msr` yields a live -/// energy counter on a kernel (`*-aws`) that ships no powercap node. Reading the -/// MSR needs `CAP_SYS_RAWIO` (root) and the `msr` module; where unavailable, -/// construction returns `None` and the executor falls back honestly. -/// -/// `sample_uj` returns the package-0 cumulative microjoules. The raw MSR is a -/// 32-bit counter that wraps about every ~40 min at ~100 W; a before/after -/// difference over a short metered window is correct, which is the executor's -/// only use of it. #[cfg(target_os = "linux")] -pub struct MsrRaplMeter { - /// Energy-status unit exponent (`ESU`): one tick is `1 / 2^esu` joules. - esu: u32, -} +pub use msr::MsrRaplMeter; -#[cfg(target_os = "linux")] -impl MsrRaplMeter { - /// Open the package-0 RAPL MSRs. `None` if `/dev/cpu/0/msr` is unreadable - /// (no `msr` module, or not root) or the registers don't respond. - pub fn package0() -> Option { - let unit = read_msr(0, 0x606)?; - let esu = ((unit >> 8) & 0x1f) as u32; - if esu == 0 { - return None; // implausible unit — treat as unavailable - } - read_msr(0, 0x611)?; // confirm the energy counter responds - Some(MsrRaplMeter { esu }) - } -} - -/// Read one 64-bit MSR via `/dev/cpu//msr` at the register offset. -#[cfg(target_os = "linux")] -fn read_msr(cpu: u32, reg: u64) -> Option { +/// RAPL package energy read **directly from the MSRs**, for hosts that expose +/// `/dev/cpu//msr` but ship no powercap driver. Built on Linux, and under +/// test on every Unix, where it reads fake trees. +#[cfg(any(target_os = "linux", all(test, unix)))] +mod msr { + use super::PowerMeter; + use sandbox_meter::probe::{counter_liveness, CounterStatus}; + use std::collections::BTreeMap; use std::os::unix::fs::FileExt; - let f = std::fs::File::open(format!("/dev/cpu/{cpu}/msr")).ok()?; - let mut buf = [0u8; 8]; - f.read_exact_at(&mut buf, reg).ok()?; - Some(u64::from_le_bytes(buf)) -} + use std::path::{Path, PathBuf}; + use std::sync::Mutex; -#[cfg(target_os = "linux")] -impl PowerMeter for MsrRaplMeter { - fn sample_uj(&self) -> Option { - let raw = read_msr(0, 0x611)? & 0xffff_ffff; - // ticks × (1e6 µJ/J) / 2^esu - Some(((raw as u128 * 1_000_000u128) >> self.esu) as u64) + /// `MSR_RAPL_POWER_UNIT`: bits 12:8 are the energy status unit (ESU). + const MSR_RAPL_POWER_UNIT: u64 = 0x606; + /// `MSR_PKG_ENERGY_STATUS`: bits 31:0 count the domain's energy in units + /// of `1 / 2^ESU` J; bits 63:32 are reserved. + const MSR_PKG_ENERGY_STATUS: u64 = 0x611; + + /// RAPL package energy read directly from the MSRs + /// (`MSR_PKG_ENERGY_STATUS`, `0x611`), each domain scaled by the energy + /// unit in its own `MSR_RAPL_POWER_UNIT` (`0x606`). This is the counter + /// [`RaplMeter`](super::RaplMeter) reads through powercap, for hosts that + /// expose the MSR but omit the `intel_rapl` sysfs driver. AWS Nitro + /// bare-metal is exactly this case: `/dev/cpu/N/msr` yields a live energy + /// counter on a kernel (`*-aws`) that ships no powercap node. Reading the + /// MSR needs `CAP_SYS_RAWIO` (root) and the `msr` module; where + /// unavailable, construction fails, saying why, and the executor falls + /// back honestly. + /// + /// **Every package.** The register is per package, and per die on a part + /// with more than one die in a package, so one CPU's read is one domain's + /// energy, not the machine's. The meter groups every CPU under + /// `/sys/devices/system/cpu` by its `topology/physical_package_id` and + /// `topology/die_id`, reads each domain through the first of its CPUs + /// whose `msr` device reads, and sums them. It does not open when a + /// domain cannot be read through any of its CPUs, or when a CPU has no + /// readable topology: an offline CPU's package is unknown and could be + /// one no online CPU covers, so the meter cannot show that the sum is + /// the whole CPU's. A host with offline CPUs (`nosmt` included) is + /// estimated. + /// + /// **Wrap.** Each domain's counter is 32 bits: a fall is a wrap at 2^32 + /// counts, its architectural range, so every delta between two reads is + /// known. More than one wrap between two reads cannot be seen: 2^32 + /// counts at the common unit (2^-14 J) is about 262 kJ, some 44 minutes at + /// 100 W per package. + /// + /// **Only when live.** It opens only once every domain's raw counter has + /// kept advancing under sandbox-meter's liveness rule + /// ([`counter_liveness`]), probed once per process for the host. + pub struct MsrRaplMeter { + domains: Mutex, + } + + impl MsrRaplMeter { + /// This host's RAPL domains through `/dev/cpu//msr`, only when + /// every one proved live while the probing thread burned CPU: probed + /// once per process and cached, about a second of CPU on the first + /// caller's thread where the MSRs read. Otherwise `Err`, saying why. + #[cfg(target_os = "linux")] + pub fn host() -> Result { + static STATUS: std::sync::OnceLock = std::sync::OnceLock::new(); + let root = Path::new("/"); + let status = *STATUS.get_or_init(|| status_over(root, sandbox_meter::probe::counter_liveness_tick)); + Self::given(root, status) + } + + /// The RAPL domains of the tree at `root` (its `sys/devices/system/cpu` + /// and `dev/cpu`), only when every one proves live, with `tick` + /// driving the load between liveness samples. For a tree other than + /// the host's: a test's fake tree. Not cached. + pub fn live_in(root: &Path, tick: impl FnMut()) -> Result { + Self::given(root, status_over(root, tick)) + } + + /// The meter over the tree at `root`, for a liveness `status` of its + /// domains: opened only when it is `Live`. + fn given(root: &Path, status: CounterStatus) -> Result { + if status != CounterStatus::Live { + let why = match MsrDomains::open_in(root) { + Err(why) => why, + Ok(_) => status.to_string(), + }; + return Err(format!("RAPL MSR: {why}")); + } + let domains = MsrDomains::open_in(root).map_err(|why| format!("RAPL MSR: {why}"))?; + Ok(MsrRaplMeter { domains: Mutex::new(domains) }) + } + + /// The domains summed, `package-

-die-`, each with the `msr` + /// device it is read through, in order. + pub fn domains(&self) -> Vec<(String, PathBuf)> { + match self.domains.lock() { + Ok(d) => d.0.iter().map(|d| (d.name.clone(), d.msr.clone())).collect(), + Err(_) => Vec::new(), + } + } + } + + impl PowerMeter for MsrRaplMeter { + /// Every domain's cumulative energy since the meter opened, summed; + /// `None` if a domain did not read (nothing is updated then). + fn sample_uj(&self) -> Option { + self.domains.lock().ok()?.cumulative_uj() + } + } + + /// The liveness run over the domains of the tree at `root`: never ticked + /// if they do not open. + fn status_over(root: &Path, tick: impl FnMut()) -> CounterStatus { + counter_liveness(|| MsrDomains::open_in(root).ok().map(|d| move || d.raw_sample()), tick) + } + + /// One RAPL energy domain: a die of a package (the package itself on a + /// single-die part). + struct MsrDomain { + /// `package-

-die-`: its rail name for the liveness rule. + name: String, + /// The `msr` device of the CPU it is read through. + msr: PathBuf, + /// Its energy status unit: one count is `1 / 2^esu` J. Non-zero. + esu: u32, + /// The counter's last read. + last_raw: u32, + /// Counts since the meter opened, wrap-corrected at 2^32. + counts: u64, + } + + struct MsrDomains(Vec); + + impl MsrDomains { + /// Every (package, die) of the CPUs under `root/sys/devices/system/cpu`, + /// each read through the first of its CPUs whose + /// `root/dev/cpu//msr` reads both registers. `Err` when there is no + /// CPU, when a CPU's package or die cannot be read, when a domain + /// cannot be read through any of its CPUs, or when a domain's energy + /// unit is zero. + fn open_in(root: &Path) -> Result { + let cpu_dir = root.join("sys/devices/system/cpu"); + let entries = std::fs::read_dir(&cpu_dir).map_err(|e| format!("{}: {e}", cpu_dir.display()))?; + let mut by_domain: BTreeMap<(u32, u32), Vec> = BTreeMap::new(); + for entry in entries.flatten() { + let Some(cpu) = cpu_number(&entry.file_name().to_string_lossy()) else { + continue; // cpufreq, cpuidle, online, ... + }; + let topology = entry.path().join("topology"); + // An offline CPU has no topology: its package is unknown, and + // could be one no online CPU covers. + let id = |file: &str| { + read_u32(&topology.join(file)).map_err(|e| { + format!("cpu{cpu} has no readable {file} ({e}): its package is unknown, and could be one no other CPU covers") + }) + }; + let key = (id("physical_package_id")?, id("die_id")?); + by_domain.entry(key).or_default().push(cpu); + } + if by_domain.is_empty() { + return Err(format!("no CPU under {}", cpu_dir.display())); + } + let mut domains = Vec::with_capacity(by_domain.len()); + for ((package, die), mut cpus) in by_domain { + cpus.sort_unstable(); + let name = format!("package-{package}-die-{die}"); + let read = cpus.iter().find_map(|cpu| { + let msr = root.join(format!("dev/cpu/{cpu}/msr")); + let unit = read_msr(&msr, MSR_RAPL_POWER_UNIT)?; + let raw = read_msr(&msr, MSR_PKG_ENERGY_STATUS)?; + Some((msr, unit, raw)) + }); + let Some((msr, unit, raw)) = read else { + return Err(format!("{name}: the RAPL MSRs of none of its CPUs {cpus:?} read")); + }; + let esu = ((unit >> 8) & 0x1f) as u32; + if esu == 0 { + return Err(format!("{name}: an implausible energy unit (ESU 0)")); + } + domains.push(MsrDomain { name, msr, esu, last_raw: low32(raw), counts: 0 }); + } + Ok(MsrDomains(domains)) + } + + /// One absolute sample for the liveness rule: every domain's name and + /// raw 32-bit count, in domain order, as read (a wrap reads as a + /// fall, which is not an advance). `None` if any domain did not read. + fn raw_sample(&self) -> Option> { + self.0 + .iter() + .map(|d| read_msr(&d.msr, MSR_PKG_ENERGY_STATUS).map(|raw| (d.name.clone(), i64::from(low32(raw))))) + .collect() + } + + /// Every domain's energy since open, summed, in microjoules: each + /// domain's counts, wrap-corrected at 2^32, at its own unit. Every + /// domain is read before any is updated: a domain that does not read + /// gives `None` and changes nothing. + fn cumulative_uj(&mut self) -> Option { + let raws = self + .0 + .iter() + .map(|d| read_msr(&d.msr, MSR_PKG_ENERGY_STATUS).map(low32)) + .collect::>>()?; + let mut total = 0u64; + for (d, raw) in self.0.iter_mut().zip(raws) { + d.counts = d.counts.saturating_add(u64::from(raw.wrapping_sub(d.last_raw))); + d.last_raw = raw; + total = total.saturating_add(counts_to_uj(d.counts, d.esu)); + } + Some(total) + } + } + + /// `cpu` → `n`; `None` for anything else under the cpu directory. + fn cpu_number(dir_name: &str) -> Option { + let n = dir_name.strip_prefix("cpu")?; + if n.is_empty() || !n.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + n.parse().ok() + } + + /// The energy counter's 32 bits; the reserved upper half is ignored. + fn low32(raw: u64) -> u32 { + (raw & 0xffff_ffff) as u32 + } + + /// `counts` at `1 / 2^esu` J each, in microjoules, rounded down. + fn counts_to_uj(counts: u64, esu: u32) -> u64 { + u64::try_from((u128::from(counts) * 1_000_000) >> esu).unwrap_or(u64::MAX) + } + + fn read_u32(p: &Path) -> Result { + let s = std::fs::read_to_string(p).map_err(|e| e.to_string())?; + s.trim().parse::().map_err(|e| format!("{:?}: {e}", s.trim())) + } + + /// Read one 64-bit MSR through an `msr` device at the register offset. + fn read_msr(dev: &Path, reg: u64) -> Option { + let f = std::fs::File::open(dev).ok()?; + let mut buf = [0u8; 8]; + f.read_exact_at(&mut buf, reg).ok()?; + Some(u64::from_le_bytes(buf)) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn the_unit_conversion_and_the_cpu_names() { + assert_eq!(counts_to_uj(16_384, 14), 1_000_000, "2^14 counts at 2^-14 J is a joule"); + assert_eq!(counts_to_uj(1, 14), 61, "one count is 61.035 µJ, rounded down"); + assert_eq!(counts_to_uj(256, 14), 15_625); + assert_eq!(counts_to_uj(u64::MAX, 1), u64::MAX, "saturates"); + assert_eq!(low32(0xabcd_ef01_0000_0102), 0x0000_0102, "the reserved half is ignored"); + assert_eq!((cpu_number("cpu0"), cpu_number("cpu17")), (Some(0), Some(17))); + for other in ["cpu", "cpufreq", "cpuidle", "cpu1a", "online", "possible"] { + assert_eq!(cpu_number(other), None, "{other}"); + } + } } } @@ -122,18 +380,21 @@ impl PowerMeter for FixedMeter { #[cfg(test)] mod tests { use super::*; + use crate::firecracker::{FirecrackerConfig, VmOutcome, VmState, Vmm, VmmError}; + use crate::MicroVmExecutor; + use sandbox_capability::{ComputeGrant, Grants}; + use sandbox_meter::{DeviceEnergy, Provenance}; + use sandbox_runtime::{Executor, Unit}; + use sandbox_schema::DeviceClass; + use std::path::PathBuf; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering::SeqCst}; + use std::time::Duration; #[test] fn nometer_is_none() { assert_eq!(NoMeter.sample_uj(), None); } - #[test] - fn missing_rapl_node_is_none() { - let m = RaplMeter::at("/nonexistent/sandbox/rapl/energy_uj"); - assert_eq!(m.sample_uj(), None); - } - #[test] fn fixed_meter_advances() { let m = FixedMeter(std::sync::atomic::AtomicU64::new(0)); @@ -141,4 +402,405 @@ mod tests { let b = m.sample_uj().unwrap(); assert!(b > a); } + + /// A fake tree in the OS temp directory, removed when dropped. + struct FakeTree { + root: PathBuf, + } + + impl FakeTree { + fn new(label: &str) -> FakeTree { + static N: AtomicUsize = AtomicUsize::new(0); + let root = std::env::temp_dir().join(format!( + "sandbox-microvm-{label}-{}-{}", + std::process::id(), + N.fetch_add(1, SeqCst) + )); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("create a fake tree"); + FakeTree { root } + } + } + + impl Drop for FakeTree { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.root); + } + } + + /// Write `bytes` to `root/rel` atomically (a reader sees the old file or + /// the new one), creating its directory. + fn put(root: &Path, rel: &str, bytes: &[u8]) { + let path = root.join(rel); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let tmp = path.with_extension("tmp"); + std::fs::write(&tmp, bytes).unwrap(); + std::fs::rename(&tmp, &path).unwrap(); + } + + /// Run `f` while another thread calls `write` with a rising step every + /// 2 ms; the thread stops when `f` returns or panics. + fn advancing(write: impl Fn(u64) + Sync, f: impl FnOnce() -> T) -> T { + struct StopOnDrop<'a>(&'a AtomicBool); + impl Drop for StopOnDrop<'_> { + fn drop(&mut self) { + self.0.store(true, SeqCst); + } + } + let stop = AtomicBool::new(false); + std::thread::scope(|s| { + s.spawn(|| { + let mut step = 0; + while !stop.load(SeqCst) { + step += 1; + write(step); + std::thread::sleep(Duration::from_millis(2)); + } + }); + let _stop = StopOnDrop(&stop); + f() + }) + } + + /// The liveness tick the tests drive: a sleep, while a writer thread + /// advances the counters. + fn sleep_tick() { + std::thread::sleep(Duration::from_millis(25)); + } + + /// A VMM whose "run" is a script over the fake tree. + struct ScriptedVmm(Box); + + impl Vmm for ScriptedVmm { + fn run(&self, _cfg: &FirecrackerConfig) -> Result { + (self.0)(); + Ok(VmOutcome { state: VmState::Completed, vcpu_seconds: 2 }) + } + } + + /// The CPU record of one executor run over `meter`, the VMM's run being + /// `during`. + fn cpu_record_of(meter: Box, during: impl Fn() + 'static) -> DeviceEnergy { + let template = FirecrackerConfig { + api_socket: "/run/fc.sock".into(), + kernel_image_path: "/img/vmlinux".into(), + rootfs_path: "/img/rootfs.ext4".into(), + boot_args: "console=ttyS0".into(), + vcpus: 2, + mem_mib: 256, + }; + let exec = MicroVmExecutor::new(Box::new(ScriptedVmm(Box::new(during))), template, meter); + let grants = Grants { compute: ComputeGrant::cpu_only(1_000_000, 1000, 256 * 1024 * 1024), ..Grants::default() }; + let report = exec.run(&Unit::new(b"vm-image", &grants)).unwrap(); + assert_eq!(report.device_energy.len(), 1); + let cpu = report.device_energy[0]; + assert_eq!(cpu.class, DeviceClass::Cpu); + cpu + } + + // ---- RAPL through powercap. + + /// A powercap zone: its directory, `name`, `energy_uj` and + /// `max_energy_range_uj` (`None`: no such file). + type Zone<'a> = (&'a str, &'a str, u64, Option); + + fn powercap(zones: &[Zone<'_>]) -> FakeTree { + let tree = FakeTree::new("powercap"); + for (dir, name, uj, range) in zones { + put(&tree.root, &format!("{dir}/name"), format!("{name}\n").as_bytes()); + if let Some(range) = range { + put(&tree.root, &format!("{dir}/max_energy_range_uj"), format!("{range}\n").as_bytes()); + } + set_uj(&tree.root, dir, *uj); + } + tree + } + + fn set_uj(root: &Path, dir: &str, uj: u64) { + put(root, &format!("{dir}/energy_uj"), format!("{uj}\n").as_bytes()); + } + + fn uj_of(root: &Path, dir: &str) -> u64 { + std::fs::read_to_string(root.join(dir).join("energy_uj")).unwrap().trim().parse().unwrap() + } + + /// Two sockets with a platform zone: packages `intel-rapl:0` and + /// `intel-rapl:2` (each wrapping at a range of its own), `psys` at + /// `intel-rapl:1`, a top-level `dram` zone at `intel-rapl:3`, subzones, + /// and the MMIO interface to package 0. + fn two_sockets() -> FakeTree { + powercap(&[ + ("intel-rapl:0", "package-0", 1_000, Some(262_143_328_850)), + ("intel-rapl:1", "psys", 90_000, Some(262_143_328_850)), + ("intel-rapl:2", "package-1", 2_000, Some(65_532_610_987)), + ("intel-rapl:3", "dram", 30_000, Some(65_532_610_987)), + ("intel-rapl:0:0", "core", 700, Some(262_143_328_850)), + ("intel-rapl:0:1", "dram", 800, Some(65_532_610_987)), + ("intel-rapl-mmio:0", "package-0", 1_000, Some(262_143_328_850)), + ]) + } + + /// `RaplMeter::live_in` over `tree` while the zones in `advancing_zones` + /// rise by 4 mJ every 2 ms and the rest hold. + fn rapl_live_in(tree: &FakeTree, advancing_zones: &[&str]) -> RaplMeter { + advancing( + |step| { + for dir in advancing_zones { + set_uj(&tree.root, dir, 100_000 + 4_000 * step); + } + }, + || RaplMeter::live_in(&tree.root, sleep_tick), + ) + } + + /// The CPU figure is every package zone's and no other zone's: over two + /// live sockets with `psys`, `dram`, subzones and the MMIO interface all + /// rising too, a difference is the two packages' rise alone; package 1 + /// wraps at its own range, not package 0's; and a run through the + /// executor seals the packages' rise as `HwShunt`. + #[test] + fn the_rapl_meter_sums_every_package_zone_and_no_other() { + let tree = two_sockets(); + let meter = rapl_live_in(&tree, &["intel-rapl:0", "intel-rapl:2"]); + assert_eq!(meter.unavailable(), None, "both packages live, yet unopened"); + + let rises = [ + ("intel-rapl:0", 3_000), + ("intel-rapl:1", 50_000), + ("intel-rapl:2", 5_000), + ("intel-rapl:3", 9_000), + ("intel-rapl:0:0", 2_000), + ("intel-rapl:0:1", 1_000), + ("intel-rapl-mmio:0", 3_000), + ]; + let a = meter.sample_uj().expect("a live meter reads"); + for (dir, by) in rises { + set_uj(&tree.root, dir, uj_of(&tree.root, dir) + by); + } + let b = meter.sample_uj().expect("a live meter reads"); + assert_eq!(b - a, 8_000, "not the two packages' rise alone"); + + // Package 1 wraps at ITS range, 65,532,610,987 µJ: 65,532,600,000 -> + // 500 is 11,487 µJ. At package 0's range it would be ~1.97e11. + set_uj(&tree.root, "intel-rapl:2", 65_532_600_000); + let a = meter.sample_uj().unwrap(); + set_uj(&tree.root, "intel-rapl:2", 500); + assert_eq!(meter.sample_uj().unwrap() - a, 11_487, "package 1 did not wrap at its own range"); + + // Through the executor: the run raises both packages and psys. + let root = tree.root.clone(); + let cpu = cpu_record_of(Box::new(meter), move || { + for (dir, by) in [("intel-rapl:0", 1_000), ("intel-rapl:1", 70_000), ("intel-rapl:2", 2_000)] { + set_uj(&root, dir, uj_of(&root, dir) + by); + } + }); + assert_eq!((cpu.measured_uj, cpu.provenance), (Some(3_000), Provenance::HwShunt)); + } + + /// The meter reads nothing, and the executor estimates, unless every + /// package counter is live and every package can be read and corrected: + /// package 1 holding beside a live `psys`, `dram`, MMIO zone and package + /// 0; a tree with no package zone; a package with no wrap range. The + /// last two never open, so they are never probed. + #[test] + fn the_rapl_meter_reads_nothing_unless_every_package_is_live() { + let tree = two_sockets(); + let meter = rapl_live_in(&tree, &["intel-rapl:0", "intel-rapl:1", "intel-rapl:3", "intel-rapl-mmio:0"]); + let why = meter.unavailable().expect("package 1 held, yet the meter opened").to_string(); + assert!(why.contains("RAPL"), "{why}"); + assert_eq!(meter.sample_uj(), None); + let cpu = cpu_record_of(Box::new(meter), || {}); + assert_eq!((cpu.measured_uj, cpu.provenance), (None, Provenance::ModelBased)); + + for (tree, expect) in [ + (powercap(&[("intel-rapl:0", "psys", 90_000, Some(262_143_328_850))]), "no RAPL package zone"), + (powercap(&[("intel-rapl:0", "package-0", 1_000, None)]), "wrap range"), + ] { + let ticks = AtomicUsize::new(0); + let meter = RaplMeter::live_in(&tree.root, || { + ticks.fetch_add(1, SeqCst); + }); + let why = meter.unavailable().expect("opened").to_string(); + assert!(why.contains("RAPL"), "{why}"); + assert_eq!((meter.sample_uj(), ticks.load(SeqCst)), (None, 0), "{expect}"); + } + } + + /// A fall from above a package's range is a delta nobody knows: that read + /// and every later one give no figure, even once the counter is back up, + /// so a run after it is estimated, never measured short of the energy + /// that went uncounted. + #[test] + fn a_fall_from_above_a_package_range_stops_the_rapl_meter_for_good() { + let tree = powercap(&[("intel-rapl:0", "package-0", 1_000, Some(1_000_000_000))]); + let meter = rapl_live_in(&tree, &["intel-rapl:0"]); + assert_eq!(meter.unavailable(), None); + set_uj(&tree.root, "intel-rapl:0", 1_500_000_000); + assert!(meter.sample_uj().is_some(), "a rise reads"); + set_uj(&tree.root, "intel-rapl:0", 10); + assert_eq!(meter.sample_uj(), None, "a fall from above the range gave a figure"); + set_uj(&tree.root, "intel-rapl:0", 2_000_000_000); + assert_eq!(meter.sample_uj(), None, "a broken count read again"); + let root = tree.root.clone(); + let cpu = cpu_record_of(Box::new(meter), move || set_uj(&root, "intel-rapl:0", 2_100_000_000)); + assert_eq!((cpu.measured_uj, cpu.provenance), (None, Provenance::ModelBased)); + } + + // ---- RAPL through the MSRs: Unix only, where the meter is built under test. + #[cfg(unix)] + mod msr_meter { + use super::*; + + /// `MSR_RAPL_POWER_UNIT` with an energy status unit of `esu` (and the + /// usual power and time units around it). + fn unit_raw(esu: u64) -> u64 { + 0x000a_0003 | (esu << 8) + } + + /// Write CPU `cpu`'s fake `msr` device: the unit register and the energy + /// status register at their offsets, the rest zero. + fn set_msr(root: &Path, cpu: u32, unit: u64, status: u64) { + let mut bytes = vec![0u8; 0x611 + 8]; + bytes[0x606..0x606 + 8].copy_from_slice(&unit.to_le_bytes()); + bytes[0x611..0x611 + 8].copy_from_slice(&status.to_le_bytes()); + put(root, &format!("dev/cpu/{cpu}/msr"), &bytes); + } + + /// A CPU's topology: `None` for no topology directory (offline). + fn set_cpu(root: &Path, cpu: u32, package_die: Option<(u32, u32)>) { + let dir = format!("sys/devices/system/cpu/cpu{cpu}"); + std::fs::create_dir_all(root.join(&dir)).unwrap(); + if let Some((package, die)) = package_die { + put(root, &format!("{dir}/topology/physical_package_id"), format!("{package}\n").as_bytes()); + put(root, &format!("{dir}/topology/die_id"), format!("{die}\n").as_bytes()); + } + } + + /// A two-socket host whose package 1 has two dies, with ESU 14: cpu0, + /// cpu1 and cpu5 on package 0; cpu2 and cpu3 on package 1 die 0, cpu2's + /// `msr` device absent (read through cpu3); cpu4 on package 1 die 1. The + /// `cpufreq` directory beside them is not a CPU. + fn three_domains() -> FakeTree { + let tree = FakeTree::new("msr"); + for (cpu, pd) in [(0, (0, 0)), (1, (0, 0)), (5, (0, 0)), (2, (1, 0)), (3, (1, 0)), (4, (1, 1))] { + set_cpu(&tree.root, cpu, Some(pd)); + } + std::fs::create_dir_all(tree.root.join("sys/devices/system/cpu/cpufreq")).unwrap(); + for cpu in [0, 1, 3, 4, 5] { + set_msr(&tree.root, cpu, unit_raw(14), 1_000_192); + } + tree + } + + /// `MsrRaplMeter::live_in` over `tree` while the CPUs in `advancing_cpus` + /// count up by 4,096 every 2 ms (a multiple of 256, so every figure here + /// is a whole number of microjoules at ESU 14) and the rest hold. + fn msr_live_in(tree: &FakeTree, advancing_cpus: &[u32]) -> Result { + advancing( + |step| { + for cpu in advancing_cpus { + set_msr(&tree.root, *cpu, unit_raw(14), 1_000_192 + 4_096 * step); + } + }, + || msr::MsrRaplMeter::live_in(&tree.root, sleep_tick), + ) + } + + fn msr_status_of(root: &Path, cpu: u32) -> u64 { + let bytes = std::fs::read(root.join(format!("dev/cpu/{cpu}/msr"))).unwrap(); + u64::from_le_bytes(bytes[0x611..0x611 + 8].try_into().unwrap()) + } + + /// The MSR meter reads every package, and every die of a multi-die + /// package, each through one CPU whose `msr` device reads, and sums + /// them at each domain's own unit; a run through the executor seals + /// that sum as `HwShunt`. + #[test] + fn the_msr_meter_reads_every_package_and_die() { + let tree = three_domains(); + let meter = msr_live_in(&tree, &[0, 3, 4]).expect("every domain live"); + let held: Vec<(String, PathBuf)> = meter.domains(); + let cpu_msr = |cpu: u32| tree.root.join(format!("dev/cpu/{cpu}/msr")); + assert_eq!( + held, + [ + ("package-0-die-0".to_string(), cpu_msr(0)), + ("package-1-die-0".to_string(), cpu_msr(3)), + ("package-1-die-1".to_string(), cpu_msr(4)), + ], + "not one CPU per package and die" + ); + + // 256, 512 and 16,384 counts at ESU 14: 15,625 + 31,250 + + // 1,000,000 µJ. The reserved upper half of package 0's register + // changes too, and is ignored. + let a = meter.sample_uj().expect("a live meter reads"); + set_msr(&tree.root, 0, unit_raw(14), (0xdead_u64 << 32) | (msr_status_of(&tree.root, 0) + 256)); + set_msr(&tree.root, 3, unit_raw(14), msr_status_of(&tree.root, 3) + 512); + set_msr(&tree.root, 4, unit_raw(14), msr_status_of(&tree.root, 4) + 16_384); + let b = meter.sample_uj().expect("a live meter reads"); + assert_eq!(b - a, 1_046_875, "not every package's and die's energy"); + + // Through the executor: the run raises package 1 die 1 by a joule. + let root = tree.root.clone(); + let cpu = cpu_record_of(Box::new(meter), move || { + set_msr(&root, 4, unit_raw(14), msr_status_of(&root, 4) + 16_384); + }); + assert_eq!((cpu.measured_uj, cpu.provenance), (Some(1_000_000), Provenance::HwShunt)); + } + + /// Each domain's 32-bit counter wraps at 2^32 counts, on its own: a fall + /// from near the top to near the bottom is the counts across the wrap. + #[test] + fn the_msr_meter_wraps_each_domain_at_32_bits() { + let tree = three_domains(); + let meter = msr_live_in(&tree, &[0, 3, 4]).expect("every domain live"); + set_msr(&tree.root, 3, unit_raw(14), 0xffff_ff00); + let a = meter.sample_uj().unwrap(); + set_msr(&tree.root, 3, unit_raw(14), 0x0000_0100); + assert_eq!(meter.sample_uj().unwrap() - a, 31_250, "512 counts across the wrap"); + } + + /// The MSR meter does not open, and the executor estimates, when it + /// cannot show that its sum is the whole CPU's or that every domain is + /// live: a domain holding beside live ones; an offline CPU (no + /// topology), whose package is unknown; a CPU with no `die_id`; a domain + /// none of whose CPUs' MSRs read; a zero energy unit; no CPU at all. The + /// ones that never open are never probed. + #[test] + fn the_msr_meter_declines_what_it_cannot_cover() { + let tree = three_domains(); + let why = match msr_live_in(&tree, &[0, 3]) { + Err(why) => why, + Ok(m) => panic!("opened {:?} with package 1 die 1 holding", m.domains()), + }; + assert!(why.contains("RAPL MSR") && why.contains("kept advancing"), "{why}"); + + let unopened = |tree: &FakeTree, expect: &str| { + let ticks = AtomicUsize::new(0); + match msr::MsrRaplMeter::live_in(&tree.root, || { + ticks.fetch_add(1, SeqCst); + }) { + Err(why) => assert!(why.contains(expect), "{expect}: {why}"), + Ok(m) => panic!("{expect}: opened {:?}", m.domains()), + } + assert_eq!(ticks.load(SeqCst), 0, "{expect}: an unopened meter was probed"); + }; + let tree = three_domains(); + set_cpu(&tree.root, 6, None); + unopened(&tree, "cpu6 has no readable physical_package_id"); + let tree = three_domains(); + std::fs::remove_file(tree.root.join("sys/devices/system/cpu/cpu5/topology/die_id")).unwrap(); + unopened(&tree, "cpu5 has no readable die_id"); + let tree = three_domains(); + std::fs::remove_file(tree.root.join("dev/cpu/3/msr")).unwrap(); + unopened(&tree, "package-1-die-0: the RAPL MSRs of none of its CPUs [2, 3] read"); + let tree = three_domains(); + set_msr(&tree.root, 4, unit_raw(0), 1_000_192); + unopened(&tree, "package-1-die-1: an implausible energy unit"); + let tree = FakeTree::new("msr-empty"); + std::fs::create_dir_all(tree.root.join("sys/devices/system/cpu")).unwrap(); + unopened(&tree, "no CPU"); + } + } } diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/Cargo.toml b/sandbox/sandbox-rs/crates/sandbox-meter/Cargo.toml index c096bd657..058110058 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/Cargo.toml +++ b/sandbox/sandbox-rs/crates/sandbox-meter/Cargo.toml @@ -16,7 +16,9 @@ thiserror.workspace = true [target.'cfg(target_os = "linux")'.dependencies] libloading = "0.8" -[target.'cfg(target_os = "macos")'.dependencies] +# macOS: IOReport and the attributed runner. Both: the thread CPU clock the +# counter liveness probes burn against. +[target.'cfg(any(target_os = "macos", target_os = "linux"))'.dependencies] libc = "0.2" [[bin]] diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/examples/meter_cmd.rs b/sandbox/sandbox-rs/crates/sandbox-meter/examples/meter_cmd.rs index e86f41f7f..ae68da726 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/examples/meter_cmd.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/examples/meter_cmd.rs @@ -1,46 +1,97 @@ -//! `meter-cmd` — a `time`-like energy bracket: measure the machine's energy -//! over the execution of an external command, with an idle baseline of equal -//! length so the workload's marginal draw is honest. +//! `meter-cmd` — a `time`-like energy bracket: the machine's energy over the +//! execution of an external command, labelled by how it was obtained. //! -//! Counters are machine-wide (not process-scoped) — that caveat is printed, -//! not hidden. On a quiet machine, `work − idle` approximates the workload. +//! Where the meter reads hardware counters, the span is a measurement, and an +//! idle baseline of equal length gives the workload's marginal draw. The +//! counters are machine-wide (not process-scoped) — that caveat is printed, +//! not hidden: on a quiet machine, `work − idle` approximates the workload. +//! Where the meter could only estimate (no live hardware counter — on Apple +//! Silicon, see `ioreport_status()`), the span is wall-clock × TDP, printed +//! with its provenance. A `work − idle` difference of two such estimates is +//! not the workload's share of anything, so none is printed. //! -//! Run: `cargo run --release --example meter-cmd -- [args…]` +//! Run: `cargo run --release --example meter_cmd -- [args…]` -use sandbox_meter::PowerMeter; +use sandbox_meter::{IoReportStatus, Measurement, PowerMeter}; use std::process::Command; +/// How a span's figure was obtained, from its own records. +fn obtained(m: &Measurement) -> &'static str { + if m.fully_measured() { + "hardware-measured" + } else if m.devices.iter().all(|d| d.measured_uj.is_none()) { + "estimated" + } else { + "partly estimated" + } +} + +/// Each record's provenance, e.g. `Cpu Estimator`. +fn grades(m: &Measurement) -> String { + m.devices + .iter() + .map(|d| format!("{:?} {}", d.class, d.provenance.label())) + .collect::>() + .join(", ") +} + fn main() { let args: Vec = std::env::args().skip(1).collect(); let (program, rest) = args.split_first().expect("usage: meter-cmd [args…]"); let mut meter = PowerMeter::detect(); + let ioreport = sandbox_meter::ioreport_status(); eprintln!( "meter-cmd: probes [{}]", - if meter.is_hardware() { meter.sources().join(", ") } else { "estimator".into() } + if meter.is_hardware() { + meter.sources().join(", ") + } else if ioreport != IoReportStatus::Unavailable { + // Apple Silicon: say why the SoC counters were declined. + format!("none — estimator fallback ({ioreport})") + } else { + "none — estimator fallback".into() + } ); // measure the command (the workload inherits this process's environment) let (status, work) = meter.measure(|| { Command::new(program).args(rest).status().expect("spawn command") }); - // an idle baseline of the same wall length, for the marginal figure - let dur = work.wall; - let ((), idle) = meter.measure(|| std::thread::sleep(dur)); - let work_j = work.total_uj() as f64 / 1e6; - let idle_j = idle.total_uj() as f64 / 1e6; - let marginal = (work_j - idle_j).max(0.0); println!("\n== meter-cmd result =="); println!("command exited: {status}"); println!("wall: {:.1} s", work.wall.as_secs_f64()); - println!("machine (work): {work_j:.3} J ({:.2} W avg)", work.avg_watts()); - println!("machine (idle): {idle_j:.3} J ({:.2} W avg)", idle.avg_watts()); - println!("marginal: {marginal:.3} J (work − idle, the workload's honest share)"); - println!( - "source: {} ({})", - work.source, - if work.fully_measured() { "hardware-measured" } else { "partly estimated" } - ); + println!("machine (work): {work_j:.3} J ({:.2} W avg) [{}]", work.avg_watts(), grades(&work)); + + if work.fully_measured() { + // An idle baseline of the same wall length, for the marginal figure. + let dur = work.wall; + let ((), idle) = meter.measure(|| std::thread::sleep(dur)); + let idle_j = idle.total_uj() as f64 / 1e6; + println!("machine (idle): {idle_j:.3} J ({:.2} W avg) [{}]", idle.avg_watts(), grades(&idle)); + if !idle.fully_measured() { + println!("marginal: none — the idle baseline was {}, not a reading", obtained(&idle)); + } else if work_j > idle_j { + println!( + "marginal: {:.3} J (work − idle over machine-wide counters: the workload's \ + draw, on a quiet machine)", + work_j - idle_j + ); + } else { + // Never a floored 0 for work that ran: the counters did not + // resolve this workload above the idle draw. + println!( + "marginal: not resolved — work ({work_j:.3} J) did not exceed idle ({idle_j:.3} J) \ + on machine-wide counters" + ); + } + } else { + println!( + "estimate: {work_j:.3} J ({}) — wall-clock × TDP, not a reading; no work − idle \ + marginal, since two such estimates differ only by duration", + obtained(&work) + ); + } + println!("source: {} ({})", work.source, obtained(&work)); } diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/bin/joule_meter.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/bin/joule_meter.rs index 0d9a035c5..5d795c965 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/bin/joule_meter.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/bin/joule_meter.rs @@ -5,7 +5,7 @@ //! Usage: `joule-meter [seconds]` (default 2). use ed25519_dalek::SigningKey; -use sandbox_meter::{PowerMeter, Receipt}; +use sandbox_meter::{IoReportStatus, PowerMeter, Receipt}; use sandbox_schema::{ContentId, Tier}; use std::time::{Duration, Instant}; @@ -37,10 +37,15 @@ fn main() { .unwrap_or(2.0); let mut meter = PowerMeter::detect(); + let ioreport = sandbox_meter::ioreport_status(); println!( "probes: {}", if meter.is_hardware() { meter.sources().join(", ") + } else if ioreport != IoReportStatus::Unavailable { + // Apple Silicon: say why the SoC counters were declined, so an + // estimated host is not misread as a missing or broken probe. + format!("none — estimator fallback ({ioreport})") } else { "none — estimator fallback".into() } @@ -74,8 +79,13 @@ fn main() { } println!(" total {:.3} J avg {:.2} W", work.total_uj() as f64 / 1e6, work.avg_watts()); println!( - " source: {} (machine-wide counters; subtract idle for the workload's marginal draw)", - work.source + " source: {} ({})", + work.source, + if work.fully_measured() { + "machine-wide counters; subtract idle for the workload's marginal draw" + } else { + "wall-clock × TDP estimate, not a reading: idle and busy differ only by duration" + } ); // Seal it: the reading becomes a signed, verifiable receipt. diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/lib.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/lib.rs index 6137b0ea2..02ad724c9 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/lib.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/lib.rs @@ -15,7 +15,10 @@ pub mod jcr1; pub mod probe; pub use contention::{StorageActivity, StorageMeterError}; -pub use probe::{run_attributed, Attribution, Measurement, PowerMeter}; +pub use probe::{ + ioreport_status, nvml_status, rapl_status, run_attributed, Attribution, CounterStatus, + IoReportStatus, Measurement, PowerMeter, CPU_RAIL_LIVENESS_BUDGET, +}; /// The receipt's JCR-1 profile tag (SPEC §6.4). pub const PROFILE: &str = "sandbox/1"; diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/attribute.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/attribute.rs index fff145366..700bd8255 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/attribute.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/attribute.rs @@ -15,6 +15,14 @@ //! side — and downstream the attributed joules grade one rung below a direct //! shunt, which is exactly right. //! +//! An attribution rests on a measurement or it is not made. `attributed` is +//! true only when the window's CPU-class figure is an `HwShunt` reading that +//! advanced, and the child and the machine both logged CPU time. When the +//! meter could only estimate the window — no hardware probe, as on an Apple +//! Silicon host whose CPU energy rail is frozen, or a CPU counter that held +//! across the window — `attributed` is false and `soc_cpu_uj` is the +//! estimator's figure: carried as context, never an apportioned reading. +//! //! macOS (Apple Silicon) is implemented natively here. On other platforms the //! attributed runner falls back to running the command unattributed and //! reporting `None` — a caller then keeps the machine-wide figure, honestly @@ -25,9 +33,13 @@ use std::time::Duration; /// One attributed run of a child process. #[derive(Clone, Debug)] pub struct Attribution { - /// SoC CPU energy over the window (the `HwShunt` total), microjoules. + /// SoC CPU energy over the window, microjoules. A hardware (`HwShunt`) + /// reading only when `attributed` is true; otherwise it may be the + /// estimator's wall × TDP figure, and a caller must label it estimated. pub soc_cpu_uj: u64, - /// Energy attributed to the child tree: `soc_cpu_uj · share`. `ModelBased`. + /// Energy attributed to the child tree: `soc_cpu_uj · share`. `ModelBased` + /// when `attributed` is true; otherwise it apportions an estimate and is + /// no better than one. pub attributed_uj: u64, /// The child tree's CPU-seconds (user + system, descendants folded in). pub proc_cpu_secs: f64, @@ -36,8 +48,11 @@ pub struct Attribution { /// `proc_cpu_secs / busy_cpu_secs`, clamped to `[0, 1]`. pub share: f64, pub wall: Duration, - /// True when the attribution rests on real per-process accounting (vs. the - /// unattributed fallback on unsupported platforms). + /// True when the attribution rests on real per-process accounting of a + /// hardware CPU reading that advanced over the window. False for the + /// unattributed fallback on unsupported platforms, and whenever the + /// window's CPU figure is not such a reading (an estimate, or zero while + /// the machine was busy). pub attributed: bool, } @@ -45,6 +60,7 @@ pub struct Attribution { mod imp { use super::*; use crate::probe::PowerMeter; + use crate::{DeviceEnergy, Provenance}; use sandbox_schema::DeviceClass; use std::ffi::CString; use std::time::Instant; @@ -178,12 +194,23 @@ mod imp { }; // SoC CPU energy this window (sum the CPU-class device records). - let soc_cpu_uj: u64 = measurement + let cpu_records: Vec<&DeviceEnergy> = measurement .devices .iter() .filter(|d| d.class == DeviceClass::Cpu) + .collect(); + let soc_cpu_uj: u64 = cpu_records + .iter() .map(|d| d.measured_uj.unwrap_or(d.estimated_uj)) .sum(); + // Only a hardware reading can be apportioned as one. The estimator's + // wall × TDP figure (no probe, or a CPU counter that held across the + // window) is not a measurement, and neither is a zero while the + // machine was busy. + let cpu_is_a_reading = !cpu_records.is_empty() + && cpu_records + .iter() + .all(|d| d.provenance == Provenance::HwShunt && d.measured_uj.is_some()); let (share, attributed_ok) = if busy_cpu_secs > 0.0 && proc_cpu_secs > 0.0 { ((proc_cpu_secs / busy_cpu_secs).clamp(0.0, 1.0), true) @@ -202,7 +229,9 @@ mod imp { busy_cpu_secs, share, wall, - attributed: attributed_ok && meter.is_hardware(), + // `attributed_ok` already requires the machine to have been + // busy, so a zero reading here is a counter that held. + attributed: attributed_ok && cpu_is_a_reading && soc_cpu_uj > 0, }, )) } @@ -274,29 +303,116 @@ mod tests { use super::*; use crate::probe::PowerMeter; + /// 2–3 s of one core, bounded by the clock — POSIX `srand()` returns the + /// previous seed, i.e. the time in seconds of the previous call — and by + /// an iteration cap, so an awk that differs can only end early, never + /// hang. Long on purpose: the machine tick counters behind `busy_cpu_secs` + /// update in bursts up to ~1 s apart (measured on an M5 Max and an M3 + /// Ultra, macOS 27.0.1), so a shorter window can see no machine busy time + /// at all and decline an attribution for that reason instead of the one + /// under test. It also outlasts the liveness probe's budget, so a CPU rail + /// the probe saw keep moving must move across this window too. + const BUSY_CHILD: &str = "awk 'BEGIN { srand(); t0 = srand(); \ + while (srand() - t0 < 3 && n++ < 2000) for (i = 0; i < 100000; i++) x += i; print x }'"; + + fn run_busy_child(meter: &mut PowerMeter) -> Attribution { + let (ok, _out, a) = + run_attributed(meter, "sh", &["-c", BUSY_CHILD], std::path::Path::new(".")) + .expect("spawn"); + assert!(ok, "the child exits cleanly"); + // Attribution never exceeds the SoC total it apportions. + assert!(a.attributed_uj <= a.soc_cpu_uj, "attributed ≤ SoC total: {a:?}"); + assert!(a.share >= 0.0 && a.share <= 1.0, "share in [0,1]: {a:?}"); + a + } + + /// On Apple Silicon both CPU-time inputs registered the busy child, so an + /// attribution that is declined is declined for the rule under test. + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + fn assert_cpu_time_registered(a: &Attribution) { + assert!(a.proc_cpu_secs > 0.0, "setup: the child's rusage shows no CPU time: {a:?}"); + assert!(a.busy_cpu_secs > 0.0, "setup: the machine tick counters did not move: {a:?}"); + } + + /// An attribution rests on a hardware CPU reading that advanced, or it is + /// not made. The platform block pins which outcome this host must give, + /// so neither branch passes without asserting. #[test] fn attributed_run_captures_a_busy_child() { let mut meter = PowerMeter::detect(); - // A child that burns ~0.4s of one core. - let (ok, _out, a) = run_attributed( - &mut meter, - "sh", - &["-c", "awk 'BEGIN{x=0;for(i=0;i<40000000;i++)x+=i; print x}'"], - std::path::Path::new("."), - ) - .expect("spawn"); - assert!(ok, "the child exits cleanly"); - // The child genuinely consumed CPU time. + let a = run_busy_child(&mut meter); if a.attributed { + assert!(meter.is_hardware(), "attributed without a hardware probe: {a:?}"); + assert!(a.soc_cpu_uj > 0, "attributed a CPU reading that did not advance: {a:?}"); assert!(a.proc_cpu_secs > 0.0, "the child used CPU"); assert!(a.busy_cpu_secs >= a.proc_cpu_secs, "busy ≥ this one process"); assert!(a.share > 0.0 && a.share <= 1.0, "share in (0,1]"); - // Attribution never exceeds the SoC total it apportions. - assert!(a.attributed_uj <= a.soc_cpu_uj, "attributed ≤ SoC total"); - if meter.is_hardware() { - assert!(a.soc_cpu_uj > 0, "a busy window costs SoC energy"); + } + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + { + use crate::probe::{ioreport_status, IoReportStatus}; + let status = ioreport_status(); + eprintln!( + "attribution: status={status:?} hardware={} attributed={} soc_cpu_uj={} \ + attributed_uj={}", + meter.is_hardware(), + a.attributed, + a.soc_cpu_uj, + a.attributed_uj + ); + assert_cpu_time_registered(&a); + if status == IoReportStatus::Live { + assert!(a.attributed, "a live CPU rail over a busy child was not attributed: {a:?}"); + } else { + // Declined rung: the window was estimated, and an estimate is + // never apportioned as a reading. + assert!(!meter.is_hardware(), "IOReport declined ({status:?}) yet hardware"); + assert!(!a.attributed, "attributed an estimate ({status:?}): {a:?}"); + assert!(a.soc_cpu_uj > 0, "the estimator still accounts for the window"); } } + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + assert!(!a.attributed, "the unattributed fallback never attributes"); + } + + /// The attribution rule on its own, deterministic on every host. The + /// M5 Max shape on macOS 27.0.1 — CPU counter held, GPU advanced — seen + /// through a hardware probe: the window is estimated, and an estimate is + /// never apportioned as a reading. (main attributed it.) + #[test] + fn a_window_whose_cpu_counter_held_is_never_attributed() { + use crate::probe::tests::{meter_of, scripted}; + use sandbox_schema::DeviceClass::{Cpu, Gpu}; + let mut meter = meter_of(vec![scripted( + "ioreport-energy-model", + &[(Cpu, 480_083_160_000), (Gpu, 97_187_370_334)], + &[(Cpu, 480_083_160_000), (Gpu, 97_188_052_877)], + )]); + let a = run_busy_child(&mut meter); + assert!(meter.is_hardware(), "the scripted probe stands in for a hardware probe"); + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + assert_cpu_time_registered(&a); + assert!(!a.attributed, "a window whose CPU counter held was attributed: {a:?}"); + } + + /// The positive path, which no host running macOS 27.0.1 can reach for + /// real: a CPU counter that advanced across a busy child is attributed. + #[test] + fn a_window_whose_cpu_counter_advanced_is_attributed() { + use crate::probe::tests::{meter_of, scripted}; + use sandbox_schema::DeviceClass::Cpu; + let mut meter = + meter_of(vec![scripted("ioreport-energy-model", &[(Cpu, 1_000_000)], &[(Cpu, 3_000_000)])]); + let a = run_busy_child(&mut meter); + assert_eq!(a.soc_cpu_uj, 2_000_000, "the window's CPU reading"); + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + { + assert_cpu_time_registered(&a); + assert!(a.attributed, "a CPU reading that advanced over a busy child: {a:?}"); + assert!(a.share > 0.0 && a.attributed_uj > 0, "{a:?}"); + } + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + assert!(!a.attributed, "the unattributed fallback never attributes"); } #[test] diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/ioreport.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/ioreport.rs index f405e4a1f..faaff3b5e 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/ioreport.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/ioreport.rs @@ -2,17 +2,54 @@ //! CPU, GPU, and ANE cumulative energy, read sudo-free from userspace (the //! same source `powermetrics` consumes). `HwShunt` provenance. //! +//! **Used only when the CPU energy rail is live.** A subscription opens, and +//! returns channels, even while the CPU rail holds a frozen value. Measured on +//! macOS 27.0.1 by dumping the channels: on an M5 Max `CPU Energy` held under +//! sustained full CPU load while `GPU Energy` kept advancing; on an M3 Ultra +//! every Energy Model rail held, `DIE_0_CPU Energy` and `DIE_1_CPU Energy` +//! included. Every span over such a probe reports GPU energy, or zero, as a +//! measurement of CPU work. So [`IoReportProbe::given`] declines unless +//! [`ioreport_status`](super::ioreport_status) is `Live` — every CPU rail +//! (`CPU Energy`, or each die's `DIE__CPU Energy` on a multi-die part) +//! kept advancing while the probing thread burned CPU time: in at least five +//! sample intervals, at least four in five of those observed, over at least +//! 300 ms — and the meter falls back to its estimator. Every rail, not one: +//! [`snapshot`](Probe::snapshot) sums every CPU rail into one `Cpu` figure, +//! and a die whose rail held is a die that figure would claim without reading +//! it. Liveness is decided per CPU rail on the raw integer, never on a summed +//! total: the live GPU rail is exactly what corrupts a total. A few advances are not enough, because +//! the frozen rail is not frozen for good: it is refreshed in rare batches, +//! and one batch can land in two steps. The rule, the sampling loop and the +//! whole probe run live in [`liveness`](super::liveness), apart from this FFI; +//! the probe here only reads every channel's group, name and raw integer from +//! one absolute sample. +//! +//! The production status is one composition with no branch of its own: +//! [`host_status`] maps [`real_run`], which is +//! [`probe_with`] over this module's real +//! subscription, its real tick and its real clocks. Nothing else in +//! production runs the probe, and [`ioreport_status`](super::ioreport_status) +//! caches it once per process. The tests run that same composition end to +//! end: on the real IOReport, bracketed by reads of their own, and on a +//! scripted IOReport (`seam`, compiled only for tests) that replaces nothing +//! but this FFI leaf. +//! //! The bindings are hand-rolled and minimal: IOReport and the few //! CoreFoundation calls needed to walk a samples dictionary. Every `Copy`/ -//! `Create` is released; channel units are normalized to microjoules via the +//! `Create` is released exactly once — including the channel dictionary that +//! [`IoReportProbe::open`] asked for, when IOReport hands back a separate +//! subscribed one; channel units are normalized to microjoules via the //! channel's unit label (nJ/uJ/mJ). #![allow(non_snake_case)] -use super::{Probe, ProbeError}; +use super::liveness::{admits, probe_with, spin, status_of, Channel, ProbeRun, ENERGY_MODEL}; +use super::{IoReportStatus, Probe, ProbeError, CPU_RAIL_LIVENESS_BUDGET}; use sandbox_schema::DeviceClass; use std::ffi::c_void; use std::ptr; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; type CFTypeRef = *const c_void; type CFStringRef = *const c_void; @@ -61,6 +98,7 @@ extern "C" { a: CFTypeRef, ) -> CFDictionaryRef; fn IOReportChannelGetChannelName(ch: CFDictionaryRef) -> CFStringRef; + fn IOReportChannelGetGroup(ch: CFDictionaryRef) -> CFStringRef; fn IOReportChannelGetUnitLabel(ch: CFDictionaryRef) -> CFStringRef; fn IOReportSimpleGetIntegerValue(ch: CFDictionaryRef, idx: i32) -> i64; } @@ -97,6 +135,59 @@ fn class_for(channel: &str) -> Option { } } +/// How many times this process ran the real liveness probe: one +/// process-wide counter, bumped by [`host_status`] before anything else it +/// does, and read through +/// [`ioreport_probe_runs`](super::ioreport_probe_runs). +pub(super) static REAL_RUNS: AtomicUsize = AtomicUsize::new(0); + +/// This host's IOReport status from one real probe run: the value +/// [`ioreport_status`](super::ioreport_status) caches, and its only source. +/// One composition with no branch of its own (count the run, run the real +/// probe, map its verdict), so the path the tests drive is the path the +/// meter is gated on. Call [`ioreport_status`](super::ioreport_status), not +/// this. +pub(super) fn host_status() -> IoReportStatus { + REAL_RUNS.fetch_add(1, Ordering::SeqCst); + status_of(real_run().verdict()) +} + +/// The real probe run: [`probe_with`] over the Energy Model subscription, +/// the tick that burns this thread's CPU time, this thread's CPU clock and a +/// monotonic clock, for [`CPU_RAIL_LIVENESS_BUDGET`]. The only production +/// caller of `probe_with`, and called in production by [`host_status`] +/// alone. The hardware tests call it too, uncounted and uncached, to check +/// the very run the status comes from. +fn real_run() -> ProbeRun { + let epoch = Instant::now(); + probe_with( + || IoReportProbe::open().ok().map(|p| move || p.read()), + probe_tick, + thread_cpu, + move || epoch.elapsed(), + CPU_RAIL_LIVENESS_BUDGET, + ) +} + +/// CPU time the calling thread has used (user + system), from +/// `CLOCK_THREAD_CPUTIME_ID`. Zero if the clock cannot be read, which ends a +/// tick at its wall cap rather than never. +fn thread_cpu() -> Duration { + let mut ts = libc::timespec { tv_sec: 0, tv_nsec: 0 }; + let rc = unsafe { libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, &mut ts) }; + if rc != 0 { + return Duration::ZERO; + } + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) +} + +/// The real probe's tick: [`spin`] on this thread's CPU clock, capped on a +/// monotonic one. +fn probe_tick() { + let t = Instant::now(); + spin(thread_cpu, || t.elapsed()); +} + /// Convert a channel reading to microjoules by its unit label. fn to_uj(value: i64, unit: &str) -> Option { if value < 0 { @@ -118,8 +209,10 @@ pub struct IoReportProbe { } // SAFETY: the IOReport subscription and channel dictionary are CoreFoundation -// objects with no thread affinity; they are only ever touched through `&mut -// self` (`snapshot`), so a caller that owns the probe exclusively — or guards a +// objects with no thread affinity. A probe the meter holds is only ever touched +// through `&mut self` (`snapshot`); the liveness run touches its own probe +// through `read`, on the one thread that owns it outright for the run, and +// drops it there. So a caller that owns the probe exclusively — or guards a // shared `PowerMeter` behind a mutex, as the joule-ui host does across its // worker pool — accesses them from one thread at a time with a happens-before // edge between accesses. Sending the probe between threads under that @@ -127,9 +220,40 @@ pub struct IoReportProbe { unsafe impl Send for IoReportProbe {} impl IoReportProbe { - pub fn new() -> Result { + /// The probe the meter uses, built for this host's IOReport `status`, + /// which production passes from the cached + /// [`ioreport_status`](super::ioreport_status) and nowhere else: **only + /// when the CPU energy rail is live** ([`admits`]). Otherwise + /// `Unavailable`, naming why, and the meter estimates instead: a frozen + /// CPU rail would seal every span's CPU energy as a measured zero, and on + /// a part whose GPU rail stays live it would pass GPU energy off as the + /// energy of the work. + pub(super) fn given(status: IoReportStatus) -> Result { + if !admits(status) { + return Err(ProbeError::Unavailable(status.to_string())); + } + let mut p = Self::open()?; + // Viability: one real read, and it must carry the CPU class — a probe + // that could only report GPU/ANE energy would be the same defect. + let snap = p.snapshot()?; + if !snap.iter().any(|(class, _)| *class == DeviceClass::Cpu) { + return Err(ProbeError::Unavailable( + "Energy Model group has no CPU energy channel in a known unit".into(), + )); + } + Ok(p) + } + + /// Open the Energy Model subscription with no liveness gate. Used by the + /// liveness probe itself and by [`given`](Self::given) once the gate + /// passed. + fn open() -> Result { + #[cfg(test)] + if let Some(scripted) = seam::open() { + return scripted; + } unsafe { - let group = cf_string("Energy Model"); + let group = cf_string(ENERGY_MODEL); let desired = IOReportCopyChannelsInGroup(group, ptr::null(), 0, 0, 0); CFRelease(group); if desired.is_null() { @@ -139,21 +263,69 @@ impl IoReportProbe { let sub = IOReportCreateSubscription(ptr::null(), desired, &mut subbed, 0, ptr::null()); if sub.is_null() { CFRelease(desired as CFTypeRef); + if !subbed.is_null() { + CFRelease(subbed as CFTypeRef); + } return Err(ProbeError::Unavailable("IOReport subscription failed".into())); } - let probe = IoReportProbe { - subscription: sub, - channels: if subbed.is_null() { desired } else { subbed }, + // The probe keeps one channel dictionary, released on drop. When + // IOReport hands back a separate subscribed one, that is the one + // kept, and the dictionary asked for (owned: a Copy) is released + // here. It used to be released by nobody: `leaks` counted one + // ~172 KB dictionary per open (200 opens, 200 root leaks). + let channels = if subbed.is_null() { + desired + } else { + CFRelease(desired as CFTypeRef); + subbed }; - // Probe viability: one real read with at least one usable channel. - let mut p = probe; - let snap = p.snapshot()?; - if snap.is_empty() { - return Err(ProbeError::Unavailable( - "Energy Model group has no CPU/GPU/ANE energy channels".into(), - )); + Ok(IoReportProbe { subscription: sub, channels }) + } + } + + /// One absolute sample of this subscription: every channel it holds, its + /// group, its name and its raw integer, in channel order, filtered by + /// nothing. `None` if the sample could not be read. What the rule is + /// handed, and what a run records, is taken from this by + /// [`split_sample`](super::liveness::split_sample). It samples this + /// probe's own subscription, which holds the Energy Model group alone: + /// a few hundred channels (364 on an M5 Max, 645 on an M3 Ultra), ~6-12 ms + /// of this thread's CPU a sample, against ~76-145 ms for a subscription to + /// every channel (~11,900 / ~14,300 channels). These are the same + /// absolute samples [`snapshot`](Probe::snapshot) reads, so the probe + /// tests the path a span will use. + fn read(&self) -> Option> { + #[cfg(test)] + if let Some(scripted) = seam::read() { + return scripted; + } + unsafe { + let samples = IOReportCreateSamples(self.subscription, self.channels, ptr::null()); + if samples.is_null() { + return None; } - Ok(p) + let key = cf_string("IOReportChannels"); + let arr = CFDictionaryGetValue(samples, key) as CFArrayRef; + CFRelease(key); + if arr.is_null() { + CFRelease(samples); + return None; + } + let n = CFArrayGetCount(arr); + let mut out = Vec::with_capacity(n.max(0) as usize); + for i in 0..n { + let ch = CFArrayGetValueAtIndex(arr, i) as CFDictionaryRef; + if ch.is_null() { + continue; + } + out.push(Channel { + group: rust_string(IOReportChannelGetGroup(ch)).unwrap_or_default(), + name: rust_string(IOReportChannelGetChannelName(ch)).unwrap_or_default(), + raw: IOReportSimpleGetIntegerValue(ch, 0), + }); + } + CFRelease(samples); + Some(out) } } } @@ -217,3 +389,1375 @@ impl Drop for IoReportProbe { } } } + +/// Does this host's Energy Model subscription open with a CPU rail in it? +/// For the selection tests: where it does, a `Live` status must build the +/// probe. +#[cfg(test)] +pub(super) fn energy_model_has_a_cpu_rail() -> bool { + IoReportProbe::open() + .ok() + .and_then(|p| p.read()) + .is_some_and(|channels| !super::liveness::split_sample(&channels).0.is_empty()) +} + +/// A scripted IOReport for the calling thread only, so the tests can run the +/// production composition end to end — [`host_status`] itself, and +/// [`real_run`] — with nothing replaced but the FFI leaf: whether the Energy +/// Model subscription opens, and what each sample holds. Every other thread, +/// and every build that ships, reads the real IOReport. A thread with a +/// script installed must never ask for the cached status: the cache would +/// keep the scripted one for the whole process. +#[cfg(test)] +mod seam { + use super::{Channel, IoReportProbe, ProbeError}; + use std::cell::RefCell; + use std::ptr; + use std::time::{Duration, Instant}; + + /// `CLOCK_THREAD_CPUTIME_ID` in macOS ``, declared by the tests + /// themselves, so their clock shares nothing with the production one. + const TEST_CLOCK_THREAD_CPUTIME_ID: libc::clockid_t = 16; + + /// This thread's CPU time, on the tests' own clock. + pub(super) fn test_thread_cpu() -> Duration { + let mut ts = libc::timespec { tv_sec: 0, tv_nsec: 0 }; + let rc = unsafe { libc::clock_gettime(TEST_CLOCK_THREAD_CPUTIME_ID, &mut ts) }; + assert_eq!(rc, 0, "the tests' own thread CPU clock did not read"); + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) + } + + /// What a scripted IOReport does, and what was done to it. + #[derive(Debug, Default)] + pub(super) struct Script { + /// Whether the Energy Model subscription opens. + opens: bool, + /// Its samples, in order; past the end, the last repeats. `None`: a + /// sample that could not be read. + samples: Vec>>, + /// How many times a subscription was asked for. + pub(super) opened: u32, + /// At every sample call, in order: this thread's CPU time on the + /// tests' own clock, and a monotonic reading. The run ticks once + /// between two calls, so these time each tick apart from the code + /// under test. + pub(super) stamps: Vec<(Duration, Instant)>, + } + + thread_local! { + static SCRIPT: RefCell> = const { RefCell::new(None) }; + } + + /// Script IOReport on this thread until the script is finished or + /// dropped. + pub(super) fn install(opens: bool, samples: Vec>>) -> Installed { + assert!(!opens || !samples.is_empty(), "a subscription that opens needs samples"); + SCRIPT.with(|s| { + let mut s = s.borrow_mut(); + assert!(s.is_none(), "a script is already installed on this thread"); + *s = Some(Script { opens, samples, ..Script::default() }); + }); + Installed(()) + } + + /// An installed script. + pub(super) struct Installed(()); + + impl Installed { + /// Remove the script, and return what was done to it. + pub(super) fn finish(self) -> Script { + SCRIPT.with(|s| s.borrow_mut().take()).expect("the script installed on this thread") + } + } + + impl Drop for Installed { + fn drop(&mut self) { + let _ = SCRIPT.try_with(|s| s.try_borrow_mut().map(|mut s| s.take())); + } + } + + /// The scripted open, when a script is installed on this thread. + pub(super) fn open() -> Option> { + SCRIPT.with(|s| { + let mut s = s.borrow_mut(); + let script = s.as_mut()?; + script.opened += 1; + Some(if script.opens { + // No FFI object behind it: every sample comes from the + // script, and dropping it releases nothing. + Ok(IoReportProbe { subscription: ptr::null_mut(), channels: ptr::null_mut() }) + } else { + Err(ProbeError::Unavailable("scripted: the Energy Model subscription did not open".into())) + }) + }) + } + + /// The next scripted sample, when a script is installed on this thread. + pub(super) fn read() -> Option>> { + SCRIPT.with(|s| { + let mut s = s.borrow_mut(); + let script = s.as_mut()?; + script.stamps.push((test_thread_cpu(), Instant::now())); + let n = script.stamps.len(); + assert!(n <= 10_000, "the probe sampled a scripted IOReport without end"); + Some(script.samples[(n - 1).min(script.samples.len() - 1)].clone()) + }) + } +} + +#[cfg(test)] +mod tests { + use super::seam::{self, test_thread_cpu}; + use super::*; + use crate::probe::liveness::{ + is_cpu_rail, split_sample, CpuRail, SampleCall, SampleShape, PROBE_TICK, TICK_WALL_CAP, + }; + use crate::probe::{ioreport_status, PowerMeter}; + use crate::Provenance; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{mpsc, Arc}; + + // The oracle subscribes to every channel. The implementation never does: + // it subscribes to the Energy Model group alone, which a test checks + // channel by channel. + #[link(name = "IOReport", kind = "dylib")] + extern "C" { + fn IOReportCopyAllChannels(a: u64, b: u64) -> CFMutableDictionaryRef; + } + + fn ms(n: u64) -> Duration { + Duration::from_millis(n) + } + + fn burn(d: Duration) { + let t = Instant::now(); + let mut x: u64 = 0; + while t.elapsed() < d { + x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + } + std::hint::black_box(x); + } + + /// Burn this thread's CPU until the tests' own clock says `d` of it went. + fn burn_cpu(d: Duration) { + let c0 = test_thread_cpu(); + let mut x: u64 = 0; + while test_thread_cpu().saturating_sub(c0) < d { + for _ in 0..10_000 { + x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + } + } + std::hint::black_box(x); + } + + fn burn_until(stop: &AtomicBool) { + let mut x: u64 = 0; + while !stop.load(Ordering::Relaxed) { + for _ in 0..10_000 { + x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + } + } + std::hint::black_box(x); + } + + /// The upper median: the middle value, or the higher of the two middle + /// ones. + fn median(d: &[Duration]) -> Duration { + let mut d = d.to_vec(); + d.sort(); + d.get(d.len() / 2).copied().unwrap_or_default() + } + + // ---- The production clock and tick, on the tests' own clock. + + /// One production tick as the tests measure it, with literals rather + /// than the production constants: it used at least 25 ms of this + /// thread's CPU, or took at least 500 ms of wall time (the only way a + /// tick may end short: a machine loaded to about twenty runnable threads + /// per core), and it used no more CPU than wall time, which one thread + /// cannot. + fn assert_tick(which: &str, cpu: Duration, wall: Duration) { + assert!( + cpu >= ms(25) || wall >= ms(500), + "{which} used {cpu:?} of this thread's CPU in {wall:?}: a tick burns until this thread's CPU clock has \ + advanced 25 ms, and only a 500 ms wall cap may end it sooner" + ); + assert!(cpu <= wall + ms(1), "{which}: {cpu:?} of one thread's CPU in {wall:?} of wall time"); + } + + /// The probe's tick burns CPU on the calling thread; it does not sleep, + /// and it burns a full tick of this thread's CPU however loaded the + /// machine is. Five ticks, each measured on the tests' own clock. + #[test] + fn the_probe_tick_burns_cpu_on_the_calling_thread() { + for i in 0..5 { + let (cpu0, t) = (test_thread_cpu(), Instant::now()); + probe_tick(); + let (cpu, wall) = (test_thread_cpu().saturating_sub(cpu0), t.elapsed()); + assert_tick(&format!("probe_tick #{i}"), cpu, wall); + } + } + + /// The production thread clock is THIS thread's CPU time, pinned on the + /// tests' own clock: it barely moves while this thread sleeps (not a wall + /// clock), barely moves while another thread burns and this one waits + /// (not the process's CPU clock), and moves with this thread's own burn. + #[test] + fn the_production_thread_clock_is_this_threads_cpu_time() { + let before = thread_cpu(); + std::thread::sleep(ms(100)); + let slept = thread_cpu().saturating_sub(before); + assert!( + slept < ms(5), + "the production thread clock advanced {slept:?} while this thread slept 100 ms: not this thread's \ + CPU time" + ); + + let before = thread_cpu(); + let other = std::thread::spawn(|| { + let c0 = test_thread_cpu(); + burn_cpu(ms(100)); + test_thread_cpu().saturating_sub(c0) + }) + .join() + .expect("the burning thread"); + let waited = thread_cpu().saturating_sub(before); + assert!(other >= ms(100), "setup: the other thread burned only {other:?}"); + assert!( + waited < ms(5), + "the production thread clock advanced {waited:?} while another thread burned {other:?} and this one \ + waited: it counts other threads' CPU" + ); + + let (prod0, own0) = (thread_cpu(), test_thread_cpu()); + burn_cpu(ms(50)); + let (own, prod) = (test_thread_cpu().saturating_sub(own0), thread_cpu().saturating_sub(prod0)); + assert!( + prod + ms(2) >= own && prod <= own + ms(2), + "this thread burned {own:?} by the tests' clock; the production thread clock saw {prod:?}" + ); + eprintln!( + "ioreport thread clock: {slept:?} over a 100 ms sleep, {waited:?} while another thread burned \ + {other:?}, {prod:?} for {own:?} of this thread's burn" + ); + } + + // ---- The production composition, on a scripted IOReport. + + /// A channel of the Energy Model group. + fn em(name: &str, raw: i64) -> Channel { + Channel { group: ENERGY_MODEL.to_string(), name: name.to_string(), raw } + } + + /// A scripted sample sequence, long enough for a whole budget: at sample + /// `i`, each named CPU rail reads its `i`-th value (then holds its last), + /// beside an unrelated channel of the group and a `GPU Energy` rail that + /// moves at every sample, as on an M5 Max. + fn samples(rails: &[(&str, &[i64])]) -> Vec>> { + (0..200) + .map(|i| { + let mut s: Vec = rails.iter().map(|(name, v)| em(name, v[i.min(v.len() - 1)])).collect(); + s.push(em("DRAM0", 42)); + s.push(em("GPU Energy", 1_000 + i as i64)); + Some(s) + }) + .collect() + } + + /// Every tick of a scripted run, from the stamps the script took at each + /// sample call on the tests' own clock (one tick sits between two calls). + fn assert_ticks_burned(script: &seam::Script) { + for (i, pair) in script.stamps.windows(2).enumerate() { + let (cpu, wall) = (pair[1].0.saturating_sub(pair[0].0), pair[1].1 - pair[0].1); + assert_tick(&format!("tick {i} of {} in the production run", script.stamps.len() - 1), cpu, wall); + } + } + + /// The production status, `host_status` itself (the composition the cache + /// holds), where the Energy Model subscription does not open: + /// `Unavailable` — never `Live`, never `CpuRailAbsent` — asked for once, + /// never sampled, and never ticked. + #[test] + fn the_production_status_of_a_subscription_that_does_not_open_is_unavailable() { + let script = seam::install(false, Vec::new()); + let cpu0 = test_thread_cpu(); + let status = host_status(); + let used = test_thread_cpu().saturating_sub(cpu0); + let script = script.finish(); + assert_eq!(status, IoReportStatus::Unavailable, "no subscription"); + assert_eq!(script.opened, 1, "the production status asks for one subscription"); + assert!(script.stamps.is_empty(), "sampled a subscription that did not open, {} times", script.stamps.len()); + assert!(used < ms(20), "no tick runs without a subscription (a tick burns 25 ms); this thread used {used:?}"); + } + + /// A CPU rail that holds for the whole budget, through the production + /// status, while the GPU rail moves at every sample (the M5 Max on macOS + /// 27.0.1): `CpuRailFrozen`, watched for the whole budget with a full tick + /// of this thread's CPU burned before every sample but the first. The + /// time bound is a lower one only (the run may not give up early); load + /// can only lengthen it. + #[test] + fn the_production_status_of_a_constant_rail_is_frozen() { + let script = seam::install(true, samples(&[("CPU Energy", &[480_083_160])])); + let status = host_status(); + let script = script.finish(); + assert_eq!(status, IoReportStatus::CpuRailFrozen, "a rail that held was called {status:?}"); + assert_eq!(script.opened, 1); + assert!(script.stamps.len() >= 6, "{} sample(s): the probe takes five intervals at least", script.stamps.len()); + let watched = script.stamps[script.stamps.len() - 1].1 - script.stamps[0].1; + assert!(watched >= ms(900), "a rail that held was watched for {watched:?} of a 1 s budget"); + assert_ticks_burned(&script); + } + + /// One batch refresh through the production status: the CPU rail steps + /// once, then holds, while the GPU rail moves at every sample — + /// `CpuRailFrozen`, with a full tick of this thread's CPU burned before + /// every sample but the first. + #[test] + fn the_production_status_of_one_batch_refresh_is_frozen() { + let script = seam::install(true, samples(&[("CPU Energy", &[480_083_160, 541_893_638])])); + let status = host_status(); + let script = script.finish(); + assert_eq!(status, IoReportStatus::CpuRailFrozen, "one refresh called {status:?}"); + assert_eq!(script.opened, 1); + assert!(script.stamps.len() >= 6, "{} sample(s): the probe takes five intervals at least", script.stamps.len()); + assert_ticks_burned(&script); + } + + /// The refresh the verifier caught on this M5 Max on 2026-09-30, through + /// the production status: `CPU Energy` 2,643,843,378 -> +61,764,485, then + /// +3,435 at the next sample, then holding, while the GPU rail moves at + /// every sample. Two advances in consecutive intervals — what the earlier + /// two-advance rule called live — are `CpuRailFrozen`, watched for the + /// whole budget. + #[test] + fn the_production_status_of_the_observed_two_step_refresh_is_frozen() { + let a: i64 = 2_643_843_378; + let rail: &[i64] = &[a, a, a, a + 61_764_485, a + 61_764_485 + 3_435]; + let script = seam::install(true, samples(&[("CPU Energy", rail)])); + let status = host_status(); + let script = script.finish(); + assert_eq!(status, IoReportStatus::CpuRailFrozen, "a two-step refresh was called {status:?}"); + assert_eq!(script.opened, 1); + assert!(script.stamps.len() >= 6, "{} sample(s): the probe takes five intervals at least", script.stamps.len()); + let watched = script.stamps[script.stamps.len() - 1].1 - script.stamps[0].1; + assert!(watched >= ms(900), "a two-step refresh was watched for {watched:?} of a 1 s budget"); + assert_ticks_burned(&script); + } + + /// A CPU rail that advances at every sample, through the production + /// status: `Live`, with a full tick of burn between each two samples. It + /// takes five intervals at least, and twelve at most: each interval holds + /// a tick of at least 25 ms of wall time, so by the twelfth the samples + /// span 300 ms (a thirteenth is allowed, for clock granularity). + /// + /// The same script through the real run, uncounted and uncached, shows + /// from the run's own record that it stopped at the first interval the + /// rule allows: five advances or more, and the samples spanning 300 ms, + /// where the sample before spanned less, or had fewer than five advances. + #[test] + fn the_production_status_of_a_rail_that_keeps_advancing_is_live() { + let advancing: Vec = (0..200).map(|i| 100 + 3 * i).collect(); + let script = seam::install(true, samples(&[("CPU Energy", &advancing)])); + let status = host_status(); + let script = script.finish(); + assert_eq!(status, IoReportStatus::Live); + let n = script.stamps.len(); + assert!((6..=14).contains(&n), "live after {} interval(s)", n - 1); + assert_ticks_burned(&script); + + let script = seam::install(true, samples(&[("CPU Energy", &advancing)])); + let run = real_run(); + let script = script.finish(); + assert_eq!(run.verdict(), Some(CpuRail::Live), "{:?}", run.watch); + let (k, at) = (run.watch.intervals as usize, &run.watch.at); + assert_eq!(at.len(), k + 1, "one clock reading per sample handed to the rule"); + assert_eq!(run.watch.rails, [("CPU Energy".to_string(), 0, k as u32)], "it advanced at every interval"); + let window = |i: usize| at[i] - at[0]; + assert!(k >= 5 && window(k) >= ms(300), "live after {k} intervals over {:?}", window(k)); + assert!(k == 5 || window(k - 1) < ms(300), "still sampling at {:?} after {} intervals", window(k - 1), k - 1); + assert_ticks_burned(&script); + } + + /// Two dies that advance in TURN for the whole budget, one in the odd + /// intervals and the other in the even ones, through the production + /// status: two rails that each advanced in half the intervals, + /// `CpuRailFrozen`. A production sampler that handed the rule each + /// rail's sum over the CPU rails would see every rail advance at every + /// interval, and call it live. + #[test] + fn the_production_status_of_two_dies_advancing_in_turn_is_frozen() { + let die = |parity: i64| -> Vec { + (0..200).scan(36_244_185, |v, i: i64| { + *v += if i > 0 && i % 2 == parity { 7 } else { 0 }; + Some(*v) + }) + .collect() + }; + let (d0, d1) = (die(1), die(0)); + let script = seam::install(true, samples(&[("DIE_0_CPU Energy", &d0), ("DIE_1_CPU Energy", &d1)])); + let status = host_status(); + let script = script.finish(); + assert_eq!(status, IoReportStatus::CpuRailFrozen, "two dies advancing in turn were called {status:?}"); + assert_ticks_burned(&script); + } + + /// (a) Two dies that both advance at every sample, through the production + /// status, while the GPU rail moves at every sample: `Live`, after five + /// intervals at least and fourteen at most, as for one rail. + #[test] + fn the_production_status_of_two_dies_that_keep_advancing_is_live() { + let advancing: Vec = (0..200).map(|i| 100 + 3 * i).collect(); + let script = + seam::install(true, samples(&[("DIE_0_CPU Energy", &advancing), ("DIE_1_CPU Energy", &advancing)])); + let status = host_status(); + let script = script.finish(); + assert_eq!(status, IoReportStatus::Live, "two dies that kept advancing were called {status:?}"); + let n = script.stamps.len(); + assert!((6..=14).contains(&n), "live after {} interval(s)", n - 1); + assert_ticks_burned(&script); + } + + /// (b) `DIE_0_CPU Energy` advances at every sample while `DIE_1_CPU + /// Energy` holds, through the production status: `CpuRailFrozen`, + /// watched for the whole budget. The meter sums both dies into its `Cpu` + /// figure, so a Live here would claim a die it does not read. The + /// one-rail rule called this live within ~310 ms. + #[test] + fn the_production_status_of_one_live_die_beside_a_frozen_one_is_frozen() { + let advancing: Vec = (0..200).map(|i| 9_107_678 + 3 * i).collect(); + let script = + seam::install(true, samples(&[("DIE_0_CPU Energy", &advancing), ("DIE_1_CPU Energy", &[9_742_153])])); + let status = host_status(); + let script = script.finish(); + assert_eq!(status, IoReportStatus::CpuRailFrozen, "one live die beside a frozen one was called {status:?}"); + let watched = script.stamps[script.stamps.len() - 1].1 - script.stamps[0].1; + assert!(watched >= ms(900), "watched for {watched:?} of a 1 s budget"); + assert_ticks_burned(&script); + } + + /// An Energy Model group with no CPU rail (a moving GPU rail and an + /// unrelated channel only), through the production status: + /// `CpuRailAbsent`. + #[test] + fn the_production_status_with_no_cpu_rail_is_absent() { + let script = seam::install(true, samples(&[])); + let status = host_status(); + let script = script.finish(); + assert_eq!(status, IoReportStatus::CpuRailAbsent); + assert_ticks_burned(&script); + } + + /// The real run, on a scripted IOReport, records each sample exactly as + /// its sampler read it and hands the rule each sample's CPU rails as + /// read: with the GPU rail moving at every sample, a sampler that handed + /// back an old sample would show here. + #[test] + fn the_real_run_records_each_sample_as_read() { + let script = + seam::install(true, samples(&[("DIE_0_CPU Energy", &[9_107_678]), ("DIE_1_CPU Energy", &[9_742_153])])); + let run = real_run(); + let script = script.finish(); + assert!(run.opened); + assert_eq!(run.calls.len(), script.stamps.len(), "one record per sample call"); + assert_eq!(run.watch.observed.len(), run.calls.len(), "every sample read was handed to the rule"); + for (i, (call, rails)) in run.calls.iter().zip(&run.watch.observed).enumerate() { + let want_rails = [("DIE_0_CPU Energy".to_string(), 9_107_678), ("DIE_1_CPU Energy".to_string(), 9_742_153)]; + assert_eq!(rails[..], want_rails, "sample {i}: the rule was handed {rails:?}"); + let gpu = vec![("GPU Energy".to_string(), 1_000 + i as i64)]; + assert_eq!(call.read, Some(SampleShape { channels: 4, group_only: true, gpu }), "sample {i}"); + } + assert_eq!(run.verdict(), Some(CpuRail::Frozen)); + assert_ticks_burned(&script); + } + + // ---- The real IOReport. + + /// The probe samples a subscription to the Energy Model group alone — the + /// reason one sample is cheap enough to follow every tick of burn. Every + /// channel in its sample must belong to that group (by the tests' own + /// walk, and by what the production read reports), and the sampler hands + /// the rule CPU rails only, one of them at least wherever the status + /// says a CPU rail exists. + #[test] + fn the_probe_samples_the_energy_model_group_alone() { + let Ok(p) = IoReportProbe::open() else { + assert_eq!(ioreport_status(), IoReportStatus::Unavailable, "the probe's subscription did not open"); + return; + }; + let t = Instant::now(); + let (mut channels, mut other_groups) = (0usize, Vec::new()); + unsafe { + let samples = IOReportCreateSamples(p.subscription, p.channels, ptr::null()); + assert!(!samples.is_null(), "an open subscription yields a sample"); + let key = cf_string("IOReportChannels"); + let arr = CFDictionaryGetValue(samples, key) as CFArrayRef; + CFRelease(key); + assert!(!arr.is_null(), "a sample carries a channel array"); + for i in 0..CFArrayGetCount(arr) { + let ch = CFArrayGetValueAtIndex(arr, i) as CFDictionaryRef; + channels += 1; + let group = rust_string(IOReportChannelGetGroup(ch)).unwrap_or_default(); + if group != "Energy Model" { + other_groups.push(group); + } + } + CFRelease(samples); + } + let sampled_in = t.elapsed(); + eprintln!("ioreport subscription: {channels} channels, one sample took {sampled_in:?}"); + assert!(channels > 0, "the Energy Model subscription is empty"); + assert!(other_groups.is_empty(), "the probe samples channels outside the Energy Model group: {other_groups:?}"); + let read = p.read().expect("a second sample"); + assert_eq!(read.len(), channels, "the production read dropped or added channels"); + assert!(read.iter().all(|c| c.group == "Energy Model"), "the production read reports another group"); + let (rails, shape) = split_sample(&read); + assert!( + rails.iter().all(|(name, _)| is_cpu_rail(name)), + "the sampler handed the rule a non-CPU rail: {rails:?}" + ); + assert_eq!((shape.channels, shape.group_only), (channels, true)); + let status = ioreport_status(); + assert_eq!(rails.is_empty(), status == IoReportStatus::CpuRailAbsent, "status {status:?}, CPU rails {rails:?}"); + } + + /// The channels a test walk kept: `(group, name, ABSOLUTE raw value)`. + type Walked = Vec<(String, String, i64)>; + + /// Where the tests read IOReport for themselves: a subscription to every + /// channel (the oracle), or to the Energy Model group (the timeline), + /// opened and walked by the tests' own code. It shares nothing with the + /// probe's but the raw FFI. + struct TestSub { + sub: IOReportSubscriptionRef, + chans: CFMutableDictionaryRef, + } + + impl TestSub { + /// Subscribe to `wanted`, a channel dictionary this takes ownership of. + fn subscribe(wanted: CFMutableDictionaryRef) -> Option { + if wanted.is_null() { + return None; + } + unsafe { + let mut subbed: CFMutableDictionaryRef = ptr::null_mut(); + let sub = IOReportCreateSubscription(ptr::null(), wanted, &mut subbed, 0, ptr::null()); + CFRelease(wanted as CFTypeRef); + if sub.is_null() || subbed.is_null() { + if !sub.is_null() { + CFRelease(sub as CFTypeRef); + } + if !subbed.is_null() { + CFRelease(subbed as CFTypeRef); + } + return None; + } + Some(TestSub { sub, chans: subbed }) + } + } + + /// The oracle: every channel. + fn every_channel() -> Option { + Self::subscribe(unsafe { IOReportCopyAllChannels(0, 0) }) + } + + /// The timeline: the Energy Model group. + fn energy_model() -> Option { + unsafe { + let group = cf_string("Energy Model"); + let wanted = IOReportCopyChannelsInGroup(group, ptr::null(), 0, 0, 0); + CFRelease(group); + Self::subscribe(wanted) + } + } + + /// One plain sample (no delta): `(group, name, ABSOLUTE raw value)` + /// for every channel whose name `keep` selects, in channel order, and + /// how many channels the sample held. `None` if it could not be taken. + fn sample(&self, keep: impl Fn(&str) -> bool) -> Option<(usize, Walked)> { + unsafe { + let smp = IOReportCreateSamples(self.sub, self.chans, ptr::null()); + if smp.is_null() { + return None; + } + let key = cf_string("IOReportChannels"); + let arr = CFDictionaryGetValue(smp, key) as CFArrayRef; + CFRelease(key); + let (mut count, mut vals) = (0usize, Vec::new()); + if !arr.is_null() { + for i in 0..CFArrayGetCount(arr) { + let ch = CFArrayGetValueAtIndex(arr, i) as CFDictionaryRef; + if ch.is_null() { + continue; + } + count += 1; + let name = rust_string(IOReportChannelGetChannelName(ch)).unwrap_or_default(); + if keep(&name) { + let group = rust_string(IOReportChannelGetGroup(ch)).unwrap_or_default(); + vals.push((group, name, IOReportSimpleGetIntegerValue(ch, 0))); + } + } + } + CFRelease(smp); + Some((count, vals)) + } + } + + /// Every channel whose name ends with `CPU Energy`, in any group. + fn read(&self) -> Option> { + self.sample(|name| name.ends_with("CPU Energy")).map(|(_, vals)| vals) + } + + /// The Energy Model group's `... CPU Energy` channels in one read: + /// each one's name, its ordinal among the same-named ones, and its + /// absolute raw value. + fn energy_model_rails(&self) -> Option> { + Some(em_keyed(&self.read()?, "CPU Energy").into_iter().map(|((n, o), v)| (n, o, v)).collect()) + } + } + + impl Drop for TestSub { + fn drop(&mut self) { + unsafe { + CFRelease(self.chans as CFTypeRef); + CFRelease(self.sub as CFTypeRef); + } + } + } + + /// The Energy Model group's channels whose names end with `suffix`, by + /// the tests' own match: each keyed by its name and its ordinal among + /// same-named ones, with its raw value, in channel order. + fn em_keyed(read: &[(String, String, i64)], suffix: &str) -> Vec<((String, u32), i64)> { + let mut out: Vec<((String, u32), i64)> = Vec::new(); + for (group, name, value) in read { + if group == "Energy Model" && name.ends_with(suffix) { + let ordinal = out.iter().filter(|((n, _), _)| n == name).count() as u32; + out.push(((name.clone(), ordinal), *value)); + } + } + out + } + + /// A probe sample's `(name, raw)` rails, keyed the same way. + fn keyed(rails: &[(String, i64)]) -> Vec<((String, u32), i64)> { + let mut out: Vec<((String, u32), i64)> = Vec::new(); + for (name, value) in rails { + let ordinal = out.iter().filter(|((n, _), _)| n == name).count() as u32; + out.push(((name.clone(), ordinal), *value)); + } + out + } + + /// What the independent oracle saw on this host. + #[derive(Debug, PartialEq)] + enum Oracle { + NoIoReport, + NoCpuRail, + CpuRailMoved(bool), + } + + /// The fewest reads (each compared with the one before it) the oracle + /// needs before its verdict says anything: a rail that keeps moving + /// changes at five reads at least ([`oracle_saw_it_keep_moving`]). + const ORACLE_MIN_READS: u32 = 5; + + /// The oracle's verdict on one watch, from each CPU rail's count of the + /// reads at which its value changed, of `reads` reads: did EVERY rail + /// keep moving, as a Live status claims? Only if there is a rail, and + /// each one changed at five reads at least, and at four reads in five at + /// least (80%): a die that held is not vouched for by the other die. A live rail changes at nearly every read, the + /// reads being ~0.15 s apart or more; a batch refresh lands in one step, + /// or a few within ~150 ms, so however it falls across the reads it + /// cannot vouch for a Live the probe got from that same refresh. Its own + /// literals, not the implementation's constants, so a change to one + /// cannot silently move the other. + fn oracle_saw_it_keep_moving(changes: &[u32], reads: u32) -> bool { + !changes.is_empty() && changes.iter().all(|&c| c >= 5 && c * 5 >= reads * 4) + } + + /// An oracle that shares nothing with the liveness probe but the raw FFI. + /// The probe samples a subscription to the Energy Model group, strips a + /// `DIE__` prefix, and burns on its own thread between samples. The + /// oracle opens its OWN subscription to every channel, reads each CPU + /// rail's ABSOLUTE value (no delta samples), matches names with + /// `ends_with("CPU Energy")` in any group, and keeps the CPU loaded from a + /// HELPER thread for as long as it reads, so its reads are under load even + /// while this thread blocks in an all-channel sample (~0.12–0.8 s). + /// + /// It reads for the whole `window`, and past it until it has + /// [`ORACLE_MIN_READS`] reads, so a live rail could meet its bar, never + /// past four windows. Fewer reads than that is no evidence either way, + /// and fails the test: never a verdict on four reads or fewer. + /// + /// `CpuRailMoved(true)` means every CPU rail kept moving + /// ([`oracle_saw_it_keep_moving`]): it changed at five reads or more, at + /// four reads in five at least. + fn oracle_cpu_rail(window: Duration) -> Oracle { + match TestSub::every_channel() { + None => Oracle::NoIoReport, + Some(oracle) => oracle_watch(&oracle, window), + } + } + + fn oracle_watch(oracle: &TestSub, window: Duration) -> Oracle { + let values = |read: Vec<(String, String, i64)>| read.into_iter().map(|(_, _, v)| v).collect::>(); + let Some(mut prev) = oracle.read().map(values) else { + return Oracle::NoIoReport; + }; + if prev.is_empty() { + return Oracle::NoCpuRail; + } + let stop = Arc::new(AtomicBool::new(false)); + let load = { + let stop = Arc::clone(&stop); + std::thread::spawn(move || burn_until(&stop)) + }; + let started = Instant::now(); + // Per rail: at how many reads its absolute value had changed. + let mut changes = vec![0u32; prev.len()]; + let mut reads = 0u32; + while (started.elapsed() < window || reads < ORACLE_MIN_READS) && started.elapsed() < window * 4 { + std::thread::sleep(ms(25)); + let Some(cur) = oracle.read().map(values) else { + continue; + }; + if cur.len() != prev.len() { + continue; + } + reads += 1; + for (count, (now, before)) in changes.iter_mut().zip(cur.iter().zip(&prev)) { + *count += u32::from(now != before); + } + prev = cur; + } + let took = started.elapsed(); + stop.store(true, Ordering::Relaxed); + load.join().expect("the oracle's load thread"); + eprintln!( + "ioreport oracle: {reads} reads over {took:?} under a helper-thread burn, \ + changes per CPU rail {changes:?}" + ); + assert!( + reads >= ORACLE_MIN_READS, + "the oracle got {reads} read(s) of the CPU rail in {took:?}: it takes {ORACLE_MIN_READS} to tell a \ + rail that keeps moving from one that held, so this run is no evidence either way" + ); + Oracle::CpuRailMoved(oracle_saw_it_keep_moving(&changes, reads)) + } + + /// The oracle's "moved" verdict: every rail changed at five reads or + /// more, at four in five at least. A batch refresh that falls across its + /// reads — in one step, in the two steps seen on this M5 Max, or on both + /// dies of an M3 Ultra — is not "moved", so it cannot vouch for a Live + /// the probe got from the same refresh. + #[test] + fn the_oracle_says_moved_only_for_a_rail_that_changed_at_four_reads_in_five() { + let moved = |changes: &[u32], reads: u32| oracle_saw_it_keep_moving(changes, reads); + // A live rail: every read, or four in five. + assert!(moved(&[20], 20)); + assert!(moved(&[16], 20), "80%"); + assert!(moved(&[5], 5)); + assert!(moved(&[20, 16], 20), "both dies of an M3 Ultra"); + // Not every rail kept moving. + assert!(!moved(&[0, 20], 20), "one die moving, the other held"); + assert!(!moved(&[20, 15], 20), "one die moving, the other at 75%"); + assert!(!moved(&[], 20), "no rail"); + // Not a live rail. + assert!(!moved(&[15], 20), "75%"); + assert!(!moved(&[4], 5), "four changes are too few"); + assert!(!moved(&[0], 20), "a rail that held"); + assert!(!moved(&[1], 20), "one refresh in one step"); + assert!(!moved(&[2], 20), "the two-step refresh seen on 2026-09-30"); + assert!(!moved(&[2], 5), "the two-step refresh, across five reads"); + assert!(!moved(&[2, 2], 20), "an M3 Ultra refresh stepping twice on each die"); + assert!(!moved(&[5], 7), "five changes in seven reads, 71%"); + } + + /// The probe's sampler reads each CPU rail's OWN absolute value — not 0, + /// not a sum over rails, not another rail's value — and exactly the + /// group's CPU rails. Every probe sample is bracketed by two of the + /// oracle's reads on its own all-channel subscription. The probe's rails + /// must be the Energy Model group's CPU rails as the oracle lists them; + /// where the oracle saw a rail hold across the bracket (a frozen rail is + /// constant), the probe's value must EQUAL it; where the rail moved, the + /// probe's value must lie between the two reads. + #[test] + fn the_probe_sampler_reads_each_cpu_rails_own_value() { + let Ok(p) = IoReportProbe::open() else { + assert_eq!(ioreport_status(), IoReportStatus::Unavailable, "the probe's subscription did not open"); + return; + }; + let oracle = TestSub::every_channel().expect("the probe's subscription opened, the oracle's did not"); + let mut before = oracle.energy_model_rails().expect("an oracle read"); + let (mut held, mut moved) = (0usize, 0usize); + for _ in 0..3 { + let sample = split_sample(&p.read().expect("a probe sample")).0; + let after = oracle.energy_model_rails().expect("an oracle read"); + let keys: Vec<(String, u32)> = keyed(&sample).into_iter().map(|(k, _)| k).collect(); + let oracle_keys: Vec<(String, u32)> = before.iter().map(|(n, o, _)| (n.clone(), *o)).collect(); + assert_eq!( + keys, oracle_keys, + "the probe's CPU rails are not the Energy Model group's, as the oracle lists them" + ); + for ((name, ordinal), value) in keyed(&sample) { + let find = |read: &[(String, u32, i64)]| { + read.iter().find(|(n, o, _)| *n == name && *o == ordinal).map(|(_, _, v)| *v) + }; + let (Some(a), Some(b)) = (find(&before), find(&after)) else { + panic!("the probe read CPU rail {name:?} #{ordinal}; the oracle's Energy Model reads {before:?} / {after:?} have no such channel"); + }; + if a == b { + held += 1; + assert_eq!( + value, a, + "{name} #{ordinal} held at {a} across the oracle reads around the probe's sample; the probe read {value}" + ); + } else { + moved += 1; + assert!( + (a.min(b)..=a.max(b)).contains(&value), + "{name} #{ordinal} moved {a} -> {b} across the oracle reads around the probe's sample; \ + the probe read {value}, outside them" + ); + } + } + if !sample.is_empty() { + eprintln!("ioreport sampler: probe {sample:?} between oracle {before:?} and {after:?}"); + } + before = after; + } + eprintln!("ioreport sampler: {held} rail value(s) equal to a held oracle rail, {moved} within a moving one"); + assert_eq!( + held + moved == 0, + ioreport_status() == IoReportStatus::CpuRailAbsent, + "status {:?}, {} CPU rail value(s) compared", + ioreport_status(), + held + moved + ); + } + + /// The property that matters: the IOReport probe is **never** used when its + /// CPU rail is not live, because then every span reports GPU-only energy, + /// zero, or a stale batch refresh as a measurement of CPU work. It asserts + /// the meter's decision against what the rail actually did, on whatever + /// host runs it, and every branch asserts — there is no early return that + /// passes silently. + /// + /// The oracle gets three times the probe's budget, so a rail too slow for + /// the probe but moving within the oracle's window is the only case whose + /// outcome is not pinned — and that one fails safe: declined, never + /// over-claimed. A refresh landing in both windows, in one step or + /// several, cannot rescue a probe that called it live: the oracle calls the + /// CPU moving only if every rail changed at five reads or more, four in + /// five. Nor can one die that keeps moving vouch for a die that held. + #[test] + fn the_ioreport_probe_is_never_selected_on_a_frozen_cpu_rail() { + let status = ioreport_status(); + let available = IoReportProbe::given(ioreport_status()).is_ok(); + let selected = PowerMeter::detect().sources().contains(&"ioreport-energy-model"); + let oracle = oracle_cpu_rail(CPU_RAIL_LIVENESS_BUDGET * 3); + // Recorded so a log says which branch below was actually asserted. + eprintln!( + "ioreport liveness: status={status:?} oracle={oracle:?} selected={selected} available={available}" + ); + assert_eq!(available, selected, "the constructor and PowerMeter::detect() disagree"); + match oracle { + Oracle::NoIoReport => { + assert_eq!(status, IoReportStatus::Unavailable, "the oracle could not open IOReport"); + assert!(!selected, "selected an IOReport probe the oracle could not open"); + } + Oracle::NoCpuRail => { + assert_eq!(status, IoReportStatus::CpuRailAbsent, "the oracle found no CPU rail"); + assert!(!selected, "selected an IOReport probe that has no CPU rail"); + } + Oracle::CpuRailMoved(false) => { + assert_eq!( + status, + IoReportStatus::CpuRailFrozen, + "the oracle did not see every CPU rail keep moving under {:?} of CPU burn", + CPU_RAIL_LIVENESS_BUDGET * 3 + ); + assert!( + !selected, + "IOReport probe selected although not every CPU rail kept moving under {:?} \ + of CPU burn — every span would report GPU-only energy, zero, or a stale \ + refresh as a measurement of CPU work", + CPU_RAIL_LIVENESS_BUDGET * 3 + ); + } + Oracle::CpuRailMoved(true) => { + // A live rail: the probe had a third of the oracle's time. If it + // saw the rail keep moving, the probe must be used; if not, + // declining is the safe side, and IOReport itself did open. + if status == IoReportStatus::Live { + assert!(selected, "liveness probe said Live but the IOReport probe was declined"); + } else { + assert_ne!(status, IoReportStatus::Unavailable, "the oracle opened IOReport"); + assert!(!selected, "selected an IOReport probe whose status is {status:?}"); + } + } + } + } + + /// A liveness claim, checked against what the oracle saw. + fn assert_agrees_with_oracle(who: &str, status: IoReportStatus, oracle: &Oracle) { + match status { + IoReportStatus::Live => assert_eq!( + *oracle, + Oracle::CpuRailMoved(true), + "{who} said Live, the oracle did not see every CPU rail keep moving" + ), + IoReportStatus::CpuRailFrozen => assert!( + matches!(oracle, Oracle::CpuRailMoved(_)), + "{who} said a CPU rail exists and is frozen; the oracle saw {oracle:?}" + ), + // The oracle looks in every group, the probe in the Energy Model + // group only: it may find a rail the probe does not. Declining + // then is the safe side. + IoReportStatus::CpuRailAbsent => { + assert_ne!(*oracle, Oracle::NoIoReport, "{who} opened IOReport, the oracle could not") + } + IoReportStatus::Unavailable => { + assert_eq!(*oracle, Oracle::NoIoReport, "{who} could not open or sample IOReport, the oracle could") + } + } + } + + /// The dangerous direction on its own: no liveness claim is Live for CPU + /// rails the oracle cannot see all keep moving. It runs the real probe run + /// itself — the composition the cached status comes from — uncounted and + /// uncached, beside the cached status the meter is gated on, and checks + /// every other claim against the oracle too. + /// + /// It also pins HOW that run sampled and burned, from the run's own + /// record and never from a wall-clock time: every sample held the Energy + /// Model group alone, as many channels as the oracle counts in that + /// group (a few hundred, against ~12-14 thousand for every channel); the + /// median sample call used under 50 ms of this thread's CPU (~6-12 ms for + /// the group, ~76-145 ms for every channel, however long another + /// process's IOReport sampling makes it wait); and every interval burned + /// at least half a tick of this thread's CPU between its samples. + #[test] + fn the_liveness_probe_never_reports_a_frozen_rail_as_live() { + let t = Instant::now(); + let run = real_run(); + let took = t.elapsed(); + let probed = status_of(run.verdict()); + let cached = ioreport_status(); + let oracle_sub = TestSub::every_channel(); + // The oracle's own count of every channel, and of the Energy Model + // group's, from one of its own walks. + let counts = oracle_sub.as_ref().map(|o| { + let (every, read) = o.sample(|_| true).expect("an oracle read"); + (every, read.iter().filter(|(group, _, _)| group == "Energy Model").count()) + }); + let oracle = match &oracle_sub { + Some(o) => oracle_watch(o, CPU_RAIL_LIVENESS_BUDGET * 3), + None => Oracle::NoIoReport, + }; + let intervals = run.watch.intervals; + let window = match (run.watch.at.first(), run.watch.at.last()) { + (Some(first), Some(last)) => *last - *first, + _ => Duration::ZERO, + }; + eprintln!( + "ioreport probe: probed={probed:?} cached={cached:?} oracle={oracle:?} took={took:?} \ + intervals={intervals} window={window:?} rails={:?}", + run.watch.rails + ); + assert_agrees_with_oracle("the uncached probe", probed, &oracle); + assert_agrees_with_oracle("the cached status", cached, &oracle); + let Some((every, group)) = counts else { + // No IOReport, as the oracle agreed above: the real run never + // opened, so it never sampled and never ticked. + assert!(!run.opened && run.calls.is_empty() && run.ticks == 0, "{run:?}"); + return; + }; + assert!(run.opened, "the oracle opened IOReport; the real run did not"); + // Group-only, from the run's own record of every sample it read. + let shapes: Vec<&SampleShape> = run.calls.iter().filter_map(|c| c.read.as_ref()).collect(); + assert!(shapes.len() >= 2, "the real run read {} sample(s) where the oracle read IOReport", shapes.len()); + for (i, shape) in shapes.iter().enumerate() { + assert!(shape.group_only, "sample {i}: the probe sampled channels outside the Energy Model group"); + assert_eq!( + shape.channels, group, + "sample {i} held {} channels; the oracle counts {group} in the Energy Model group (every \ + channel: {every})", + shape.channels + ); + } + assert!( + group * 10 <= every, + "the Energy Model group ({group} channels) is not far below every channel ({every})" + ); + let cpu = median(&run.calls.iter().map(|c| c.cpu).collect::>()); + let wall = median(&run.calls.iter().map(|c| c.ended.saturating_sub(c.started)).collect::>()); + eprintln!( + "ioreport probe structure: {} samples, each {group} channels, all in the Energy Model group (every \ + channel: {every}); median thread CPU per sample call {cpu:?} over {} calls (median wall {wall:?}, \ + not asserted)", + shapes.len(), + run.calls.len() + ); + assert!( + cpu < ms(50), + "the probe's median sample call used {cpu:?} of this thread's CPU over {} calls: it is not sampling the \ + Energy Model group alone (~6-12 ms of CPU a sample; every channel takes ~76-145 ms)", + run.calls.len() + ); + // Burn between samples: the thread CPU the run's ticks burned in + // each interval, measured around the ticks and apart from the samples. + assert_eq!( + run.interval_burn.len(), + intervals as usize, + "one burn figure per interval: {:?}", + run.interval_burn + ); + let least = run.interval_burn.iter().min().copied().unwrap_or_default(); + eprintln!( + "ioreport probe burn: {intervals} intervals, the least thread CPU burned in one {least:?} (tick \ + {PROBE_TICK:?})" + ); + for (i, burned) in run.interval_burn.iter().enumerate() { + assert!( + *burned >= PROBE_TICK / 2, + "interval {i} of {intervals}: the probe burned {burned:?} of this thread's CPU between its samples, \ + under half its {PROBE_TICK:?} tick. A tick burns until this thread's CPU clock has advanced a full \ + tick, however loaded the machine is; only its {TICK_WALL_CAP:?} wall cap can cut it short" + ); + } + } + + /// One read of the tests' own Energy Model subscription, stamped before + /// and after: the group's CPU and GPU rails, keyed by name and ordinal. + struct Stamped { + started: Instant, + ended: Instant, + cpu: Vec<((String, u32), i64)>, + gpu: Vec<((String, u32), i64)>, + } + + /// Run `work` on this thread while a helper thread reads the tests' own + /// Energy Model subscription over and over, each read stamped before and + /// after it. The helper finishes one read before `work` starts and one + /// that began after `work` returned, so every moment of `work` lies + /// between two reads. Returns `work`'s value, the reads, and the instants + /// just before and just after `work`. + fn with_timeline(work: impl FnOnce() -> T) -> (T, Vec, Instant, Instant) { + let stop = Arc::new(AtomicBool::new(false)); + let (first_read, ready) = mpsc::channel::<()>(); + let reader = { + let stop = Arc::clone(&stop); + std::thread::spawn(move || { + let sub = TestSub::energy_model().expect("the timeline's own Energy Model subscription"); + let mut reads = Vec::new(); + loop { + let stopping = stop.load(Ordering::SeqCst); + let started = Instant::now(); + let (_, read) = sub + .sample(|name| name.ends_with("CPU Energy") || name.ends_with("GPU Energy")) + .expect("a timeline read"); + let ended = Instant::now(); + reads.push(Stamped { + started, + ended, + cpu: em_keyed(&read, "CPU Energy"), + gpu: em_keyed(&read, "GPU Energy"), + }); + if reads.len() == 1 { + let _ = first_read.send(()); + } + if stopping { + return reads; + } + std::thread::sleep(ms(5)); + } + }) + }; + ready.recv().expect("the timeline's first read"); + let t0 = Instant::now(); + let out = work(); + let t1 = Instant::now(); + stop.store(true, Ordering::SeqCst); + let reads = reader.join().expect("the timeline thread"); + (out, reads, t0, t1) + } + + /// The values the real probe hands its rule are each CPU rail's OWN, + /// CURRENT absolute value, in the very run the cached status comes from + /// (`real_run`, uncounted and uncached here). A helper thread reads the + /// tests' own Energy Model subscription over and over while the run goes, + /// so each sample the run took is bracketed by a timeline read that + /// surely came before it and one that surely came after it. The counters + /// only rise, so: + /// + /// - every CPU rail the run handed the rule, and the GPU rail it + /// recorded, lies within its bracket: EQUAL to it where the rail held + /// across the bracket, as a frozen rail does; + /// - the run's CPU-rail keys are the Energy Model group's own, as the + /// all-channel oracle lists them: none missing, none added; + /// - where the timeline saw `GPU Energy` rise between the run's first + /// sample and its last, the run's GPU values rose too. A sampler that + /// hands back an old sample, per probe or process-wide, fails here: GPU + /// Energy moves about every 20 ms on an M5 Max. Where it did not move + /// inside the run (an M3 Ultra can be quieter), the log says so, and + /// the brackets are what is asserted. + #[test] + fn the_real_probe_hands_the_rule_each_cpu_rails_own_fresh_value() { + let Some(oracle) = TestSub::every_channel() else { + let run = real_run(); + assert!( + !run.opened && run.calls.is_empty() && run.ticks == 0, + "the oracle could not open IOReport, the real run did: {run:?}" + ); + return; + }; + let (_, every) = oracle.sample(|name| name.ends_with("CPU Energy")).expect("an oracle read"); + drop(oracle); + let em_cpu: Vec<(String, u32)> = em_keyed(&every, "CPU Energy").into_iter().map(|(key, _)| key).collect(); + let (run, timeline, t0, t1) = with_timeline(real_run); + assert!(run.opened, "the oracle opened IOReport; the real run did not"); + let read: Vec<&SampleCall> = run.calls.iter().filter(|c| c.read.is_some()).collect(); + assert_eq!(read.len(), run.watch.observed.len(), "every sample read was handed to the rule, once, in order"); + assert!(read.len() >= 2, "the real run read {} sample(s)", read.len()); + // The run's clock starts inside `real_run`: after `t0`, and early + // enough that its last call ended by `t1`. So sample `k` was read + // after `t0 + started` and before `t1 - (last_end - ended)`. + let last_end = run.calls.iter().map(|c| c.ended).max().expect("a sample call"); + let bracket = |call: &SampleCall| { + let (after, before) = (t0 + call.started, t1 - (last_end - call.ended)); + let lower = timeline.iter().rev().find(|r| r.ended <= after).expect("a timeline read before the run"); + let upper = timeline.iter().find(|r| r.started >= before).expect("a timeline read after the run"); + (lower, upper) + }; + let value = |rails: &[((String, u32), i64)], key: &(String, u32)| { + rails + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| *v) + .unwrap_or_else(|| panic!("the timeline read no {key:?}")) + }; + let gpu_keys: Vec<(String, u32)> = timeline[0].gpu.iter().map(|(key, _)| key.clone()).collect(); + let (mut held, mut moved) = (0usize, 0usize); + let mut gpu_seen = Vec::new(); + for (k, (call, rails)) in read.iter().zip(&run.watch.observed).enumerate() { + let (lower, upper) = bracket(call); + let cpu = keyed(rails); + let keys: Vec<(String, u32)> = cpu.iter().map(|(key, _)| key.clone()).collect(); + assert_eq!( + keys, em_cpu, + "sample {k}: the run handed the rule CPU rails {keys:?}; the Energy Model group has {em_cpu:?}" + ); + let gpu = keyed(&call.read.as_ref().expect("a read sample").gpu); + let keys: Vec<(String, u32)> = gpu.iter().map(|(key, _)| key.clone()).collect(); + assert_eq!( + keys, gpu_keys, + "sample {k}: the run recorded GPU rails {keys:?}; the Energy Model group has {gpu_keys:?}" + ); + for (what, probe, lo_rails, hi_rails) in + [("CPU", &cpu, &lower.cpu, &upper.cpu), ("GPU", &gpu, &lower.gpu, &upper.gpu)] + { + for (key, p) in probe { + let (lo, hi) = (value(lo_rails, key), value(hi_rails, key)); + assert!( + lo <= *p && *p <= hi, + "sample {k}: {what} rail {key:?} read {p}; the timeline read {lo} just before that sample \ + and {hi} just after it: not the rail's own current value" + ); + if what == "CPU" { + if lo == hi { + held += 1; + } else { + moved += 1; + } + } + } + } + gpu_seen.push(gpu); + } + // Freshness, stated on its own: where the timeline saw GPU Energy + // rise after the run's first sample and before its last, the run's + // own GPU values rose too. + let (after_first, before_last) = (bracket(read[0]).1, bracket(read[read.len() - 1]).0); + let (first, last) = (&gpu_seen[0], &gpu_seen[gpu_seen.len() - 1]); + let mut rose = Vec::new(); + for (key, p_first) in first { + let p_last = value(last, key); + let (seen_after_first, seen_before_last) = (value(&after_first.gpu, key), value(&before_last.gpu, key)); + if seen_after_first < seen_before_last { + assert!( + *p_first < p_last, + "the timeline saw {key:?} rise from {seen_after_first} to {seen_before_last} between the run's \ + first sample and its last, yet the run's own values went {p_first} -> {p_last}: a sampler that \ + does not re-read IOReport" + ); + rose.push(format!("{}: {p_first} -> {p_last}", key.0)); + } + } + let distinct = |i: usize| { + let mut v: Vec = gpu_seen.iter().filter_map(|g| g.get(i).map(|(_, x)| *x)).collect(); + v.dedup(); + v.len() + }; + eprintln!( + "ioreport freshness: {} samples of the real run, each bracketed by the timeline's {} reads; CPU rail \ + values {held} equal to a held rail, {moved} within a moving one; GPU Energy {}", + read.len(), + timeline.len(), + if rose.is_empty() { + format!( + "did not rise inside the run as the timeline saw it, so freshness rests on the brackets alone \ + ({} distinct value(s) in the run)", + distinct(0) + ) + } else { + format!( + "rose inside the run, and so did the run's own values ({}; {} distinct)", + rose.join(", "), + distinct(0) + ) + } + ); + } + + /// When the probe IS used, CPU work meters as a CPU measurement that + /// advanced — not a zero, and not a GPU figure that merely happened to + /// move. When it is not, the status says why, and real CPU work meters as + /// the estimator's own figure: non-zero, wall × TDP, `Estimator`. + #[test] + fn a_selected_ioreport_probe_meters_cpu_work() { + let mut meter = PowerMeter::detect(); + if meter.sources().contains(&"ioreport-energy-model") { + assert_eq!(ioreport_status(), IoReportStatus::Live); + assert_eq!(real_run().verdict(), Some(CpuRail::Live), "an uncached real run disagrees with the cache"); + let ((), m) = meter.measure(|| burn(CPU_RAIL_LIVENESS_BUDGET)); + let cpu = m + .devices + .iter() + .find(|d| d.class == DeviceClass::Cpu) + .expect("a live IOReport probe measures the CPU class"); + assert_eq!(cpu.provenance, Provenance::HwShunt, "{:?}", m.devices); + assert!( + cpu.measured_uj.is_some_and(|uj| uj > 0), + "a live CPU rail metered {CPU_RAIL_LIVENESS_BUDGET:?} of CPU work as {:?}", + cpu.measured_uj + ); + } else { + assert_ne!( + ioreport_status(), + IoReportStatus::Live, + "status Live but PowerMeter::detect() did not use the IOReport probe" + ); + assert!(!meter.is_hardware(), "no other probe exists on Apple Silicon"); + // The declined backend's fallback meters real CPU work as what it + // is: the estimator's wall × TDP figure, never 0 for work that + // ran, and never a hardware or measured tag. + let ((), m) = meter.measure(|| burn(ms(120))); + eprintln!("declined meter: {:?} over {:?} ({})", m.devices, m.wall, ioreport_status()); + assert_eq!(m.source, "estimator"); + assert_eq!(m.devices.len(), 1, "{:?}", m.devices); + let d = m.devices[0]; + assert_eq!((d.class, d.provenance, d.measured_uj), (DeviceClass::Cpu, Provenance::Estimator, None)); + assert!(!m.fully_measured()); + d.check_provenance().unwrap(); + assert_eq!(d.estimated_uj, (m.wall.as_secs_f64() * meter.tdp_watts * 1e6) as u64, "wall × TDP"); + assert!(m.wall >= ms(120) && d.estimated_uj > 0, "{d:?} for {:?} of work", m.wall); + assert_eq!(m.total_uj(), d.estimated_uj, "the span's figure is the estimate"); + assert_eq!(meter.sample_uj(), None, "an estimator has no cumulative counter"); + } + } + + /// The constructor is gated by the status it is given, and only `Live` + /// admits it: every other status declines, naming itself; `Live` builds + /// the probe wherever the Energy Model subscription opens with a CPU + /// rail. The status is given, so nothing here probes. + #[test] + fn the_constructor_admits_only_a_live_status() { + for status in [IoReportStatus::Unavailable, IoReportStatus::CpuRailAbsent, IoReportStatus::CpuRailFrozen] { + match IoReportProbe::given(status) { + Err(ProbeError::Unavailable(why)) => assert_eq!(why, status.to_string(), "declined, naming why"), + Err(other) => panic!("{status:?} was declined for another reason: {other}"), + Ok(_) => panic!("{status:?} built the IOReport probe"), + } + } + let live = IoReportProbe::given(IoReportStatus::Live); + assert_eq!( + live.is_ok(), + energy_model_has_a_cpu_rail(), + "a Live status builds the probe exactly where its subscription opens with a CPU rail: {:?}", + live.err() + ); + } + + /// A span's per-class sum (`snapshot`) counts each class's rail itself, + /// never a sub-rail beside it: checked on every Energy Model channel name + /// that says CPU, GPU or ANE on both chips (macOS 27.0.1, one absolute + /// sample each; the CPU sub-rails sum to the aggregate, commit 7aa42418). + /// Only `CPU Energy` / `DIE__CPU Energy` count as CPU and only + /// `GPU Energy` as GPU. ANE has no aggregate on either chip: `ANE0` on an + /// M5 Max, one `ANE0_` per die on an M3 Ultra, so their sum counts + /// each die once. + #[test] + fn class_for_counts_each_class_rail_once_on_both_chips() { + let cpu_rails = ["CPU Energy", "DIE_0_CPU Energy", "DIE_1_CPU Energy"]; + let cpu_subrails = [ + // M5 Max: CPU Energy = MCPU0 + MCPU1 + PCPU, each of its cores, + // their SRAM and its power manager. Every name 7aa42418 lists, + // and more. + "MCPU0", + "MCPU1", + "PCPU", + "MCPU0_0", + "MCPU0_3", + "MCPU1_5", + "MCPU1_5_SRAM", + "PCPU2_SRAM", + "PACC_0", + "PACC_5", + "MCPM0", + "MCPM1", + "PCPM", + "MCPU0_0_SRAM", + "PCPU5_SRAM", + "MCPM0_SRAM", + "MCPM1_SRAM", + "PCPM_SRAM", + "MCPU0DTL00", + "MCPU0DTL0e", + "PCPUDTL513", + // M3 Ultra: DIE_n_CPU Energy = EACC_CPU + PACC0_CPU + PACC1_CPU. + "DIE_0_EACC_CPU", + "DIE_1_PACC0_CPU", + "DIE_0_PACC1_CPU", + "DIE_0_PACC1_CPU3", + "DIE_1_EACC_CPU2_SRAM", + "DIE_0_EACC_CPM", + "DIE_1_PACC0_CPM", + "DIE_1_PACC1_CPM_SRAM", + "DIE_0_PACC1_CPM_SRAM", + "DIE_1_EACC_CPM_SRAM", + "ECPUDTL00", + "PCPU1DTL513", + ]; + let gpu_subrails = ["GPU0", "GPU0_0", "GPU CS0_0", "GPU SRAM0_0", "GPU CS SRAM0_0"]; + // The domains 7aa42418 lists as no aggregate's summand: no class here. + let other_domains = [ + "AFR0", "ISP0", "AVE0", "MSR0", "AMCC0", "DCS0", "DRAM0", "DISP0", "DISPEXT0", "FAB0", + "PCIe Port 0 Energy", "apciec0 Energy", "ISP0_1", "DRAM0_0", "AMCC0_1", "DISPEXT0_1", + ]; + for name in cpu_rails { + assert_eq!(class_for(name), Some(DeviceClass::Cpu), "{name}"); + } + for name in cpu_subrails.iter().chain(&gpu_subrails) { + assert_eq!(class_for(name), None, "{name} would be summed beside its own aggregate"); + } + for name in other_domains { + assert_eq!(class_for(name), None, "{name} is no CPU, GPU or ANE rail"); + } + assert_eq!(class_for("GPU Energy"), Some(DeviceClass::Gpu)); + for name in ["ANE0", "ANE0_0", "ANE0_1"] { + assert_eq!(class_for(name), Some(DeviceClass::Npu), "{name}"); + } + } +} diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs new file mode 100644 index 000000000..06b04e495 --- /dev/null +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/liveness.rs @@ -0,0 +1,1885 @@ +//! The CPU-rail liveness rule behind [`ioreport_status`](super::ioreport_status), +//! kept apart from the IOReport FFI so every part of it is tested on its own, +//! on every platform: the per-sample rule ([`RailWatch`]), the sampling loop +//! ([`watch`]), the tick that burns CPU between samples ([`spin`]), one whole +//! probe run over an injected subscription, tick and clocks ([`probe_with`]), +//! the status a run gives the host ([`status_of`]), and the one gate between a +//! status and the probe's use ([`admits`]). The FFI side only opens the +//! subscription, reads every channel's group, name and raw integer from one +//! absolute sample, and reads this thread's CPU clock. The production status +//! is `status_of` of `probe_with` over those real parts, one composition with +//! no branch of its own, so the parts the tests drive are the parts the meter +//! is gated on. +//! +//! IOReport's CPU energy is live only when EVERY CPU rail in its samples +//! keeps advancing, each one on its own: `CPU Energy`, or on a multi-die part +//! each die's `DIE__CPU Energy` (an M3 Ultra has two). The meter sums +//! every CPU rail into one `Cpu` figure, and a sum claims every rail in it, +//! so a die whose rail held is a die the figure would claim without reading +//! it. A rail is present if it is in any sample the probe read, even one, and +//! is then judged over every interval observed. For each rail, three things +//! must hold at once, checked after every interval between two CONSECUTIVE +//! samples, and the CPU is live as soon as they hold for every rail: +//! +//! - it advanced in at least [`MIN_LIVE_ADVANCES`] (five) separate intervals; +//! - those advancing intervals are at least four in five (80%) of all the +//! intervals observed so far ([`sustained`]); +//! - the samples so far span at least [`MIN_LIVE_WINDOW`] (300 ms) of wall +//! time, from the first sample. +//! +//! A few advances are not liveness, because on macOS 27.0.1 an unprivileged +//! reader's CPU rail is not frozen for good: it is refreshed in rare batches, +//! and one batch can be published in more than one step. An M3 Ultra's +//! `DIE_0_CPU Energy` and `DIE_1_CPU Energy` held for at least 22 minutes, +//! then stepped +1,344,869 and +1,391,637 mJ within one 100 ms sample, and +//! held again. An M5 Max's `CPU Energy` read 480,083,160 mJ at ~10:30 and +//! 541,893,638 mJ by 10:47 (2026-09-29), yet did not change once in 13,614 +//! samples of a 25-minute watch from 10:51. On 2026-09-30 a group-only watcher +//! sampling that M5 Max every ~16 ms saw a refresh land in TWO steps 64 ms +//! apart: `CPU Energy` 2,643,843,378 -> 2,705,607,863 (+61,764,485), then -> +//! 2,705,611,298 (+3,435), with `MCPU0`, `MCPU1` and `PCPU` each stepping +//! twice beside it. This probe's intervals are ~31 ms, so a probe window that +//! holds such a refresh sees two advancing intervals. The earlier rule, two +//! advances, called that live, and two real probes did while an independent +//! reader saw no sustained movement. Called live, a refresh selects the probe +//! for the life of the process, and every later refresh (~1.3 kJ per die) +//! lands whole in whichever span straddled it, sealed as a measurement. +//! +//! A refresh is a burst: a few steps inside ~150 ms, then nothing. A live +//! rail advances in nearly every interval for as long as it is watched (the +//! M5 Max's `GPU Energy` changes about every 28 ms). So the rule asks for many +//! advances, in nearly every interval (one missed interval in five is allowed, +//! for a live rail that updates a little slower than the probe samples), over +//! a window twice a refresh's span. A refresh of fewer than five steps can +//! never be live, whatever the timing. One of five or more steps inside +//! 150 ms cannot be at any steady sampling pace: five advances in 150 ms need +//! intervals of 37.5 ms or less, and then a 300 ms window holds eight +//! intervals or more, of which the burst advanced in under two thirds. +//! +//! Each interval is read against the PREVIOUS sample, never the first one: a +//! rail compared with its first value would count one refresh again at every +//! later sample, which is exactly how the original probe called a +//! batch-refreshed rail live. +//! +//! **The same rule gates every other vendor counter** the meter reads: Linux +//! RAPL's package domains and NVML's GPUs (see [`counter_run`]). A counter +//! the probe cannot see keep advancing is not a reading of the work, whatever +//! its vendor. The meter sums each counter's rails into one figure per class, +//! as it sums IOReport's CPU rails (every package into `Cpu`, every GPU into +//! `Gpu`), so each is held to the same every-rail rule. The sampling +//! interval is the probe's choice and is not part of the rule: NVML samples +//! at a longer tick than IOReport and RAPL ([`NVML_PROBE_TICK`]), because a +//! GPU energy counter is published at a coarser cadence than a CPU one. + +// Off Apple Silicon macOS the IOReport-only items are unused (the real sampler +// lives in `ioreport`); on Linux the counter items are used by `rapl`/`nvml`. +#![cfg_attr(not(all(target_os = "macos", target_arch = "aarch64")), allow(dead_code))] + +use super::{CounterStatus, IoReportStatus, ProbeError}; +use std::cell::{Cell, RefCell}; +use std::time::Duration; + +/// What the liveness probe saw of the CPU energy rail(s) in its subscription. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum CpuRail { + /// Every CPU energy rail in the samples kept advancing, each one on its + /// own, while this thread burned CPU: in at least [`MIN_LIVE_ADVANCES`] + /// separate sample intervals, at least four in five of those observed, + /// over at least [`MIN_LIVE_WINDOW`]. + Live, + /// A CPU energy rail exists, but not every one kept advancing: at least + /// one held, or stepped in a burst (a batch refresh, in one step or + /// several, not a reading). + Frozen, + /// No `CPU Energy` channel exists in the Energy Model group. + Absent, +} + +/// In how many separate sample intervals each CPU rail must advance before it +/// can count as live. Two is not enough: a batch refresh can land in two +/// steps (see the module doc). +pub(super) const MIN_LIVE_ADVANCES: u32 = 5; + +/// How much wall time the samples must span, from the first, before a rail +/// can count as live: about twice the span of a batch refresh's steps. +pub(super) const MIN_LIVE_WINDOW: Duration = Duration::from_millis(300); + +/// Did a rail that advanced in `advances` of `intervals` intervals keep +/// advancing? At least [`MIN_LIVE_ADVANCES`] advances, in at least four of +/// every five intervals observed (80%). The window is checked apart, by +/// [`RailWatch::observe`]. +pub(super) fn sustained(advances: u32, intervals: u32) -> bool { + advances >= MIN_LIVE_ADVANCES && u64::from(advances) * 5 >= u64::from(intervals) * 4 +} + +/// CPU time the probing thread burns between two liveness samples: its own +/// CPU clock must advance this much, however loaded the machine is. +pub(super) const PROBE_TICK: Duration = Duration::from_millis(25); + +/// The NVML probe's tick: 150 ms of this thread's CPU between two samples, +/// so every interval spans at least 150 ms of wall time. NVML does not +/// document how often its total-energy counter is published, and a GPU's +/// power telemetry is commonly coarser than a CPU rail's: a live counter +/// updated every 100 ms, sampled every 25 ms, advances in about one interval +/// in four and would be declined. At 150 ms every interval of a counter +/// published at up to that period holds an update. A counter published more +/// slowly fails safe: declined and estimated, never over-claimed. Five +/// intervals take 750 ms, inside the 1 s budget. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `rapl` and `nvml` +pub(super) const NVML_PROBE_TICK: Duration = Duration::from_millis(150); + +/// The most wall-clock time one tick may take. A tick ends when this thread +/// has burned [`PROBE_TICK`] of CPU; only a machine loaded to about twenty +/// runnable threads per core keeps it from doing so within this cap. +pub(super) const TICK_WALL_CAP: Duration = Duration::from_millis(500); + +/// The probe never runs longer than this many budgets, however slow or +/// failing its samples are. +pub(super) const PROBE_CAP_BUDGETS: u32 = 5; + +/// The IOReport group the probe subscribes to, and the group every channel of +/// its samples must belong to. +pub(super) const ENERGY_MODEL: &str = "Energy Model"; + +/// `DIE__CPU Energy` → `CPU Energy`: multi-die parts (the Ultra) publish +/// one CPU rail per die. +pub(super) fn strip_die_prefix(name: &str) -> &str { + if let Some(rest) = name.strip_prefix("DIE_") { + let after = rest.trim_start_matches(|c: char| c.is_ascii_digit()); + if after.len() < rest.len() { + if let Some(r) = after.strip_prefix('_') { + return r; + } + } + } + name +} + +/// The CPU energy rail itself — `CPU Energy`, or `DIE__CPU Energy` on +/// multi-die parts — never a sub-rail and never another class's rail. +pub(super) fn is_cpu_rail(channel: &str) -> bool { + strip_die_prefix(channel).eq_ignore_ascii_case("CPU Energy") +} + +/// The GPU energy rail itself: `GPU Energy`, or `DIE__GPU Energy`. Never +/// handed to the rule. A probe run records it beside the CPU rails because it +/// keeps moving while the CPU rail holds (on macOS 27.0.1, about every 20 ms +/// on an M5 Max and several times a second on an M3 Ultra), so a test can +/// tell a sampler that re-reads IOReport from one that hands back an old +/// sample. +pub(super) fn is_gpu_rail(channel: &str) -> bool { + strip_die_prefix(channel).eq_ignore_ascii_case("GPU Energy") +} + +/// One channel of one absolute IOReport sample, as the FFI reads it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Channel { + /// The IOReport group the channel belongs to. + pub(super) group: String, + /// Its channel name. + pub(super) name: String, + /// Its raw cumulative integer, before any unit conversion. + pub(super) raw: i64, +} + +/// Each CPU rail's channel name and raw cumulative integer in one absolute +/// sample, in channel order: what the rule is handed. +pub(super) type Sample = Vec<(String, i64)>; + +/// What one sample that could be read held, beyond the CPU rails the rule +/// was handed (the rule keeps those itself, in [`RailWatch::observed`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SampleShape { + /// How many channels the sample held. + pub(super) channels: usize, + /// Whether every one of them belongs to the [`ENERGY_MODEL`] group. + pub(super) group_only: bool, + /// The GPU energy rail(s) ([`is_gpu_rail`]): name and raw integer, in + /// channel order. + pub(super) gpu: Sample, +} + +/// One sample, split: the CPU rails for the rule — every channel that is the +/// CPU rail itself ([`is_cpu_rail`]), its name and raw integer as read, in +/// channel order, never summed or converted — and the sample's shape for the +/// run's record. +pub(super) fn split_sample(channels: &[Channel]) -> (Sample, SampleShape) { + let rails = channels.iter().filter(|c| is_cpu_rail(&c.name)).map(|c| (c.name.clone(), c.raw)).collect(); + let gpu = channels.iter().filter(|c| is_gpu_rail(&c.name)).map(|c| (c.name.clone(), c.raw)).collect(); + let shape = + SampleShape { channels: channels.len(), group_only: channels.iter().all(|c| c.group == ENERGY_MODEL), gpu }; + (rails, shape) +} + +/// The liveness rule over a stream of samples. +/// +/// It consumes each SAMPLE whole, with the wall-clock time it was read, and +/// compares it with the previous one itself: the first sample is the +/// baseline, and every later one closes one interval. A rail is identified by +/// its channel name and its ordinal among the same-named channels in the +/// sample, so two dies (or two channels that share a name — the M3 Ultra +/// publishes 264 duplicated Energy Model names) are two rails, and each rail +/// counts at most one advance per interval. Two rails that each stepped, or +/// that advanced in turn, do not add up to a live rail. +/// +/// The watch is live only when EVERY rail it has seen kept advancing, and it +/// has seen at least one. Every rail it is handed is summed into one figure +/// by the meter — IOReport's CPU rails (one per die on a multi-die part), +/// RAPL's packages, NVML's GPUs — and a sum claims every rail in it: one rail +/// that held is a domain the figure would claim without reading it. A rail +/// seen in any sample, the baseline included, is a rail the watch needs, +/// counted over every interval observed: one missing from the other samples +/// advanced in none of their intervals. +#[derive(Debug, Default)] +pub(super) struct RailWatch { + /// Every sample handed to the rule, in order, exactly as handed: the + /// record a test checks against IOReport. The last one opens the next + /// interval. + pub(super) observed: Vec, + /// The wall-clock time each of those samples was handed with, in order: + /// the window is the last one less the first. + pub(super) at: Vec, + /// Sample intervals observed so far. + pub(super) intervals: u32, + /// Each rail seen in any sample, the first included, in the order first + /// seen: its channel name, its ordinal among same-named channels, and in + /// how many intervals it advanced. + pub(super) rails: Vec<(String, u32, u32)>, + /// Whether, after some interval, every rail seen had kept advancing + /// ([`sustained`]) over at least [`MIN_LIVE_WINDOW`]. Once set it stays + /// set: the rails are live from that interval on, and the probe stops. + live: bool, +} + +impl RailWatch { + /// Fold in the next absolute sample, read at wall-clock time `at` (any + /// origin; monotonic). The first sample is the baseline: its rails are + /// seen, with no advance. After it, each sample closes the interval that + /// the previous sample opened, and the rule is evaluated: live as soon as + /// every rail seen advanced in at least [`MIN_LIVE_ADVANCES`] intervals, + /// in at least four in five of the intervals so far, and the samples so + /// far span at least [`MIN_LIVE_WINDOW`]. + pub(super) fn observe(&mut self, sample: Sample, at: Duration) { + let prev = self.observed.last(); + if prev.is_some() { + self.intervals += 1; + } + // Against no previous sample every rail is new: seen, not advanced. + for (name, ordinal, advanced) in rail_advances(prev.map_or(&[][..], Vec::as_slice), &sample) { + match self.rails.iter_mut().find(|(n, o, _)| n == name && *o == ordinal) { + Some((_, _, count)) => *count += u32::from(advanced), + None => self.rails.push((name.to_string(), ordinal, u32::from(advanced))), + } + } + self.observed.push(sample); + self.at.push(at); + if self.intervals > 0 && !self.live { + let window = at.saturating_sub(self.at[0]); + let kept = |(_, _, advances): &(String, u32, u32)| sustained(*advances, self.intervals); + self.live = window >= MIN_LIVE_WINDOW && !self.rails.is_empty() && self.rails.iter().all(kept); + } + } + + /// Has every rail kept advancing, long enough to be live? + pub(super) fn is_live(&self) -> bool { + self.live + } + + /// Keep sampling? Until the rails prove live, or `budget` has elapsed AND a + /// live rail has had the [`MIN_LIVE_ADVANCES`] intervals it needs to show + /// itself — so slow samples cannot make a live rail look frozen — and + /// never past [`PROBE_CAP_BUDGETS`] budgets. + pub(super) fn wants_more(&self, elapsed: Duration, budget: Duration) -> bool { + !self.is_live() + && elapsed < budget * PROBE_CAP_BUDGETS + && (elapsed < budget || self.intervals < MIN_LIVE_ADVANCES) + } + + /// The verdict, or `None` if not one interval was observed (IOReport + /// could not be sampled twice). + pub(super) fn verdict(&self) -> Option { + if self.intervals == 0 { + None + } else if self.is_live() { + Some(CpuRail::Live) + } else if self.rails.is_empty() { + Some(CpuRail::Absent) + } else { + Some(CpuRail::Frozen) + } + } +} + +/// One sample interval, from two consecutive absolute samples: each CPU rail +/// of the later sample — its name, its ordinal among same-named channels — +/// and whether its raw integer rose. Raw integers, before unit conversion, so +/// a small advance in a fine unit cannot truncate to zero. Never summed. A +/// channel list that changed between the samples (reordered, grown, shrunk, +/// renamed) is not evidence of anything, so no rail advanced in that interval +/// (the safe side: declined, never over-claimed). +pub(super) fn rail_advances<'a>(before: &[(String, i64)], after: &'a [(String, i64)]) -> Vec<(&'a str, u32, bool)> { + let paired = before.len() == after.len() + && before.iter().zip(after).all(|((n0, _), (n1, _))| n0 == n1); + // `paired` is checked first, so `before[i]` is only read when both samples + // hold the same channels in the same order. + after + .iter() + .enumerate() + .map(|(i, (name, a))| { + let ordinal = after[..i].iter().filter(|(n, _)| n == name).count() as u32; + (name.as_str(), ordinal, paired && *a > before[i].1) + }) + .collect() +} + +/// The probe loop, generic over its sampler, its tick and its clock so it is +/// tested with scripted ones. The real probe passes the FFI sampler, a +/// tick that burns [`PROBE_TICK`] of the thread's CPU time ([`spin`]), and a +/// monotonic clock. +/// +/// It takes a baseline sample, then ticks and samples again, feeding every +/// sample that could be read to [`RailWatch`], with the clock's reading just +/// after the sample returned, for as long as [`RailWatch::wants_more`] says. +/// A sample that could not be read is skipped, not fatal: the next one that +/// can be read closes the interval, and a host whose samples never read runs +/// to the cap and gets no verdict. +pub(super) fn watch( + mut sample: impl FnMut() -> Option, + mut tick: impl FnMut(), + now: impl Fn() -> Duration, + budget: Duration, +) -> RailWatch { + let started = now(); + let mut w = RailWatch::default(); + if let Some(s) = sample() { + w.observe(s, now()); + } + while w.wants_more(now().saturating_sub(started), budget) { + tick(); + if let Some(s) = sample() { + w.observe(s, now()); + } + } + w +} + +/// The probe's tick: burn CPU on the calling thread until its CPU clock `cpu` +/// has advanced [`PROBE_TICK`], or [`TICK_WALL_CAP`] has passed on the wall +/// clock `wall`. Returns the CPU time `cpu` saw burned. +/// +/// The probe spins rather than sleeps, because a rail that advances only +/// under load must see load between the samples. It counts THIS thread's CPU +/// time, not wall time, so a loaded machine that takes CPU from the thread +/// makes the tick longer, never lighter: every interval carries a full tick +/// of burn. The clocks are passed in so the rule is tested with scripted +/// ones; the real probe passes this thread's CPU clock and a monotonic one. +pub(super) fn spin(cpu: impl Fn() -> Duration, wall: impl Fn() -> Duration) -> Duration { + spin_for(PROBE_TICK, cpu, wall) +} + +/// [`spin`] for a tick of `tick` of this thread's CPU time, still capped at +/// [`TICK_WALL_CAP`] of wall time. +pub(super) fn spin_for(tick: Duration, cpu: impl Fn() -> Duration, wall: impl Fn() -> Duration) -> Duration { + let (cpu0, wall0) = (cpu(), wall()); + let mut acc: u64 = 0; + loop { + for _ in 0..1_000 { + acc = acc.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + } + std::hint::black_box(acc); + let burned = cpu().saturating_sub(cpu0); + if burned >= tick || wall().saturating_sub(wall0) >= TICK_WALL_CAP { + return burned; + } + } +} + +/// The status a probe verdict gives the host. No verdict means IOReport +/// could not be opened, or opened but could not be sampled twice: +/// `Unavailable`, not `Absent`, and never `Live`. +pub(super) fn status_of(verdict: Option) -> IoReportStatus { + match verdict { + None => IoReportStatus::Unavailable, + Some(CpuRail::Live) => IoReportStatus::Live, + Some(CpuRail::Frozen) => IoReportStatus::CpuRailFrozen, + Some(CpuRail::Absent) => IoReportStatus::CpuRailAbsent, + } +} + +/// The one gate between a status and the IOReport probe's use: only a CPU +/// rail that proved live admits it. Every selection and constructor path +/// asks this, with the cached status and nothing else. +pub(super) fn admits(status: IoReportStatus) -> bool { + status == IoReportStatus::Live +} + +// ---- Every other vendor counter: RAPL, NVML. + +/// One liveness run over a vendor energy counter whose rails the meter sums +/// into one figure (Linux RAPL's package domains; NVML's GPUs), generic over +/// everything it touches so the run is tested with scripted parts: `open` +/// opens the counter and hands back its sampler (`None`: it did not open), +/// which reads every rail's name and raw cumulative integer from one +/// absolute sample (`None`: this read failed); `tick` burns between samples; +/// `now` is a monotonic clock (any origin). The production status of each +/// such counter is [`counter_status_of`] this over the real counter, the +/// real tick and the real clock, and nothing else. +/// +/// The rule is the IOReport rule, [`RailWatch`]'s: every rail the counter +/// sums must advance in at least five intervals, four in five of those +/// observed, over at least 300 ms, every interval read against the sample +/// before it. A counter that does not open is never sampled or ticked, and +/// gives no verdict. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `rapl` and `nvml` +pub(super) fn counter_run( + open: impl FnOnce() -> Option, + tick: impl FnMut(), + now: impl Fn() -> Duration, + budget: Duration, +) -> RailWatch +where + R: FnMut() -> Option, +{ + match open() { + None => RailWatch::default(), + Some(read) => watch(read, tick, now, budget), + } +} + +/// The status a counter run's verdict gives the counter. No verdict (it did +/// not open, or was never read twice) and no rail at all are `Unavailable`; +/// only every rail kept advancing is `Live`. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `rapl` and `nvml` +pub(super) fn counter_status_of(verdict: Option) -> CounterStatus { + match verdict { + Some(CpuRail::Live) => CounterStatus::Live, + Some(CpuRail::Frozen) => CounterStatus::Frozen, + None | Some(CpuRail::Absent) => CounterStatus::Unavailable, + } +} + +/// The one gate between a counter's status and its probe's use: only a +/// counter that proved live admits it. Every constructor asks this, with the +/// counter's cached status and nothing else. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `rapl` and `nvml` +pub(super) fn counter_admits(status: CounterStatus) -> bool { + status == CounterStatus::Live +} + +/// A counter probe's constructor behind its gate: `open` runs only when +/// [`counter_admits`] the `status`; otherwise `Unavailable`, naming the +/// counter (`what`) and why, and the counter is never touched. Every counter +/// probe's `given` is this and nothing else. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `rapl` and `nvml` +pub(super) fn gated

( + what: &str, + status: CounterStatus, + open: impl FnOnce() -> Result, +) -> Result { + if !counter_admits(status) { + return Err(ProbeError::Unavailable(format!("{what}: {status}"))); + } + open() +} + +/// CPU time the calling thread has used (user + system), from +/// `CLOCK_THREAD_CPUTIME_ID`: the clock a counter probe's tick burns +/// against. Zero if the clock cannot be read, which ends a tick at its wall +/// cap rather than never. +#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `rapl` and `nvml` +pub(super) fn thread_cpu() -> Duration { + let mut ts = libc::timespec { tv_sec: 0, tv_nsec: 0 }; + // SAFETY: `ts` is a valid, writable timespec for the call's duration. + let rc = unsafe { libc::clock_gettime(libc::CLOCK_THREAD_CPUTIME_ID, &mut ts) }; + if rc != 0 { + return Duration::ZERO; + } + Duration::new(ts.tv_sec as u64, ts.tv_nsec as u32) +} + +/// A counter probe's real tick: [`spin_for`] `tick` of this thread's CPU +/// time, capped on a monotonic clock. +#[cfg(any(target_os = "macos", target_os = "linux"))] +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] // used by `rapl` and `nvml` +pub(super) fn counter_tick(tick: Duration) { + let t = std::time::Instant::now(); + spin_for(tick, thread_cpu, || t.elapsed()); +} + +/// What one sample call of a probe run cost, and what the sample held. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SampleCall { + /// The run's wall clock when the call began, and when it returned. + pub(super) started: Duration, + pub(super) ended: Duration, + /// The CPU time this thread used in the call, on the run's CPU clock. + pub(super) cpu: Duration, + /// The sample's shape, or `None` if it could not be read. + pub(super) read: Option, +} + +/// One probe run: the rule's state, and what the run did to reach it. The +/// production status reads only the verdict; the rest is recorded in the +/// same run so the tests can check the very run a status comes from. +#[derive(Debug, Default)] +#[cfg_attr(not(test), allow(dead_code))] +pub(super) struct ProbeRun { + /// Whether the subscription opened. One that did not was never sampled + /// or ticked. + pub(super) opened: bool, + /// The rule, with every sample it was handed ([`RailWatch::observed`]). + pub(super) watch: RailWatch, + /// Every sample call, in order, read or not. + pub(super) calls: Vec, + /// How many ticks ran. + pub(super) ticks: u32, + /// For each interval the rule observed (between two consecutive samples + /// that were read), the CPU time the ticks burned in it, measured around + /// the ticks, apart from the samples' own cost. + pub(super) interval_burn: Vec, +} + +impl ProbeRun { + /// The rule's verdict: `None` if the subscription did not open, or never + /// gave two samples to compare. + pub(super) fn verdict(&self) -> Option { + self.watch.verdict() + } +} + +/// One whole liveness probe run, generic over everything it touches so the +/// run itself, not only the rule, is tested with scripted parts: `open` +/// opens the subscription and hands back its sampler (`None`: it did not +/// open), `tick` burns between samples, `cpu` is this thread's CPU clock and +/// `wall` a monotonic clock (any origin for either). The production status is +/// [`status_of`] this over the real subscription, the real tick and the real +/// clocks, and nothing else. +/// +/// A subscription that does not open is never sampled and never ticked: no +/// verdict, so `Unavailable`. Otherwise the run is [`watch`] over the +/// sampler, each sample split ([`split_sample`]) into the CPU rails the rule +/// is handed and the shape the run records. Every sample call is timed on +/// both clocks, and each interval's tick CPU is measured around the ticks, +/// apart from the samples. +pub(super) fn probe_with( + open: impl FnOnce() -> Option, + mut tick: impl FnMut(), + cpu: impl Fn() -> Duration, + wall: impl Fn() -> Duration, + budget: Duration, +) -> ProbeRun +where + R: FnMut() -> Option>, +{ + let Some(mut read) = open() else { + return ProbeRun::default(); + }; + let (calls, interval_burn) = (RefCell::new(Vec::new()), RefCell::new(Vec::new())); + // Tick CPU since the last sample that was read, how many samples were + // read, and how many ticks ran. + let (burn, reads, ticks) = (Cell::new(Duration::ZERO), Cell::new(0u32), Cell::new(0u32)); + let watch = watch( + || { + let (cpu0, started) = (cpu(), wall()); + let channels = read(); + let (cpu1, ended) = (cpu(), wall()); + let (rails, shape) = match channels { + Some(channels) => { + let (rails, shape) = split_sample(&channels); + (Some(rails), Some(shape)) + } + None => (None, None), + }; + calls.borrow_mut().push(SampleCall { started, ended, cpu: cpu1.saturating_sub(cpu0), read: shape }); + if rails.is_some() { + if reads.get() > 0 { + interval_burn.borrow_mut().push(burn.get()); + } + reads.set(reads.get() + 1); + burn.set(Duration::ZERO); + } + rails + }, + || { + let before = cpu(); + tick(); + ticks.set(ticks.get() + 1); + burn.set(burn.get() + cpu().saturating_sub(before)); + }, + &wall, + budget, + ); + ProbeRun { + opened: true, + watch, + calls: calls.into_inner(), + ticks: ticks.get(), + interval_burn: interval_burn.into_inner(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + + fn raw(v: &[(&str, i64)]) -> Sample { + v.iter().map(|(n, x)| (n.to_string(), *x)).collect() + } + + /// The probe's interval on an idle M5 Max: a 25 ms tick and a group-only + /// sample, ~31 ms between two samples. + const STEP: Duration = Duration::from_millis(31); + + /// Feed `samples` to a [`RailWatch`] in order, `step` apart on the rule's + /// clock (the first at 0). Returns the watch, and whether it was live + /// once each interval closed (index 0: after the first interval). + fn paced(samples: &[Sample], step: Duration) -> (RailWatch, Vec) { + let mut w = RailWatch::default(); + let mut live = Vec::new(); + for (i, s) in samples.iter().enumerate() { + w.observe(s.clone(), step * i as u32); + if i > 0 { + live.push(w.is_live()); + } + } + (w, live) + } + + /// The interval (counting from 1) after which the watch first said live. + fn first_live(live: &[bool]) -> Option { + live.iter().position(|&l| l).map(|i| i + 1) + } + + /// One `CPU Energy` rail's samples. + fn rails(values: &[i64]) -> Vec { + values.iter().map(|v| raw(&[("CPU Energy", *v)])).collect() + } + + /// A rail's values over `intervals` intervals: it advances in interval + /// `i` (counting from 1) exactly when `advances(i)`, and holds otherwise. + fn rail_where(intervals: usize, advances: impl Fn(usize) -> bool) -> Vec { + let mut v = vec![1_000_000]; + for i in 1..=intervals { + let last = v[i - 1]; + v.push(if advances(i) { last + 7 } else { last }); + } + v + } + + /// A [`RailWatch`] fed these samples in order, [`STEP`] apart. + fn fed(samples: &[&[(&str, i64)]]) -> RailWatch { + paced(&samples.iter().map(|s| raw(s)).collect::>(), STEP).0 + } + + /// One rail, sample by sample, [`STEP`] apart. + fn one_rail(values: &[i64]) -> RailWatch { + paced(&rails(values), STEP).0 + } + + #[test] + fn a_cpu_rail_is_the_rail_itself_on_one_die_or_many() { + assert!(is_cpu_rail("CPU Energy")); // M5 Max + assert!(is_cpu_rail("DIE_0_CPU Energy")); // M3 Ultra, per die + assert!(is_cpu_rail("DIE_1_CPU Energy")); + assert!(is_cpu_rail("DIE_12_CPU Energy")); + assert!(is_cpu_rail("cpu energy")); + assert!(!is_cpu_rail("GPU Energy")); + assert!(!is_cpu_rail("ANE0")); + assert!(!is_cpu_rail("MCPU0_0")); + assert!(!is_cpu_rail("DIE_0_PACC0_CPU0")); + // A sub-rail is not the rail, however it is spelled. + assert!(!is_cpu_rail("PCPU Energy")); + // Not a die prefix. + assert!(!is_cpu_rail("DIE__CPU Energy")); + assert!(!is_cpu_rail("DIE_0CPU Energy")); + } + + /// The per-interval input to the rule: each CPU rail of the later sample, + /// paired with the earlier one by position and name, advanced or not on + /// the raw integer, with its ordinal among same-named channels. + #[test] + fn an_interval_is_read_per_cpu_rail_on_the_raw_integer() { + // M5 Max, macOS 27.0.1: `CPU Energy` (mJ) held across 3 s of burn. + let held = raw(&[("CPU Energy", 480_083_160)]); + assert_eq!(rail_advances(&held, &held), [("CPU Energy", 0, false)]); + // One raw unit is an advance: no unit conversion can truncate it away. + assert_eq!(rail_advances(&held, &raw(&[("CPU Energy", 480_083_161)])), [("CPU Energy", 0, true)]); + // A counter that went backwards did not advance. + assert_eq!(rail_advances(&held, &raw(&[("CPU Energy", 480_083_159)])), [("CPU Energy", 0, false)]); + // M3 Ultra, same OS: each die is its own rail. + let dies = raw(&[("DIE_0_CPU Energy", 9_107_678), ("DIE_1_CPU Energy", 9_742_153)]); + assert_eq!( + rail_advances(&dies, &raw(&[("DIE_0_CPU Energy", 9_107_678), ("DIE_1_CPU Energy", 9_742_154)])), + [("DIE_0_CPU Energy", 0, false), ("DIE_1_CPU Energy", 0, true)] + ); + // Two channels that share a name are told apart by their ordinal. + let twins = raw(&[("CPU Energy", 10), ("CPU Energy", 20)]); + assert_eq!( + rail_advances(&twins, &raw(&[("CPU Energy", 10), ("CPU Energy", 21)])), + [("CPU Energy", 0, false), ("CPU Energy", 1, true)] + ); + // A reordered or resized channel list is not evidence (crosswise, + // 9_742_153 > 9_107_678 would read as a move). + let swapped = raw(&[("DIE_1_CPU Energy", 9_742_153), ("DIE_0_CPU Energy", 9_107_678)]); + assert!(rail_advances(&dies, &swapped).iter().all(|(_, _, adv)| !adv)); + let grown = raw(&[("CPU Energy", 480_083_161), ("CPU Energy", 9)]); + assert!(rail_advances(&held, &grown).iter().all(|(_, _, adv)| !adv)); + assert!(rail_advances(&grown, &held).iter().all(|(_, _, adv)| !adv)); + } + + /// A rail that steps once and then holds — a batch refresh caught mid-probe — + /// is not live, however large the step. The values are the M5 Max rail's, + /// before and after the refresh seen at ~10:46. Every interval is read + /// against the sample before it: read against the FIRST sample, the step + /// would count again at every later sample and look live. + #[test] + fn a_single_jump_is_a_batch_refresh_not_a_live_rail() { + let w = one_rail(&[480_083_160, 480_083_160, 541_893_638, 541_893_638, 541_893_638]); + assert_eq!(w.intervals, 4); + assert!(!w.is_live(), "one refresh counted as a live rail: {w:?}"); + assert_eq!(w.rails, [("CPU Energy".to_string(), 0, 1)], "one advance, in one interval"); + assert_eq!(w.verdict(), Some(CpuRail::Frozen)); + } + + /// Both dies of an M3 Ultra stepping once — in the same interval, as + /// observed (+1,344,869 and +1,391,637 mJ in one 100 ms sample), or in + /// different ones — do not add up to a live rail: liveness is counted per + /// rail. + #[test] + fn one_jump_on_each_die_is_not_a_live_rail() { + let (d0, d1) = ("DIE_0_CPU Energy", "DIE_1_CPU Energy"); + let w = fed(&[ + &[(d0, 9_107_678), (d1, 9_742_153)], + &[(d0, 9_107_678), (d1, 9_742_153)], + &[(d0, 10_452_547), (d1, 11_133_790)], + &[(d0, 10_452_547), (d1, 11_133_790)], + ]); + assert_eq!(w.verdict(), Some(CpuRail::Frozen), "the observed refresh: {w:?}"); + + let w = fed(&[&[(d0, 1), (d1, 1)], &[(d0, 2), (d1, 1)], &[(d0, 2), (d1, 2)], &[(d0, 2), (d1, 2)]]); + assert_eq!(w.verdict(), Some(CpuRail::Frozen), "one step per die, apart: {w:?}"); + } + + /// Two channels that share a name are still two rails: one step each is + /// not one rail stepping twice, whether they step together or apart. + #[test] + fn two_channels_that_share_a_name_are_two_rails() { + let w = fed(&[&[("CPU Energy", 10), ("CPU Energy", 20)], &[("CPU Energy", 11), ("CPU Energy", 21)]]); + assert_eq!(w.verdict(), Some(CpuRail::Frozen), "together: {w:?}"); + let w = fed(&[ + &[("CPU Energy", 10), ("CPU Energy", 20)], + &[("CPU Energy", 11), ("CPU Energy", 20)], + &[("CPU Energy", 11), ("CPU Energy", 21)], + ]); + assert_eq!(w.verdict(), Some(CpuRail::Frozen), "apart: {w:?}"); + } + + /// Two rails that advance in TURN — one in the odd intervals, the other + /// in the even ones, for a whole 1 s budget — each advanced in half the + /// intervals, so neither kept advancing, though between them some CPU + /// rail moved in every interval. Liveness is counted per rail, never as + /// "a CPU rail moved": two dies, and two channels that share a name. + #[test] + fn two_rails_that_advance_in_turn_are_not_one_live_rail() { + for (r0, r1) in [("DIE_0_CPU Energy", "DIE_1_CPU Energy"), ("CPU Energy", "CPU Energy")] { + let (v0, v1) = (rail_where(32, |i| i % 2 == 1), rail_where(32, |i| i % 2 == 0)); + let samples: Vec = v0.iter().zip(&v1).map(|(a, b)| raw(&[(r0, *a), (r1, *b)])).collect(); + let (w, live) = paced(&samples, STEP); + assert_eq!(first_live(&live), None, "{r0} / {r1} in turn: {w:?}"); + assert_eq!(w.verdict(), Some(CpuRail::Frozen), "{r0} / {r1}"); + assert_eq!(w.rails, [(r0.to_string(), 0, 16), (r1.to_string(), u32::from(r0 == r1), 16)]); + } + } + + // ---- Every CPU rail: the meter sums the dies of a multi-die part into + // ---- one `Cpu` figure, so each die's rail must keep advancing. + + /// The two CPU rails of an M3 Ultra. + const DIES: (&str, &str) = ("DIE_0_CPU Energy", "DIE_1_CPU Energy"); + + /// Samples of two rails, `r0` and `r1`: one value of each per sample. + fn two_rails(r0: &str, v0: &[i64], r1: &str, v1: &[i64]) -> Vec { + assert_eq!(v0.len(), v1.len()); + v0.iter().zip(v1).map(|(a, b)| raw(&[(r0, *a), (r1, *b)])).collect() + } + + /// (a) Both dies keep advancing: live, at the interval where the slower + /// die is, never before. Both at every interval: the tenth, where the + /// window reaches 300 ms, as for one rail. DIE_1 missing one interval in + /// five: the tenth too (eight in ten). DIE_1 missing the first three: + /// the fifteenth (twelve in fifteen), though DIE_0 alone was ready at the + /// tenth. + #[test] + fn two_dies_that_both_keep_advancing_are_live() { + let (d0, d1) = DIES; + let every = rail_where(32, |_| true); + for (what, die1, at) in [ + ("every interval", every.clone(), 10), + ("missing one interval in five", rail_where(32, |i| i % 5 != 0), 10), + ("missing the first three intervals", rail_where(32, |i| i > 3), 15), + ] { + for (first, second) in [((d0, &every), (d1, &die1)), ((d1, &die1), (d0, &every))] { + let (w, live) = paced(&two_rails(first.0, first.1, second.0, second.1), STEP); + assert_eq!(first_live(&live), Some(at), "DIE_1 {what}, {} listed first: {w:?}", first.0); + assert!(live[at - 1..].iter().all(|&l| l), "DIE_1 {what}: live, then not: {live:?}"); + assert_eq!(w.verdict(), Some(CpuRail::Live), "DIE_1 {what}"); + assert_eq!(w.rails.len(), 2, "two dies, two rails: {w:?}"); + } + } + } + + /// (b) One die keeps advancing at every interval while the other does + /// not: frozen, and never live at any interval, whichever die is listed + /// first. The other die held for the whole budget (its energy is in the + /// `Cpu` sum, unread), stepped once or five times in a batch refresh, or + /// advanced in three intervals in five. The earlier rule, one rail's + /// liveness, called each of these live at the tenth interval. The same + /// for two channels that share the name `CPU Energy`. + #[test] + fn one_die_that_keeps_advancing_beside_a_frozen_one_is_frozen() { + let (d0, d1) = DIES; + let every = rail_where(32, |_| true); + for (what, die1) in [ + ("held", rail_where(32, |_| false)), + ("stepped once", rail_where(32, |i| i == 4)), + ("stepped five times in a refresh", rail_where(32, |i| i <= 5)), + ("advanced in three intervals in five", rail_where(32, |i| i % 5 != 0 && i % 5 != 4)), + ] { + for (first, second) in [((d0, &every), (d1, &die1)), ((d1, &die1), (d0, &every))] { + let (w, live) = paced(&two_rails(first.0, first.1, second.0, second.1), STEP); + assert_eq!(first_live(&live), None, "DIE_1 {what}, {} listed first: {w:?}", first.0); + assert_eq!(w.verdict(), Some(CpuRail::Frozen), "DIE_1 {what}"); + let die0 = w.rails.iter().find(|(n, _, _)| n == d0).expect("DIE_0 is a rail"); + assert_eq!(die0.2, 32, "DIE_0 advanced in every interval: {w:?}"); + } + } + let (w, live) = paced(&two_rails("CPU Energy", &every, "CPU Energy", &rail_where(32, |_| false)), STEP); + assert_eq!(first_live(&live), None, "twin channels, one held: {w:?}"); + assert_eq!(w.rails, [("CPU Energy".to_string(), 0, 32), ("CPU Energy".to_string(), 1, 0)]); + assert_eq!(w.verdict(), Some(CpuRail::Frozen)); + } + + /// (c) One die that keeps advancing, and no other die in any sample: a + /// host whose `Cpu` figure sums that one rail. Live, exactly as `CPU + /// Energy` alone is: at the tenth interval. + #[test] + fn one_die_that_keeps_advancing_with_no_other_die_in_the_samples_is_live() { + let (d0, _) = DIES; + let samples: Vec = rail_where(32, |_| true).iter().map(|v| raw(&[(d0, *v)])).collect(); + let (w, live) = paced(&samples, STEP); + assert_eq!(first_live(&live), Some(10), "{w:?}"); + assert_eq!(w.rails, [(d0.to_string(), 0, 32)]); + assert_eq!(w.verdict(), Some(CpuRail::Live)); + } + + /// A die in only some of the samples is present: a rail the watch needs, + /// judged over every interval observed, so the intervals it was missing + /// from count against it. DIE_1 in the baseline alone, in the first five + /// samples alone, or from the third sample on and holding: frozen, never + /// live. DIE_1 from the fifth sample on and advancing at every interval + /// after: live only once its advances are four in five of every + /// interval, at the twentieth (sixteen in twenty), though DIE_0 was ready + /// at the tenth. + #[test] + fn a_die_in_only_some_samples_is_a_rail_the_watch_needs() { + let (d0, d1) = DIES; + let v = rail_where(32, |_| true); + let held = 9_742_153; + let script = |die1: &dyn Fn(usize) -> Option| -> Vec { + (0..=32) + .map(|i| match die1(i) { + Some(x) => raw(&[(d0, v[i]), (d1, x)]), + None => raw(&[(d0, v[i])]), + }) + .collect() + }; + for (what, samples) in [ + ("in the baseline alone", script(&|i| (i == 0).then_some(held))), + ("in the first five samples alone", script(&|i| (i < 5).then(|| v[i]))), + ("from the third sample on, holding", script(&|i| (i >= 2).then_some(held))), + ] { + let (w, live) = paced(&samples, STEP); + assert_eq!(first_live(&live), None, "DIE_1 {what}: {w:?}"); + assert_eq!(w.verdict(), Some(CpuRail::Frozen), "DIE_1 {what}"); + assert!(w.rails.iter().any(|(n, _, _)| n == d1), "DIE_1 {what} is a rail: {w:?}"); + } + let (w, _) = paced(&script(&|i| (i == 0).then_some(held)), STEP); + assert_eq!(w.rails, [(d0.to_string(), 0, 31), (d1.to_string(), 0, 0)], "the baseline's rails are rails"); + let (w, live) = paced(&script(&|i| (i >= 4).then(|| v[i])), STEP); + assert_eq!(first_live(&live), Some(20), "DIE_1 from the fifth sample on, advancing: {w:?}"); + assert_eq!(w.verdict(), Some(CpuRail::Live)); + } + + /// A rail that advances in every interval is live as soon as it has + /// advanced five times AND the samples span 300 ms — not one interval + /// before either, and it stays live after. At ~31 ms an interval five + /// advances come by 155 ms, but the window reaches 300 ms only at the + /// tenth interval; at 60 ms both arrive together, at the fifth; at 75 ms + /// the window is 300 ms at the fourth, with four advances, so the fifth. + #[test] + fn a_rail_advancing_every_interval_is_live_not_before_five_advances_and_300_ms() { + for (step, at) in [(STEP, 10), (Duration::from_millis(60), 5), (Duration::from_millis(75), 5)] { + let (w, live) = paced(&rails(&rail_where(32, |_| true)), step); + assert_eq!(first_live(&live), Some(at), "{step:?} an interval: {live:?}"); + assert!(live[at - 1..].iter().all(|&l| l), "{step:?}: live, then not: {live:?}"); + assert_eq!(w.verdict(), Some(CpuRail::Live)); + } + } + + /// (*) The refresh the verifier caught on the M5 Max on 2026-09-30, + /// macOS 27.0.1, from a group-only watcher sampling every ~16 ms: + /// `CPU Energy` 2,643,843,378 -> 2,705,607,863 (+61,764,485), then +3,435 + /// 64 ms later, then holding. At the probe's ~31 ms intervals the two + /// steps fall in consecutive intervals, or two apart; the rail then holds + /// for the rest of a 1 s budget. Frozen, and never live at any interval. + /// The earlier rule (two advances) called both live. + #[test] + fn the_observed_two_step_refresh_is_frozen() { + let a: i64 = 2_643_843_378; + let (b, c) = (a + 61_764_485, a + 61_764_485 + 3_435); + for script in [vec![a, a, a, b, c], vec![a, a, a, b, b, c]] { + let mut v = script.clone(); + v.resize(33, c); + let (w, live) = paced(&rails(&v), STEP); + assert_eq!(first_live(&live), None, "{script:?}: {w:?}"); + assert_eq!(w.rails, [("CPU Energy".to_string(), 0, 2)], "two advances, each in its own interval"); + assert_eq!(w.verdict(), Some(CpuRail::Frozen), "{script:?}"); + } + } + + /// (*) A refresh published in three, four or five steps inside 150 ms + /// at the very start of the probe — each step in its own consecutive + /// interval, as dense as a refresh can look to this probe — then holding + /// for the rest of a 1 s budget: Frozen, and never live at any interval. + /// With five steps it has five advances at 100% by 155 ms, but the window + /// is short then; by the time it is 300 ms the five are half the + /// intervals. + #[test] + fn a_refresh_in_three_to_five_steps_at_the_start_is_frozen() { + for steps in 3..=5usize { + assert!(STEP * (steps as u32 - 1) <= Duration::from_millis(150), "the steps lie within 150 ms"); + let (w, live) = paced(&rails(&rail_where(32, |i| i <= steps)), STEP); + assert_eq!(first_live(&live), None, "{steps} steps: {w:?}"); + assert_eq!(w.rails, [("CPU Energy".to_string(), 0, steps as u32)]); + assert_eq!(w.verdict(), Some(CpuRail::Frozen), "{steps} steps"); + } + } + + /// (*) The M3 Ultra's shape: `DIE_0_CPU Energy` and `DIE_1_CPU Energy` + /// each step twice within 64 ms (two ~31 ms intervals), splitting the + /// observed +1,344,869 and +1,391,637 mJ, in step or apart, then hold for + /// the rest of a 1 s budget. Each die has two advances: Frozen. The + /// earlier rule called every one of these live. + #[test] + fn an_m3_ultra_refresh_stepping_twice_per_die_is_frozen() { + let (d0, d1) = ("DIE_0_CPU Energy", "DIE_1_CPU Energy"); + // The sample index at which each die takes its big step and its small one. + for ((b0, s0), (b1, s1)) in [((3, 5), (3, 5)), ((3, 4), (4, 5)), ((3, 5), (4, 6)), ((3, 4), (3, 4))] { + let die = |base: i64, big: i64, small: i64, at_big: usize, at_small: usize, i: usize| { + base + if i >= at_big { big } else { 0 } + if i >= at_small { small } else { 0 } + }; + let samples: Vec = (0..33) + .map(|i| { + let (v0, v1) = (die(9_107_678, 1_344_000, 869, b0, s0, i), die(9_742_153, 1_391_000, 637, b1, s1, i)); + raw(&[(d0, v0), (d1, v1)]) + }) + .collect(); + let (w, live) = paced(&samples, STEP); + assert_eq!(first_live(&live), None, "steps at {:?}: {w:?}", ((b0, s0), (b1, s1))); + assert_eq!(w.rails, [(d0.to_string(), 0, 2), (d1.to_string(), 0, 2)]); + assert_eq!(w.verdict(), Some(CpuRail::Frozen)); + } + } + + /// A live rail that misses one interval in five (80%, a rail updating a + /// little slower than the probe samples) is live, wherever the miss + /// falls: at ~31 ms after ten intervals (eight advances), at 100 ms after + /// six (five advances, 83%). + #[test] + fn a_live_rail_missing_one_interval_in_five_is_live() { + for miss in 0..5 { + let v = rail_where(32, |i| i % 5 != miss); + let (w, live) = paced(&rails(&v), STEP); + assert_eq!(first_live(&live), Some(10), "~31 ms, missing interval {miss} of five: {live:?}"); + assert_eq!(w.verdict(), Some(CpuRail::Live)); + } + let (w, live) = paced(&rails(&rail_where(12, |i| i % 5 != 0)), Duration::from_millis(100)); + assert_eq!(first_live(&live), Some(6), "100 ms: {live:?}"); + assert_eq!(w.verdict(), Some(CpuRail::Live)); + } + + /// A rail that misses two intervals in five (60%) is not live: not at + /// the fifth interval, and not after a whole budget of that, though it + /// has five advances and more over 300 ms and more. One that missed two + /// of its first five and then advanced every time is not YET live while + /// its share is under four in five: at 100 ms an interval it has five + /// advances at the seventh (700 ms), and is live only at the tenth, + /// eight in ten. + #[test] + fn a_rail_missing_two_intervals_in_five_is_not_yet_live() { + for step in [STEP, Duration::from_millis(100)] { + let (w, live) = paced(&rails(&rail_where(32, |i| i % 5 != 0 && i % 5 != 4)), step); + assert_eq!(first_live(&live), None, "{step:?}, three in five: {w:?}"); + assert!(w.rails[0].2 >= 5 && step * w.intervals >= Duration::from_millis(300)); + assert_eq!(w.verdict(), Some(CpuRail::Frozen)); + } + let (w, live) = paced(&rails(&rail_where(12, |i| i > 2)), Duration::from_millis(100)); + assert_eq!(first_live(&live), Some(10), "missed two of the first five: {live:?}"); + assert_eq!(w.verdict(), Some(CpuRail::Live)); + } + + /// A loaded host: the probe's thread gets about a quarter of a core, so a + /// 25 ms tick takes ~94 ms of wall time and an interval ~100 ms. A live + /// rail is live after five intervals, 500 ms: not at the third (300 ms, + /// three advances) nor the fourth. + #[test] + fn a_loaded_host_with_a_live_rail_is_live_after_five_intervals() { + let (w, live) = paced(&rails(&rail_where(10, |_| true)), Duration::from_millis(100)); + assert_eq!(first_live(&live), Some(5), "{live:?}"); + assert_eq!(w.verdict(), Some(CpuRail::Live)); + // The probe loop on that host stops there: six samples, five ticks. + let r = scripted(&rail(&(0..40).map(|i| 100 + 7 * i).collect::>()), ms(94), ms(6)); + assert_eq!(r.watch.verdict(), Some(CpuRail::Live)); + assert_eq!((r.watch.intervals, r.samples, r.ticks), (5, 6, 5), "{:?}", r.watch); + assert_eq!(r.ended, ms(506)); + } + + /// No interval (no sample, or one) is no verdict — the caller reports + /// IOReport as unavailable; intervals with no CPU channel are `Absent`. + #[test] + fn no_interval_gives_no_verdict_and_no_cpu_rail_is_absent() { + let mut w = RailWatch::default(); + assert_eq!(w.verdict(), None); + w.observe(raw(&[("CPU Energy", 1)]), Duration::ZERO); + assert_eq!(w.verdict(), None, "one sample is a baseline, not an interval"); + let w = fed(&[&[], &[], &[]]); + assert_eq!(w.verdict(), Some(CpuRail::Absent)); + } + + /// The probe keeps sampling within its budget until a rail proves live, + /// and past the budget until a live rail could have shown five advances — + /// but never past five budgets. + #[test] + fn the_probe_samples_until_live_or_long_enough_to_tell() { + let budget = Duration::from_millis(1000); + let w = RailWatch::default(); + assert!(w.wants_more(Duration::ZERO, budget)); + // Slow samples: past the budget with fewer than five intervals, keep going. + assert!(w.wants_more(budget * 2, budget)); + let four = one_rail(&[1; 5]); + assert_eq!(four.intervals, 4); + assert!(four.wants_more(budget * 2, budget), "four intervals are not enough to tell"); + let five = one_rail(&[1; 6]); + assert!(five.wants_more(budget - Duration::from_millis(1), budget), "within budget: keep watching"); + assert!(!five.wants_more(budget, budget), "budget spent, five intervals: done"); + // The hard cap holds with no interval at all, and with four. + let none = RailWatch::default(); + assert!(none.wants_more(budget * 5 - Duration::from_millis(1), budget)); + assert!(!none.wants_more(budget * 5, budget), "hard cap"); + assert!(!four.wants_more(budget * 5, budget), "hard cap"); + // Advances that are not yet liveness do not stop the probe early; + // liveness does. + let advancing = one_rail(&[1, 2, 3, 4, 5, 6]); + assert!(!advancing.is_live(), "five advances in 155 ms: {advancing:?}"); + assert!(advancing.wants_more(Duration::ZERO, budget), "not yet live: keep watching"); + let live = one_rail(&rail_where(10, |_| true)); + assert!(live.is_live(), "{live:?}"); + assert!(!live.wants_more(Duration::ZERO, budget), "live: stop at once"); + } + + /// The verdict's status: no verdict is `Unavailable` (IOReport opened but + /// could not be sampled twice), never `CpuRailAbsent`. + #[test] + fn a_verdict_maps_to_its_status() { + assert_eq!(status_of(None), IoReportStatus::Unavailable); + assert_eq!(status_of(Some(CpuRail::Absent)), IoReportStatus::CpuRailAbsent); + assert_eq!(status_of(Some(CpuRail::Frozen)), IoReportStatus::CpuRailFrozen); + assert_eq!(status_of(Some(CpuRail::Live)), IoReportStatus::Live); + } + + /// No verdict — IOReport never gave two samples to compare — is + /// `Unavailable`: never `Live`, and never `CpuRailAbsent`, which would + /// claim a subscription that was read and held no CPU rail. + #[test] + fn no_verdict_is_unavailable() { + assert_eq!(status_of(None), IoReportStatus::Unavailable); + } + + /// A rail that went DOWN between consecutive samples did not advance: a + /// counter reset or wrap is not energy spent. A rail that falls at every + /// sample, for a whole budget, is not live, and a reset after one advance + /// adds none. + #[test] + fn a_decrease_is_not_an_advance() { + let before = raw(&[("CPU Energy", 480_083_160)]); + assert_eq!(rail_advances(&before, &raw(&[("CPU Energy", 480_083_159)])), [("CPU Energy", 0, false)]); + assert_eq!(rail_advances(&before, &raw(&[("CPU Energy", 0)])), [("CPU Energy", 0, false)]); + let falling: Vec = (0..33).map(|i| 480_083_160 - 10 * i).collect(); + let (w, live) = paced(&rails(&falling), STEP); + assert_eq!(w.rails, [("CPU Energy".to_string(), 0, 0)], "{w:?}"); + assert_eq!(first_live(&live), None); + assert_eq!(w.verdict(), Some(CpuRail::Frozen)); + let w = one_rail(&[100, 101, 5, 5]); + assert_eq!(w.rails, [("CPU Energy".to_string(), 0, 1)], "one advance, then a reset: {w:?}"); + assert_eq!(w.verdict(), Some(CpuRail::Frozen)); + } + + /// A rail absent from the previous sample has no interval yet: its first + /// value only sets its baseline, whatever that value is. Appearing is not + /// advancing. + #[test] + fn a_rail_new_in_this_sample_only_sets_its_baseline() { + assert_eq!(rail_advances(&raw(&[]), &raw(&[("CPU Energy", 5)])), [("CPU Energy", 0, false)]); + let w = fed(&[&[], &[("CPU Energy", 5)], &[("CPU Energy", 6)]]); + assert_eq!(w.rails, [("CPU Energy".to_string(), 0, 1)], "appeared, then stepped once: {w:?}"); + assert_eq!(w.verdict(), Some(CpuRail::Frozen)); + // A second die that appears mid-probe, then steps once. + let (d0, d1) = ("DIE_0_CPU Energy", "DIE_1_CPU Energy"); + let w = fed(&[&[(d0, 1)], &[(d0, 1), (d1, 7)], &[(d0, 1), (d1, 8)]]); + assert_eq!(w.rails, [(d0.to_string(), 0, 0), (d1.to_string(), 0, 1)], "{w:?}"); + assert_eq!(w.verdict(), Some(CpuRail::Frozen)); + } + + /// The path where the Energy Model subscription cannot be opened, or + /// opens and never yields a sample, goes through the status function to + /// `Unavailable` — never `Live`, never `CpuRailAbsent`. A subscription that + /// did not open is never sampled or ticked. These go through + /// [`probe_with`], the run the production status is made of. + #[test] + fn a_subscription_that_cannot_be_opened_or_sampled_is_unavailable() { + let r = run(None, TICK, SAMPLE); + assert_eq!(status_of(r.run.verdict()), IoReportStatus::Unavailable, "no subscription"); + assert_eq!((r.samples, r.ticks), (0, 0), "sampled or ticked a subscription that did not open"); + let r = run(Some(&[None]), TICK, SAMPLE); + assert_eq!( + status_of(r.run.verdict()), + IoReportStatus::Unavailable, + "a subscription that never yields a sample" + ); + // The paths that do read map as their verdicts do. + let frozen = run(Some(&cpu_rail(&[1, 1])), TICK, SAMPLE); + assert_eq!(status_of(frozen.run.verdict()), IoReportStatus::CpuRailFrozen); + let live = run(Some(&cpu_rail(&rail_where(40, |_| true))), TICK, SAMPLE); + assert_eq!(status_of(live.run.verdict()), IoReportStatus::Live); + let absent = run(Some(&[Some(vec![em("GPU Energy", 7)])]), TICK, SAMPLE); + assert_eq!(status_of(absent.run.verdict()), IoReportStatus::CpuRailAbsent); + } + + // ---- The tick, on scripted clocks. + + fn ms(n: u64) -> Duration { + Duration::from_millis(n) + } + + /// The tick burns until this thread's CPU clock has advanced a full + /// [`PROBE_TICK`], however much wall time that takes below the cap. Here + /// the thread gets a third of a core: 1 ms of CPU per 3 ms of wall time. + #[test] + fn the_tick_burns_until_the_thread_cpu_clock_advanced_a_full_tick() { + let (cpu, wall) = (Cell::new(Duration::ZERO), Cell::new(Duration::ZERO)); + let burned = spin( + || { + cpu.set(cpu.get() + ms(1)); + cpu.get() + }, + || { + wall.set(wall.get() + ms(3)); + wall.get() + }, + ); + assert!(burned >= PROBE_TICK, "the tick stopped after {burned:?} of this thread's CPU"); + assert!(burned < PROBE_TICK + ms(2), "the tick burned past {PROBE_TICK:?}: {burned:?}"); + assert!(wall.get() >= ms(70) && wall.get() < TICK_WALL_CAP, "wall {:?}", wall.get()); + } + + /// A thread that gets no CPU at all cannot hold the probe: the tick ends + /// at its wall cap, having burned nothing. + #[test] + fn the_tick_ends_at_its_wall_cap_when_the_thread_gets_no_cpu() { + let wall = Cell::new(Duration::ZERO); + let burned = spin( + || Duration::from_secs(7), + || { + wall.set(wall.get() + ms(10)); + wall.get() + }, + ); + assert_eq!(burned, Duration::ZERO); + assert!(wall.get() >= TICK_WALL_CAP && wall.get() < TICK_WALL_CAP + ms(30), "wall {:?}", wall.get()); + } + + // ---- The probe loop, on scripted samples and a scripted clock. + + /// A 1 s budget, as the real probe's. + const BUDGET: Duration = Duration::from_millis(1000); + + /// What one scripted run of [`watch`] did. + struct Run { + watch: RailWatch, + samples: usize, + ticks: usize, + /// The scripted clock when the loop stopped. + ended: Duration, + /// `S` for a sample, `T` for a tick, in call order. + calls: String, + } + + /// Run [`watch`] over `script` (`None` = a sample that could not be read; + /// past the end, the last entry repeats) on a scripted clock that each + /// tick advances by `tick` and each sample by `sample`. A loop that + /// samples without end fails instead of hanging. + fn scripted(script: &[Option>], tick: Duration, sample: Duration) -> Run { + let clock = Cell::new(Duration::ZERO); + let calls = std::cell::RefCell::new(String::new()); + let (mut samples, mut ticks) = (0usize, 0usize); + let watch = watch( + || { + let s = script[samples.min(script.len() - 1)].as_ref().map(|v| raw(v)); + samples += 1; + assert!(samples <= 10_000, "the probe loop sampled without end"); + clock.set(clock.get() + sample); + calls.borrow_mut().push('S'); + s + }, + || { + ticks += 1; + clock.set(clock.get() + tick); + calls.borrow_mut().push('T'); + }, + || clock.get(), + BUDGET, + ); + Run { watch, samples, ticks, ended: clock.get(), calls: calls.into_inner() } + } + + fn rail(values: &[i64]) -> Vec>> { + values.iter().map(|v| Some(vec![("CPU Energy", *v)])).collect() + } + + /// The real probe's cadence on an idle machine: a 25 ms tick, then a + /// group-only sample, ~31 ms an interval. + const TICK: Duration = PROBE_TICK; + const SAMPLE: Duration = Duration::from_millis(6); + + /// (a) The M5 Max refresh caught mid-probe: one step, then the rail holds + /// at its new value for the rest of the budget. Frozen. A loop that read + /// every sample against the first would call it live at the next sample. + #[test] + fn the_probe_loop_calls_a_refresh_frozen() { + let r = scripted(&rail(&[480_083_160, 480_083_160, 541_893_638]), TICK, SAMPLE); + assert_eq!(r.watch.verdict(), Some(CpuRail::Frozen), "{:?}", r.watch); + assert_eq!(r.watch.rails, [("CPU Energy".to_string(), 0, 1)], "one advance in the whole budget"); + assert!(r.watch.intervals >= 25, "a 1 s budget at ~31 ms an interval: {}", r.watch.intervals); + assert!(r.ended >= BUDGET && r.ended < BUDGET + TICK + SAMPLE * 2, "stopped at the budget: {:?}", r.ended); + } + + /// (a2) The observed two-step refresh through the loop: two advances in + /// consecutive intervals, then the rail holds. The loop watches the whole + /// budget and calls it Frozen. + #[test] + fn the_probe_loop_calls_a_two_step_refresh_frozen() { + let a: i64 = 2_643_843_378; + let r = scripted(&rail(&[a, a, a, a + 61_764_485, a + 61_764_485 + 3_435]), TICK, SAMPLE); + assert_eq!(r.watch.verdict(), Some(CpuRail::Frozen), "{:?}", r.watch); + assert_eq!(r.watch.rails, [("CPU Energy".to_string(), 0, 2)]); + assert!(r.ended >= BUDGET && r.ended < BUDGET + TICK + SAMPLE * 2, "stopped at the budget: {:?}", r.ended); + } + + /// (b) The M3 Ultra refresh: both dies step once, in the same sample. + /// Frozen. + #[test] + fn the_probe_loop_calls_a_two_die_refresh_frozen() { + let (d0, d1) = ("DIE_0_CPU Energy", "DIE_1_CPU Energy"); + let before = Some(vec![(d0, 9_107_678), (d1, 9_742_153)]); + let after = Some(vec![(d0, 9_107_678 + 1_344_869), (d1, 9_742_153 + 1_391_637)]); + let r = scripted(&[before.clone(), before, after], TICK, SAMPLE); + assert_eq!(r.watch.verdict(), Some(CpuRail::Frozen), "{:?}", r.watch); + assert_eq!(r.watch.rails, [(d0.to_string(), 0, 1), (d1.to_string(), 0, 1)]); + } + + /// (c) Two channels that share the name `CPU Energy` step together once. + /// Two rails, one advance each: Frozen. + #[test] + fn the_probe_loop_calls_twin_channels_stepping_once_frozen() { + let r = scripted( + &[ + Some(vec![("CPU Energy", 10), ("CPU Energy", 20)]), + Some(vec![("CPU Energy", 11), ("CPU Energy", 21)]), + ], + TICK, + SAMPLE, + ); + assert_eq!(r.watch.verdict(), Some(CpuRail::Frozen), "{:?}", r.watch); + assert_eq!(r.watch.rails, [("CPU Energy".to_string(), 0, 1), ("CPU Energy".to_string(), 1, 1)]); + } + + /// (d) A rail that steps at every sample, ~31 ms apart, is live after + /// exactly ten intervals — the first whose samples span 300 ms — and the + /// loop stops sampling then, with a tick of burn before every sample but + /// the baseline. The rule is handed each sample with the clock's reading + /// just after it. + #[test] + fn the_probe_loop_stops_as_soon_as_a_rail_is_live() { + let r = scripted(&rail(&rail_where(40, |_| true)), TICK, SAMPLE); + assert_eq!(r.watch.verdict(), Some(CpuRail::Live)); + assert_eq!(r.watch.intervals, 10); + assert_eq!((r.samples, r.ticks), (11, 10)); + assert_eq!(r.calls, format!("S{}", "TS".repeat(10)), "a tick between every two samples"); + let at: Vec = (0..11u32).map(|k| SAMPLE + (TICK + SAMPLE) * k).collect(); + assert_eq!(r.watch.at, at, "each sample handed with the clock just after it"); + } + + /// (e) Samples that cannot be read keep the loop sampling, up to the + /// five-budget cap and no further, and give no verdict: `Unavailable`. + #[test] + fn the_probe_loop_gives_no_verdict_when_no_sample_reads() { + let r = scripted(&[None], TICK, SAMPLE); + assert_eq!(r.watch.verdict(), None); + assert_eq!(status_of(r.watch.verdict()), IoReportStatus::Unavailable); + assert!(r.ended >= BUDGET * PROBE_CAP_BUDGETS, "gave up at {:?}, before the cap", r.ended); + assert!(r.ended < BUDGET * PROBE_CAP_BUDGETS + TICK + SAMPLE, "ran past the cap: {:?}", r.ended); + assert_eq!(r.samples, r.ticks + 1); + // A read that fails once is skipped: the next one closes the interval. + let mut script = rail(&[1, 1]); + script.insert(1, None); + let r = scripted(&script, TICK, SAMPLE); + assert_eq!(r.watch.verdict(), Some(CpuRail::Frozen), "{:?}", r.watch); + } + + /// (f) Past the budget with five intervals seen, the loop stops (Frozen, + /// for a rail that held). Past the budget with fewer, it keeps sampling + /// until it has five — and the five-budget cap stops it however few it + /// has. + #[test] + fn the_probe_loop_needs_the_budget_and_five_intervals_to_stop() { + // Every tick takes half the budget: at the budget there are two + // intervals; it samples on to the fifth, at 2.5 budgets, and stops. + let r = scripted(&rail(&[5]), BUDGET / 2, Duration::ZERO); + assert_eq!(r.watch.verdict(), Some(CpuRail::Frozen)); + assert_eq!((r.watch.intervals, r.samples), (5, 6), "stopped at five intervals"); + assert_eq!(r.ended, BUDGET * 5 / 2); + // Every tick takes two budgets: the cap, checked before each tick, + // stops it with three intervals. + let r = scripted(&rail(&[5]), BUDGET * 2, Duration::ZERO); + assert_eq!(r.watch.verdict(), Some(CpuRail::Frozen)); + assert_eq!((r.watch.intervals, r.samples), (3, 4), "stopped at the cap"); + assert_eq!(r.ended, BUDGET * 6); + // Past the budget after the first tick, and samples fail until the + // fourth: it keeps sampling toward five intervals, and the cap stops + // it at three. + let r = scripted(&[Some(vec![("CPU Energy", 5)]), None, None, Some(vec![("CPU Energy", 5)])], BUDGET, Duration::ZERO); + assert_eq!(r.watch.verdict(), Some(CpuRail::Frozen), "{:?}", r.watch); + assert_eq!((r.watch.intervals, r.samples), (3, 6)); + assert_eq!(r.ended, BUDGET * PROBE_CAP_BUDGETS); + } + + /// (g) One die steps once, then the other die steps once, in different + /// intervals: two rails with one advance each, not one live rail. + #[test] + fn the_probe_loop_counts_advances_per_rail_never_summed() { + let (d0, d1) = ("DIE_0_CPU Energy", "DIE_1_CPU Energy"); + let r = scripted( + &[ + Some(vec![(d0, 1), (d1, 1)]), + Some(vec![(d0, 2), (d1, 1)]), + Some(vec![(d0, 2), (d1, 2)]), + ], + TICK, + SAMPLE, + ); + assert_eq!(r.watch.verdict(), Some(CpuRail::Frozen), "{:?}", r.watch); + assert_eq!(r.watch.rails, [(d0.to_string(), 0, 1), (d1.to_string(), 0, 1)]); + } + // ---- One whole probe run, on a scripted subscription, tick and clocks. + + /// A channel of the Energy Model group. + fn em(name: &str, raw: i64) -> Channel { + Channel { group: ENERGY_MODEL.to_string(), name: name.to_string(), raw } + } + + /// Samples of one `CPU Energy` rail, beside a `GPU Energy` rail that + /// moves at every sample, as it does on an M5 Max. + fn cpu_rail(values: &[i64]) -> Vec>> { + values + .iter() + .enumerate() + .map(|(i, v)| Some(vec![em("CPU Energy", *v), em("GPU Energy", 1_000 + i as i64)])) + .collect() + } + + /// What one scripted [`probe_with`] run did, counted by the fakes + /// themselves rather than by the run's own record. + struct Scripted { + run: ProbeRun, + opened: usize, + samples: usize, + ticks: usize, + } + + /// Run [`probe_with`] over `script` — `None`: the subscription does not + /// open; otherwise its samples in order (`None` = a sample that could not + /// be read; past the end, the last repeats) — with scripted clocks. Each + /// tick advances both clocks by `tick`; each sample advances the wall + /// clock by `sample` and the CPU clock by a third of it. A run that + /// samples without end fails instead of hanging. + fn run(script: Option<&[Option>]>, tick: Duration, sample: Duration) -> Scripted { + let (cpu, wall) = (Cell::new(Duration::ZERO), Cell::new(Duration::ZERO)); + let (opened, samples, ticks) = (Cell::new(0usize), Cell::new(0usize), Cell::new(0usize)); + let (cpu_at, wall_at, taken) = (&cpu, &wall, &samples); + let run = probe_with( + || { + opened.set(opened.get() + 1); + script.map(|script| { + move || { + let s = script[taken.get().min(script.len() - 1)].clone(); + taken.set(taken.get() + 1); + assert!(taken.get() <= 10_000, "the probe run sampled without end"); + wall_at.set(wall_at.get() + sample); + cpu_at.set(cpu_at.get() + sample / 3); + s + } + }) + }, + || { + ticks.set(ticks.get() + 1); + cpu.set(cpu.get() + tick); + wall.set(wall.get() + tick); + }, + || cpu.get(), + || wall.get(), + BUDGET, + ); + Scripted { run, opened: opened.get(), samples: samples.get(), ticks: ticks.get() } + } + + /// A subscription that does not open: opened once, then no sample call + /// and no tick — no verdict, so `Unavailable`. + #[test] + fn a_probe_run_whose_subscription_does_not_open_samples_and_ticks_nothing() { + let r = run(None, TICK, SAMPLE); + assert_eq!(r.opened, 1); + assert_eq!((r.samples, r.ticks), (0, 0), "sampled or ticked a subscription that did not open"); + assert!(!r.run.opened && r.run.calls.is_empty() && r.run.ticks == 0, "{:?}", r.run); + assert!(r.run.watch.observed.is_empty() && r.run.interval_burn.is_empty(), "{:?}", r.run); + assert_eq!(r.run.verdict(), None); + assert_eq!(status_of(r.run.verdict()), IoReportStatus::Unavailable); + } + + /// One batch jump — the M5 Max refresh caught mid-run — then the rail + /// holds, while the GPU rail moves at every sample: `CpuRailFrozen`. + #[test] + fn a_probe_run_calls_one_batch_jump_frozen() { + let r = run(Some(&cpu_rail(&[480_083_160, 541_893_638])), TICK, SAMPLE); + assert_eq!(r.run.verdict(), Some(CpuRail::Frozen), "{:?}", r.run.watch); + assert_eq!(r.run.watch.rails, [("CPU Energy".to_string(), 0, 1)], "one advance in the whole budget"); + assert!(r.run.watch.intervals >= 25, "a 1 s budget at ~31 ms an interval: {}", r.run.watch.intervals); + assert_eq!(status_of(r.run.verdict()), IoReportStatus::CpuRailFrozen); + } + + /// A rail that keeps advancing at every sample, ~31 ms apart: `Live` + /// once its samples span 300 ms (the tenth interval), and the run stops + /// at once — eleven samples, a tick between each two. + #[test] + fn a_probe_run_calls_a_sustained_rail_live() { + let r = run(Some(&cpu_rail(&rail_where(40, |_| true))), TICK, SAMPLE); + assert_eq!(r.run.verdict(), Some(CpuRail::Live), "{:?}", r.run.watch); + assert_eq!((r.samples, r.ticks), (11, 10)); + assert_eq!((r.run.calls.len(), r.run.ticks), (11, 10), "the run's own record agrees with the fakes"); + assert_eq!(status_of(r.run.verdict()), IoReportStatus::Live); + } + + /// Two dies that advance in TURN for the whole budget, one in the odd + /// intervals and the other in the even ones: two rails that each + /// advanced in half the intervals, never one live rail. A transform that + /// handed the rule each rail's sum over the CPU rails would see every + /// rail advance at every interval, and call it live. + #[test] + fn a_probe_run_never_sums_two_dies_that_advance_in_turn() { + let (d0, d1) = ("DIE_0_CPU Energy", "DIE_1_CPU Energy"); + let (v0, v1) = (rail_where(40, |i| i % 2 == 1), rail_where(40, |i| i % 2 == 0)); + let script: Vec>> = (0..=40) + .map(|i| Some(vec![em(d0, 36_244_185 + v0[i]), em(d1, 39_369_238 + v1[i]), em("GPU Energy", i as i64)])) + .collect(); + let r = run(Some(&script), TICK, SAMPLE); + assert_eq!(r.run.verdict(), Some(CpuRail::Frozen), "{:?}", r.run.watch); + let n = r.run.watch.intervals; + assert!(n >= 25, "a 1 s budget at ~31 ms an interval: {n}"); + assert_eq!(r.run.watch.rails, [(d0.to_string(), 0, n.div_ceil(2)), (d1.to_string(), 0, n / 2)]); + // Each die was handed to the rule with its own value, never a sum. + assert_eq!( + r.run.watch.observed[1], + [(d0.to_string(), 36_244_185 + v0[1]), (d1.to_string(), 39_369_238 + v1[1])] + ); + } + + /// Samples of two dies, each at its own values (holding its last past + /// the end), beside a `GPU Energy` rail that moves at every sample. + fn two_die_script(v0: &[i64], v1: &[i64]) -> Vec>> { + let (d0, d1) = DIES; + (0..200) + .map(|i| { + let at = |v: &[i64]| v[i.min(v.len() - 1)]; + Some(vec![em(d0, 36_244_185 + at(v0)), em(d1, 39_369_238 + at(v1)), em("GPU Energy", i as i64)]) + }) + .collect() + } + + /// (a) Through a whole probe run: both dies advance at every sample, + /// ~31 ms apart. `Live` at the tenth interval, and the run stops there. + #[test] + fn a_probe_run_calls_two_dies_that_keep_advancing_live() { + let every = rail_where(200, |_| true); + let r = run(Some(&two_die_script(&every, &every)), TICK, SAMPLE); + assert_eq!(r.run.verdict(), Some(CpuRail::Live), "{:?}", r.run.watch); + assert_eq!((r.samples, r.ticks), (11, 10)); + assert_eq!(status_of(r.run.verdict()), IoReportStatus::Live); + } + + /// (b) Through a whole probe run: DIE_0 advances at every sample, DIE_1 + /// holds, and the GPU rail moves at every sample. `CpuRailFrozen`, + /// watched for the whole budget: DIE_0 advanced in every interval, DIE_1 + /// in none. + #[test] + fn a_probe_run_calls_one_live_die_beside_a_frozen_one_frozen() { + let (d0, d1) = DIES; + let r = run(Some(&two_die_script(&rail_where(200, |_| true), &[0])), TICK, SAMPLE); + assert_eq!(r.run.verdict(), Some(CpuRail::Frozen), "{:?}", r.run.watch); + assert_eq!(status_of(r.run.verdict()), IoReportStatus::CpuRailFrozen); + let n = r.run.watch.intervals; + assert!(n >= 25, "a 1 s budget at ~31 ms an interval: {n}"); + assert_eq!(r.run.watch.rails, [(d0.to_string(), 0, n), (d1.to_string(), 0, 0)]); + } + + /// (c) Through a whole probe run: one die that advances at every sample, + /// and no other die in the samples. `Live` at the tenth interval. + #[test] + fn a_probe_run_calls_one_die_alone_that_keeps_advancing_live() { + let (d0, _) = DIES; + let v = rail_where(40, |_| true); + let script: Vec>> = + (0..=40).map(|i| Some(vec![em(d0, v[i]), em("GPU Energy", i as i64)])).collect(); + let r = run(Some(&script), TICK, SAMPLE); + assert_eq!(r.run.verdict(), Some(CpuRail::Live), "{:?}", r.run.watch); + assert_eq!((r.samples, r.ticks), (11, 10)); + assert_eq!(r.run.watch.rails, [(d0.to_string(), 0, 10)]); + } + + /// The run records every sample call, read or not: when it began and + /// ended and the CPU it used, on the run's clocks; each read sample's + /// channel count, whether every channel was in the Energy Model group, and + /// its GPU rail. The rule keeps every CPU sample it was handed, exactly as + /// handed, and each interval's tick CPU is measured apart from the + /// samples: a failed read leaves its interval open, so that interval + /// carries two ticks. + #[test] + fn a_probe_run_records_what_each_sample_held_and_cost() { + let foreign = + Channel { group: "CPU Stats".to_string(), name: "CPU Core Performance States".to_string(), raw: 0 }; + let script = [ + Some(vec![em("CPU Energy", 10), em("MCPU0", 4), em("GPU Energy", 500)]), + None, + Some(vec![em("CPU Energy", 10), foreign, em("GPU Energy", 507)]), + Some(vec![em("CPU Energy", 11), em("GPU Energy", 512)]), + ]; + let r = run(Some(&script), TICK, SAMPLE); + let run = &r.run; + assert!(run.opened); + assert_eq!(run.calls.len(), r.samples, "one record per sample call"); + assert_eq!(run.ticks as usize, r.ticks); + // Call timing, on the scripted clocks: each sample takes SAMPLE of + // wall time and a third of it of CPU; a tick sits between two calls. + assert_eq!((run.calls[0].started, run.calls[0].ended, run.calls[0].cpu), (ms(0), SAMPLE, SAMPLE / 3)); + assert_eq!(run.calls[1].started, SAMPLE + TICK, "the second call follows one tick"); + assert!(run.calls.iter().all(|c| c.cpu == SAMPLE / 3 && c.ended - c.started == SAMPLE), "{:?}", run.calls); + // What each read held. + assert_eq!(run.calls[1].read, None, "a sample that could not be read"); + let shape = |i: usize| run.calls[i].read.clone().expect("a read sample"); + assert_eq!(shape(0), SampleShape { channels: 3, group_only: true, gpu: vec![("GPU Energy".to_string(), 500)] }); + assert_eq!( + shape(2), + SampleShape { channels: 3, group_only: false, gpu: vec![("GPU Energy".to_string(), 507)] } + ); + assert_eq!(shape(3), SampleShape { channels: 2, group_only: true, gpu: vec![("GPU Energy".to_string(), 512)] }); + // The rule was handed each read sample's CPU rails, and only those. + let cpu = |v: i64| vec![("CPU Energy".to_string(), v)]; + assert_eq!(run.watch.observed[..3], [cpu(10), cpu(10), cpu(11)]); + assert_eq!(run.watch.observed.len(), run.calls.iter().filter(|c| c.read.is_some()).count()); + // Tick CPU per interval, apart from the samples: the failed read + // left the first interval open across two ticks. + assert_eq!(run.interval_burn[..2], [TICK * 2, TICK]); + assert_eq!(run.interval_burn.len(), run.watch.intervals as usize); + } + + /// Only a live CPU rail admits the IOReport probe; every other status + /// declines it. + #[test] + fn only_a_live_status_admits_the_probe() { + assert!(admits(IoReportStatus::Live)); + assert!(!admits(IoReportStatus::CpuRailFrozen)); + assert!(!admits(IoReportStatus::CpuRailAbsent)); + assert!(!admits(IoReportStatus::Unavailable)); + } + + /// The GPU rail the run records, on one die or many — never a sub-rail. + #[test] + fn a_gpu_rail_is_the_rail_itself() { + assert!(is_gpu_rail("GPU Energy")); // both chips + assert!(is_gpu_rail("DIE_1_GPU Energy")); + for sub in ["GPU0", "GPU0_0", "GPU CS0_0", "GPU SRAM0_0", "GPU CS SRAM0_0", "CPU Energy"] { + assert!(!is_gpu_rail(sub), "{sub}"); + } + } + + // ---- Every other vendor counter (RAPL, NVML): the same rule, every rail. + + /// What one scripted [`counter_run`] did. + struct CounterRun { + watch: RailWatch, + samples: usize, + ticks: usize, + } + + /// Run [`counter_run`] on a scripted clock that each tick advances by + /// `tick` and each sample by `sample`; `read(t)` is the counter's sample + /// at clock time `t` (`None`: that read failed). `opens: false` is a + /// counter that does not open. A loop that samples without end fails + /// instead of hanging. + fn counter_scripted( + opens: bool, + read: impl Fn(Duration) -> Option>, + tick: Duration, + sample: Duration, + ) -> CounterRun { + let clock = Cell::new(Duration::ZERO); + let (samples, ticks) = (Cell::new(0usize), Cell::new(0usize)); + let watch = counter_run( + || { + opens.then_some(|| { + samples.set(samples.get() + 1); + assert!(samples.get() <= 10_000, "the counter probe sampled without end"); + clock.set(clock.get() + sample); + read(clock.get()).map(|v| raw(&v)) + }) + }, + || { + ticks.set(ticks.get() + 1); + clock.set(clock.get() + tick); + }, + || clock.get(), + BUDGET, + ); + CounterRun { watch, samples: samples.get(), ticks: ticks.get() } + } + + /// A counter published every `period` of wall time, `per` raw units a + /// step: its value at `t` (a GPU's NVML counter, in mJ). + fn published(period: Duration, per: i64, t: Duration) -> i64 { + 1_000_000 + per * (t.as_nanos() / period.as_nanos()) as i64 + } + + /// A RAPL package counter on Linux: `energy_uj` moves about every + /// millisecond, so it advances in every ~25 ms interval of the probe and is + /// live once the samples span 300 ms: the twelfth interval at 25 ms + a + /// negligible sysfs read. + #[test] + fn a_counter_whose_only_rail_keeps_advancing_is_live() { + let r = counter_scripted( + true, + |t| Some(vec![("intel-rapl:0", published(Duration::from_millis(1), 61, t))]), + PROBE_TICK, + Duration::ZERO, + ); + assert_eq!(r.watch.verdict(), Some(CpuRail::Live), "{:?}", r.watch); + assert_eq!(counter_status_of(r.watch.verdict()), CounterStatus::Live); + assert_eq!((r.watch.intervals, r.ticks, r.samples), (12, 12, 13), "stops as soon as it is live"); + } + + /// Two packages, one of which held: the summed `Cpu` figure would claim a + /// package its counter does not read, so the counter is not live. The + /// IOReport loop holds the same samples to the same rule. + #[test] + fn a_counter_with_one_rail_that_held_is_frozen() { + let both = |t| { + Some(vec![("intel-rapl:0", published(Duration::from_millis(1), 61, t)), ("intel-rapl:1", 4_242)]) + }; + let r = counter_scripted(true, both, PROBE_TICK, Duration::ZERO); + assert_eq!(r.watch.verdict(), Some(CpuRail::Frozen), "{:?}", r.watch); + assert_eq!(counter_status_of(r.watch.verdict()), CounterStatus::Frozen); + assert_eq!(r.watch.rails[1].2, 0, "the second package never advanced"); + assert!(r.watch.rails[0].2 >= MIN_LIVE_ADVANCES, "the first one kept advancing: {:?}", r.watch); + + let clock = Cell::new(Duration::ZERO); + let ioreport = watch( + || { + clock.set(clock.get() + PROBE_TICK); + both(clock.get()).map(|v| raw(&v)) + }, + || {}, + || clock.get(), + BUDGET, + ); + assert_eq!(ioreport.verdict(), Some(CpuRail::Frozen), "the IOReport loop: every rail, as here"); + } + + /// Both packages advancing, one a little slower (four intervals in five): + /// live, at the interval where the slower one is. + #[test] + fn a_counter_whose_every_rail_keeps_advancing_is_live() { + let r = counter_scripted( + true, + |t| { + let i = (t.as_millis() / 25) as i64; + Some(vec![("intel-rapl:0", 100 + 7 * i), ("intel-rapl:1", 100 + 7 * (i - i / 5))]) + }, + PROBE_TICK, + Duration::ZERO, + ); + assert_eq!(r.watch.verdict(), Some(CpuRail::Live), "{:?}", r.watch); + let slow = r.watch.rails[1].2; + assert!(sustained(slow, r.watch.intervals), "{:?}", r.watch); + } + + /// A counter republished in a burst — a few steps inside 150 ms, then + /// holding for the rest of the budget — is not live, on any rail count. + #[test] + fn a_counter_that_steps_in_a_burst_is_frozen() { + for steps in 1..=5i64 { + let r = counter_scripted( + true, + move |t| { + let i = (t.as_millis() / 25) as i64; + Some(vec![("nvml:0", 1_000 + 500 * i.clamp(0, steps))]) + }, + PROBE_TICK, + Duration::ZERO, + ); + assert_eq!(r.watch.verdict(), Some(CpuRail::Frozen), "{steps} steps: {:?}", r.watch); + } + } + + /// A counter that wraps once (RAPL's `energy_uj` within its + /// `max_energy_range_uj`) misses one interval among many: still live. A + /// counter that only falls is not. + #[test] + fn a_counter_that_wraps_once_is_still_live_and_one_that_falls_is_not() { + let r = counter_scripted( + true, + |t| { + let i = (t.as_millis() / 25) as i64; + Some(vec![("intel-rapl:0", if i < 6 { 262_000_000 + 61 * i } else { 61 * i })]) + }, + PROBE_TICK, + Duration::ZERO, + ); + assert_eq!(r.watch.verdict(), Some(CpuRail::Live), "{:?}", r.watch); + let r = counter_scripted( + true, + |t| Some(vec![("intel-rapl:0", 1_000_000 - (t.as_millis() as i64))]), + PROBE_TICK, + Duration::ZERO, + ); + assert_eq!(r.watch.verdict(), Some(CpuRail::Frozen), "{:?}", r.watch); + } + + /// A counter that does not open is never sampled or ticked; one that + /// opens and never reads, or reads no rail, gives no verdict a status + /// could call live: `Unavailable`. + #[test] + fn a_counter_that_cannot_be_opened_or_read_is_unavailable() { + let r = counter_scripted(false, |_| unreachable!("a counter that did not open was read"), PROBE_TICK, Duration::ZERO); + assert_eq!((r.samples, r.ticks), (0, 0), "sampled or ticked a counter that did not open"); + assert_eq!(counter_status_of(r.watch.verdict()), CounterStatus::Unavailable); + let r = counter_scripted(true, |_| None, PROBE_TICK, Duration::ZERO); + assert_eq!(counter_status_of(r.watch.verdict()), CounterStatus::Unavailable, "{:?}", r.watch); + let r = counter_scripted(true, |_| Some(vec![]), PROBE_TICK, Duration::ZERO); + assert_eq!(r.watch.verdict(), Some(CpuRail::Absent)); + assert_eq!(counter_status_of(r.watch.verdict()), CounterStatus::Unavailable, "no rail is no counter"); + } + + /// Why NVML samples at its own tick: a live GPU counter published every + /// 100 ms advances in about one 25 ms interval in four, and the rule + /// declines it; at [`NVML_PROBE_TICK`] every interval holds an update and + /// it is live. One published every 400 ms is declined even there — it + /// fails safe, estimated, never over-claimed. + #[test] + fn an_nvml_counter_is_judged_at_its_own_tick() { + let every_100ms = |t| Some(vec![("nvml:0", published(Duration::from_millis(100), 3_500, t))]); + let fast = counter_scripted(true, every_100ms, PROBE_TICK, Duration::from_micros(200)); + assert_eq!(fast.watch.verdict(), Some(CpuRail::Frozen), "at 25 ms: {:?}", fast.watch); + let nvml = counter_scripted(true, every_100ms, NVML_PROBE_TICK, Duration::from_micros(200)); + assert_eq!(nvml.watch.verdict(), Some(CpuRail::Live), "at 150 ms: {:?}", nvml.watch); + assert_eq!(nvml.watch.intervals, MIN_LIVE_ADVANCES, "live at its fifth interval, ~750 ms"); + let every_400ms = |t| Some(vec![("nvml:0", published(Duration::from_millis(400), 14_000, t))]); + let slow = counter_scripted(true, every_400ms, NVML_PROBE_TICK, Duration::from_micros(200)); + assert_eq!(slow.watch.verdict(), Some(CpuRail::Frozen), "{:?}", slow.watch); + } + + /// The tick a counter probe burns is the tick it was asked for. + #[test] + fn a_counter_tick_burns_its_own_length() { + let cpu = Cell::new(Duration::ZERO); + let wall = Cell::new(Duration::ZERO); + let burned = spin_for( + NVML_PROBE_TICK, + || { + cpu.set(cpu.get() + ms(1)); + cpu.get() + }, + || { + wall.set(wall.get() + ms(1)); + wall.get() + }, + ); + assert!(burned >= NVML_PROBE_TICK && burned < NVML_PROBE_TICK + ms(2), "{burned:?}"); + } + + /// The gate as a constructor: a counter that is not live is never + /// opened, and the refusal names it and why; a live one is opened, and + /// its own error passes through. + #[test] + fn a_gated_counter_is_opened_only_when_live() { + for status in [CounterStatus::Unavailable, CounterStatus::Frozen] { + let opened = Cell::new(false); + let r = gated("RAPL", status, || { + opened.set(true); + Ok(()) + }); + assert!(!opened.get(), "{status:?} opened the counter"); + match r { + Err(ProbeError::Unavailable(why)) => { + assert!(why.starts_with("RAPL: ") && why.contains(&status.to_string()), "{why}") + } + other => panic!("{status:?}: {other:?}"), + } + } + assert!(matches!(gated("NVML", CounterStatus::Live, || Ok(7)), Ok(7))); + let failed: Result<(), _> = gated("NVML", CounterStatus::Live, || Err(ProbeError::Unavailable("no NVML".into()))); + assert!(matches!(failed, Err(ProbeError::Unavailable(why)) if why == "no NVML")); + } + + /// Only a live counter admits its probe, and each verdict maps to its + /// status. + #[test] + fn only_a_live_counter_admits_its_probe() { + assert!(counter_admits(CounterStatus::Live)); + assert!(!counter_admits(CounterStatus::Frozen)); + assert!(!counter_admits(CounterStatus::Unavailable)); + assert_eq!(counter_status_of(None), CounterStatus::Unavailable); + assert_eq!(counter_status_of(Some(CpuRail::Absent)), CounterStatus::Unavailable); + assert_eq!(counter_status_of(Some(CpuRail::Frozen)), CounterStatus::Frozen); + assert_eq!(counter_status_of(Some(CpuRail::Live)), CounterStatus::Live); + } + + /// The thread CPU clock the counter ticks burn against moves while this + /// thread burns, and a real tick burns at least its length. + #[cfg(any(target_os = "macos", target_os = "linux"))] + #[test] + fn the_real_counter_tick_burns_this_threads_cpu() { + let before = thread_cpu(); + counter_tick(PROBE_TICK); + let burned = thread_cpu().saturating_sub(before); + assert!(burned >= PROBE_TICK, "a {PROBE_TICK:?} tick burned {burned:?} of this thread's CPU"); + } +} diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs index 2836993df..6b5474e7a 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/mod.rs @@ -7,16 +7,27 @@ //! //! - **Apple Silicon (macOS/aarch64):** the IOReport "Energy Model" counters //! (CPU/GPU/ANE energy, the same source `powermetrics` reads, sudo-free) — -//! `HwShunt`. +//! `HwShunt` — **only when every CPU energy rail keeps advancing** (one +//! per die on a multi-die part, all summed into the `Cpu` figure) +//! ([`ioreport_status`]). +//! On macOS 27.0.1 an unprivileged reader sees that rail frozen on both an +//! M5 Max and an M3 Ultra, so those hosts get the estimator below, and +//! [`ioreport_status`] says why. //! - **Linux:** Intel/AMD RAPL via powercap (`energy_uj`, wrap-corrected) — -//! `HwShunt`; NVIDIA GPUs via NVML's total-energy counter when -//! `libnvidia-ml` is present at runtime — `HwShunt`. +//! `HwShunt` — summing the zones whose `name` is `package-` (never +//! `psys`, which already includes them, or `dram`), **only when every +//! package counter keeps advancing** ([`rapl_status`]); NVIDIA GPUs via NVML's total-energy counter when +//! `libnvidia-ml` is present at runtime — `HwShunt` — **only when every +//! GPU's counter keeps advancing** ([`nvml_status`]). The rule is +//! IOReport's: every rail a figure sums must keep advancing. //! - **Anywhere else (or unreadable counters):** wall-clock × configured TDP — //! `Estimator`, with `measured_uj` absent, exactly as SPEC §6.3 demands. //! //! A probe can degrade but it can never lie: if the hardware path fails //! mid-span, the reading comes back `Estimator`, not a zero dressed as a -//! measurement. +//! measurement. The same holds when the CPU energy counter did not advance +//! across a span: this process ran during it, so a zero there is not a +//! reading of the span, and the whole span comes back `Estimator`. use crate::{DeviceEnergy, Provenance}; use sandbox_schema::DeviceClass; @@ -24,10 +35,18 @@ use std::time::{Duration, Instant}; #[cfg(all(target_os = "macos", target_arch = "aarch64"))] mod ioreport; +// The vendor-counter liveness rule, apart from any FFI: used by `ioreport`, +// `rapl` and `nvml`, and through [`counter_liveness`] by a caller that reads a +// counter of its own; built and tested on every platform. +mod liveness; #[cfg(target_os = "linux")] mod nvml; -#[cfg(target_os = "linux")] +// Built everywhere: its zone listing, reads and liveness run take the +// powercap root as a path, so a caller's [`RaplPackages::live_in`] and the +// tests run them on fake powercap trees on any platform. Only Linux opens the +// sysfs tree. mod rapl; +pub use rapl::RaplPackages; pub mod attribute; pub use attribute::{run_attributed, Attribution}; @@ -37,6 +56,247 @@ pub use attribute::{run_attributed, Attribution}; /// provenance is that nobody mistakes this for a measurement. const DEFAULT_TDP_WATTS: f64 = 20.0; +/// Why the Apple Silicon IOReport probe was or was not used on this host — +/// so a caller, a CLI or a receipt can say *why* an Apple Silicon host is +/// estimated instead of misdiagnosing it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IoReportStatus { + /// No IOReport Energy Model subscription could be opened, or fewer than + /// two of its samples could be read before the probe's cap: not Apple + /// Silicon macOS (always the case on other platforms), or the surface + /// changed. + Unavailable, + /// A subscription opened, but no `CPU Energy` rail (nor `DIE__CPU + /// Energy`) exists in the Energy Model group — an unfamiliar channel + /// naming. Never used. + CpuRailAbsent, + /// A CPU energy rail exists, but not every one kept advancing while this + /// process burned CPU: at least one rail (on a multi-die part, one die's) + /// did not advance in at least five sample intervals, four in five of + /// those observed, over at least 300 ms. It held, or stepped in a burst — + /// a batch refresh of energy spent long before, in one step or a few, not + /// a reading of the work. The platform is not exposing live energy for + /// the whole CPU to this process, so every span would report GPU-only + /// energy, zero, a stale refresh, or some dies' energy as the whole + /// CPU's, as a measurement of CPU work. Never used. Observed on macOS + /// 27.0.1 on both an M5 Max and an M3 Ultra. + CpuRailFrozen, + /// Every CPU energy rail kept advancing under load, each one on its own: + /// in at least five sample intervals, at least four in five of those + /// observed, over at least 300 ms of wall time; the probe is used. + Live, +} + +impl std::fmt::Display for IoReportStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + IoReportStatus::Unavailable => { + "no IOReport Energy Model subscription could be opened or sampled" + } + IoReportStatus::CpuRailAbsent => { + "the IOReport Energy Model group has no `CPU Energy` rail" + } + IoReportStatus::CpuRailFrozen => { + "not every IOReport `CPU Energy` rail (one per die on a multi-die part) kept \ + advancing while this process burned CPU (one held, or stepped only in a batch \ + refresh), so it cannot meter CPU work" + } + IoReportStatus::Live => { + "every IOReport `CPU Energy` rail keeps advancing under CPU load" + } + }) + } +} + +/// How long (wall time) the liveness probe watches the CPU rail, burning CPU +/// on its thread and sampling after every 25 ms of that thread's CPU time, +/// before it may conclude the rail is frozen. A live rail must advance in at +/// least five intervals, four in five of those observed, over at least +/// 300 ms, and a live Energy Model rail moves many times a second (the M5 +/// Max's `GPU Energy` about every 28 ms), so at ~31 ms an interval a live +/// rail proves itself in about 310 ms and a second is generous; a slower +/// live rail fails safe — estimated, never over-claimed. If a second holds +/// fewer than five sample intervals, the probe samples on past it, never +/// beyond five budgets. Paid once per process, on the thread that first needs +/// it; see [`ioreport_status`]. +pub const CPU_RAIL_LIVENESS_BUDGET: Duration = Duration::from_millis(1000); + +/// This host's IOReport status, probed once per process and cached. +/// +/// [`PowerMeter::detect`] uses the IOReport probe only when this is +/// [`IoReportStatus::Live`]: every CPU energy rail — `CPU Energy`, or each +/// die's `DIE__CPU Energy` on multi-die parts — must keep advancing, each +/// one on its own, while the probing thread burns CPU: every rail, because +/// the probe sums them all into one `Cpu` figure, and a rail is present if it +/// is in any sample the probe read. Each must advance in at least five +/// sample intervals, those being at least four in five of the intervals +/// observed, over at least 300 ms of wall time from the first sample. It is +/// decided on each rail's raw integer between consecutive samples, after +/// every interval, and never on a summed total. A step, or a few steps close together, is a batch refresh, not +/// liveness: on macOS 27.0.1 one refresh was seen to land in two steps 64 ms +/// apart. The probe samples a subscription to the Energy Model group alone +/// (a few hundred channels, ~6–12 ms of this thread's CPU a sample; its wall +/// time can run far longer while another process samples every channel at +/// once) after every 25 ms of this thread's CPU time burned, however loaded +/// the machine (a tick's wall time is capped at 500 ms), until the rails +/// prove live or [`CPU_RAIL_LIVENESS_BUDGET`] has passed with at least five +/// intervals seen (never past five budgets). +/// Always [`IoReportStatus::Unavailable`] off Apple Silicon macOS. +pub fn ioreport_status() -> IoReportStatus { + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + { + static STATUS: std::sync::OnceLock = std::sync::OnceLock::new(); + *STATUS.get_or_init(ioreport::host_status) + } + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + { + IoReportStatus::Unavailable + } +} + +/// Whether a vendor energy counter the meter sums into one figure per class — +/// Linux RAPL's package domains ([`rapl_status`]), NVML's GPUs +/// ([`nvml_status`]) — is used on this host, and if not, why. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CounterStatus { + /// The counter could not be opened, or not read twice before the probe's + /// cap: not this platform, no driver, no permission (RAPL's `energy_uj` is + /// root-only on many kernels), or no device that supports it (for RAPL, no + /// zone named `package-`). + Unavailable, + /// The counter opened and read, but not every rail it sums kept advancing + /// while this process burned CPU: one held, or stepped only in a burst. + /// A figure summed over it would claim a domain it does not read, or a + /// stale refresh as a reading. Never used. + Frozen, + /// Every rail it sums kept advancing: in at least five sample intervals, + /// at least four in five of those observed, over at least 300 ms of wall + /// time, while the probing thread burned CPU. The probe is used. + Live, +} + +impl std::fmt::Display for CounterStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + CounterStatus::Unavailable => "the counter could not be opened or read twice", + CounterStatus::Frozen => { + "not every rail of the counter kept advancing while this process burned CPU (one \ + held, or stepped only in a burst), so it cannot meter the work" + } + CounterStatus::Live => "every rail of the counter keeps advancing under load", + }) + } +} + +/// This host's Linux RAPL status, probed once per process and cached. +/// +/// [`PowerMeter::detect`] uses the RAPL probe only when this is +/// [`CounterStatus::Live`]: the `energy_uj` of every top-level zone whose +/// `name` is `package-` must keep advancing while the probing thread burns +/// CPU — the IOReport rule, [`CPU_RAIL_LIVENESS_BUDGET`] and its ~25 ms tick, +/// applied to every package, since the probe sums them into one `Cpu` figure. +/// Other zones (`psys`, `dram`) are neither summed nor probed. A host where +/// powercap is absent or unreadable, or has no package zone, is `Unavailable` +/// at once, with no burn. Always `Unavailable` off Linux. +pub fn rapl_status() -> CounterStatus { + #[cfg(target_os = "linux")] + { + static STATUS: std::sync::OnceLock = std::sync::OnceLock::new(); + *STATUS.get_or_init(rapl::host_status) + } + #[cfg(not(target_os = "linux"))] + { + CounterStatus::Unavailable + } +} + +/// The liveness rule [`rapl_status`] and [`nvml_status`] apply, run over a +/// vendor energy counter this crate does not open itself (the S5 executor's +/// per-socket RAPL MSRs), so such a counter is held to the same rule before +/// its figure is called a hardware reading. `open` opens the counter and +/// hands back its sampler (`None`: it did not open); the sampler reads every +/// rail the caller sums into one figure — its name and raw cumulative +/// integer, before any unit conversion or wrap correction, in a stable order +/// — from one absolute sample (`None`: this read failed). `tick` drives the +/// load between samples: production passes [`counter_liveness_tick`]. +/// +/// `Live` only if every rail advanced in at least five sample intervals, at +/// least four in five of those observed, over at least 300 ms of wall time, +/// each interval read against the sample before it, within +/// [`CPU_RAIL_LIVENESS_BUDGET`] (never past five of them). `Frozen` if one +/// held or stepped only in a burst; `Unavailable` if the counter did not open +/// (never sampled or ticked), could not be read twice, or has no rail. The +/// caller caches the status as it sees fit; nothing here is cached. +pub fn counter_liveness(open: impl FnOnce() -> Option, tick: impl FnMut()) -> CounterStatus +where + R: FnMut() -> Option>, +{ + let epoch = Instant::now(); + liveness::counter_status_of( + liveness::counter_run(open, tick, move || epoch.elapsed(), CPU_RAIL_LIVENESS_BUDGET).verdict(), + ) +} + +/// The tick the production counter probes burn between two liveness samples: +/// 25 ms of the calling thread's CPU time, capped at 500 ms of wall time — +/// the `tick` to hand [`counter_liveness`] for a real counter. +#[cfg(any(target_os = "macos", target_os = "linux"))] +pub fn counter_liveness_tick() { + liveness::counter_tick(liveness::PROBE_TICK); +} + +/// This host's NVML status, probed once per process and cached. +/// +/// [`PowerMeter::detect`] uses the NVML probe only when this is +/// [`CounterStatus::Live`]: every GPU's total-energy counter must keep +/// advancing — the same rule, at a 150 ms tick, since a GPU energy counter is +/// published more coarsely than a CPU rail — while the probing thread burns +/// CPU. A host with no `libnvidia-ml`, or no GPU that supports the counter, is +/// `Unavailable` at once, with no burn. Always `Unavailable` off Linux. +pub fn nvml_status() -> CounterStatus { + #[cfg(target_os = "linux")] + { + static STATUS: std::sync::OnceLock = std::sync::OnceLock::new(); + *STATUS.get_or_init(nvml::host_status) + } + #[cfg(not(target_os = "linux"))] + { + CounterStatus::Unavailable + } +} + +/// How many times this process has run the real RAPL and NVML liveness +/// probes, `(rapl, nvml)`: for the test that pins their once-per-process +/// caches from a process of its own. Always `(0, 0)` off Linux. +#[doc(hidden)] +pub fn counter_probe_runs() -> (usize, usize) { + #[cfg(target_os = "linux")] + { + use std::sync::atomic::Ordering::SeqCst; + (rapl::REAL_RUNS.load(SeqCst), nvml::REAL_RUNS.load(SeqCst)) + } + #[cfg(not(target_os = "linux"))] + { + (0, 0) + } +} + +/// How many times this process has run the real IOReport liveness probe: +/// for the test that pins [`ioreport_status`]'s once-per-process cache from +/// a process of its own. Always 0 off Apple Silicon macOS, which never +/// probes. +#[doc(hidden)] +pub fn ioreport_probe_runs() -> usize { + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + { + ioreport::REAL_RUNS.load(std::sync::atomic::Ordering::SeqCst) + } + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + { + 0 + } +} + /// One measured span of work. #[derive(Clone, Debug)] pub struct Measurement { @@ -62,6 +322,20 @@ impl Measurement { self.devices.iter().all(|d| d.measured_uj.is_some()) } + /// True when the span has a hardware reading of the CPU: a `Cpu` record + /// that is `HwShunt` with a measured figure. Per class, a span's records + /// are honest as they stand; this is for a caller that grades a whole unit + /// of work by ONE figure and has no field to say the figure covers part of + /// it. A span with no CPU reading — an NVML-only probe set, which reads + /// the GPU alone — is not a reading of work that ran on the CPU, and such + /// a caller must grade it as an estimate ([`PowerMeter::estimate`]), never + /// as a hardware reading of the whole. + pub fn reads_cpu(&self) -> bool { + self.devices + .iter() + .any(|d| d.class == DeviceClass::Cpu && d.provenance == Provenance::HwShunt && d.measured_uj.is_some()) + } + /// Average power over the span, in watts. pub fn avg_watts(&self) -> f64 { let secs = self.wall.as_secs_f64(); @@ -89,6 +363,30 @@ pub enum ProbeError { Read(String), } +/// The statuses the meter's probe selection is gated on: one per vendor +/// counter, each from its own cached probe in production. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Statuses { + ioreport: IoReportStatus, + rapl: CounterStatus, + nvml: CounterStatus, +} + +impl Statuses { + /// This host's: each counter probed once per process and cached. + fn host() -> Statuses { + Statuses { ioreport: ioreport_status(), rapl: rapl_status(), nvml: nvml_status() } + } + + /// No counter usable: the base a test sets one status on. + #[cfg(test)] + const NONE: Statuses = Statuses { + ioreport: IoReportStatus::Unavailable, + rapl: CounterStatus::Unavailable, + nvml: CounterStatus::Unavailable, + }; +} + /// The platform power meter. Detect once, measure many spans. pub struct PowerMeter { probes: Vec>, @@ -98,22 +396,40 @@ pub struct PowerMeter { impl PowerMeter { /// Detect the best available energy sources for this platform. Always /// succeeds: with no hardware source, the meter is an honest estimator. + /// Each vendor counter is included only when its cached status is live: + /// IOReport on Apple Silicon ([`ioreport_status`]), RAPL and NVML on + /// Linux ([`rapl_status`], [`nvml_status`]). The first call in a process + /// pays each probe that applies on this thread: about + /// [`CPU_RAIL_LIVENESS_BUDGET`] of wall time or less, most of it burning + /// CPU, and never much more than five times it; a counter that cannot be + /// opened costs nothing. pub fn detect() -> PowerMeter { + Self::detect_given(Statuses::host()) + } + + /// [`detect`](Self::detect) for given counter statuses, which production + /// takes from the cached status functions and nowhere else: each probe is + /// built only if its gate ([`admits`](liveness::admits), + /// [`counter_admits`](liveness::counter_admits)) passes its status. + /// Separate so the gates are tested for every status without probing. + fn detect_given(statuses: Statuses) -> PowerMeter { let mut probes: Vec> = Vec::new(); #[cfg(all(target_os = "macos", target_arch = "aarch64"))] - if let Ok(p) = ioreport::IoReportProbe::new() { + if let Ok(p) = ioreport::IoReportProbe::given(statuses.ioreport) { probes.push(Box::new(p)); } #[cfg(target_os = "linux")] { - if let Ok(p) = rapl::RaplProbe::new() { + if let Ok(p) = rapl::RaplProbe::given(statuses.rapl) { probes.push(Box::new(p)); } - if let Ok(p) = nvml::NvmlProbe::new() { + if let Ok(p) = nvml::NvmlProbe::given(statuses.nvml) { probes.push(Box::new(p)); } } + // A platform without a given probe has nothing to gate. + let _ = statuses; let tdp_watts = std::env::var("SANDBOX_METER_TDP_W") .ok() @@ -129,7 +445,11 @@ impl PowerMeter { self.probes.iter().map(|p| p.name()).collect() } - /// True when at least one hardware energy source is live. + /// True when at least one hardware energy source is live. This says what + /// the meter *can* read, not what a given span *did*: a span whose CPU + /// counter held comes back `Estimator` even from a hardware meter, so + /// label a span by its own records ([`Measurement::fully_measured`], each + /// record's provenance), never by this. pub fn is_hardware(&self) -> bool { !self.probes.is_empty() } @@ -137,9 +457,15 @@ impl PowerMeter { /// The current cumulative SoC energy in microjoules, summed across every /// live hardware probe and device class — a monotonic counter for callers /// that want before/after *sampling* rather than a managed [`Span`] (e.g. - /// the joule-ui `Meter` edge). `None` when no hardware probe is active: the - /// estimator path has no cumulative counter, and a sampler must say so - /// rather than fabricate one. + /// the joule-ui `Meter` edge). `None` when no hardware probe is active + /// (on Apple Silicon that includes a host whose CPU energy rail is not + /// live — see [`ioreport_status`]): the estimator path has no cumulative + /// counter, and a sampler must say so rather than fabricate one. + /// + /// Being a sum over every class, a before/after difference of this value + /// cannot tell a CPU counter that held from one that advanced; a span that + /// must be a measurement of CPU work should use [`measure`](Self::measure) + /// or [`start`](Self::start), which apply that rule. pub fn sample_uj(&mut self) -> Option { if self.probes.is_empty() { return None; @@ -174,6 +500,20 @@ impl PowerMeter { } } + /// The estimator's figure for `wall` of work: the single `Estimator` CPU + /// record (wall-clock × this meter's TDP, never a measured figure) that a + /// span with no usable hardware reading comes back as. For a caller that + /// must replace a span with no CPU reading by an estimate of the whole + /// ([`Measurement::reads_cpu`]): the figure is the one [`Span::stop`] + /// itself would give. + pub fn estimate(&self, wall: Duration) -> Measurement { + Measurement { + devices: vec![self.estimator_record(wall)], + wall, + source: "estimator", + } + } + fn read_all(&mut self) -> Vec<(usize, Vec<(DeviceClass, u64)>)> { let mut out = Vec::new(); for (i, p) in self.probes.iter_mut().enumerate() { @@ -228,7 +568,17 @@ impl Span { } } - if totals.is_empty() { + // A CPU energy counter that did not advance across a span in which + // this process ran is not a measurement of that span — a frozen rail, + // or one too coarse to resolve it. Then the span has no CPU reading, + // and whatever another class did (a GPU rail that kept compositing) + // is not the energy of this work. The whole span is estimated, + // exactly as when every probe vanished. A probe set with no CPU class + // (NVML alone) is not subject to this. + let cpu_counter_held = totals + .iter() + .any(|(class, uj)| *class == DeviceClass::Cpu && *uj == 0); + if totals.is_empty() || cpu_counter_held { return Measurement { devices: vec![meter.estimator_record(wall)], wall, @@ -277,26 +627,299 @@ mod tests { reading.devices[0].check_provenance().unwrap(); } + fn busy(d: Duration) { + let t = Instant::now(); + let mut x = 0u64; + while t.elapsed() < d { + x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + } + std::hint::black_box(x); + } + + /// The span is the single estimator record — no measured figure anywhere. + fn assert_estimated(r: &Measurement) { + assert_eq!(r.source, "estimator", "{:?}", r.devices); + assert_eq!(r.devices.len(), 1, "one estimator record: {:?}", r.devices); + let d = r.devices[0]; + assert_eq!(d.class, DeviceClass::Cpu); + assert_eq!(d.provenance, Provenance::Estimator); + assert_eq!(d.measured_uj, None); + assert!(!r.fully_measured()); + d.check_provenance().unwrap(); + } + + /// Whatever the host: `detect()` succeeds, and a span is either a + /// hardware measurement whose CPU record advanced or the single estimator + /// record — never a zero CPU reading sealed as a measurement, and never a + /// total that advanced only because another class (a GPU still + /// compositing) did. Asserts the CPU record, not the summed total. #[test] fn detect_never_fails_and_records_validate() { let mut meter = PowerMeter::detect(); - let ((), reading) = meter.measure(|| { - // A short busy spin so hardware counters actually advance. - let t = Instant::now(); - let mut x = 0u64; - while t.elapsed() < Duration::from_millis(120) { - x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); - } - std::hint::black_box(x); - }); + let ((), reading) = meter.measure(|| busy(Duration::from_millis(120))); assert!(!reading.devices.is_empty()); for d in &reading.devices { d.check_provenance().unwrap(); } - // On Apple Silicon / Linux-with-RAPL this is a real measurement. - if meter.is_hardware() { - assert!(reading.fully_measured(), "hardware probe must measure"); - assert!(reading.total_uj() > 0, "a busy span consumes energy"); + if reading.fully_measured() { + assert!(meter.is_hardware(), "only a hardware probe produces a measured record"); + for d in &reading.devices { + assert_eq!(d.provenance, Provenance::HwShunt, "{:?}", reading.devices); + } + if let Some(cpu) = reading.devices.iter().find(|d| d.class == DeviceClass::Cpu) { + assert!( + cpu.measured_uj.is_some_and(|uj| uj > 0), + "a measured CPU record read zero: {cpu:?}" + ); + } + } else { + assert_estimated(&reading); + } + if !meter.is_hardware() { + assert_estimated(&reading); + } + // On Apple Silicon the IOReport probe exists exactly when its CPU rail + // is live (the probe's own tests pin that against an oracle). + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + assert_eq!( + meter.is_hardware(), + ioreport_status() == IoReportStatus::Live, + "hardware={} status={:?}", + meter.is_hardware(), + ioreport_status() + ); + } + + /// A probe that replays scripted cumulative snapshots: the rail behaviour + /// measured on real hosts, deterministic on every platform. + struct Scripted { + name: &'static str, + snaps: std::collections::VecDeque>, + } + + impl Probe for Scripted { + fn name(&self) -> &'static str { + self.name + } + fn snapshot(&mut self) -> Result, ProbeError> { + self.snaps + .pop_front() + .ok_or_else(|| ProbeError::Read("script exhausted".into())) } } + + pub(super) fn scripted( + name: &'static str, + before: &[(DeviceClass, u64)], + after: &[(DeviceClass, u64)], + ) -> Box { + Box::new(Scripted { + name, + snaps: [before.to_vec(), after.to_vec()].into(), + }) + } + + pub(super) fn meter_of(probes: Vec>) -> PowerMeter { + PowerMeter { + probes, + tdp_watts: 10.0, + } + } + + use DeviceClass::{Cpu, Gpu, Npu}; + + /// The M5 Max on macOS 27.0.1, from a channel dump over 3 s of burn: + /// `CPU Energy` and `ANE0` held, `GPU Energy` advanced. main sealed this as + /// Cpu 0 µJ, Npu 0 µJ and Gpu 682 543 µJ, all `HwShunt` — GPU energy passed + /// off as the energy of the work. + #[test] + fn a_span_whose_cpu_counter_held_is_estimated_even_though_the_gpu_advanced() { + let before = [(Cpu, 480_083_160_000), (Npu, 191_479_000), (Gpu, 97_187_370_334)]; + let after = [(Cpu, 480_083_160_000), (Npu, 191_479_000), (Gpu, 97_188_052_877)]; + let mut m = meter_of(vec![scripted("ioreport-energy-model", &before, &after)]); + let ((), r) = m.measure(|| ()); + assert_estimated(&r); + } + + /// The M3 Ultra on macOS 27.0.1: every rail held, both dies' CPU rails + /// included. main sealed three measured zeros. + #[test] + fn a_span_where_every_counter_held_is_estimated() { + let snap = [(Cpu, 18_849_831_000), (Npu, 21_704_000), (Gpu, 221_414_992)]; + let mut m = meter_of(vec![scripted("ioreport-energy-model", &snap, &snap)]); + let ((), r) = m.measure(|| ()); + assert_estimated(&r); + } + + /// The rule is about the CPU counter, not about IOReport: a RAPL package + /// counter that held is no measurement either, and it takes a GPU that + /// advanced on another probe down with it — the span has no CPU reading. + #[test] + fn a_held_cpu_counter_estimates_the_whole_composite_span() { + let rapl = scripted("rapl-powercap", &[(Cpu, 5_000)], &[(Cpu, 5_000)]); + let nvml = scripted("nvml-total-energy", &[(Gpu, 9_000)], &[(Gpu, 12_000)]); + let mut m = meter_of(vec![rapl, nvml]); + let ((), r) = m.measure(|| ()); + assert_estimated(&r); + } + + /// A CPU counter that advanced is a measurement, per class. A class that + /// held beside it is left as it is (see the rule's scope in `Span::stop`). + #[test] + fn a_span_whose_cpu_counter_advanced_is_measured() { + let before = [(Cpu, 1_000), (Gpu, 7_000)]; + let after = [(Cpu, 2_234), (Gpu, 7_000)]; + let mut m = meter_of(vec![scripted("ioreport-energy-model", &before, &after)]); + let ((), r) = m.measure(|| ()); + assert!(r.fully_measured(), "{:?}", r.devices); + assert_eq!(r.source, "ioreport-energy-model"); + let cpu = r.devices.iter().find(|d| d.class == Cpu).unwrap(); + assert_eq!(cpu.provenance, Provenance::HwShunt); + assert_eq!(cpu.measured_uj, Some(1_234)); + for d in &r.devices { + d.check_provenance().unwrap(); + } + } + + /// A probe set with no CPU class (NVML alone) measures its GPU as before. + #[test] + fn a_probe_set_without_a_cpu_class_is_not_subject_to_the_cpu_rule() { + let nvml = scripted("nvml-total-energy", &[(Gpu, 9_000)], &[(Gpu, 12_000)]); + let mut m = meter_of(vec![nvml]); + let ((), r) = m.measure(|| ()); + assert!(r.fully_measured(), "{:?}", r.devices); + assert_eq!(r.devices.len(), 1); + assert_eq!(r.devices[0].class, Gpu); + assert_eq!(r.devices[0].measured_uj, Some(3_000)); + } + + /// `reads_cpu` is true only for a span with a hardware CPU reading. An + /// NVML-only span (a GPU reading, no CPU record) is not one, and neither + /// is the estimator's record. `estimate` is the very record `Span::stop` + /// gives a span with no reading. + #[test] + fn a_span_reads_the_cpu_only_with_a_hardware_cpu_record() { + let nvml = scripted("nvml-total-energy", &[(Gpu, 9_000)], &[(Gpu, 12_000)]); + let mut m = meter_of(vec![nvml]); + let ((), gpu_only) = m.measure(|| ()); + assert!(gpu_only.fully_measured(), "the GPU record stands as a reading: {:?}", gpu_only.devices); + assert!(!gpu_only.reads_cpu(), "a GPU-only span read the CPU: {:?}", gpu_only.devices); + + let mut m = meter_of(vec![scripted("rapl-powercap", &[(Cpu, 1_000)], &[(Cpu, 2_000)])]); + let ((), cpu) = m.measure(|| ()); + assert!(cpu.reads_cpu(), "{:?}", cpu.devices); + + let mut none = meter_of(Vec::new()); + let ((), est) = none.measure(|| ()); + assert!(!est.reads_cpu(), "the estimator's record is no reading: {:?}", est.devices); + let again = none.estimate(est.wall); + assert_eq!(again.devices, est.devices, "estimate() is the record Span::stop gives"); + assert_eq!(again.source, "estimator"); + let e = none.estimate(Duration::from_millis(8)); + assert_estimated(&e); + assert_eq!(e.devices[0].estimated_uj, 80_000, "8 ms at 10 W"); + } + + /// Selection goes through the one gate: an injected status that is not + /// `Live` never selects the IOReport probe, and `Live` does wherever its + /// subscription opens with a CPU rail (both Apple Silicon hosts here). + /// Nothing here probes: the status is given. + #[test] + fn detect_selects_the_ioreport_probe_only_for_a_live_status() { + let selects = |status| { + PowerMeter::detect_given(Statuses { ioreport: status, ..Statuses::NONE }) + .sources() + .contains(&"ioreport-energy-model") + }; + for status in [IoReportStatus::Unavailable, IoReportStatus::CpuRailAbsent, IoReportStatus::CpuRailFrozen] { + assert!(!selects(status), "{status:?} selected the IOReport probe"); + } + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + assert_eq!( + selects(IoReportStatus::Live), + ioreport::energy_model_has_a_cpu_rail(), + "a Live status must select the probe wherever its subscription opens with a CPU rail" + ); + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + assert!(!selects(IoReportStatus::Live), "no IOReport probe exists off Apple Silicon macOS"); + } + + /// RAPL and NVML each go through their own gate: a status that is not + /// `Live` never selects its probe, wherever the counter exists, and one + /// live counter does not admit the other. `Live` selects each wherever it + /// opens — on Linux; neither exists elsewhere. Nothing here probes: the + /// statuses are given. + #[test] + fn detect_selects_rapl_and_nvml_only_for_a_live_status() { + use CounterStatus::{Frozen, Live, Unavailable}; + let sources = |rapl, nvml| PowerMeter::detect_given(Statuses { rapl, nvml, ..Statuses::NONE }).sources(); + for status in [Unavailable, Frozen] { + let s = sources(status, status); + assert!(!s.contains(&"rapl-powercap") && !s.contains(&"nvml-total-energy"), "{status:?} selected {s:?}"); + let s = sources(Live, status); + assert!(!s.contains(&"nvml-total-energy"), "NVML {status:?} selected: {s:?}"); + let s = sources(status, Live); + assert!(!s.contains(&"rapl-powercap"), "RAPL {status:?} selected: {s:?}"); + } + let live = sources(Live, Live); + #[cfg(target_os = "linux")] + { + assert_eq!(live.contains(&"rapl-powercap"), rapl::RaplProbe::open().is_ok(), "{live:?}"); + assert_eq!(live.contains(&"nvml-total-energy"), nvml::NvmlProbe::open().is_ok(), "{live:?}"); + } + #[cfg(not(target_os = "linux"))] + assert!(live.is_empty(), "no RAPL or NVML probe exists off Linux: {live:?}"); + } + + /// The counter statuses agree with themselves, and off Linux there is no + /// RAPL or NVML to probe. + #[test] + fn the_counter_statuses_explain_themselves() { + assert_eq!(rapl_status(), rapl_status()); + assert_eq!(nvml_status(), nvml_status()); + #[cfg(not(target_os = "linux"))] + { + assert_eq!((rapl_status(), nvml_status()), (CounterStatus::Unavailable, CounterStatus::Unavailable)); + assert_eq!(counter_probe_runs(), (0, 0), "no RAPL or NVML probe runs off Linux"); + } + assert!(CounterStatus::Frozen.to_string().contains("kept advancing")); + } + + #[test] + fn the_ioreport_status_explains_itself() { + // Off Apple Silicon macOS there is no IOReport to probe. + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + assert_eq!(ioreport_status(), IoReportStatus::Unavailable); + // It agrees with itself. That the probe behind it runs once per + // process is pinned by `the_liveness_probe_runs_once_per_process`, + // which counts the real probe's runs. + assert_eq!(ioreport_status(), ioreport_status()); + assert!(IoReportStatus::CpuRailFrozen.to_string().contains("CPU Energy")); + assert!(IoReportStatus::CpuRailAbsent.to_string().contains("CPU Energy")); + } + + /// [`counter_liveness`] holds a caller's own counter to the rule, on + /// every rail: two rails advancing at every sample are `Live`; one rail + /// holding beside one advancing is `Frozen`; a counter that does not + /// open is `Unavailable` and never ticked. + #[test] + fn counter_liveness_holds_a_callers_counter_to_the_every_rail_rule() { + let tick = || std::thread::sleep(Duration::from_millis(25)); + let n = std::cell::Cell::new(0i64); + let both = || { + n.set(n.get() + 1); + Some(vec![("package-0".to_string(), n.get()), ("package-1".to_string(), 3 * n.get())]) + }; + assert_eq!(counter_liveness(|| Some(both), tick), CounterStatus::Live); + let m = std::cell::Cell::new(0i64); + let one_held = || { + m.set(m.get() + 1); + Some(vec![("package-0".to_string(), m.get()), ("package-1".to_string(), 7)]) + }; + assert_eq!(counter_liveness(|| Some(one_held), tick), CounterStatus::Frozen); + let ticks = std::cell::Cell::new(0); + type Sampler = fn() -> Option>; + let none = counter_liveness(|| None::, || ticks.set(ticks.get() + 1)); + assert_eq!((none, ticks.get()), (CounterStatus::Unavailable, 0)); + } } diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs index edd4f20f8..af06a2d35 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/nvml.rs @@ -2,11 +2,33 @@ //! (`nvmlDeviceGetTotalEnergyConsumption`, millijoules since driver load), //! loaded at runtime from `libnvidia-ml.so.1` — no link-time dependency, the //! probe simply doesn't exist on machines without the driver. `HwShunt`. +//! +//! **Used only when every GPU's counter is live.** A counter that answers is +//! not yet a reading of the work: one that holds, or is republished in rare +//! lumps, would seal zero or a stale refresh as a span's GPU energy. So +//! [`NvmlProbe::given`] declines unless [`nvml_status`](super::nvml_status) is +//! `Live` — every supported GPU's raw counter advanced in at least five +//! sample intervals, at least four in five of those observed, over at least +//! 300 ms, while the probing thread burned CPU — and the meter has no GPU +//! reading. Every GPU must be live, not one: the probe sums them into one +//! `Gpu` figure. The probe samples at [`NVML_PROBE_TICK`] (150 ms), because a +//! GPU energy counter is published more coarsely than a CPU rail. The rule +//! and the run live in [`liveness`](super::liveness) ([`counter_run`]); this +//! module only opens the devices and reads their raw integers. +//! +//! A GPU-only figure is the GPU's energy, not the work's: the meter's span +//! rule (a CPU counter that held estimates the span) does not apply to a +//! probe set with no CPU class, so a caller that grades a whole unit of work +//! by one figure must not grade a span with no CPU reading as a reading of +//! that work ([`Measurement::reads_cpu`](super::Measurement::reads_cpu)). -use super::{Probe, ProbeError}; +use super::liveness::{counter_run, counter_status_of, counter_tick, gated, RailWatch, Sample, NVML_PROBE_TICK}; +use super::{CounterStatus, Probe, ProbeError, CPU_RAIL_LIVENESS_BUDGET}; use libloading::{Library, Symbol}; use sandbox_schema::DeviceClass; use std::ffi::c_void; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Instant; type NvmlReturn = i32; type Device = *mut c_void; @@ -21,8 +43,48 @@ pub struct NvmlProbe { // the probe is used from one thread at a time by the meter. unsafe impl Send for NvmlProbe {} +/// How many times this process ran the real NVML liveness probe: bumped by +/// [`host_status`] before anything else it does, read through +/// [`counter_probe_runs`](super::counter_probe_runs). +pub(super) static REAL_RUNS: AtomicUsize = AtomicUsize::new(0); + +/// This host's NVML status from one real probe run: the value +/// [`nvml_status`](super::nvml_status) caches, and its only source. One +/// composition with no branch of its own (count the run, run the real probe, +/// map its verdict). Call [`nvml_status`](super::nvml_status), not this. +pub(super) fn host_status() -> CounterStatus { + REAL_RUNS.fetch_add(1, Ordering::SeqCst); + counter_status_of(real_run().verdict()) +} + +/// The real probe run: [`counter_run`] over every supported GPU's raw +/// counter, a tick that burns [`NVML_PROBE_TICK`] of this thread's CPU time +/// and a monotonic clock, for [`CPU_RAIL_LIVENESS_BUDGET`]. A host with no +/// NVML, or no GPU that supports the counter, does not open, and is never +/// ticked. +fn real_run() -> RailWatch { + let epoch = Instant::now(); + counter_run( + || NvmlProbe::open().ok().map(|p| move || p.raw_sample()), + || counter_tick(NVML_PROBE_TICK), + move || epoch.elapsed(), + CPU_RAIL_LIVENESS_BUDGET, + ) +} + impl NvmlProbe { - pub fn new() -> Result { + /// The probe the meter uses, built for this host's NVML `status`, which + /// production passes from the cached [`nvml_status`](super::nvml_status) + /// and nowhere else: **only when every GPU's counter is live** + /// ([`gated`]). Otherwise `Unavailable`, naming why. + pub fn given(status: CounterStatus) -> Result { + gated("NVML", status, Self::open) + } + + /// Load NVML and every GPU that supports the energy counter, with no + /// liveness gate. Used by the liveness probe itself and by + /// [`given`](Self::given) once the gate passed. + pub(super) fn open() -> Result { unsafe { let lib = Library::new("libnvidia-ml.so.1") .map_err(|e| ProbeError::Unavailable(format!("no NVML: {e}")))?; @@ -71,6 +133,23 @@ impl NvmlProbe { }) } } + + /// One absolute sample for the liveness rule: every supported GPU's raw + /// counter (mJ), named by its position, in device order, as read — never + /// summed. `None` if any device could not be read. + fn raw_sample(&self) -> Option { + self.devices + .iter() + .enumerate() + .map(|(i, d)| { + let mut mj = 0u64; + // SAFETY: `d` is a handle NVML returned for the library this + // probe keeps loaded, and `mj` is writable for the call. + let rc = unsafe { (self.get_energy)(*d, &mut mj) }; + (rc == 0).then(|| (format!("nvml:{i}"), i64::try_from(mj).unwrap_or(i64::MAX))) + }) + .collect() + } } impl Probe for NvmlProbe { @@ -91,3 +170,52 @@ impl Probe for NvmlProbe { Ok(vec![(DeviceClass::Gpu, total_mj.saturating_mul(1_000))]) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::probe::nvml_status; + + /// The gate declines every status but `Live`, without loading NVML, and + /// `Live` builds the probe wherever NVML opens with a supported GPU. + #[test] + fn only_a_live_status_builds_the_nvml_probe() { + for status in [CounterStatus::Unavailable, CounterStatus::Frozen] { + match NvmlProbe::given(status) { + Err(ProbeError::Unavailable(why)) => assert!(why.contains("NVML"), "{why}"), + Err(e) => panic!("{status:?}: {e}"), + Ok(_) => panic!("{status:?} built the NVML probe"), + } + } + assert_eq!(NvmlProbe::given(CounterStatus::Live).is_ok(), NvmlProbe::open().is_ok()); + } + + /// Against an independent oracle that reads every GPU's ABSOLUTE counter + /// before and after three budgets, with no rule of its own: the NVML rung + /// is never selected over a GPU whose counter did not move, and a selected + /// rung's GPUs all move (an idle GPU still draws power). A host with no + /// NVML must report `Unavailable`. It logs what it observed. + #[test] + fn the_nvml_rung_is_never_selected_over_a_counter_that_held() { + let status = nvml_status(); + let selected = crate::probe::PowerMeter::detect().sources().contains(&"nvml-total-energy"); + let Ok(p) = NvmlProbe::open() else { + eprintln!("nvml: status={status:?} oracle=no NVML device selected={selected}"); + assert_eq!(status, CounterStatus::Unavailable); + assert!(!selected); + return; + }; + let before = p.raw_sample().expect("oracle read"); + std::thread::sleep(CPU_RAIL_LIVENESS_BUDGET * 3); + let after = p.raw_sample().expect("oracle read"); + let every_moved = before.iter().zip(&after).all(|((_, b), (_, a))| a != b); + eprintln!("nvml: status={status:?} oracle every_moved={every_moved} selected={selected} {before:?} -> {after:?}"); + assert_eq!(selected, status == CounterStatus::Live, "selection disagrees with the status"); + if !every_moved { + assert_ne!(status, CounterStatus::Live, "NVML live although a GPU counter held"); + } + if status == CounterStatus::Live { + assert!(every_moved, "NVML live, yet the oracle saw a GPU hold"); + } + } +} diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/rapl.rs b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/rapl.rs index ecf5d9f31..e41d77a05 100644 --- a/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/rapl.rs +++ b/sandbox/sandbox-rs/crates/sandbox-meter/src/probe/rapl.rs @@ -1,14 +1,66 @@ //! Intel/AMD RAPL via the Linux powercap interface — package-domain -//! cumulative energy from `/sys/class/powercap/intel-rapl:N/energy_uj`, -//! wrap-corrected against `max_energy_range_uj`. `HwShunt` provenance. +//! cumulative energy from `/sys/class/powercap/intel-rapl:N/energy_uj`, for +//! every top-level zone whose `name` is `package-`, each wrap-corrected +//! against its own `max_energy_range_uj`. `HwShunt` provenance. +//! +//! **The package zones and no other.** A top-level powercap zone is not +//! necessarily a package. Many Intel hosts expose `psys`, the platform zone, +//! whose count already includes the packages', so summing it beside them +//! counts their energy twice; some expose a top-level `dram` zone, which is not +//! CPU energy at all. The `Cpu` figure is the sum of every zone whose `name` +//! file reads `package-` — every socket's — and nothing else: not +//! `psys`, not `dram`, not a package's subzones (`intel-rapl:0:0`, its `core`, +//! `uncore` or `dram`), and not the second interface to the same packages +//! (`intel-rapl-mmio:N`). Every package or none: a package zone whose counter +//! or wrap range cannot be read, or a top-level zone whose `name` cannot be +//! read (it may be a package), does not open the probe, rather than leave a +//! package out of a figure that claims the CPU. +//! +//! **Used only when every package counter is live.** A readable `energy_uj` +//! is not yet a reading of the work: a counter that holds, or that is +//! republished in rare lumps, would seal zero or a stale refresh as the energy +//! of a span. So [`RaplProbe::given`] declines unless +//! [`rapl_status`](super::rapl_status) is `Live` — every package zone's raw +//! `energy_uj` advanced in at least five sample intervals, at least four in +//! five of those observed, over at least 300 ms, while the probing thread +//! burned CPU — and the meter falls back to its estimator. Every package must +//! be live, not one: the probe sums them into one `Cpu` figure, and a package +//! whose counter held is a domain that figure would claim without reading it. +//! Only the package zones are probed, as only they are summed: a live `psys` +//! beside a held package does not make the counter live. The rule and the run +//! live in [`liveness`](super::liveness) ([`counter_run`]); this module only +//! opens the package zones and reads their raw integers. +//! +//! Built everywhere: the zone listing, the reads and the run take the +//! powercap root as a path, so the tests, and a caller's +//! [`RaplPackages::live_in`], run them on fake trees on any platform. Only +//! Linux opens the sysfs tree ([`RaplPackages::host`], the meter's probe). +//! [`RaplPackages`] is this probe for a caller outside the crate that takes +//! its own before/after differences. -use super::{Probe, ProbeError}; +use super::liveness::{counter_run, counter_status_of, gated, RailWatch, Sample}; +#[cfg(target_os = "linux")] +use super::liveness::{counter_tick, PROBE_TICK}; +#[cfg(target_os = "linux")] +use super::CounterStatus; +use super::{Probe, ProbeError, CPU_RAIL_LIVENESS_BUDGET}; use sandbox_schema::DeviceClass; use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +#[cfg(target_os = "linux")] +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Instant; + +/// The powercap tree the production probe reads. +#[cfg(target_os = "linux")] +const POWERCAP: &str = "/sys/class/powercap"; struct Domain { + /// The package zone's directory name, `intel-rapl:N`: its rail name for + /// the liveness rule. + name: String, energy_path: PathBuf, + /// The value this zone's counter wraps at, read at open: non-zero. max_range_uj: u64, last_raw: u64, /// Accumulated energy since probe creation, wrap-corrected. @@ -17,28 +69,116 @@ struct Domain { pub struct RaplProbe { domains: Vec, + /// Why the cumulative count can no longer be trusted, once a package's + /// delta could not be known: every snapshot from then on fails, so every + /// span is estimated rather than one missing that energy. + broken: Option, +} + +/// How many times this process ran the real RAPL liveness probe: bumped by +/// [`host_status`] before anything else it does, read through +/// [`counter_probe_runs`](super::counter_probe_runs). +#[cfg(target_os = "linux")] +pub(super) static REAL_RUNS: AtomicUsize = AtomicUsize::new(0); + +/// This host's RAPL status from one real probe run: the value +/// [`rapl_status`](super::rapl_status) caches, and its only source. One +/// composition with no branch of its own (count the run, run the real probe, +/// map its verdict). Call [`rapl_status`](super::rapl_status), not this. +#[cfg(target_os = "linux")] +pub(super) fn host_status() -> CounterStatus { + REAL_RUNS.fetch_add(1, Ordering::SeqCst); + counter_status_of(real_run().verdict()) +} + +/// The real probe run: [`run_over`] the host's powercap tree with a tick that +/// burns [`PROBE_TICK`] of this thread's CPU time. +#[cfg(target_os = "linux")] +fn real_run() -> RailWatch { + run_over(Path::new(POWERCAP), || counter_tick(PROBE_TICK)) +} + +/// [`counter_run`] over the package zones' raw counters under `root`, the +/// given `tick` and a monotonic clock, for [`CPU_RAIL_LIVENESS_BUDGET`]. A +/// tree that does not open (absent, unreadable, no package zone) is never +/// ticked. +fn run_over(root: &Path, tick: impl FnMut()) -> RailWatch { + let epoch = Instant::now(); + counter_run( + || RaplProbe::open_in(root).ok().map(|p| move || p.raw_sample()), + tick, + move || epoch.elapsed(), + CPU_RAIL_LIVENESS_BUDGET, + ) +} + +/// Whether a powercap zone's `name` file names a CPU package: `package-`, +/// `` one or more ASCII digits, surrounding whitespace allowed. Not `psys`, +/// `dram`, `core`, `uncore`, or anything else. +fn is_package_name(name: &str) -> bool { + name.trim() + .strip_prefix("package-") + .is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit())) +} + +/// Whether a powercap directory is a top-level RAPL zone, `intel-rapl:`: +/// not the `intel-rapl` control type itself, a subzone (`intel-rapl:0:0`), or +/// another interface (`intel-rapl-mmio:0`). +fn is_top_level_zone(dir_name: &str) -> bool { + dir_name + .strip_prefix("intel-rapl:") + .is_some_and(|n| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit())) } impl RaplProbe { - pub fn new() -> Result { - let base = PathBuf::from("/sys/class/powercap"); - let entries = fs::read_dir(&base) - .map_err(|e| ProbeError::Unavailable(format!("no powercap: {e}")))?; + /// The probe the meter uses, built for this host's RAPL `status`, which + /// production passes from the cached [`rapl_status`](super::rapl_status) + /// and nowhere else: **only when every package counter is live** + /// ([`gated`]). Otherwise `Unavailable`, naming why, and the + /// meter estimates instead. + #[cfg(target_os = "linux")] + pub fn given(status: CounterStatus) -> Result { + gated("RAPL", status, Self::open) + } + + /// Open every package zone of the host's powercap tree, with no liveness + /// gate. Used by the liveness probe itself and by [`given`](Self::given) + /// once the gate passed. + #[cfg(target_os = "linux")] + pub(super) fn open() -> Result { + Self::open_in(Path::new(POWERCAP)) + } + + /// Open every package zone of the powercap tree at `root`: each top-level + /// `intel-rapl:` zone whose `name` is `package-`, with its readable + /// `energy_uj` and non-zero `max_energy_range_uj`. `Unavailable` when + /// there is no such zone, and when a package zone, or a top-level zone + /// that may be one, cannot be read whole. + fn open_in(root: &Path) -> Result { + let entries = fs::read_dir(root).map_err(|e| ProbeError::Unavailable(format!("no powercap: {e}")))?; let mut domains = Vec::new(); for entry in entries.flatten() { let name = entry.file_name().to_string_lossy().into_owned(); - // Top-level packages only: "intel-rapl:0", "intel-rapl:1", … - // (sub-domains like :0:0 double-count the package figure). - if !name.starts_with("intel-rapl:") || name.matches(':').count() != 1 { + if !is_top_level_zone(&name) { continue; } let dir = entry.path(); + // A zone that cannot say what it is may be a package. + let zone = fs::read_to_string(dir.join("name")) + .map_err(|e| ProbeError::Unavailable(format!("RAPL zone {name} has no readable name: {e}")))?; + if !is_package_name(&zone) { + continue; // psys, a top-level dram zone, ...: not the packages' energy + } let energy_path = dir.join("energy_uj"); - let Ok(raw) = read_u64(&energy_path) else { - continue; // unreadable without privileges on some kernels - }; - let max_range_uj = read_u64(&dir.join("max_energy_range_uj")).unwrap_or(u64::MAX); + // Unreadable without privileges on many kernels; a package left out + // would leave its energy out of a figure that claims the CPU. + let raw = read_u64(&energy_path).map_err(|e| ProbeError::Unavailable(format!("RAPL package {name}: {e}")))?; + let max_range_uj = read_u64(&dir.join("max_energy_range_uj")) + .ok() + .filter(|&r| r > 0) + .ok_or_else(|| ProbeError::Unavailable(format!("RAPL package {name}: no readable wrap range")))?; domains.push(Domain { + name, energy_path, max_range_uj, last_raw: raw, @@ -46,10 +186,103 @@ impl RaplProbe { }); } if domains.is_empty() { - return Err(ProbeError::Unavailable("no readable RAPL package domains".into())); + return Err(ProbeError::Unavailable("no RAPL package zone".into())); } - Ok(RaplProbe { domains }) + // A stable rail order, so consecutive samples pair domain by domain. + domains.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(RaplProbe { domains, broken: None }) } + + /// One absolute sample for the liveness rule: every package domain's name + /// and raw `energy_uj`, in domain order, as read — never wrap-corrected or + /// summed (a wrap reads as a fall, which is not an advance). `None` if any + /// domain could not be read. + fn raw_sample(&self) -> Option { + self.domains + .iter() + .map(|d| read_u64(&d.energy_path).ok().map(|raw| (d.name.clone(), raw_i64(raw)))) + .collect() + } +} + +/// The RAPL package counters as one cumulative CPU energy counter, for a +/// caller that takes its own before/after differences rather than metering +/// spans through [`PowerMeter`](super::PowerMeter) — the S5 executor's +/// `RaplMeter`. It is the probe the meter itself uses, under the same rules: +/// every top-level zone whose `name` is `package-` and nothing else (not +/// `psys`, `dram`, subzones or `intel-rapl-mmio`), every package or none, +/// each wrap-corrected at its own `max_energy_range_uj`, a package with no +/// readable range never opened, and a fall from above a package's range +/// breaking the count for good. There is no ungated constructor: it opens only +/// once every package counter has proved live. +pub struct RaplPackages { + probe: RaplProbe, +} + +impl RaplPackages { + /// This host's package zones (`/sys/class/powercap`), only when the + /// cached [`rapl_status`](super::rapl_status) is `Live`: every package + /// counter kept advancing while the probing thread burned CPU. Otherwise + /// `Unavailable`, naming why; always so off Linux. + pub fn host() -> Result { + #[cfg(target_os = "linux")] + { + RaplProbe::given(super::rapl_status()).map(|probe| RaplPackages { probe }) + } + #[cfg(not(target_os = "linux"))] + { + Err(ProbeError::Unavailable("RAPL: no powercap off Linux".into())) + } + } + + /// The package zones of the powercap tree at `root`, only when every one + /// proves live: the liveness run [`rapl_status`](super::rapl_status) makes + /// over the host's tree, made here over `root`, with `tick` driving the + /// load between samples (production's tick burns CPU, + /// [`counter_liveness_tick`](super::counter_liveness_tick)). For a tree + /// other than the host's sysfs: a test's fake tree. Not cached; the run + /// takes about 300 ms over live counters and the whole + /// [`CPU_RAIL_LIVENESS_BUDGET`] over a held one. + pub fn live_in(root: &Path, tick: impl FnMut()) -> Result { + let status = counter_status_of(run_over(root, tick).verdict()); + gated("RAPL", status, || RaplProbe::open_in(root)).map(|probe| RaplPackages { probe }) + } + + /// The package zones summed, by directory name (`intel-rapl:`), in + /// order. + pub fn zones(&self) -> Vec<&str> { + self.probe.domains.iter().map(|d| d.name.as_str()).collect() + } + + /// The packages' cumulative energy since this opened, in microjoules: + /// each package's deltas, wrap-corrected at its own range, summed. Every + /// package is read before any is updated: a package whose counter does + /// not read fails the read and changes nothing. Once a package's delta + /// could not be known (a fall from above its range), this read and every + /// later one fail: a difference taken across that read would come back + /// short of the energy that went uncounted. + pub fn cumulative_uj(&mut self) -> Result { + Ok(self.probe.snapshot()?.iter().map(|(_, uj)| *uj).sum()) + } +} + +/// One package's energy between two reads, modulo its own wrap: a rise is the +/// rise, and a fall wrapped at `max_range_uj`, the energy left to the wrap +/// plus what it counted since. `None` for a fall from above the range: the +/// counter contradicts its declared range, and its delta is unknown. Cannot +/// see more than one wrap between reads — minutes at package power. +fn zone_delta(last_raw: u64, raw: u64, max_range_uj: u64) -> Option { + if raw >= last_raw { + return Some(raw - last_raw); + } + max_range_uj.checked_sub(last_raw)?.checked_add(raw) +} + +/// A raw counter as the rule's signed integer, saturating: `energy_uj` never +/// approaches `i64::MAX` (its range is far below it), and a value that did +/// would read as held, never as an advance. +fn raw_i64(raw: u64) -> i64 { + i64::try_from(raw).unwrap_or(i64::MAX) } fn read_u64(p: &std::path::Path) -> Result { @@ -65,15 +298,26 @@ impl Probe for RaplProbe { "rapl-powercap" } + /// The packages' cumulative energy since the probe opened: each package's + /// deltas, wrap-corrected at its own range, summed. A package whose delta + /// cannot be known breaks the count for good — this read and every later + /// one fail, and the meter estimates — because a span bracketing that + /// read from both sides would otherwise come back measured, short of the + /// energy that went uncounted. fn snapshot(&mut self) -> Result, ProbeError> { + if let Some(why) = &self.broken { + return Err(ProbeError::Read(why.clone())); + } + let raws = self.domains.iter().map(|d| read_u64(&d.energy_path)).collect::, _>>()?; let mut total = 0u64; - for d in &mut self.domains { - let raw = read_u64(&d.energy_path)?; - let delta = if raw >= d.last_raw { - raw - d.last_raw - } else { - // Counter wrapped within its declared range. - d.max_range_uj.saturating_sub(d.last_raw).saturating_add(raw) + for (d, raw) in self.domains.iter_mut().zip(raws) { + let Some(delta) = zone_delta(d.last_raw, raw, d.max_range_uj) else { + let why = format!( + "RAPL package {} fell from {} to {} µJ, above its {} µJ wrap range", + d.name, d.last_raw, raw, d.max_range_uj + ); + self.broken = Some(why.clone()); + return Err(ProbeError::Read(why)); }; d.accumulated_uj = d.accumulated_uj.saturating_add(delta); d.last_raw = raw; @@ -82,3 +326,397 @@ impl Probe for RaplProbe { Ok(vec![(DeviceClass::Cpu, total)]) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::probe::liveness::counter_status_of; + #[cfg(target_os = "linux")] + use crate::probe::rapl_status; + use crate::probe::{CounterStatus, PowerMeter}; + use crate::Provenance; + use std::time::Duration; + + /// The gate declines every status but `Live`, without touching powercap, + /// and `Live` builds the probe wherever powercap opens. + #[cfg(target_os = "linux")] + #[test] + fn only_a_live_status_builds_the_rapl_probe() { + for status in [CounterStatus::Unavailable, CounterStatus::Frozen] { + match RaplProbe::given(status) { + Err(ProbeError::Unavailable(why)) => assert!(why.contains("RAPL"), "{why}"), + Err(e) => panic!("{status:?}: {e}"), + Ok(_) => panic!("{status:?} built the RAPL probe"), + } + } + assert_eq!(RaplProbe::given(CounterStatus::Live).is_ok(), RaplProbe::open().is_ok()); + } + + /// A raw sample is every domain's own integer, named, in a stable order. + #[test] + fn a_raw_sample_is_each_domains_integer() { + assert_eq!(raw_i64(262_143_328_850), 262_143_328_850); + assert_eq!(raw_i64(u64::MAX), i64::MAX, "saturates, never wraps negative"); + #[cfg(target_os = "linux")] + if let Ok(p) = RaplProbe::open() { + let s = p.raw_sample().expect("a readable domain reads"); + assert_eq!(s.len(), p.domains.len()); + assert!(s.windows(2).all(|w| w[0].0 < w[1].0), "domain order: {s:?}"); + } + } + + #[cfg(target_os = "linux")] + fn burn(d: Duration) { + let t = Instant::now(); + let mut x = 0u64; + while t.elapsed() < d { + x = x.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + } + std::hint::black_box(x); + } + + /// The property that matters, against an independent oracle that reads + /// every package's ABSOLUTE `energy_uj` before and after three budgets of + /// burn, with no rule of its own: the RAPL rung is never selected over a + /// package whose counter did not move, and a selected rung's packages all + /// move. A host with no readable powercap must report `Unavailable`. It + /// logs what it observed, so a CI log shows which branch was asserted. + #[cfg(target_os = "linux")] + #[test] + fn the_rapl_rung_is_never_selected_over_a_counter_that_held() { + let status = rapl_status(); + let selected = crate::probe::PowerMeter::detect().sources().contains(&"rapl-powercap"); + let Ok(p) = RaplProbe::open() else { + eprintln!("rapl: status={status:?} oracle=no readable powercap selected={selected}"); + assert_eq!(status, CounterStatus::Unavailable); + assert!(!selected); + return; + }; + let before = p.raw_sample().expect("oracle read"); + burn(CPU_RAIL_LIVENESS_BUDGET * 3); + let after = p.raw_sample().expect("oracle read"); + let every_moved = before.iter().zip(&after).all(|((_, b), (_, a))| a != b); + eprintln!("rapl: status={status:?} oracle every_moved={every_moved} selected={selected} {before:?} -> {after:?}"); + assert_eq!(selected, status == CounterStatus::Live, "selection disagrees with the status"); + if !every_moved { + assert_ne!(status, CounterStatus::Live, "RAPL live although a package counter held"); + } + if status == CounterStatus::Live { + assert!(every_moved, "RAPL live, yet the oracle saw a package hold"); + } + } + + /// A fake powercap tree in the OS temp directory, removed when dropped: + /// zone directories each with a `name`, an `energy_uj` and a + /// `max_energy_range_uj`, as sysfs lays them out. + struct FakePowercap { + root: PathBuf, + } + + /// One zone of a [`FakePowercap`]: its directory, its `name` file (`None`: + /// no such file), its `energy_uj`, and its `max_energy_range_uj` (`None`: + /// no such file). + type FakeZone<'a> = (&'a str, Option<&'a str>, &'a str, Option<&'a str>); + + impl FakePowercap { + fn with_zones(zones: &[FakeZone<'_>]) -> Self { + static N: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let n = N.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let root = std::env::temp_dir().join(format!("sandbox-meter-powercap-{}-{n}", std::process::id())); + let fake = FakePowercap { root }; + fs::create_dir_all(&fake.root).expect("create a fake powercap tree"); + for (dir, name, value, range) in zones { + let zone = fake.root.join(dir); + fs::create_dir_all(&zone).expect("create a fake zone"); + if let Some(name) = name { + fs::write(zone.join("name"), format!("{name}\n")).unwrap(); + } + if let Some(range) = range { + fs::write(zone.join("max_energy_range_uj"), format!("{range}\n")).unwrap(); + } + fake.write(dir, value); + } + fake + } + + /// Replace zone `dir`'s `energy_uj`, atomically, so a reader sees the + /// old value or the new one. + fn write(&self, dir: &str, value: &str) { + let dir = self.root.join(dir); + fs::write(dir.join("energy_uj.tmp"), format!("{value}\n")).unwrap(); + fs::rename(dir.join("energy_uj.tmp"), dir.join("energy_uj")).unwrap(); + } + } + + impl Drop for FakePowercap { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.root); + } + } + + /// A tree laid out as a two-socket Intel host with a platform zone: two + /// packages (`intel-rapl:0`, `intel-rapl:2`), `psys` (`intel-rapl:1`), a + /// top-level `dram` zone (`intel-rapl:3`), each package's `core` and + /// `dram` subzones, and the MMIO interface to package 0. Each package + /// wraps at a range of its own. + fn two_sockets_with_psys_and_dram() -> FakePowercap { + FakePowercap::with_zones(&[ + ("intel-rapl:0", Some("package-0"), "1000", Some("262143328850")), + ("intel-rapl:1", Some("psys"), "90000", Some("262143328850")), + ("intel-rapl:2", Some("package-1"), "2000", Some("65532610987")), + ("intel-rapl:3", Some("dram"), "30000", Some("65532610987")), + ("intel-rapl:0:0", Some("core"), "700", Some("262143328850")), + ("intel-rapl:0:1", Some("dram"), "800", Some("65532610987")), + ("intel-rapl:2:0", Some("core"), "900", Some("262143328850")), + ("intel-rapl-mmio:0", Some("package-0"), "1000", Some("262143328850")), + ]) + } + + /// The package zones by their `name` file, and no other: `package-`; + /// the top-level zones by their directory: `intel-rapl:`. + #[test] + fn a_package_is_a_top_level_zone_named_package_n() { + for name in ["package-0", "package-1\n", "package-12", " package-3 \n"] { + assert!(is_package_name(name), "{name:?}"); + } + for name in ["psys", "dram", "core", "uncore", "package-", "package-x", "package-0a", "Package-0", "package0", ""] { + assert!(!is_package_name(name), "{name:?}"); + } + for dir in ["intel-rapl:0", "intel-rapl:12"] { + assert!(is_top_level_zone(dir), "{dir:?}"); + } + for dir in ["intel-rapl", "intel-rapl:", "intel-rapl:0:0", "intel-rapl:1:2", "intel-rapl-mmio:0", "intel-rapl:x"] { + assert!(!is_top_level_zone(dir), "{dir:?}"); + } + } + + /// The probe opens, samples and sums the package zones and nothing else. + /// On a tree with two packages, `psys`, a top-level `dram` zone, subzones + /// and an MMIO interface: the probe holds the two packages, each with its + /// own range; the liveness sample names those two; a span over which every + /// zone rose is metered, through the meter itself, as the two packages' + /// rise alone; a package wrapping at its own range is corrected at that + /// range, while `psys` and `dram` fall too. + #[test] + fn the_probe_sums_and_samples_the_package_zones_only() { + let tree = two_sockets_with_psys_and_dram(); + let p = RaplProbe::open_in(&tree.root).expect("a tree with two package zones opens"); + let held: Vec<(&str, u64)> = p.domains.iter().map(|d| (d.name.as_str(), d.max_range_uj)).collect(); + assert_eq!( + held, + [("intel-rapl:0", 262_143_328_850), ("intel-rapl:2", 65_532_610_987)], + "the probe did not hold exactly the two package zones" + ); + let sample = p.raw_sample().expect("the packages read"); + assert_eq!(sample, [("intel-rapl:0".to_string(), 1_000), ("intel-rapl:2".to_string(), 2_000)], "the probe sampled other zones"); + + let mut meter = PowerMeter { probes: vec![Box::new(p)], tdp_watts: 10.0 }; + let ((), m) = meter.measure(|| { + tree.write("intel-rapl:0", "4000"); + tree.write("intel-rapl:1", "190000"); + tree.write("intel-rapl:2", "7000"); + tree.write("intel-rapl:3", "80000"); + tree.write("intel-rapl:0:0", "2700"); + tree.write("intel-rapl:0:1", "1800"); + tree.write("intel-rapl-mmio:0", "4000"); + }); + let cpu: Vec<_> = m.devices.iter().map(|d| (d.class, d.measured_uj, d.provenance)).collect(); + assert_eq!(cpu, [(DeviceClass::Cpu, Some(8_000), Provenance::HwShunt)], "not the packages' rise alone"); + // Package 1 wraps at its own range, 65,532,610,987 µJ, as psys and + // dram fall too: 65,532,600,000 -> 500 is 11,487 µJ, plus package 0's + // 1,000. + tree.write("intel-rapl:2", "65532600000"); + let _ = meter.sample_uj(); + let ((), m) = meter.measure(|| { + tree.write("intel-rapl:0", "5000"); + tree.write("intel-rapl:2", "500"); + tree.write("intel-rapl:1", "10"); + tree.write("intel-rapl:3", "10"); + }); + let cpu: Vec<_> = m.devices.iter().map(|d| (d.class, d.measured_uj, d.provenance)).collect(); + assert_eq!(cpu, [(DeviceClass::Cpu, Some(12_487), Provenance::HwShunt)], "{:?}", m.devices); + } + + /// The liveness run — the one `rapl_status` runs on Linux, here over a + /// fake tree with a sleeping tick — watches the package zones only: Live + /// while both packages advance with `psys` and `dram` holding; Frozen + /// while `psys`, `dram`, the MMIO zone and package 0 advance and package 1 + /// holds; and a tree with no package zone is never ticked. + #[test] + fn the_liveness_run_watches_the_package_zones_only() { + /// The run over `tree` while another thread advances the zones in + /// `advancing` by 4 mJ every 2 ms. + fn run_advancing(tree: &FakePowercap, advancing: &[&str]) -> CounterStatus { + let stop = std::sync::atomic::AtomicBool::new(false); + std::thread::scope(|s| { + s.spawn(|| { + let mut uj: u64 = 100_000; + while !stop.load(std::sync::atomic::Ordering::SeqCst) { + uj += 4_000; + for dir in advancing { + tree.write(dir, &uj.to_string()); + } + std::thread::sleep(Duration::from_millis(2)); + } + }); + let watch = run_over(&tree.root, || std::thread::sleep(Duration::from_millis(25))); + let status = counter_status_of(watch.verdict()); + eprintln!("rapl run: zones {advancing:?} advancing, the rest holding -> {status:?}"); + stop.store(true, std::sync::atomic::Ordering::SeqCst); + status + }) + } + let tree = two_sockets_with_psys_and_dram(); + assert_eq!( + run_advancing(&tree, &["intel-rapl:0", "intel-rapl:2"]), + CounterStatus::Live, + "both packages advancing, psys and dram holding, was not live" + ); + let tree = two_sockets_with_psys_and_dram(); + assert_eq!( + run_advancing(&tree, &["intel-rapl:0", "intel-rapl:1", "intel-rapl:3", "intel-rapl-mmio:0"]), + CounterStatus::Frozen, + "package 1 holding beside a live psys, dram and package 0 was not declined" + ); + let psys_only = FakePowercap::with_zones(&[("intel-rapl:0", Some("psys"), "90000", Some("262143328850"))]); + let ticks = std::cell::Cell::new(0); + let watch = run_over(&psys_only.root, || ticks.set(ticks.get() + 1)); + assert_eq!((counter_status_of(watch.verdict()), ticks.get()), (CounterStatus::Unavailable, 0)); + } + + /// No package zone, no probe: a host exposing only `psys` and `dram` (a + /// `package-0` subzone does not count) does not open. Nor does a tree + /// that would leave a package out, or could not correct its wrap: a + /// package whose `energy_uj` does not read beside one that does, a + /// top-level zone whose `name` does not read (it may be a package), a + /// package with no `max_energy_range_uj`, or a zero one. + #[test] + fn the_probe_never_leaves_a_package_out_or_takes_another_zone() { + let unavailable = |zones: &[FakeZone<'_>]| { + let tree = FakePowercap::with_zones(zones); + match RaplProbe::open_in(&tree.root) { + Err(ProbeError::Unavailable(why)) => why, + Err(e) => panic!("{zones:?}: {e}"), + Ok(p) => panic!("{zones:?} opened {:?}", p.domains.iter().map(|d| &d.name).collect::>()), + } + }; + let why = unavailable(&[ + ("intel-rapl:0", Some("psys"), "90000", Some("262143328850")), + ("intel-rapl:1", Some("dram"), "30000", Some("65532610987")), + ("intel-rapl:0:0", Some("package-0"), "1000", Some("262143328850")), + ]); + assert!(why.contains("no RAPL package zone"), "{why}"); + let why = unavailable(&[ + ("intel-rapl:0", Some("package-0"), "1000", Some("262143328850")), + ("intel-rapl:1", Some("package-1"), "not a number", Some("262143328850")), + ]); + assert!(why.contains("intel-rapl:1"), "{why}"); + let why = unavailable(&[ + ("intel-rapl:0", Some("package-0"), "1000", Some("262143328850")), + ("intel-rapl:1", None, "2000", Some("262143328850")), + ]); + assert!(why.contains("no readable name"), "{why}"); + for range in [None, Some("0")] { + let why = unavailable(&[("intel-rapl:0", Some("package-0"), "1000", range)]); + assert!(why.contains("wrap range"), "{range:?}: {why}"); + } + let gone = FakePowercap::with_zones(&[]); + assert!(matches!(RaplProbe::open_in(&gone.root.join("absent")), Err(ProbeError::Unavailable(_)))); + } + + /// One package's delta at its own range; a fall from above the range is + /// unknown, and breaks the probe's count for good: that read and every + /// later one fail — even once the counter is back above where it was — so + /// a span that brackets the unknown delta from both sides is estimated, + /// never measured short of the energy that went uncounted. + #[test] + fn a_package_wraps_at_its_own_range_and_a_contradiction_breaks_the_count() { + assert_eq!(zone_delta(100, 400, 1_000), Some(300)); + assert_eq!(zone_delta(7, 7, 1_000), Some(0)); + assert_eq!(zone_delta(900, 50, 1_000), Some(150)); + assert_eq!(zone_delta(65_532_600_000, 500, 65_532_610_987), Some(11_487)); + assert_eq!(zone_delta(2_000, 50, 1_000), None, "a fall from above the range gave a figure"); + + let tree = FakePowercap::with_zones(&[("intel-rapl:0", Some("package-0"), "900", Some("1000"))]); + let mut p = RaplProbe::open_in(&tree.root).unwrap(); + tree.write("intel-rapl:0", "950"); + assert_eq!(p.snapshot().unwrap(), [(DeviceClass::Cpu, 50)]); + tree.write("intel-rapl:0", "20"); + assert_eq!(p.snapshot().unwrap(), [(DeviceClass::Cpu, 120)], "a wrap at 1,000: 50 + 20 more"); + tree.write("intel-rapl:0", "5000"); + assert_eq!(p.snapshot().unwrap(), [(DeviceClass::Cpu, 5_100)]); + + // A span opens at 5,000; mid-span a sampler reads the counter at 10, + // below where it was and above nothing it could have wrapped from; + // the span closes with the counter back up at 6,000. + let mut meter = PowerMeter { probes: vec![Box::new(p)], tdp_watts: 10.0 }; + let span = meter.start(); + tree.write("intel-rapl:0", "10"); + assert_eq!(meter.sample_uj(), None, "the contradictory read gave a count"); + tree.write("intel-rapl:0", "6000"); + let m = span.stop(&mut meter); + assert_eq!(m.source, "estimator", "a span over an unknown delta was measured: {:?}", m.devices); + assert!(m.devices.iter().all(|d| d.provenance == Provenance::Estimator && d.measured_uj.is_none()), "{:?}", m.devices); + assert_eq!(meter.sample_uj(), None, "a broken count read again"); + let ((), m) = meter.measure(|| tree.write("intel-rapl:0", "7000")); + assert_eq!(m.source, "estimator", "{:?}", m.devices); + + let tree = FakePowercap::with_zones(&[("intel-rapl:0", Some("package-0"), "5000", Some("1000"))]); + let mut p = RaplProbe::open_in(&tree.root).unwrap(); + tree.write("intel-rapl:0", "10"); + assert!(matches!(p.snapshot(), Err(ProbeError::Read(why)) if why.contains("above its 1000"))); + } + + /// The public counter is the meter's probe behind the liveness gate: + /// over a tree whose two packages advance (with `psys` and `dram` + /// holding) it opens holding exactly the package zones, and its + /// cumulative figure is their rise alone; with package 1 holding beside + /// a live `psys`, `dram` and package 0 it does not open, naming RAPL; a + /// broken count stays broken. + #[test] + fn the_public_package_counter_opens_only_over_live_packages() { + /// `RaplPackages::live_in(tree)` while another thread advances the + /// zones in `advancing` by 4 mJ every 2 ms. + fn live_in_advancing(tree: &FakePowercap, advancing: &[&str]) -> Result { + let stop = std::sync::atomic::AtomicBool::new(false); + std::thread::scope(|s| { + s.spawn(|| { + let mut uj: u64 = 100_000; + while !stop.load(std::sync::atomic::Ordering::SeqCst) { + uj += 4_000; + for dir in advancing { + tree.write(dir, &uj.to_string()); + } + std::thread::sleep(Duration::from_millis(2)); + } + }); + let opened = RaplPackages::live_in(&tree.root, || std::thread::sleep(Duration::from_millis(25))); + stop.store(true, std::sync::atomic::Ordering::SeqCst); + opened + }) + } + let tree = two_sockets_with_psys_and_dram(); + let mut packages = live_in_advancing(&tree, &["intel-rapl:0", "intel-rapl:2"]).expect("both packages live"); + assert_eq!(packages.zones(), ["intel-rapl:0", "intel-rapl:2"]); + let before = packages.cumulative_uj().unwrap(); + for (dir, by) in [("intel-rapl:0", 3_000), ("intel-rapl:1", 50_000), ("intel-rapl:2", 5_000), ("intel-rapl:3", 9_000)] { + let now = read_u64(&tree.root.join(dir).join("energy_uj")).unwrap(); + tree.write(dir, &(now + by).to_string()); + } + assert_eq!(packages.cumulative_uj().unwrap() - before, 8_000, "not the packages' rise alone"); + // Package 1 reads above its 65,532,610,987 µJ range, then falls. + tree.write("intel-rapl:2", "70000000000"); + packages.cumulative_uj().expect("a rise reads"); + tree.write("intel-rapl:2", "10"); + assert!(packages.cumulative_uj().is_err(), "a fall from above the range gave a figure"); + tree.write("intel-rapl:2", "20"); + assert!(packages.cumulative_uj().is_err(), "a broken count read again"); + + let tree = two_sockets_with_psys_and_dram(); + match live_in_advancing(&tree, &["intel-rapl:0", "intel-rapl:1", "intel-rapl:3", "intel-rapl-mmio:0"]) { + Err(ProbeError::Unavailable(why)) => assert!(why.contains("RAPL"), "{why}"), + Err(e) => panic!("{e}"), + Ok(p) => panic!("opened {:?} with package 1 holding", p.zones()), + } + } +} diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/tests/counter_probes_once_per_process.rs b/sandbox/sandbox-rs/crates/sandbox-meter/tests/counter_probes_once_per_process.rs new file mode 100644 index 000000000..f85e8e4ca --- /dev/null +++ b/sandbox/sandbox-rs/crates/sandbox-meter/tests/counter_probes_once_per_process.rs @@ -0,0 +1,52 @@ +//! The RAPL and NVML liveness probes run once per PROCESS. This file holds +//! this one test, so cargo runs it as a process of its own and its threads' +//! first calls are the process's first use of the caches. + +use sandbox_meter::probe::{counter_probe_runs, nvml_status, rapl_status, CounterStatus}; +use sandbox_meter::PowerMeter; +use std::sync::{Arc, Barrier}; + +/// However many threads ask at once, and whichever they ask first (a status, +/// or a meter whose selection is gated on it), each real probe runs once, and +/// every caller gets its one status; selection agrees with it. +#[test] +fn the_counter_liveness_probes_run_once_per_process() { + assert_eq!(counter_probe_runs(), (0, 0), "a probe ran before this test's first use of the caches"); + const THREADS: usize = 8; + let start = Arc::new(Barrier::new(THREADS)); + let askers: Vec<_> = (0..THREADS) + .map(|i| { + let start = Arc::clone(&start); + std::thread::spawn(move || { + let selects = || { + let s = PowerMeter::detect().sources(); + (s.contains(&"rapl-powercap"), s.contains(&"nvml-total-energy")) + }; + start.wait(); + if i % 2 == 0 { + let status = (rapl_status(), nvml_status()); + (status, selects()) + } else { + let selected = selects(); + ((rapl_status(), nvml_status()), selected) + } + }) + }) + .collect(); + let seen: Vec<_> = askers.into_iter().map(|t| t.join().expect("a thread that asked")).collect(); + let first = seen[0].0; + for (status, (rapl, nvml)) in &seen { + assert_eq!(*status, first, "threads saw different statuses: {seen:?}"); + assert_eq!(*rapl, first.0 == CounterStatus::Live, "RAPL selection disagrees with its status: {seen:?}"); + assert_eq!(*nvml, first.1 == CounterStatus::Live, "NVML selection disagrees with its status: {seen:?}"); + } + let runs = counter_probe_runs(); + eprintln!("counters once per process: {THREADS} threads, (rapl, nvml)={first:?}, real probe runs={runs:?}"); + #[cfg(target_os = "linux")] + assert_eq!(runs, (1, 1), "each real probe runs once in this process ({THREADS} threads asked at once)"); + #[cfg(not(target_os = "linux"))] + { + assert_eq!(runs, (0, 0), "no RAPL or NVML probe runs off Linux"); + assert_eq!(first, (CounterStatus::Unavailable, CounterStatus::Unavailable)); + } +} diff --git a/sandbox/sandbox-rs/crates/sandbox-meter/tests/ioreport_probe_once_per_process.rs b/sandbox/sandbox-rs/crates/sandbox-meter/tests/ioreport_probe_once_per_process.rs new file mode 100644 index 000000000..68ba663e6 --- /dev/null +++ b/sandbox/sandbox-rs/crates/sandbox-meter/tests/ioreport_probe_once_per_process.rs @@ -0,0 +1,77 @@ +//! The IOReport liveness probe runs once per PROCESS. This file holds this one +//! test, so cargo runs it as a process of its own and its threads' first +//! calls are the process's first use of the cache: the case a per-thread +//! cache, or one that checks and then sets, gets wrong. + +use sandbox_meter::probe::ioreport_probe_runs; +use sandbox_meter::{ioreport_status, IoReportStatus, PowerMeter}; +use std::sync::{Arc, Barrier}; + +/// However many threads ask at once, and whichever they ask first (the +/// status, or a meter whose selection is gated on it), the real probe runs +/// once, and every caller gets its one status. +#[test] +fn the_liveness_probe_runs_once_per_process() { + assert_eq!( + ioreport_probe_runs(), + 0, + "the probe ran before this test's first use of the cache" + ); + const THREADS: usize = 8; + let start = Arc::new(Barrier::new(THREADS)); + let askers: Vec<_> = (0..THREADS) + .map(|i| { + let start = Arc::clone(&start); + std::thread::spawn(move || { + let selects = || { + PowerMeter::detect() + .sources() + .contains(&"ioreport-energy-model") + }; + start.wait(); + // Half the threads ask for the status first and half build a + // meter first, so the first use is both paths at once. + if i % 2 == 0 { + let status = ioreport_status(); + (status, selects(), ioreport_status()) + } else { + let selected = selects(); + let status = ioreport_status(); + (status, selected, ioreport_status()) + } + }) + }) + .collect(); + let seen: Vec<_> = askers + .into_iter() + .map(|t| t.join().expect("a thread that asked")) + .collect(); + let first = seen[0].0; + for (status, selected, again) in &seen { + assert_eq!( + (*status, *again), + (first, first), + "threads saw different statuses: {seen:?}" + ); + assert_eq!( + *selected, + first == IoReportStatus::Live, + "selection disagrees with the status: {seen:?}" + ); + } + assert_eq!(ioreport_status(), first); + let runs = ioreport_probe_runs(); + eprintln!( + "ioreport once per process: {THREADS} threads, status={first:?}, real probe runs={runs}" + ); + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + assert_eq!( + runs, 1, + "the real probe ran {runs} times in this process ({THREADS} threads asked at once): {seen:?}" + ); + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + { + assert_eq!(runs, 0, "no IOReport probe runs off Apple Silicon macOS"); + assert_eq!(first, IoReportStatus::Unavailable); + } +} diff --git a/sandbox/sandbox-rs/crates/sandbox-proof/src/lib.rs b/sandbox/sandbox-rs/crates/sandbox-proof/src/lib.rs index ee043f524..8b83196eb 100644 --- a/sandbox/sandbox-rs/crates/sandbox-proof/src/lib.rs +++ b/sandbox/sandbox-rs/crates/sandbox-proof/src/lib.rs @@ -367,7 +367,8 @@ pub fn prove_microvm_with( /// Prove the **S5 (microVM) substrate** on this host with a real Firecracker /// boot. Requires `/dev/kvm` and a staged firecracker binary, kernel, and /// rootfs; writes a Firecracker `--no-api` config, supervises the boot to -/// completion, and seals an S5 cell. CPU energy is read from RAPL. +/// completion, and seals an S5 cell. CPU energy is read from RAPL (every +/// package zone, once every package counter has proved live), else estimated. pub fn prove_microvm( inputs: &MicroVmInputs, cloud: &str, @@ -419,7 +420,7 @@ pub fn prove_microvm( }; prove_microvm_with( Box::new(vmm), - Box::new(RaplMeter::intel_package0()), + Box::new(RaplMeter::host()), template, cloud, at_unix_us, @@ -1150,9 +1151,12 @@ fn resolve_weight_hash( /// the call goes over the wire. Here the silicon is the prover's own, which is /// the one case where the joules can actually be *measured* — so the device /// records come straight off [`PowerMeter`] with whatever provenance the -/// hardware honestly supports (`HwShunt` on Apple Silicon IOReport and Linux -/// RAPL/NVML; `Estimator` where no counter exists). The meter can degrade; it -/// does not invent a reading, and this function does not re-tag one. +/// hardware honestly supports: `HwShunt` from a live counter (Linux RAPL/NVML, +/// or Apple Silicon IOReport when its CPU energy rail is live), and +/// `Estimator` where no live counter exists, which on macOS 27.0.1 includes +/// every Apple Silicon host checked. The meter can degrade; it does not invent +/// a reading, and this function does not re-tag one — in what it seals or in +/// what it says. /// /// Three properties the cell attests, each checked rather than asserted: /// @@ -1163,8 +1167,9 @@ fn resolve_weight_hash( /// empty outbound allowlist, inbound false — and that emptiness is asserted /// before the engine is spawned. Deny-by-default is the standard's posture; /// this makes it a checked precondition of the proof. -/// 3. **The energy is this host's.** Measured over the engine's execution span -/// by the best counter the platform offers. +/// 3. **The energy is this host's.** Metered over the engine's execution span +/// by the best source the platform offers — a live counter, else the +/// estimator — and labelled by the span's own records. /// /// The engine is supplied by the operator and driven as a subprocess, exactly /// as [`prove_accelerator`] drives a GPU workload. That keeps the sandbox @@ -1228,7 +1233,6 @@ pub fn prove_local( // (3) Meter this host's own counters across the engine's execution. let mut meter = PowerMeter::detect(); let sources = meter.sources().join(","); - let is_hw = meter.is_hardware(); let (status, measurement) = meter.measure(|| { std::process::Command::new(&inputs.engine) .args(&inputs.args) @@ -1318,7 +1322,7 @@ pub fn prove_local( HashSource::Fresh => "hashed", HashSource::Cached => "cache hit: trusts size+mtime", }, - if is_hw { "measured" } else { "ESTIMATED" }, + energy_word(&measurement), measurement.wall.as_secs_f64(), measurement.avg_watts(), sources, @@ -1348,10 +1352,65 @@ pub fn prove_local( sign(cell, proof_key) } +/// How [`prove_local`] describes the energy it sealed, decided by the span's +/// own records. The records are sealed verbatim, so this is the receipt's +/// claim in one word: "measured" only when every record is a hardware +/// reading. A hardware meter can still return an estimate — the meter falls +/// back to its estimator for a span whose CPU counter held — so whether the +/// meter *has* a hardware source says nothing about this span. +fn energy_word(m: &sandbox_meter::Measurement) -> &'static str { + use sandbox_meter::Provenance; + let read = !m.devices.is_empty() + && m + .devices + .iter() + .all(|d| d.provenance == Provenance::HwShunt && d.measured_uj.is_some()); + if read { + "measured" + } else { + "ESTIMATED" + } +} + #[cfg(test)] mod tests { use super::*; + /// `prove_local` says "measured" only for a span whose every record is a + /// hardware reading — never for the estimator's record, which a hardware + /// meter also returns when the span's CPU counter held. main printed + /// "measured" for every span of a hardware meter, and sealed the same + /// records. + #[test] + fn a_local_proof_says_estimated_for_an_estimated_span() { + use sandbox_meter::{DeviceEnergy, Measurement, Provenance}; + let rec = |class, measured_uj, estimated_uj, provenance| DeviceEnergy { + class, + measured_uj, + estimated_uj, + provenance, + }; + let span = |devices| Measurement { + devices, + wall: std::time::Duration::from_secs(1), + source: "test", + }; + // The meter's estimator record: no live counter, or a CPU counter that held. + let est = span(vec![rec(DeviceClass::Cpu, None, 20_000_000, Provenance::Estimator)]); + assert_eq!(energy_word(&est), "ESTIMATED"); + // A live CPU counter, with the GPU and ANE beside it. + let live = span(vec![ + rec(DeviceClass::Cpu, Some(9_000_000), 9_000_000, Provenance::HwShunt), + rec(DeviceClass::Npu, Some(0), 0, Provenance::HwShunt), + rec(DeviceClass::Gpu, Some(700_000), 700_000, Provenance::HwShunt), + ]); + assert_eq!(energy_word(&live), "measured"); + // Anything short of a reading on every record is not a measurement. + let modeled = span(vec![rec(DeviceClass::Cpu, None, 5_000, Provenance::ModelBased)]); + assert_eq!(energy_word(&modeled), "ESTIMATED"); + assert_eq!(energy_word(&span(Vec::new())), "ESTIMATED"); + } + #[test] fn prove_here_produces_a_verifiable_cell() { let key = SigningKey::from_bytes(&[99u8; 32]);