From 1fb670a512267c3bc025a684d5d65c48507e113f Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 8 Oct 2026 20:21:15 -0400 Subject: [PATCH] mesh: an in-process engine is metered by this process's own CPU and GPU energy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mesh's headline figure, microjoules per accepted token, is to be measured at the meter. Yet every real serve through mesh was estimated: the GGUF serving binaries were hardwired to EstimatorMeter, and HwMeter bridges the host meter, which on macOS 27 is the estimator (IOReport's CPU rails are frozen). - mesh-meter `ProcessMeter`, behind `hw-meter`, for an engine that runs in this process. Each serve is metered by jouleclaw's ProcessSpanMeter: this process's own CPU energy plus its own GPU energy. - A span that covers the whole request is graded as its counters grade it: HwShunt for a live reading. Whole means the process was effectively the GPU's sole user and used no Neural Engine; a process with no GPU time spent exactly 0 J on it, so CPU-only engines qualify. - Any other span is an estimate of the whole request, Estimator: the reading plus EstimatorMeter for the rest of the host, as HwMeter bills a counter that reads part of a request. `mesh/1` has no coverage field. - `Meter::open_span` and `Window`. The default is None, so existing meters keep snapshot semantics. ProcessMeter grades over the whole span, because its GPU grade depends on the process's share over it. Session uses a span window where a meter keeps one. - mesh-engine-gguf, with `metal`: serve_falcon and serve_openai_falcon meter with ProcessMeter (the Metal liveness load comes from jouleclaw-backend-apple). New `serve_gemma_metered` serves Gemma 4 through mesh with ProcessMeter. - Fixes mesh-engine-gguf under `gguf`: it no longer compiled on main. ServeOutcome and ServeRequest gained a `decision` field and three initializers lacked it, unseen because CI never built the feature. standards-build now runs `cargo test -p mesh-engine-gguf --features gguf` on the mesh-rs leg. Measured on an M5 Max: - mesh-core tests/process_meter.rs: a ReferenceEngine serve sealed HwShunt 1,051,668 µJ (4,108 µJ per accepted token), within the 1,090,946 µJ of this process's CPU energy the test read around the serve. - serve_gemma_metered, Gemma 4 E2B on CPU, 3 x 16 tokens: HwShunt 12.52 J, 12.16 J and 12.54 J (0.78, 0.76 and 0.78 J per accepted token). Every envelope verifies. Mutation caught: billing a whole span's figure plus the rest-of-host estimate fails the unit test and the end-to-end bracket ("the receipt read 55691805 µJ, more than this process's 966177 µJ"). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/standards-build.yml | 2 +- mesh/README.md | 2 +- mesh/mesh-rs/Cargo.lock | 133 ++++++++++++ mesh/mesh-rs/Cargo.toml | 1 + mesh/mesh-rs/crates/mesh-core/Cargo.toml | 4 + .../crates/mesh-core/tests/process_meter.rs | 83 ++++++++ .../crates/mesh-engine-gguf/Cargo.toml | 11 +- .../mesh-engine-gguf/src/bin/serve_falcon.rs | 7 + .../src/bin/serve_gemma_metered.rs | 64 ++++++ .../src/bin/serve_openai_falcon.rs | 6 + .../crates/mesh-engine-gguf/src/lib.rs | 5 + mesh/mesh-rs/crates/mesh-meter/src/lib.rs | 197 +++++++++++++++++- 12 files changed, 505 insertions(+), 10 deletions(-) create mode 100644 mesh/mesh-rs/crates/mesh-core/tests/process_meter.rs create mode 100644 mesh/mesh-rs/crates/mesh-engine-gguf/src/bin/serve_gemma_metered.rs diff --git a/.github/workflows/standards-build.yml b/.github/workflows/standards-build.yml index e256b0302..ecb3ef3b3 100644 --- a/.github/workflows/standards-build.yml +++ b/.github/workflows/standards-build.yml @@ -99,7 +99,7 @@ jobs: - { name: proof-rs, dir: proof/proof-rs, runner: linux, z3: true, conformance: proof-conformance } - { name: joulehook-rs, dir: joulehook/joulehook-rs, runner: linux } - { name: eoc-rs, dir: eoc/eoc-rs, runner: linux, feature_tests: "-p eoc-meter --features cuda" } - - { name: mesh-rs, dir: mesh/mesh-rs, runner: linux-arm64 } + - { name: mesh-rs, dir: mesh/mesh-rs, runner: linux-arm64, feature_tests: "-p mesh-engine-gguf --features gguf" } - { name: joule-ui-rs, dir: joule-ui/joule-ui-rs, runner: linux } # jouleclaw-rs is the one entry whose runner IS a correctness choice. # It reaches candle → ug → gemm, which uses inline aarch64 assembly diff --git a/mesh/README.md b/mesh/README.md index 007ff10ef..f40161bda 100644 --- a/mesh/README.md +++ b/mesh/README.md @@ -42,7 +42,7 @@ per serve: authorise, route, meter, seal. See [`SPEC.md`](SPEC.md) §2. | `mesh-router` | energy-weighted cheapest-sufficient routing with a full candidate audit | | `mesh-engine` | the `ServingEngine` interface + a deterministic, weights-free `ReferenceEngine` | | `mesh-engine-gguf` | a **real** backend (behind `gguf`): wraps the family's pure-Rust GGUF/Metal inference substrate so the mesh routes and meters actual model tokens | -| `mesh-meter` | the meter window; an always-available estimator + an optional bridge to the family hardware meter (RAPL / NVML / Apple) behind `hw-meter` | +| `mesh-meter` | the meter window; an always-available estimator + an optional bridge to the family hardware meter (RAPL / NVML / Apple) behind `hw-meter`, and `ProcessMeter`: this process's own CPU and GPU energy for an engine that runs in process, measured when a span covers the whole request | | `mesh-receipt` | the `mesh/1` receipt, JCR-1 sealing (det-CBOR + COSE_Sign1), and a hash-chained log | | `mesh-core` | the orchestration that ties authorise → route → meter → seal together | | `mesh-conformance` | the deterministic reference scenario, its checks, and the `gen-vectors` bin | diff --git a/mesh/mesh-rs/Cargo.lock b/mesh/mesh-rs/Cargo.lock index 3089fc71a..faf260bfc 100644 --- a/mesh/mesh-rs/Cargo.lock +++ b/mesh/mesh-rs/Cargo.lock @@ -78,6 +78,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "bs58" version = "0.5.1" @@ -277,6 +286,16 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.1", + "objc2", +] + [[package]] name = "ed25519" version = "2.2.3" @@ -426,6 +445,17 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "jouleclaw-backend-apple" +version = "0.1.0" +dependencies = [ + "jouleclaw-core", + "objc2", + "objc2-foundation", + "objc2-metal", + "objc2-metal-performance-shaders", +] + [[package]] name = "jouleclaw-core" version = "0.1.0" @@ -531,6 +561,7 @@ dependencies = [ "hex", "jcp-grant", "jcp-schema", + "jouleclaw-backend-apple", "mesh-engine", "mesh-meter", "mesh-receipt", @@ -554,6 +585,7 @@ dependencies = [ "ed25519-dalek", "jcp-grant", "jcp-schema", + "jouleclaw-backend-apple", "jouleclaw-loader-gguf", "mesh-core", "mesh-engine", @@ -666,6 +698,107 @@ dependencies = [ "malloc_buf", ] +[[package]] +name = "objc2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08849bbd4767dfae9457696856ae1c84fe4e0281bbe4a7abff2d0e06fb7981f8" +dependencies = [ + "objc2-encode", + "objc2-exception-helper", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.1", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-core-graphics" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" +dependencies = [ + "dispatch2", + "objc2", + "objc2-core-foundation", + "objc2-io-surface", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-exception-helper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7a1c5fbb72d7735b076bb47b578523aedc40f3c439bea6dfd595c089d79d98a" +dependencies = [ + "cc", +] + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.1", + "block2", + "libc", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-io-surface" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180788110936d59bab6bd83b6060ffdfffb3b922ba1396b312ae795e1de9d81d" +dependencies = [ + "bitflags 2.13.1", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-metal" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" +dependencies = [ + "bitflags 2.13.1", + "block2", + "dispatch2", + "objc2", + "objc2-core-foundation", + "objc2-foundation", +] + +[[package]] +name = "objc2-metal-performance-shaders" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a10f15ff4e2045f947ffca8a17fc351b155c321de16f04278a008a1f446784" +dependencies = [ + "bitflags 2.13.1", + "block2", + "objc2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation", + "objc2-metal", +] + [[package]] name = "once_cell" version = "1.21.4" diff --git a/mesh/mesh-rs/Cargo.toml b/mesh/mesh-rs/Cargo.toml index 19fde0f0a..1a3d1e310 100644 --- a/mesh/mesh-rs/Cargo.toml +++ b/mesh/mesh-rs/Cargo.toml @@ -33,6 +33,7 @@ jcp-schema = { path = "../../jcp/jcp-rs/crates/jcp-schema" } jcp-grant = { path = "../../jcp/jcp-rs/crates/jcp-grant" } # Optional: the family's honest energy meter (RAPL / NVML / Apple), behind `hw-meter`. jouleclaw-energy = { path = "../../jouleclaw/jouleclaw-rs/crates/jouleclaw-energy" } +jouleclaw-backend-apple = { path = "../../jouleclaw/jouleclaw-rs/crates/jouleclaw-backend-apple" } # Optional: the family's pure-Rust GGUF/Metal inference substrate, behind `gguf`. jouleclaw-loader-gguf = { path = "../../jouleclaw/jouleclaw-rs/crates/jouleclaw-loader-gguf" } diff --git a/mesh/mesh-rs/crates/mesh-core/Cargo.toml b/mesh/mesh-rs/crates/mesh-core/Cargo.toml index c6653f6e1..dceaec7cc 100644 --- a/mesh/mesh-rs/crates/mesh-core/Cargo.toml +++ b/mesh/mesh-rs/crates/mesh-core/Cargo.toml @@ -22,6 +22,10 @@ thiserror = { workspace = true } hex = { workspace = true } serde_json = { workspace = true } +# The GPU load for `ProcessMeter`'s liveness probe in tests/process_meter.rs. +[target.'cfg(target_os = "macos")'.dev-dependencies] +jouleclaw-backend-apple = { workspace = true } + [features] default = [] # Forwarded to mesh-meter: use the family's hardware-backed meter. Each platform diff --git a/mesh/mesh-rs/crates/mesh-core/tests/process_meter.rs b/mesh/mesh-rs/crates/mesh-core/tests/process_meter.rs new file mode 100644 index 000000000..99dc3d84e --- /dev/null +++ b/mesh/mesh-rs/crates/mesh-core/tests/process_meter.rs @@ -0,0 +1,83 @@ +//! An in-process engine metered by this process's own CPU and GPU energy, +//! end to end: served through the mesh and sealed into a `mesh/1` receipt. +//! +//! The engine here works on the CPU alone, so its GPU energy is exactly 0 (it +//! accrued no GPU time) and the span covers the whole request: the receipt is +//! `HwShunt`, its figure this process's own CPU energy over the serve, and it +//! must lie within the CPU energy the test reads around the serve itself. Where +//! the GPU counter could not be proved live (a GPU busy with other work fails +//! the probe), the span covers part of the request, and the receipt must say +//! so: an estimate, with no measured figure. + +#![cfg(all(target_os = "macos", feature = "hw-meter-apple"))] + +use ed25519_dalek::SigningKey; +use mesh_core::Mesh; +use mesh_engine::ReferenceEngine; +use mesh_meter::{Meter, ProcessMeter, Provenance, SpanUses}; +use mesh_router::BackendSpec; +use mesh_schema::{ContentId, Fidelity, ServeRequest}; + +unsafe extern "C" { + fn proc_pid_rusage(pid: i32, flavor: i32, buffer: *mut std::ffi::c_void) -> i32; +} + +/// This process's CPU energy, nJ: `ri_energy_nj` of `rusage_info_v6`, read +/// here into a raw buffer at the SDK's offset (336), not through the meter. +fn cpu_nj() -> u64 { + let mut words = [0u64; 464 / 8]; + let rc = unsafe { proc_pid_rusage(std::process::id() as i32, 6, words.as_mut_ptr().cast()) }; + assert_eq!(rc, 0, "proc_pid_rusage"); + words[336 / 8] +} + +#[test] +fn an_in_process_engine_is_sealed_with_this_processs_own_energy() { + let meter = ProcessMeter::new(jouleclaw_backend_apple::gpu_liveness_load, SpanUses::CpuGpu); + let mut mesh = Mesh::new(&meter, SigningKey::from_bytes(&[11u8; 32]), "did:example:mesh-steward"); + mesh.register( + BackendSpec::new("tiny", "ssm-1b", Fidelity::F1, 50_000), + Box::new(ReferenceEngine::new("tiny", "ssm-1b", Fidelity::F1, 50_000, 400_000)), + ); + let req = ServeRequest { + prompt_hash: ContentId::of_bytes(b"an in-process serve"), + prompt_tokens: 4, + max_tokens: 256, + fidelity_floor: Fidelity::F1, + joule_ceiling_uj: 10_000_000_000, + decision: None, + }; + + let before = cpu_nj(); + let resp = mesh.serve(&req, &[], None).expect("serve"); + let around_uj = (cpu_nj() - before) / 1_000; + let e = resp.energy; + eprintln!( + "process meter: {} (metered {}); receipt {:?} measured {:?} estimated {} µJ; {:?} µJ per accepted token; \ + this process's CPU energy around the serve {around_uj} µJ", + meter.name(), + meter.is_metered(), + e.provenance, + e.measured_uj, + e.estimated_uj, + resp.uj_per_accepted_token() + ); + + assert!(e.check_provenance().is_ok(), "{e:?}"); + if cfg!(target_arch = "aarch64") { + assert!(meter.is_metered(), "this process's CPU energy counter is live on Apple Silicon"); + } + match e.provenance { + Provenance::HwShunt => { + let uj = e.measured_uj.expect("a HwShunt figure is measured"); + assert!(uj > 0, "a served span read no energy: {e:?}"); + assert!( + uj <= around_uj + 1, + "the receipt read {uj} µJ, more than this process's {around_uj} µJ of CPU energy around a CPU-only serve" + ); + } + Provenance::Estimator => assert_eq!(e.measured_uj, None, "{e:?}"), + Provenance::ModelBased => panic!("no model grades this span: {e:?}"), + } + assert!(resp.receipt.verify_envelope(&resp.envelope, mesh.steward(), &mesh.verifying_key()).is_ok()); +} diff --git a/mesh/mesh-rs/crates/mesh-engine-gguf/Cargo.toml b/mesh/mesh-rs/crates/mesh-engine-gguf/Cargo.toml index 5e55ad819..ae507f32b 100644 --- a/mesh/mesh-rs/crates/mesh-engine-gguf/Cargo.toml +++ b/mesh/mesh-rs/crates/mesh-engine-gguf/Cargo.toml @@ -21,6 +21,8 @@ jcp-grant = { workspace = true, optional = true } ed25519-dalek = { workspace = true, optional = true } serde_json = { workspace = true, optional = true } mesh-openai = { path = "../mesh-openai", optional = true } +# The GPU load for `ProcessMeter`'s liveness probe, with the `metal` feature. +jouleclaw-backend-apple = { workspace = true, optional = true } [features] default = [] @@ -40,7 +42,9 @@ gguf = [ "mesh-openai/server", ] # Apple Silicon UMA Metal overlay (zero-copy GGUF → GPU). Forwarded to the loader. -metal = ["gguf", "jouleclaw-loader-gguf/metal"] +# Decode on the GPU, in process, and meter each serve by this process's own CPU +# and GPU energy (`mesh_meter::ProcessMeter`) rather than the load estimator. +metal = ["gguf", "jouleclaw-loader-gguf/metal", "mesh-meter/hw-meter-apple", "dep:jouleclaw-backend-apple"] # Data-parallel matmul in the loader (faster CPU decode). rayon = ["gguf", "jouleclaw-loader-gguf/rayon"] @@ -54,5 +58,10 @@ name = "serve_openai_falcon" path = "src/bin/serve_openai_falcon.rs" required-features = ["gguf"] +[[bin]] +name = "serve_gemma_metered" +path = "src/bin/serve_gemma_metered.rs" +required-features = ["metal"] + [lints] workspace = true diff --git a/mesh/mesh-rs/crates/mesh-engine-gguf/src/bin/serve_falcon.rs b/mesh/mesh-rs/crates/mesh-engine-gguf/src/bin/serve_falcon.rs index 577eddc83..965d39e1c 100644 --- a/mesh/mesh-rs/crates/mesh-engine-gguf/src/bin/serve_falcon.rs +++ b/mesh/mesh-rs/crates/mesh-engine-gguf/src/bin/serve_falcon.rs @@ -16,6 +16,7 @@ use jcp_schema::{Capability, Did, Grant}; use jouleclaw_loader_gguf::bpe_tokenizer::BpeTokenizer; use mesh_core::{Mesh, MeshError, INFER_ACTION}; use mesh_engine_gguf::FalconEngine; +#[cfg_attr(feature = "metal", allow(unused_imports))] use mesh_meter::{EstimatorMeter, Meter}; use mesh_router::BackendSpec; use mesh_schema::{ContentId, Fidelity, ServeRequest}; @@ -45,6 +46,7 @@ fn request(prompt_text: &str, prompt_len: usize, n_tokens: u32) -> ServeRequest max_tokens: n_tokens, fidelity_floor: Fidelity::F3, joule_ceiling_uj: 10_000_000_000, // 10 kJ ceiling + decision: None, } } @@ -66,6 +68,11 @@ fn main() { // Steward (seals receipts) + caller (mints grants). let steward_sk = SigningKey::from_bytes(&[7u8; 32]); let caller_sk = SigningKey::from_bytes(&[42u8; 32]); + // The engine decodes here, in this process: with `metal` each serve is + // metered by this process's own CPU and GPU energy, else estimated. + #[cfg(feature = "metal")] + let meter = mesh_meter::ProcessMeter::new(jouleclaw_backend_apple::gpu_liveness_load, mesh_meter::SpanUses::CpuGpu); + #[cfg(not(feature = "metal"))] let meter = EstimatorMeter::new(15.0); println!("meter: {}\n", meter.name()); diff --git a/mesh/mesh-rs/crates/mesh-engine-gguf/src/bin/serve_gemma_metered.rs b/mesh/mesh-rs/crates/mesh-engine-gguf/src/bin/serve_gemma_metered.rs new file mode 100644 index 000000000..acbdae823 --- /dev/null +++ b/mesh/mesh-rs/crates/mesh-engine-gguf/src/bin/serve_gemma_metered.rs @@ -0,0 +1,64 @@ +//! Serve a real Gemma 4 checkpoint through Mesh, in this process, and meter +//! each serve by this process's own CPU and GPU energy: the `mesh/1` receipt +//! carries a measured figure (`HwShunt`) whenever the span covers the whole +//! request, and says it is an estimate when it does not. +//! +//! ```bash +//! cargo run --release -p mesh-engine-gguf --features metal --bin serve_gemma_metered -- \ +//! /path/to/gemma-4-E2B [n_tokens] [n_requests] +//! ``` +//! +//! The prompt is a fixed run of token ids: this measures the energy of serving, +//! and the ids are hashed into the request like any prompt. + +use std::time::Instant; + +use ed25519_dalek::SigningKey; +use mesh_core::Mesh; +use mesh_engine_gguf::GgufEngine; +use mesh_meter::{Meter, ProcessMeter, SpanUses}; +use mesh_router::BackendSpec; +use mesh_schema::{ContentId, Fidelity, ServeRequest}; + +fn main() { + let mut args = std::env::args().skip(1); + let dir = args.next().expect("usage: serve_gemma_metered [n_tokens] [n_requests]"); + let n_tokens: u32 = args.next().and_then(|s| s.parse().ok()).unwrap_or(16); + let n_requests: usize = args.next().and_then(|s| s.parse().ok()).unwrap_or(3); + + let t0 = Instant::now(); + let engine = GgufEngine::load(&dir, "gemma-4-e2b", "gemma-4-E2B", Fidelity::F2, 200_000).expect("load Gemma 4"); + println!("loaded {dir} in {:.1}s", t0.elapsed().as_secs_f64()); + + let meter = ProcessMeter::new(jouleclaw_backend_apple::gpu_liveness_load, SpanUses::CpuGpu); + println!("meter: {} (this process's counters live: {})\n", meter.name(), meter.is_metered()); + + let steward_sk = SigningKey::from_bytes(&[7u8; 32]); + let mut mesh = Mesh::new(&meter, steward_sk, "did:example:mesh-gemma-metered"); + mesh.register(BackendSpec::new("gemma-4-e2b", "gemma-4-E2B", Fidelity::F2, 200_000), Box::new(engine)); + + let prompt: Vec = vec![2, 818, 5279, 529, 7001, 563, 236881]; + for i in 0..n_requests { + let req = ServeRequest { + prompt_hash: ContentId::of_bytes(format!("serve_gemma_metered request {i}").as_bytes()), + prompt_tokens: prompt.len() as u32, + max_tokens: n_tokens, + fidelity_floor: Fidelity::F2, + joule_ceiling_uj: 100_000_000_000, + decision: None, + }; + let t = Instant::now(); + let resp = mesh.serve(&req, &prompt, None).expect("serve"); + let e = resp.energy; + println!( + "request {i}: {} tokens in {:.2}s; receipt {:?}, measured {:?} µJ, estimated {} µJ; {:?} µJ per accepted token; envelope verifies: {}", + resp.receipt.outcome.tokens_accepted, + t.elapsed().as_secs_f64(), + e.provenance, + e.measured_uj, + e.estimated_uj, + resp.uj_per_accepted_token(), + resp.receipt.verify_envelope(&resp.envelope, mesh.steward(), &mesh.verifying_key()).is_ok() + ); + } +} diff --git a/mesh/mesh-rs/crates/mesh-engine-gguf/src/bin/serve_openai_falcon.rs b/mesh/mesh-rs/crates/mesh-engine-gguf/src/bin/serve_openai_falcon.rs index 110d778e9..eafcacd22 100644 --- a/mesh/mesh-rs/crates/mesh-engine-gguf/src/bin/serve_openai_falcon.rs +++ b/mesh/mesh-rs/crates/mesh-engine-gguf/src/bin/serve_openai_falcon.rs @@ -19,6 +19,7 @@ use ed25519_dalek::SigningKey; use jouleclaw_loader_gguf::bpe_tokenizer::BpeTokenizer; use mesh_core::Mesh; use mesh_engine_gguf::FalconEngine; +#[cfg_attr(feature = "metal", allow(unused_imports))] use mesh_meter::EstimatorMeter; use mesh_openai::ChatBackend; use mesh_router::BackendSpec; @@ -58,6 +59,11 @@ fn main() -> std::io::Result<()> { ) .expect("load Falcon-H1"); + // The engine decodes here, in this process: with `metal` each serve is + // metered by this process's own CPU and GPU energy, else estimated. + #[cfg(feature = "metal")] + let meter = mesh_meter::ProcessMeter::new(jouleclaw_backend_apple::gpu_liveness_load, mesh_meter::SpanUses::CpuGpu); + #[cfg(not(feature = "metal"))] let meter = EstimatorMeter::new(15.0); let steward_sk = SigningKey::from_bytes(&[7u8; 32]); let mut mesh = Mesh::new(&meter, steward_sk, "did:example:mesh-falcon-openai"); diff --git a/mesh/mesh-rs/crates/mesh-engine-gguf/src/lib.rs b/mesh/mesh-rs/crates/mesh-engine-gguf/src/lib.rs index 39face501..ad7d400ea 100644 --- a/mesh/mesh-rs/crates/mesh-engine-gguf/src/lib.rs +++ b/mesh/mesh-rs/crates/mesh-engine-gguf/src/lib.rs @@ -116,6 +116,8 @@ mod engine { cache_hit_tokens: 0, spec_proposed: 0, spec_accepted: 0, + // A token-generating engine answers no typed decision. + decision: None, }; Served { tokens, outcome } } @@ -247,6 +249,8 @@ mod falcon { cache_hit_tokens: 0, spec_proposed: 0, spec_accepted: 0, + // A token-generating engine answers no typed decision. + decision: None, }; Served { tokens, outcome } } @@ -281,6 +285,7 @@ mod tests { max_tokens: 8, fidelity_floor: Fidelity::F3, joule_ceiling_uj: u64::MAX, + decision: None, }; let served = e.serve(&req, &[2u32, 3u32]); assert!(served.outcome.tokens_out > 0); diff --git a/mesh/mesh-rs/crates/mesh-meter/src/lib.rs b/mesh/mesh-rs/crates/mesh-meter/src/lib.rs index 131c52a87..42f77daf1 100644 --- a/mesh/mesh-rs/crates/mesh-meter/src/lib.rs +++ b/mesh/mesh-rs/crates/mesh-meter/src/lib.rs @@ -19,6 +19,14 @@ //! model, `Estimator` where the host exposes no live counter — as on Apple Silicon //! whose IOReport CPU rail is frozen (macOS 27.0.1). //! +//! Also with `hw-meter`, [`ProcessMeter`] meters an engine that runs in this +//! process (a GGUF decode, in process) by this process's own CPU energy and its +//! own GPU energy over the span. Its spans are measured (`HwShunt`) when they +//! cover the whole request: the process was effectively the GPU's sole user +//! over the span, and it used no Neural Engine. A process that used no GPU time +//! spent exactly 0 J on it, so CPU-only engines qualify. Every other span is +//! graded an estimate of the whole request. +//! //! A `mesh/1` receipt has no coverage field (SPEC §3.3), so its figure is read //! as the energy of the whole request. A counter whose domain does not cover the //! whole request — an NVML GPU counter, with the CPU that drives the GPU @@ -53,20 +61,43 @@ pub trait Meter { fn name(&self) -> &str; /// Snapshot the running cumulative energy now. fn snapshot(&self) -> Sample; + /// Open a window that the meter grades over the whole span, for a meter + /// whose grade is not a property of either end alone (a [`ProcessMeter`] + /// grades a span by this process's share of the GPU over it). `None`, the + /// default, and a [`Session`] takes a snapshot at each end instead. + fn open_span(&self) -> Option> { + None + } } -/// An open measurement window. Built from a baseline snapshot; `stop()` closes -/// it and returns the provenance-tagged delta. +/// A measurement window a meter keeps for itself ([`Meter::open_span`]). +pub trait Window { + /// Close the window: the provenance-tagged energy over it. + fn close(self: Box) -> Energy; +} + +/// An open measurement window. Built from a baseline snapshot, or from the +/// meter's own window where it keeps one; `stop()` closes it and returns the +/// provenance-tagged figure. pub struct Session<'m> { meter: &'m dyn Meter, - start: Sample, + open: Open<'m>, +} + +enum Open<'m> { + Snapshot(Sample), + Span(Box), } impl<'m> Session<'m> { - /// Open a window by snapshotting `meter` now. + /// Open a window on `meter` now: its own window where it keeps one, else a + /// snapshot. pub fn start(meter: &'m dyn Meter) -> Self { - let start = meter.snapshot(); - Self { meter, start } + let open = match meter.open_span() { + Some(window) => Open::Span(window), + None => Open::Snapshot(meter.snapshot()), + }; + Self { meter, open } } /// Close the window. Returns an [`Energy`] whose provenance matches the @@ -75,8 +106,12 @@ impl<'m> Session<'m> { /// provenance and `measured_uj` is established here and re-checked at seal /// time, so a dishonest receipt cannot be produced downstream. pub fn stop(self) -> Energy { + let start = match self.open { + Open::Span(window) => return window.close(), + Open::Snapshot(start) => start, + }; let end = self.meter.snapshot(); - let delta = end.uj.saturating_sub(self.start.uj); + let delta = end.uj.saturating_sub(start.uj); match end.provenance { Provenance::HwShunt => Energy::measured(delta), Provenance::ModelBased => Energy::modeled(delta), @@ -708,6 +743,154 @@ mod hw { #[cfg(feature = "hw-meter")] pub use hw::HwMeter; +#[cfg(feature = "hw-meter")] +pub use process::ProcessMeter; + +/// What an in-process engine's work runs on, for a [`ProcessMeter`]: CPU and +/// GPU, or CPU, GPU and the Neural Engine (no unprivileged counter reads it). +#[cfg(feature = "hw-meter")] +pub use jouleclaw_energy::span::SpanUses; + +/// The meter for an engine that runs in this process: this process's own CPU +/// and GPU energy over the span ([`jouleclaw_energy::span::ProcessSpanMeter`]). +#[cfg(feature = "hw-meter")] +mod process { + use jouleclaw_energy::span::{ProcessSpanMeter, SpanEnergy, SpanStart, SpanUses}; + use jouleclaw_energy::{Coverage, EnergyError, Provenance as JpProvenance}; + use mesh_schema::Energy; + + use super::{EstimatorMeter, Meter, Sample, Window}; + + /// Meters a serve by this process's own CPU energy and its own GPU energy + /// over the span, for an engine that runs here (a GGUF decode on Metal, in + /// process). It is the caller's word that the engine runs in this process; + /// construct it only for one that does. + /// + /// A `mesh/1` receipt has no coverage field, so its figure stands for the + /// whole request (SPEC §3.3), and a span is graded that way: + /// - a span over which the process was effectively the GPU's sole user and + /// used no Neural Engine covers the whole request, with CPU and GPU both + /// read. It is graded as the counters grade it: `HwShunt` for a live + /// hardware reading. + /// - a span that covers only part of the request is an estimate of it, + /// `Estimator`. That is a span whose GPU was shared, whose GPU had no + /// figure, or that declared Neural Engine use, which no unprivileged + /// counter reads. Its figure is the reading plus this crate's + /// [`EstimatorMeter`] for the rest of the host over the same window, as + /// [`super::HwMeter`] bills a counter that reads part of a request. + /// - a span with no figure at all is the estimator's. + /// + /// Off macOS, or where the process counters are not live, the span meter + /// falls back to the family's host meter, whose grade it carries. + pub struct ProcessMeter { + span: ProcessSpanMeter, + uses: SpanUses, + rest: EstimatorMeter, + } + + impl ProcessMeter { + /// Meter an in-process engine whose work runs on what `uses` says. + /// `gpu_load` must put a few tens of milliseconds of work on the GPU + /// from this process and wait for it: the GPU counter's liveness probe + /// runs it once per process. + pub fn new(gpu_load: impl FnMut(), uses: SpanUses) -> Self { + Self { span: ProcessSpanMeter::new(gpu_load), uses, rest: EstimatorMeter::default() } + } + + /// Whether this process's own counters meter its spans; else each span + /// is the host meter's. + pub fn is_metered(&self) -> bool { + self.span.is_metered() + } + } + + impl Meter for ProcessMeter { + fn name(&self) -> &str { + if self.span.is_metered() { "process-cpu+gpu" } else { self.span.fallback_name() } + } + + /// A span meter has no running total to point-read: its grade is a + /// property of the whole span. A snapshot is the estimator's. + fn snapshot(&self) -> Sample { + self.rest.snapshot() + } + + fn open_span(&self) -> Option> { + Some(Box::new(ProcessWindow { meter: self, rest_at: self.rest.snapshot().uj, start: Some(self.span.start()) })) + } + } + + struct ProcessWindow<'m> { + meter: &'m ProcessMeter, + rest_at: u64, + start: Option, + } + + impl Window for ProcessWindow<'_> { + fn close(mut self: Box) -> Energy { + let rest_uj = self.meter.rest.snapshot().uj.saturating_sub(self.rest_at); + let span = match self.start.take() { + Some(start) => self.meter.span.finish(start, self.meter.uses), + None => Err(EnergyError::NotCurrent), + }; + energy_of(span, rest_uj) + } + } + + /// The `mesh/1` energy of a span: see [`ProcessMeter`] for the rule. Pure, + /// so the tests drive every case. `rest_uj` is the estimator's figure for + /// the host over the same window. + pub(crate) fn energy_of(span: Result, rest_uj: u64) -> Energy { + match span { + Ok(s) if s.coverage == Coverage::Whole => match s.provenance { + JpProvenance::HwShunt => Energy::measured(s.uj), + JpProvenance::ModelBased => Energy::modeled(s.uj), + JpProvenance::Estimator => Energy::estimator(s.uj), + }, + Ok(s) => Energy::estimator(s.uj.saturating_add(rest_uj)), + Err(_) => Energy::estimator(rest_uj), + } + } + + #[cfg(test)] + mod tests { + use jouleclaw_energy::span::{SpanEnergy, SpanSource}; + use jouleclaw_energy::{Coverage, EnergyError, Provenance as JpProvenance}; + use mesh_schema::Provenance; + + use super::energy_of; + + fn span(uj: u64, provenance: JpProvenance, coverage: Coverage) -> Result { + Ok(SpanEnergy { + uj, + provenance, + coverage, + source: SpanSource::CpuOnly { cpu_uj: uj, why: "test".to_string() }, + }) + } + + /// Only a whole span keeps its hardware grade; a partial one is an + /// estimate of the whole request, the reading plus the estimator's + /// figure for the rest of the host; no figure is the estimator's. + #[test] + fn a_span_keeps_its_hardware_grade_only_when_it_covers_the_whole_request() { + let whole = energy_of(span(9_000, JpProvenance::HwShunt, Coverage::Whole), 40_000); + assert_eq!((whole.provenance, whole.measured_uj, whole.estimated_uj), (Provenance::HwShunt, Some(9_000), 9_000)); + assert!(whole.check_provenance().is_ok()); + + let partial = energy_of(span(9_000, JpProvenance::HwShunt, Coverage::Partial), 40_000); + assert_eq!((partial.provenance, partial.measured_uj, partial.estimated_uj), (Provenance::Estimator, None, 49_000)); + assert!(partial.check_provenance().is_ok()); + + let modeled = energy_of(span(7_000, JpProvenance::ModelBased, Coverage::Whole), 40_000); + assert_eq!(modeled.provenance, Provenance::ModelBased); + + let none = energy_of(Err(EnergyError::NotCurrent), 40_000); + assert_eq!((none.provenance, none.measured_uj, none.estimated_uj), (Provenance::Estimator, None, 40_000)); + } + } +} + #[cfg(test)] mod tests { use super::*; -- 2.54.0 (Apple Git-157)