diff --git a/mesh/mesh-rs/crates/mesh-router/src/lib.rs b/mesh/mesh-rs/crates/mesh-router/src/lib.rs index d689a4dc9..cd9905309 100644 --- a/mesh/mesh-rs/crates/mesh-router/src/lib.rs +++ b/mesh/mesh-rs/crates/mesh-router/src/lib.rs @@ -41,7 +41,7 @@ use mesh_schema::{BackendId, Fidelity, RouteCandidate, RouteDecision, ServeReque /// without inheriting that confound. So this type carries the *shape* of the /// cost and the operator supplies the numbers — the same reason /// `BandingScale` ships no canonical scale. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct CostProfile { /// Fixed microjoules to serve the request at all, independent of size. pub base_uj: u64, @@ -53,8 +53,113 @@ pub struct CostProfile { /// says this backend gets more expensive per token as the prefix grows, /// which is what makes a long-context request refusable *before* it runs. pub uj_per_token_per_kctx: u64, + /// The operating point these constants were measured at, if the operator + /// declared one. See [`Calibration`]. + /// + /// `None` is the historical shape and stays the default, so every existing + /// constructor and every sealed `wire()` string is unchanged. It is not a + /// safe default, though — it is an *undeclared* one, and + /// [`Self::require_applicable_at`] refuses it. + pub calibration: Option, } +/// The operating point a [`CostProfile`]'s constants were measured at. +/// +/// # Why a cost model has to say where it fits +/// +/// A profile is a fit, and a fit is only a fit *somewhere*. The per-token +/// energy of a serving backend is not a property of the backend; it is a +/// property of the backend **at an occupancy**. The same weights on the same +/// silicon cost dramatically less per token when batches are full — fixed +/// per-step overheads amortise across more tokens, and the memory traffic that +/// dominates decode is shared. Published figures are almost always measured at +/// saturation, because that is the number a vendor wants quoted. +/// +/// # The two errors are not symmetric at a gate +/// +/// [`CostProfile`] feeds a *ceiling*. That makes the direction of an error +/// decisive: +/// +/// - **Over-estimating refuses** work that would in fact have fit. Visible, +/// annoying, and safe — the operator sees a refusal and can raise the budget. +/// - **Under-estimating admits** work the ceiling cannot cover. Silent, and +/// the overspend is discovered after the joules are gone. +/// +/// A profile fit at saturation and applied at low concurrency under-estimates, +/// which is the unsafe direction. So the model records the point it was +/// measured at, and [`CostProfile::require_applicable_at`] refuses exactly the +/// cases that under-estimate — while permitting the ones that over-estimate. +/// +/// # This ships no constants +/// +/// `source` names the measurement, it does not substitute for it. Importing a +/// published coefficient would put an unfalsified number inside a gate, and the +/// only energy-calibration harness in this tree measures Apple silicon — so +/// there is no honest cross-vendor constant to ship. A profile's numbers are +/// the operator's to measure; this type only makes the model say where they +/// came from. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Calibration { + /// Concurrent requests in flight when the constants were measured. + pub concurrency: u32, + /// Where the measurement came from, so a reader can go and check it rather + /// than trust it. A harness run id, a receipt content-id, a document — + /// anything that resolves to the actual observation. + pub source: String, +} + +/// How far a profile is being applied from the point it was measured at. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Applicability { + /// The profile does not say where its constants were measured. Its error + /// direction is therefore unknown, which at a gate must be read as + /// possibly-under-estimating. + Unstated, + /// Applied at the concurrency it was measured at. + AtCalibration { concurrency: u32 }, + /// Applied **below** the calibration point, where per-token cost is higher + /// than the fit predicts. This is the under-estimating direction: the gate + /// admits work the ceiling cannot cover. + BelowCalibration { measured_at: u32, applied_at: u32 }, + /// Applied **above** the calibration point, where per-token cost is lower + /// than the fit predicts. This over-estimates, so the gate refuses work + /// that would have fit — the safe direction, and not an error worth + /// blocking on. + AboveCalibration { measured_at: u32, applied_at: u32 }, +} + +/// Why a cost model may not be used to gate at a given operating point. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CostModelError { + /// The profile declares no calibration point. + Uncalibrated, + /// The profile was fit at a higher occupancy than it is being applied at, + /// so it under-estimates here. + UnderEstimatesAtThisOccupancy { measured_at: u32, applied_at: u32 }, +} + +impl core::fmt::Display for CostModelError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + CostModelError::Uncalibrated => write!( + f, + "cost profile declares no calibration point, so its error \ + direction is unknown; an unknown direction at a ceiling must \ + be treated as under-estimating" + ), + CostModelError::UnderEstimatesAtThisOccupancy { measured_at, applied_at } => write!( + f, + "cost profile was measured at concurrency {measured_at} and is \ + being applied at {applied_at}; a saturation fit under-estimates \ + below its measurement point, and under-estimating admits work \ + the ceiling cannot cover" + ), + } + } +} + +impl std::error::Error for CostModelError {} + /// Tokens of context per unit of [`CostProfile::uj_per_token_per_kctx`]. const CTX_UNIT: u64 = 1024; @@ -64,12 +169,12 @@ impl CostProfile { /// Retained so existing callers are unchanged and the scalar case stays /// honestly expressible — some backends really are flat. pub fn linear(uj_per_token: u64) -> Self { - CostProfile { base_uj: 0, uj_per_token, uj_per_token_per_kctx: 0 } + CostProfile { base_uj: 0, uj_per_token, uj_per_token_per_kctx: 0, calibration: None } } /// A profile whose per-token cost grows with the context length. pub fn context_scaled(base_uj: u64, uj_per_token: u64, uj_per_token_per_kctx: u64) -> Self { - CostProfile { base_uj, uj_per_token, uj_per_token_per_kctx } + CostProfile { base_uj, uj_per_token, uj_per_token_per_kctx, calibration: None } } /// Whether this profile can represent a context-dependent cost at all. @@ -95,13 +200,72 @@ impl CostProfile { self.base_uj.saturating_add(flat).saturating_add(ctx) } + /// Record the operating point these constants were measured at. + pub fn measured_at(mut self, concurrency: u32, source: impl Into) -> Self { + self.calibration = + Some(Calibration { concurrency, source: source.into() }); + self + } + + /// How far this profile is being applied from where it was measured. + /// + /// Reports rather than decides — a caller that wants the *decision* uses + /// [`Self::require_applicable_at`]. Kept separate because "this is an + /// extrapolation" is worth surfacing in a receipt even where it is not + /// worth refusing over. + pub fn applicability_at(&self, concurrency: u32) -> Applicability { + match &self.calibration { + None => Applicability::Unstated, + Some(c) if c.concurrency == concurrency => { + Applicability::AtCalibration { concurrency } + } + Some(c) if concurrency < c.concurrency => Applicability::BelowCalibration { + measured_at: c.concurrency, + applied_at: concurrency, + }, + Some(c) => Applicability::AboveCalibration { + measured_at: c.concurrency, + applied_at: concurrency, + }, + } + } + + /// Refuse to gate with this profile at `concurrency` unless its error + /// direction is known to be safe. + /// + /// Refuses [`Applicability::Unstated`] and + /// [`Applicability::BelowCalibration`] — the two cases that under-estimate, + /// and so admit work the ceiling cannot cover. **Permits** + /// [`Applicability::AboveCalibration`]: over-estimating refuses work that + /// would have fit, which is visible to the operator and costs no joules. + /// + /// The asymmetry is the whole point. A guard that refused both directions + /// would be treating a conservative estimate as equally dangerous as an + /// optimistic one, at a ceiling where only one of them can overspend. + pub fn require_applicable_at(&self, concurrency: u32) -> Result { + match self.applicability_at(concurrency) { + Applicability::Unstated => Err(CostModelError::Uncalibrated), + Applicability::BelowCalibration { measured_at, applied_at } => { + Err(CostModelError::UnderEstimatesAtThisOccupancy { measured_at, applied_at }) + } + ok => Ok(ok), + } + } + /// The profile as it is sealed into a receipt, so a route decision can be /// re-derived rather than trusted. pub fn wire(&self) -> String { - format!( + let base = format!( "base={};per_token={};per_token_per_kctx={}", self.base_uj, self.uj_per_token, self.uj_per_token_per_kctx - ) + ); + // Appended only when declared, so a profile without a calibration + // seals byte-identically to every receipt written before this field + // existed and published vectors keep verifying. + match &self.calibration { + None => base, + Some(c) => format!("{base};calibrated_at={};source={}", c.concurrency, c.source), + } } } @@ -424,4 +588,100 @@ mod tests { assert_eq!(d.candidates.len(), 1); assert!(!d.candidates[0].admissible); } + +#[cfg(test)] +mod calibration_tests { + use super::*; + + /// A profile with no calibration seals exactly as before, so receipts and + /// published vectors written before this field existed keep verifying. + #[test] + fn an_uncalibrated_profile_seals_byte_identically() { + let p = CostProfile::linear(7); + assert_eq!(p.wire(), "base=0;per_token=7;per_token_per_kctx=0"); + assert!(!p.wire().contains("calibrated_at")); + + // And declaring one is additive — the historical prefix is untouched. + let c = p.clone().measured_at(64, "harness-run:abc123"); + assert!(c.wire().starts_with("base=0;per_token=7;per_token_per_kctx=0")); + assert!(c.wire().contains("calibrated_at=64")); + assert!(c.wire().contains("source=harness-run:abc123")); + } + + /// Declaring a calibration point changes no estimate. This type records + /// where the numbers came from; it does not adjust them, because adjusting + /// them would require a scaling law this tree has not measured. + #[test] + fn calibration_is_provenance_not_a_correction() { + let plain = CostProfile::context_scaled(100, 7, 3); + let declared = plain.clone().measured_at(64, "run:x"); + for (tokens, ctx) in [(0, 0), (100, 0), (1_000, 4_096), (10, 1_000_000)] { + assert_eq!( + plain.estimate_uj(tokens, ctx), + declared.estimate_uj(tokens, ctx), + "estimate must not move when provenance is added" + ); + } + } + + /// An undeclared calibration point is not a safe default. The profile's + /// error direction is unknown, and unknown at a ceiling has to be read as + /// possibly-under-estimating. + #[test] + fn an_uncalibrated_profile_may_not_gate() { + let p = CostProfile::linear(7); + assert_eq!(p.applicability_at(1), Applicability::Unstated); + assert_eq!(p.require_applicable_at(1), Err(CostModelError::Uncalibrated)); + } + + /// The unsafe direction: a fit taken at saturation, applied where batches + /// are thin. Per-token cost is higher than the fit predicts, so the gate + /// admits work the ceiling cannot cover. + #[test] + fn applying_a_saturation_fit_at_low_concurrency_is_refused() { + let p = CostProfile::linear(7).measured_at(64, "run:saturated"); + assert_eq!( + p.applicability_at(2), + Applicability::BelowCalibration { measured_at: 64, applied_at: 2 } + ); + assert_eq!( + p.require_applicable_at(2), + Err(CostModelError::UnderEstimatesAtThisOccupancy { measured_at: 64, applied_at: 2 }) + ); + } + + /// The asymmetry, stated as a test: over-estimating is **permitted**. + /// + /// A guard that refused both directions would treat a conservative estimate + /// as equally dangerous as an optimistic one. At a ceiling only one of them + /// can overspend — the other merely refuses work that would have fit, which + /// the operator can see and correct. + #[test] + fn over_estimating_is_permitted_because_only_under_estimating_overspends() { + let p = CostProfile::linear(7).measured_at(2, "run:thin"); + assert_eq!( + p.applicability_at(64), + Applicability::AboveCalibration { measured_at: 2, applied_at: 64 } + ); + assert!( + p.require_applicable_at(64).is_ok(), + "a profile that over-estimates here must still be usable to gate" + ); + + // At the measurement point it is simply applicable. + assert!(matches!( + p.require_applicable_at(2), + Ok(Applicability::AtCalibration { concurrency: 2 }) + )); + } + + /// The gate's existing saturating discipline is untouched: nothing added + /// here can make an estimate wrap and come back small. + #[test] + fn estimates_still_saturate_rather_than_wrap() { + let p = CostProfile::context_scaled(u64::MAX, u64::MAX, u64::MAX) + .measured_at(1, "run:absurd"); + assert_eq!(p.estimate_uj(u64::MAX, u64::MAX), u64::MAX); + } +} }