jcp-trace: an undefined wedge metric is not zero #292
3 changed files with 79 additions and 15 deletions
|
|
@ -77,7 +77,14 @@ fn main() {
|
|||
println!("\n── sealed eval/1 receipt ──");
|
||||
println!(" tokens : {} in / {} out", t.input_tokens, t.output_tokens);
|
||||
println!(" energy : {} µJ ({})", t.uj_total, t.energy_provenance.jcr1_label());
|
||||
println!(" uj_per_accepted_token: {}", t.uj_per_accepted_token);
|
||||
println!(
|
||||
" uj_per_accepted_token: {}",
|
||||
// "—" when no tokens were accepted: 0 is this figure's optimum, so
|
||||
// printing it would rank a run that delivered nothing above every run
|
||||
// that did.
|
||||
t.uj_per_accepted_token
|
||||
.map_or_else(|| "—".to_string(), |uj| uj.to_string())
|
||||
);
|
||||
println!(" envelope : {} bytes, content id {}", envelope.len(),
|
||||
&ContentId::of_bytes(&envelope).to_hex()[..16]);
|
||||
|
||||
|
|
|
|||
|
|
@ -277,8 +277,12 @@ impl GenAiTrace {
|
|||
max_end = max_end.max(span.end_ns);
|
||||
}
|
||||
t.total_tokens = t.input_tokens.saturating_add(t.output_tokens);
|
||||
// The wedge metric: joules per accepted output token.
|
||||
t.uj_per_accepted_token = t.uj_total.checked_div(t.output_tokens).unwrap_or(0);
|
||||
// The wedge metric: joules per accepted output token. `None` when no
|
||||
// tokens were accepted — the division is undefined then, and 0 is this
|
||||
// figure's optimum, so returning it reported spend that bought nothing
|
||||
// as the cheapest run in the ledger. `mesh/SPEC.md` §6 states the family
|
||||
// convention: "omitted when no tokens were accepted".
|
||||
t.uj_per_accepted_token = t.uj_total.checked_div(t.output_tokens);
|
||||
// The total is only as trustworthy as its weakest measured component.
|
||||
t.energy_provenance = prov.unwrap_or(Provenance::Estimator);
|
||||
t.wall_ms = if max_end > min_start && min_start != u64::MAX {
|
||||
|
|
@ -301,8 +305,9 @@ pub struct Totals {
|
|||
pub total_tokens: u64,
|
||||
/// Total microjoules across all spans that carry a reading.
|
||||
pub uj_total: u64,
|
||||
/// Microjoules per accepted output token — the headline figure.
|
||||
pub uj_per_accepted_token: u64,
|
||||
/// Microjoules per accepted output token — the headline figure. `None` when
|
||||
/// no tokens were accepted (the division is undefined, not zero-cost).
|
||||
pub uj_per_accepted_token: Option<u64>,
|
||||
/// The weakest provenance among the spans (the honest aggregate trust).
|
||||
pub energy_provenance: Provenance,
|
||||
/// Wall-clock span of the run in milliseconds.
|
||||
|
|
@ -316,7 +321,7 @@ impl Default for Totals {
|
|||
output_tokens: 0,
|
||||
total_tokens: 0,
|
||||
uj_total: 0,
|
||||
uj_per_accepted_token: 0,
|
||||
uj_per_accepted_token: None,
|
||||
energy_provenance: Provenance::Estimator,
|
||||
wall_ms: 0,
|
||||
}
|
||||
|
|
@ -768,7 +773,7 @@ fn eval_wire(
|
|||
steward: &str,
|
||||
) -> Value {
|
||||
let trace_cid = trace.content_id();
|
||||
json!({
|
||||
let mut o = json!({
|
||||
"eval_receipt": "eval/1",
|
||||
"trace_id": trace_cid.to_hex(),
|
||||
"gate_spec_id": gate_spec_id,
|
||||
|
|
@ -777,7 +782,6 @@ fn eval_wire(
|
|||
"output_tokens": totals.output_tokens,
|
||||
"total_tokens": totals.total_tokens,
|
||||
"uj_total": totals.uj_total,
|
||||
"uj_per_accepted_token": totals.uj_per_accepted_token,
|
||||
"energy_provenance": totals.energy_provenance.jcr1_label(),
|
||||
"wall_ms": totals.wall_ms,
|
||||
"assessments": assessments.iter().map(Assessment::wire).collect::<Vec<_>>(),
|
||||
|
|
@ -792,7 +796,13 @@ fn eval_wire(
|
|||
"seq": seq,
|
||||
"prev": prev.to_hex(),
|
||||
"steward": steward,
|
||||
})
|
||||
});
|
||||
// Omitted when undefined rather than sealed as 0. Canonical CBOR sorts keys,
|
||||
// so inserting here does not change the wire for a receipt that has one.
|
||||
if let Some(uj) = totals.uj_per_accepted_token {
|
||||
o["uj_per_accepted_token"] = json!(uj);
|
||||
}
|
||||
o
|
||||
}
|
||||
|
||||
/// A sealed eval receipt plus the projections that went into it.
|
||||
|
|
@ -850,8 +860,9 @@ pub struct EvalReceipt {
|
|||
pub output_tokens: u64,
|
||||
/// Total microjoules.
|
||||
pub uj_total: u64,
|
||||
/// Microjoules per accepted token.
|
||||
pub uj_per_accepted_token: u64,
|
||||
/// Microjoules per accepted token. `None` when the receipt sealed no figure
|
||||
/// because no tokens were accepted.
|
||||
pub uj_per_accepted_token: Option<u64>,
|
||||
/// The aggregate energy provenance.
|
||||
pub energy_provenance: String,
|
||||
/// Did the quality gate pass?
|
||||
|
|
@ -899,7 +910,7 @@ pub fn verify(envelope: &[u8], steward: &str, pubkey: &[u8; 32]) -> Result<EvalR
|
|||
gate_spec_id: s("gate_spec_id")?,
|
||||
output_tokens: u("output_tokens")?,
|
||||
uj_total: u("uj_total")?,
|
||||
uj_per_accepted_token: u("uj_per_accepted_token")?,
|
||||
uj_per_accepted_token: v.get("uj_per_accepted_token").and_then(Value::as_u64),
|
||||
energy_provenance: s("energy_provenance")?,
|
||||
gate_passed: u("gate_passed")? != 0,
|
||||
gate_failures: failures,
|
||||
|
|
@ -1013,10 +1024,56 @@ mod tests {
|
|||
// (800+120 + 1500+200) tokens * 10 µJ = 26_200 µJ
|
||||
assert_eq!(t.uj_total, 26_200);
|
||||
// 26_200 / 320 accepted = 81 µJ/accepted-token (integer)
|
||||
assert_eq!(t.uj_per_accepted_token, 81);
|
||||
assert_eq!(t.uj_per_accepted_token, Some(81));
|
||||
assert_eq!(t.energy_provenance, Provenance::Estimator);
|
||||
}
|
||||
|
||||
/// A run that burned prompt tokens and accepted no output has no wedge
|
||||
/// metric — the division is undefined. It is not 0, which is this figure's
|
||||
/// optimum: that reported the run as the cheapest in the ledger, and the
|
||||
/// figure is sealed into `eval/1` and read straight back out rather than
|
||||
/// recomputed. `mesh/SPEC.md` §6 states the family convention: "omitted
|
||||
/// when no tokens were accepted".
|
||||
#[test]
|
||||
fn a_run_that_accepted_nothing_seals_no_wedge_metric() {
|
||||
let (sk, pk, steward) = steward_key();
|
||||
let mut trace = GenAiTrace::new(
|
||||
"trace-nothing-accepted",
|
||||
vec![GenAiSpan::model_span(
|
||||
"s1",
|
||||
"plan",
|
||||
SpanType::ChatModel,
|
||||
"ssm-1b",
|
||||
800,
|
||||
0,
|
||||
)],
|
||||
);
|
||||
trace.attach_estimator_energy(10);
|
||||
|
||||
let t = trace.totals();
|
||||
assert_eq!(t.output_tokens, 0, "nothing was accepted");
|
||||
assert_eq!(t.uj_total, 8_000, "but the spend is real");
|
||||
assert_eq!(
|
||||
t.uj_per_accepted_token, None,
|
||||
"0 uJ/token would read as free, and as cheaper than any real run"
|
||||
);
|
||||
|
||||
let assessments = vec![Assessment::judge_pass("correctness", "openai:/gpt-4o")];
|
||||
let gate = Gate::require_pass(["correctness"]);
|
||||
let sealed =
|
||||
seal_eval(&trace, &assessments, &gate, &sk, &steward, 0, &genesis_prev()).unwrap();
|
||||
assert!(
|
||||
sealed.wire.get("uj_per_accepted_token").is_none(),
|
||||
"an undefined figure is omitted from the signed core: {}",
|
||||
sealed.wire
|
||||
);
|
||||
|
||||
let r = verify(&sealed.envelope, &steward, &pk)
|
||||
.expect("a receipt without the figure still verifies");
|
||||
assert_eq!(r.uj_per_accepted_token, None);
|
||||
assert_eq!(r.uj_total, 8_000, "and the spend is still on it");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggregate_provenance_is_the_weakest() {
|
||||
let mut trace = sample_trace();
|
||||
|
|
@ -1060,7 +1117,7 @@ mod tests {
|
|||
|
||||
let r = verify(&sealed.envelope, &steward, &pk).unwrap();
|
||||
assert_eq!(r.output_tokens, 320);
|
||||
assert_eq!(r.uj_per_accepted_token, 81);
|
||||
assert_eq!(r.uj_per_accepted_token, Some(81));
|
||||
assert_eq!(r.energy_provenance, "Estimator");
|
||||
assert!(r.gate_passed);
|
||||
assert_eq!(r.sample_size, 2);
|
||||
|
|
|
|||
|
|
@ -242,7 +242,7 @@ mod tests {
|
|||
// Only the one chat span carries tokens: (1200+180) * 5 = 6_900 µJ.
|
||||
assert_eq!(t.uj_total, 6_900);
|
||||
assert_eq!(t.output_tokens, 180);
|
||||
assert_eq!(t.uj_per_accepted_token, 38); // 6_900 / 180
|
||||
assert_eq!(t.uj_per_accepted_token, Some(38)); // 6_900 / 180
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
Loading…
Reference in a new issue