From 30370b199687300f8dabee59dd55c98b2761f1c9 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 1 Oct 2026 01:19:03 -0400 Subject: [PATCH] eoc-agent: a local model's measured step is a measured span LlmReply carried a bare `microjoules`, and charge_llm_step tagged every model step Estimated. A backend that ran a model in this process and read it on a live counter (an eoc-local backend's MeterSpan, now backed by the per-process counter) had no way to say so. Its measured figure was recorded as an estimate. - LlmReply carries a JouleCost. LlmReply::new is the provider's estimate, as before, and LlmReply::with_cost gives a figure with its grade. - charge_llm_step records the provider's cost as graded and charges the budget that figure. - NeuralProvider wraps any in-process NeuralBackend as an LlmProvider. The reply's energy is the backend's own JouleCost. Tokens are counted from the text (approx_tokens, or the model's tokenizer through with_token_counter) and feed only the token budget. A model served over the network still reports an estimate: it ran on hardware no counter here reads. Tests: an MlcBackend metered by an advancing counter gives a measured span, and a held, failed or stub counter gives the estimate. A whole ReAct run records the model step measured. Reverting charge_llm_step to always-estimated fails both end-to-end tests. Co-Authored-By: Claude Opus 5.5 --- eoc/eoc-rs/Cargo.lock | 2 + eoc/eoc-rs/crates/eoc-agent/Cargo.toml | 2 + eoc/eoc-rs/crates/eoc-agent/src/agent.rs | 45 +++-- eoc/eoc-rs/crates/eoc-agent/src/lib.rs | 9 +- eoc/eoc-rs/crates/eoc-agent/src/provider.rs | 206 ++++++++++++++++++++ 5 files changed, 244 insertions(+), 20 deletions(-) create mode 100644 eoc/eoc-rs/crates/eoc-agent/src/provider.rs diff --git a/eoc/eoc-rs/Cargo.lock b/eoc/eoc-rs/Cargo.lock index 02cf854f8..2c0f25329 100644 --- a/eoc/eoc-rs/Cargo.lock +++ b/eoc/eoc-rs/Cargo.lock @@ -734,7 +734,9 @@ version = "0.1.0" dependencies = [ "async-trait", "eoc-core", + "eoc-local", "eoc-meter", + "eoc-neural", "serde", "serde_json", "thiserror 2.0.20", diff --git a/eoc/eoc-rs/crates/eoc-agent/Cargo.toml b/eoc/eoc-rs/crates/eoc-agent/Cargo.toml index 7cea95528..30e0585df 100644 --- a/eoc/eoc-rs/crates/eoc-agent/Cargo.toml +++ b/eoc/eoc-rs/crates/eoc-agent/Cargo.toml @@ -11,6 +11,7 @@ description = "EOC — agentic loops (ReAct, Reflexion, ToT, LATS, Plan-and-Exec [dependencies] eoc-core.workspace = true eoc-meter.workspace = true +eoc-neural.workspace = true serde = { workspace = true } serde_json = "1" @@ -19,4 +20,5 @@ async-trait.workspace = true tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } [dev-dependencies] +eoc-local = { path = "../eoc-local", features = ["mlc"] } tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/eoc/eoc-rs/crates/eoc-agent/src/agent.rs b/eoc/eoc-rs/crates/eoc-agent/src/agent.rs index 594624aa1..a88faabc3 100644 --- a/eoc/eoc-rs/crates/eoc-agent/src/agent.rs +++ b/eoc/eoc-rs/crates/eoc-agent/src/agent.rs @@ -25,13 +25,19 @@ use crate::trace::{Span, SpanKind, Trace}; /// A bare-bones vendor-neutral LLM completion port. /// -/// Production deployments wire this to [`eoc_vendor_api`] backends; tests -/// supply scripted implementations. +/// A provider that calls a model over the network wires this to an +/// `eoc-vendor-api` client. One that runs a model in this process wraps an +/// [`eoc_neural::NeuralBackend`] in a +/// [`NeuralProvider`](crate::provider::NeuralProvider). Tests supply +/// scripted implementations. #[async_trait] pub trait LlmProvider: Send + Sync { - /// Run a single completion. Implementations *should* return their - /// best estimate of `prompt_tokens + completion_tokens` and the - /// estimated micro-joules attributable to the call. + /// Run a single completion. Implementations return + /// `prompt_tokens + completion_tokens` and the energy of the call with + /// its provenance: [`JouleCost::measured`] only when a live counter read + /// the call and covers where the model ran, [`JouleCost::estimated`] + /// otherwise. A model served over the network ran on hardware no counter + /// here reads, so a remote call's figure is an estimate. async fn complete(&self, prompt: &str) -> AgentResult; } @@ -42,18 +48,20 @@ pub struct LlmReply { pub text: String, /// Total tokens (prompt + completion). pub tokens: u64, - /// Energy estimate (micro-joules). - pub microjoules: u64, + /// The energy of the call and how the provider came by it. + pub cost: JouleCost, } impl LlmReply { - /// Convenience constructor. + /// A reply whose energy is the provider's estimate, `microjoules`. pub fn new(text: impl Into, tokens: u64, microjoules: u64) -> Self { - Self { - text: text.into(), - tokens, - microjoules, - } + Self::with_cost(text, tokens, JouleCost::estimated(microjoules)) + } + + /// A reply whose energy is `cost`, with the provenance the provider + /// gives it. + pub fn with_cost(text: impl Into, tokens: u64, cost: JouleCost) -> Self { + Self { text: text.into(), tokens, cost } } } @@ -338,9 +346,10 @@ impl AgentLoop { /// Charge an LLM step against the budget *and* record the span. /// Returns the stop reason if the budget is exhausted. /// - /// The step's figure is the provider's own, [`LlmReply::microjoules`], - /// an estimate by [`LlmProvider`]'s contract that no counter here read, - /// so the span is [`JouleSource::Estimated`](eoc_core::JouleSource). + /// The step's figure is the provider's own, [`LlmReply::cost`], with + /// the provenance the provider gave it: a span is + /// [`JouleSource::Measured`](eoc_core::JouleSource) only when the + /// provider's counter read the call. pub fn charge_llm_step( &mut self, kind: SpanKind, @@ -350,9 +359,9 @@ impl AgentLoop { ) -> Option { let span = Span::new(self.trace.len(), kind, label, input, reply.text.clone()) .with_tokens(reply.tokens) - .with_joule_cost(JouleCost::estimated(reply.microjoules)); + .with_joule_cost(reply.cost); self.push_span(span); - self.budget.charge_llm(reply.tokens, reply.microjoules) + self.budget.charge_llm(reply.tokens, reply.cost.microjoules) } /// Check the hard iteration cap. diff --git a/eoc/eoc-rs/crates/eoc-agent/src/lib.rs b/eoc/eoc-rs/crates/eoc-agent/src/lib.rs index 58e06ea77..72b083c77 100644 --- a/eoc/eoc-rs/crates/eoc-agent/src/lib.rs +++ b/eoc/eoc-rs/crates/eoc-agent/src/lib.rs @@ -4,8 +4,11 @@ //! through *think → act → observe* cycles, sometimes branching, sometimes //! reflecting, sometimes planning ahead. EOC takes that loop and threads //! a **joule meter** through every step, so an agent's total cost is the -//! sum of (a) provider-reported token costs, (b) tool-execution wall -//! energy, and (c) any local inference performed between steps. +//! sum of (a) its model steps, each at the figure and grade its provider +//! reports: measured when a model running in this process was read by a +//! live counter ([`NeuralProvider`]), estimated for a model served over the +//! network; (b) tool-execution energy; and (c) any local inference performed +//! between steps. //! //! ## Loops shipped //! @@ -37,6 +40,7 @@ pub mod codeact; pub mod error; pub mod lats; pub mod plan_execute; +pub mod provider; pub mod react; pub mod reflexion; pub mod tot; @@ -48,6 +52,7 @@ pub use codeact::CodeActLoop; pub use error::{AgentError, AgentResult}; pub use lats::{LatsConfig, LatsLoop}; pub use plan_execute::{PlanExecuteLoop, Planner, Worker}; +pub use provider::NeuralProvider; pub use react::{ReActLoop, ReActStep}; pub use reflexion::{ReflexionLoop, ReflexionMemory}; pub use tot::{Search, ToTLoop, Thought}; diff --git a/eoc/eoc-rs/crates/eoc-agent/src/provider.rs b/eoc/eoc-rs/crates/eoc-agent/src/provider.rs new file mode 100644 index 000000000..19f730ea4 --- /dev/null +++ b/eoc/eoc-rs/crates/eoc-agent/src/provider.rs @@ -0,0 +1,206 @@ +//! [`NeuralProvider`]: a model that runs in this process, as an +//! [`LlmProvider`]. +//! +//! A backend that runs inference here (an `eoc-local` backend, or any other +//! [`NeuralBackend`]) meters its own span and grades the figure: measured +//! when its counter read the inference and advanced over it, estimated +//! otherwise. A loop driven through this provider records that figure with +//! that grade, so a model step a live counter measured is a measured span. + +use async_trait::async_trait; +use eoc_core::Query; +use eoc_neural::NeuralBackend; + +use crate::agent::{LlmProvider, LlmReply}; +use crate::error::AgentResult; + +/// An in-process [`NeuralBackend`] as an [`LlmProvider`]. The reply's text is +/// the backend's payload and its energy is the backend's own +/// [`JouleCost`](eoc_core::JouleCost), grade and all. +/// +/// A [`eoc_core::Response`] carries no token count, so the reply's tokens +/// are counted from the text, prompt and completion, by `count_tokens`: +/// [`approx_tokens`] unless [`Self::with_token_counter`] gives the backend's +/// own tokenizer. Tokens feed the loop's token budget, never its energy +/// figure. +pub struct NeuralProvider { + backend: B, + count_tokens: fn(&str) -> u64, +} + +impl NeuralProvider { + /// Drive `backend`, counting tokens with [`approx_tokens`]. + pub fn new(backend: B) -> Self { + Self { + backend, + count_tokens: approx_tokens, + } + } + + /// Count tokens with `count` instead, such as the model's tokenizer. + pub fn with_token_counter(mut self, count: fn(&str) -> u64) -> Self { + self.count_tokens = count; + self + } + + /// The backend. + pub fn backend(&self) -> &B { + &self.backend + } +} + +#[async_trait] +impl LlmProvider for NeuralProvider { + async fn complete(&self, prompt: &str) -> AgentResult { + let response = self.backend.infer(&Query::new(prompt)).await; + let tokens = + (self.count_tokens)(prompt).saturating_add((self.count_tokens)(&response.payload)); + Ok(LlmReply::with_cost( + response.payload, + tokens, + response.joule_cost, + )) + } +} + +/// About one token per four characters, rounded up, and at least one for +/// any text that is not empty: the usual rule of thumb for English with a +/// BPE vocabulary. +pub fn approx_tokens(text: &str) -> u64 { + (text.chars().count() as u64).div_ceil(4) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use eoc_core::{JouleCost, JouleSource, Response, Stage}; + use eoc_local::mlc::MlcBackend; + use eoc_meter::{JouleCounter, StubCounter}; + use eoc_neural::EchoBackend; + + use super::*; + use crate::agent::scripted::Scripted; + use crate::agent::{Agent, AgentLoop, Budget, StopReason, ToolBox}; + use crate::react::ReActLoop; + use crate::trace::SpanKind; + + /// A backend that answers `text` at a fixed cost. + struct Fixed { + text: &'static str, + cost: JouleCost, + } + + #[async_trait] + impl NeuralBackend for Fixed { + async fn infer(&self, q: &Query) -> Response { + Response::new(q.id, self.text.to_string(), Stage::Neural, self.cost) + } + } + + fn mlc(meter: Arc) -> NeuralProvider { + let backend = MlcBackend::new(std::env::temp_dir(), meter) + .expect("the temp dir exists") + .with_max_tokens(16); + NeuralProvider::new(backend) + } + + #[tokio::test] + async fn the_reply_carries_the_backends_figure_and_grade() { + let measured = NeuralProvider::new(Fixed { + text: "hi", + cost: JouleCost::measured(450), + }); + let reply = measured.complete("say hi").await.unwrap(); + assert_eq!( + (reply.text.as_str(), reply.cost), + ("hi", JouleCost::measured(450)) + ); + assert_eq!(reply.tokens, approx_tokens("say hi") + approx_tokens("hi")); + + let echo = NeuralProvider::new(EchoBackend { + estimated_microjoules: 7_000, + prefix: String::new(), + }); + assert_eq!( + echo.complete("x").await.unwrap().cost, + JouleCost::estimated(7_000) + ); + } + + /// A local backend metered by a counter that advanced over the + /// inference: the loop records a measured span of what the counter read, + /// and charges the budget that. Where the counter held, failed or is the + /// stub, the span is the backend's estimate, graded estimated. + #[tokio::test] + async fn a_local_backends_measured_inference_is_a_measured_span() { + let cases: [(Arc, Option); 4] = [ + ( + Arc::new(Scripted::new(&[Some(1_000), Some(1_450)])), + Some(450), + ), + (Arc::new(Scripted::new(&[Some(7_000)])), None), + (Arc::new(Scripted::new(&[None, Some(9_000_000)])), None), + (Arc::new(StubCounter), None), + ]; + for (meter, expected) in cases { + let reply = mlc(meter).complete("prompt").await.unwrap(); + let mut lp = AgentLoop::new(Budget::unlimited(4)); + assert_eq!( + lp.charge_llm_step(SpanKind::Think, "think", "prompt", &reply), + None + ); + let span = &lp.trace.spans[0]; + match expected { + Some(uj) => { + assert_eq!( + (span.microjoules, span.joule_source), + (uj, JouleSource::Measured), + "{span:?}" + ); + } + None => { + assert_eq!(span.joule_source, JouleSource::Estimated, "{span:?}"); + assert!( + span.microjoules > 0, + "an inference that ran estimated at 0: {span:?}" + ); + } + } + assert_eq!(lp.budget.joules.used_microjoules, span.microjoules); + assert_eq!(lp.budget.tokens.used, reply.tokens); + } + } + + /// Through a whole ReAct run: the model's step is measured when the + /// backend's counter measured it, and the loop's total is that figure. + #[tokio::test] + async fn a_react_run_on_a_metered_local_model_records_the_model_step_measured() { + let provider = NeuralProvider::new(Fixed { + text: "Thought: done\nAction: Finish[42]", + cost: JouleCost::measured(2_500), + }); + let mut agent = + ReActLoop::new(Arc::new(provider), ToolBox::new(), Budget::unlimited(4), ""); + let (answer, stop) = agent.run("the answer").await.unwrap(); + assert_eq!((answer.as_str(), stop), ("42", StopReason::Done)); + let model: Vec<_> = agent + .trace() + .spans + .iter() + .filter(|s| s.joule_source == JouleSource::Measured) + .collect(); + assert_eq!(model.len(), 1, "{:?}", agent.trace().spans); + assert_eq!(model[0].microjoules, 2_500); + assert_eq!(agent.budget().joules.used_microjoules, 2_500); + } + + #[test] + fn approx_tokens_rounds_up() { + assert_eq!(approx_tokens(""), 0); + assert_eq!(approx_tokens("a"), 1); + assert_eq!(approx_tokens("abcd"), 1); + assert_eq!(approx_tokens("abcde"), 2); + assert_eq!(approx_tokens("ééééé"), 2, "characters, not bytes"); + } +} -- 2.54.0 (Apple Git-157)