//! In-process L3 backend: JouleClaw's **own from-scratch GGUF tensor runtime**. //! //! Where [`crate::llama_backend::LlamaCompletionBackend`] shells out to //! `llama.cpp`, this backend runs the model **inside the process** on //! `jouleclaw-runtime` — the family's own kernel graph engine //! (`jouleclaw-backend-reference` / `jouleclaw-backend-apple`) over //! `jouleclaw-loader-gguf`. No external binary, no FFI: tokenize → build the //! decode-step graph → compile against the registered kernels → autoregressive //! decode → detokenize, all in Rust the family wrote. //! //! It implements the same [`LlmBackend`] trait as every other backend, so it //! drops into [`crate::jouleclaw_l3_stage`] unchanged. The model loads once at //! construction (memory-mapped GGUF); each `complete` call is one `generate`. use jouleclaw_llm::{FinishReason, LlmBackend, LlmError, LlmRequest, LlmResponse}; use jouleclaw_loader_gguf::sample::SamplingConfig; use jouleclaw_loader_gguf::tokenizer::Vocab; use jouleclaw_loader_gguf::{read_gguf_file, GgufModel}; use jouleclaw_runtime::{generate, GenerateConfig, KvCacheKind, TokenizerKind}; use std::path::Path; /// An L3 backend backed by an in-process GGUF model on `jouleclaw-runtime`. pub struct JouleclawRuntimeBackend { model: GgufModel, vocab: Vocab, name: String, /// Fallback joule cost reported when no host meter wraps the call. typical_joules: f64, } impl JouleclawRuntimeBackend { /// Load a GGUF model from disk and prepare its vocabulary. The model is /// memory-mapped; `complete` runs entirely in-process thereafter. pub fn from_path(path: impl AsRef) -> Result { let path = path.as_ref(); let model = read_gguf_file(path) .map_err(|e| LlmError::Unavailable(format!("read GGUF {}: {e:?}", path.display())))?; let vocab = Vocab::from_gguf(&model) .map_err(|e| LlmError::Unavailable(format!("load vocab: {e:?}")))?; let name = model .metadata_string("general.name") .map(|s| s.to_string()) .unwrap_or_else(|| { path.file_stem() .and_then(|s| s.to_str()) .unwrap_or("gguf-model") .to_string() }); Ok(Self { model, vocab, name, typical_joules: 1.0, }) } } impl LlmBackend for JouleclawRuntimeBackend { fn model_name(&self) -> &str { &self.name } fn complete(&self, request: &LlmRequest) -> Result { // temperature 0 → deterministic greedy (the runtime's determinism // oracle path); >0 → seeded temperature sampling. let sampling = if request.temperature <= 0.0 { SamplingConfig::greedy() } else { SamplingConfig::temperature(request.temperature, 0) }; let cfg = GenerateConfig { max_new_tokens: request.max_tokens as usize, sampling, add_bos: true, tokenizer_kind: TokenizerKind::Auto, cache_kind: KvCacheKind::InPlace, max_seq: None, stop_strings: request.stop.clone(), }; let result = generate(&self.model, &self.vocab, &request.prompt, &cfg) .map_err(|e| LlmError::Other(format!("generate: {e}")))?; Ok(LlmResponse { text: result.text, finish_reason: if result.stopped_at_eos { FinishReason::Stop } else { FinishReason::Length }, input_tokens: result.prompt_token_count as u32, output_tokens: result.tokens.len() as u32, // The `generate` path itself carries no energy meter; the run binary // meters host energy over the whole call and stamps the receipt. So // honestly: no per-call meter here (`None`), not a fabricated value. energy_joules: None, }) } fn typical_joules_per_call(&self) -> f64 { self.typical_joules } }