From 6b782b5ef64e7ae833e2e43f40ac31dcef85f5f4 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Fri, 16 Oct 2026 04:56:35 -0400 Subject: [PATCH] jouleclaw-diagnose: a cross-model panel, so a verdict can rest on concurring models, or say they did not concur `Backing::Concurring` and `Backing::Contested` were in the schema, and `require_substantive_grounding` already accepted them. But nothing produced either: diagnose only ever declared one entailer's fixed backing. A verdict could never say "two independent models agreed", nor record that they did not. - `PanelEntailer`: at least two members, each a different model. Each consulted (claim, item) pair is put to every member, and the report carries every member's result under its own model id, so `require_substantive_grounding`'s check that the claim matches the evidence holds. - Its backing is read from the results (`Entailer::backing_of`): `Concurring` when every member gave the same label on every pair, and `Contested` when any two differed. A panel is only as substantive as its members. A fixture member makes the whole panel a fixture, and an unstated member leaves it unstated, so a stand-in's labels never become corroboration. - Grounding is unanimous per item: an item grounds a claim only when every result for that pair entails it. One model's "entails" no longer outvotes another's "neutral", and a claim some model entailed, without every model's agreement on an item, says `models_disagree: true`. A pair one member entails and another contradicts is the existing consistency conflict. - Single-model behaviour is unchanged. Each pair has one result, so "every result entails" is "it entails". The new detail key appears only where a panel split, so a single model's report keeps its bytes. The trait's two new methods are provided, so existing entailers are untouched. Co-Authored-By: Claude Opus 5.5 --- .../crates/jouleclaw-diagnose/src/entailer.rs | 234 +++++++++++++++++- .../crates/jouleclaw-diagnose/src/lib.rs | 2 +- .../crates/jouleclaw-diagnose/src/report.rs | 107 +++++++- .../jouleclaw-diagnose/src/valid_answer.rs | 47 ++-- 4 files changed, 362 insertions(+), 28 deletions(-) diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/entailer.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/entailer.rs index 2712abaf9..7b06548f7 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/entailer.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/entailer.rs @@ -16,8 +16,11 @@ //! - [`FixtureEntailer`] — a programmable mock keyed by `(premise, //! hypothesis)` tuple, used in unit tests of the conflict search //! so the tests don't need to load DeBERTa. +//! +//! [`PanelEntailer`] puts each pair to several independent backends, so a +//! verdict can rest on models that concurred, or record that they did not. -use std::collections::HashMap; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Mutex; use std::time::Instant; @@ -80,6 +83,25 @@ pub trait Entailer: Send + Sync { Backing::Unstated } + /// What `results` — this backend's output for one verification — rest + /// on. For a single backend that is [`Self::backing`], whatever its labels + /// were; a [`PanelEntailer`]'s depends on whether its members agreed. + fn backing_of(&self, _results: &[EntailmentResult]) -> Backing { + self.backing() + } + + /// One result per model this backend consults for `(claim, item)`. A + /// single model returns its one result ([`Self::entail_claim_against`]); + /// a [`PanelEntailer`] returns one from each member, each under its own + /// model id, so the report carries every model's label. + fn entail_claim_against_each( + &self, + claim: &AtomicClaim, + item: &RetrievedItem, + ) -> Result, EntailError> { + Ok(vec![self.entail_claim_against(claim, item)?]) + } + /// Wrap a raw entailment for (claim, item) in a schema-shaped /// [`EntailmentResult`] with attribution. Implementations /// usually don't override this; the default builds the result @@ -301,7 +323,173 @@ impl Entailer for FixtureEntailer { } } -/// Run [`Entailer::entail_claim_against`] for the cross product of +// ────────────────────────────────────────────────────────────────── +// Cross-model panel +// ────────────────────────────────────────────────────────────────── + +/// Independent entailment backends judging the same pairs (§6.4). Each +/// consulted (claim, item) pair is put to every member, and the report +/// carries every member's result under its own model id. The panel's backing +/// is read from those results: [`Backing::Concurring`] when every member gave +/// the same label on every pair, [`Backing::Contested`] when any two differed +/// on any pair. A disagreement is a finding, kept, never collapsed to one +/// model's answer. +/// +/// A panel is only as substantive as its members. If any member is not a +/// substantive single model, the panel declares that floor instead: a +/// fixture member makes the whole panel a fixture, and an unstated member (or +/// a panel nested as a member) leaves it unstated. A fixture's labels do not +/// become corroboration by sitting beside a model's. +/// +/// What a panel's labels mean for the verdict is set by the checks, not here: +/// a claim is grounded by an item only when every result for that pair +/// entails it ([`crate::valid_answer::grounding_violations`]), and a pair one +/// member entails while another contradicts it is a surfaced conflict +/// ([`crate::valid_answer::consistency_violations`]). +pub struct PanelEntailer { + members: Vec>, + id: String, +} + +/// Why a [`PanelEntailer`] could not be formed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PanelError { + /// A panel needs at least two members to corroborate anything. + TooFewMembers(usize), + /// Two members share a model id, so their agreement would be one model + /// agreeing with itself. + DuplicateModel(String), +} + +impl std::fmt::Display for PanelError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::TooFewMembers(n) => write!(f, "a panel needs at least two members, got {n}"), + Self::DuplicateModel(id) => write!(f, "model {id:?} sits on the panel twice"), + } + } +} + +impl std::error::Error for PanelError {} + +impl PanelEntailer { + /// A panel of `members`: at least two, each a different model. + pub fn new(members: Vec>) -> Result { + if members.len() < 2 { + return Err(PanelError::TooFewMembers(members.len())); + } + let mut seen = BTreeSet::new(); + for m in &members { + if !seen.insert(m.model_id().to_string()) { + return Err(PanelError::DuplicateModel(m.model_id().to_string())); + } + } + let id = format!("panel[{}]", members.iter().map(|m| m.model_id()).collect::>().join(",")); + Ok(Self { members, id }) + } + + /// The members' model ids, sorted. + fn member_ids(&self) -> Vec { + let mut ids: Vec = self.members.iter().map(|m| m.model_id().to_string()).collect(); + ids.sort(); + ids + } + + /// The floor a member puts under the panel, or `None` when every member + /// is a substantive single model. + fn member_floor(&self) -> Option { + let backings: Vec = self.members.iter().map(|m| m.backing()).collect(); + if backings.iter().any(|b| matches!(b, Backing::Fixture { .. })) { + return Some(Backing::Fixture { model_ids: self.member_ids() }); + } + if backings.iter().all(|b| matches!(b, Backing::SingleModel { .. })) { + None + } else { + Some(Backing::Unstated) + } + } +} + +impl Entailer for PanelEntailer { + /// A panel has no single label: each member's result is taken on its own + /// ([`Entailer::entail_claim_against_each`]). + fn entail_raw(&self, _premise: &str, _hypothesis: &str) -> Result { + Err(EntailError::Backend(format!( + "{} has no single label; its members' results are taken one by one", + self.id + ))) + } + + fn model_id(&self) -> &str { + &self.id + } + + /// Before any result, a panel can claim nothing beyond its members' + /// floor: what it rests on depends on whether they agree + /// ([`Entailer::backing_of`]). + fn backing(&self) -> Backing { + self.member_floor().unwrap_or(Backing::Unstated) + } + + fn backing_of(&self, results: &[EntailmentResult]) -> Backing { + if let Some(floor) = self.member_floor() { + return floor; + } + let mut by_pair: BTreeMap<(Uuid, &[Uuid]), Vec<&EntailmentLabel>> = BTreeMap::new(); + for r in results { + by_pair.entry((r.claim_id, r.premise_item_ids.as_slice())).or_default().push(&r.label); + } + let contested = by_pair.values().any(|labels| labels.windows(2).any(|w| w[0] != w[1])); + if contested { + Backing::Contested { model_ids: self.member_ids() } + } else { + Backing::Concurring { model_ids: self.member_ids() } + } + } + + fn entail_claim_against_each( + &self, + claim: &AtomicClaim, + item: &RetrievedItem, + ) -> Result, EntailError> { + let mut out = Vec::with_capacity(self.members.len()); + for m in &self.members { + out.extend(m.entail_claim_against_each(claim, item)?); + } + Ok(out) + } +} + +/// A test stand-in that declares itself a single model, so the tests can +/// exercise a panel of substantive members without loading one. Test-only: +/// it never ships, which is what keeps its claim harmless. +#[cfg(test)] +pub(crate) struct ScriptedModel { + pub(crate) inner: FixtureEntailer, + id: String, +} + +#[cfg(test)] +impl ScriptedModel { + pub(crate) fn new(id: &str) -> Self { + Self { inner: FixtureEntailer::new(), id: id.into() } + } +} + +#[cfg(test)] +impl Entailer for ScriptedModel { + fn entail_raw(&self, premise: &str, hypothesis: &str) -> Result { + self.inner.entail_raw(premise, hypothesis) + } + fn model_id(&self) -> &str { + &self.id + } + fn backing(&self) -> Backing { + Backing::SingleModel { model_id: self.id.clone() } + } +} + +/// Run [`Entailer::entail_claim_against_each`] for the cross product of /// `claims` × `items` in parallel via `std::thread::scope`. Each /// (claim, item) pair runs on its own OS thread so Accelerate's /// sgemm calls can interleave across cores via GCD's shared @@ -353,19 +541,19 @@ pub fn entail_batch( // Fast path: single pair → no thread scope overhead. if n == 1 { let (_, claim, item) = pairs[0]; - let r = entailer.entail_claim_against(claim, item)?; - let joules = r.joules_spent; + let rs = entailer.entail_claim_against_each(claim, item)?; + let joules = rs.iter().map(|r| r.joules_spent).sum(); let _ = start.elapsed(); - return Ok((vec![r], joules)); + return Ok((rs, joules)); } // Parallel path: one OS thread per pair, joined in order. - let mut indexed_results: Vec<(usize, Result)> = + let mut indexed_results: Vec<(usize, Result, EntailError>)> = std::thread::scope(|scope| { let mut handles = Vec::with_capacity(n); for (idx, claim, item) in pairs { let h = - scope.spawn(move || (idx, entailer.entail_claim_against(claim, item))); + scope.spawn(move || (idx, entailer.entail_claim_against_each(claim, item))); handles.push(h); } handles @@ -382,10 +570,11 @@ pub fn entail_batch( // cleanly without losing the rest's work for diagnostics. let mut results = Vec::with_capacity(n); let mut joules_total = 0.0; - for (_, r) in indexed_results { - let r = r?; - joules_total += r.joules_spent; - results.push(r); + for (_, rs) in indexed_results { + for r in rs? { + joules_total += r.joules_spent; + results.push(r); + } } let _ = start.elapsed(); Ok((results, joules_total)) @@ -525,4 +714,27 @@ mod tests { assert_eq!(results.len(), 4); assert!(joules >= 4.0); // 4 calls * 1.0 per call default } + + /// A panel needs two different models: one member corroborates nothing, + /// and the same model twice would be one model agreeing with itself. + #[test] + fn a_panel_needs_two_different_models() { + let one: Vec> = vec![Box::new(ScriptedModel::new("a"))]; + assert_eq!(PanelEntailer::new(one).err(), Some(PanelError::TooFewMembers(1))); + let twice: Vec> = vec![Box::new(ScriptedModel::new("a")), Box::new(ScriptedModel::new("a"))]; + assert_eq!(PanelEntailer::new(twice).err(), Some(PanelError::DuplicateModel("a".into()))); + } + + /// A panel's floor comes from its weakest member: an unstated member (here + /// a nested panel, which states nothing before its results) leaves it + /// unstated; only substantive single models make a panel that can concur. + #[test] + fn a_panel_is_only_as_substantive_as_its_members() { + let nested = PanelEntailer::new(vec![Box::new(ScriptedModel::new("x")), Box::new(ScriptedModel::new("y"))]).unwrap(); + let outer = PanelEntailer::new(vec![Box::new(ScriptedModel::new("a")), Box::new(nested)]).unwrap(); + assert_eq!(outer.backing_of(&[]), Backing::Unstated); + let models = PanelEntailer::new(vec![Box::new(ScriptedModel::new("b")), Box::new(ScriptedModel::new("a"))]).unwrap(); + assert_eq!(models.backing_of(&[]), Backing::Concurring { model_ids: vec!["a".into(), "b".into()] }); + assert_eq!(models.model_id(), "panel[b,a]"); + } } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/lib.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/lib.rs index 9dbc58030..ea1f61a00 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/lib.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/lib.rs @@ -26,7 +26,7 @@ pub mod valid_answer; pub mod verdict; pub use atomizer::{atomize_sentences, AtomizeError, Atomizer, SentenceAtomizer}; -pub use entailer::{DebertaEntailer, EntailError, Entailer, FixtureEntailer}; +pub use entailer::{DebertaEntailer, EntailError, Entailer, FixtureEntailer, PanelEntailer, PanelError}; pub use recovery::recovery_actions_for_violations; pub use report::{verify, VerifyError, VerifyInputs}; pub use valid_answer::{ diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/report.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/report.rs index d7dd4ce17..480e2f80f 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/report.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/report.rs @@ -128,7 +128,7 @@ pub fn verify( let grounding_backing = if entailments.is_empty() { Backing::Unconsulted } else { - entailer.backing() + entailer.backing_of(&entailments) }; let report = VerificationReport { @@ -492,4 +492,109 @@ mod tests { assert!(!can_compose(VerificationAction::ReRouteWithRefinement)); assert!(!can_compose(VerificationAction::Refuse)); } + + // ── Cross-model panel ──────────────────────────────────────────── + + use crate::entailer::{PanelEntailer, ScriptedModel}; + + const PARIS: &str = "Paris is the capital of France."; + + /// One claim ("Paris is the capital of France.") against one item saying + /// the same, verified by a panel of two scripted models whose labels for + /// that pair are set by `a` and `b`. + fn verify_with_panel(a: fn(&ScriptedModel), b: fn(&ScriptedModel)) -> VerificationReport { + let plan = simple_plan(); + let items = vec![item_for("q0", "wikidata:Q90", 1, PARIS)]; + let segs = vec![("s0".into(), PARIS.into())]; + let claims = atomize_sentences(&segs, &axes_with_tier(1)).unwrap(); + let (ma, mb) = (ScriptedModel::new("nli-a"), ScriptedModel::new("nli-b")); + a(&ma); + b(&mb); + let panel = PanelEntailer::new(vec![Box::new(ma), Box::new(mb)]).unwrap(); + verify(&VerifyInputs::new(&plan, &items, &[], &claims), &panel).unwrap() + } + + fn entails(m: &ScriptedModel) { + m.inner.set_entails(PARIS, PARIS); + } + fn neutral(m: &ScriptedModel) { + m.inner.set_neutral(PARIS, PARIS); + } + fn contradicts(m: &ScriptedModel) { + m.inner.set_contradicts(PARIS, PARIS); + } + + /// Two models that both entail the claim ground it, and the report says it + /// rests on two concurring models, each of whose results it carries. + #[test] + fn a_panel_that_agrees_grounds_the_claim_as_concurring() { + let report = verify_with_panel(entails, entails); + assert!(report.violations.is_empty(), "{:?}", report.violations); + let ids: Vec<&str> = report.entailments_consulted.iter().map(|e| e.model_id.as_str()).collect(); + assert_eq!(ids, ["nli-a", "nli-b"], "one result per member, under its own model"); + let concurring = Backing::Concurring { model_ids: vec!["nli-a".into(), "nli-b".into()] }; + assert_eq!(report.grounding_backing, Some(concurring.clone())); + assert_eq!(report.require_substantive_grounding(), Ok(&concurring)); + } + + /// One model entails, the other is neutral: the claim is not grounded (one + /// model cannot outvote the other), the violation says the models + /// disagreed, and the report rests, substantively, on a contested panel. + #[test] + fn a_panel_that_disagrees_is_contested_and_does_not_ground_the_claim() { + let report = verify_with_panel(entails, neutral); + let grounding: Vec<_> = + report.violations.iter().filter(|v| v.violation_id.starts_with("grounding")).collect(); + assert_eq!(grounding.len(), 1, "{:?}", report.violations); + assert_eq!(grounding[0].detail["models_disagree"], serde_json::json!(true)); + let contested = Backing::Contested { model_ids: vec!["nli-a".into(), "nli-b".into()] }; + assert_eq!(report.require_substantive_grounding(), Ok(&contested)); + assert!(!matches!(report.verdict, VerificationAction::ProceedToComposition)); + } + + /// One model entails what the other contradicts: a surfaced conflict, not + /// a grounded claim. + #[test] + fn a_panel_split_between_entails_and_contradicts_surfaces_a_conflict() { + let report = verify_with_panel(entails, contradicts); + assert!(report.violations.iter().any(|v| v.violation_id.starts_with("consistency")), "{:?}", report.violations); + assert!(report.violations.iter().any(|v| v.violation_id.starts_with("grounding")), "{:?}", report.violations); + assert!(matches!(report.grounding_backing, Some(Backing::Contested { .. }))); + } + + /// A single model's grounding violation keeps its bytes: the disagreement + /// note appears only where a panel split. + #[test] + fn a_single_models_grounding_violation_is_unchanged() { + let plan = simple_plan(); + let items = vec![item_for("q0", "wikidata:Q90", 1, PARIS)]; + let segs = vec![("s0".into(), "Berlin is the capital of France.".into())]; + let claims = atomize_sentences(&segs, &axes_with_tier(1)).unwrap(); + let report = verify(&VerifyInputs::new(&plan, &items, &[], &claims), &FixtureEntailer::new()).unwrap(); + let g = report.violations.iter().find(|v| v.violation_id.starts_with("grounding")).expect("grounding"); + assert!(g.detail.get("models_disagree").is_none(), "{}", g.detail); + } + + /// A fixture on the panel makes the whole panel a fixture, however the + /// labels fell: a stand-in's labels do not become corroboration beside a + /// model's. + #[test] + fn a_fixture_on_the_panel_is_not_corroboration() { + let plan = simple_plan(); + let items = vec![item_for("q0", "wikidata:Q90", 1, PARIS)]; + let segs = vec![("s0".into(), PARIS.into())]; + let claims = atomize_sentences(&segs, &axes_with_tier(1)).unwrap(); + let model = ScriptedModel::new("nli-a"); + model.inner.set_entails(PARIS, PARIS); + let fixture = FixtureEntailer::new(); + fixture.set_entails(PARIS, PARIS); + let panel = PanelEntailer::new(vec![Box::new(model), Box::new(fixture)]).unwrap(); + let report = verify(&VerifyInputs::new(&plan, &items, &[], &claims), &panel).unwrap(); + assert!(report.violations.is_empty()); + assert!(matches!(report.grounding_backing, Some(Backing::Fixture { .. })), "{:?}", report.grounding_backing); + assert!(matches!( + report.require_substantive_grounding(), + Err(jouleclaw_schema::GroundingError::Fixture { .. }) + )); + } } diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/valid_answer.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/valid_answer.rs index b0512dc80..b545a7cc8 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/valid_answer.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/valid_answer.rs @@ -54,10 +54,13 @@ pub fn coverage_violations(plan: &QueryPlan, items: &[RetrievedItem]) -> Vec Vec { let mut out = Vec::new(); for claim in claims { - let entailing: Vec<&EntailmentResult> = entailments - .iter() - .filter(|e| e.claim_id == claim.claim_id) - .filter(|e| matches!(e.label, EntailmentLabel::Entails)) - .collect(); - if entailing.is_empty() { + let mut by_item: std::collections::BTreeMap<&[uuid::Uuid], Vec<&EntailmentLabel>> = + std::collections::BTreeMap::new(); + for e in entailments.iter().filter(|e| e.claim_id == claim.claim_id) { + by_item.entry(e.premise_item_ids.as_slice()).or_default().push(&e.label); + } + let grounded = by_item + .values() + .any(|labels| labels.iter().all(|l| matches!(l, EntailmentLabel::Entails))); + // Some model entailed it, but no item had every model's agreement. + let contested = !grounded + && by_item.values().flatten().any(|l| matches!(l, EntailmentLabel::Entails)); + if !grounded { let severity = severity_from_stakes(claim.stakes); out.push(Violation { violation_id: format!("grounding:{}", claim.claim_id), @@ -82,13 +91,21 @@ pub fn grounding_violations( severity, claim_id: Some(claim.claim_id), item_id: None, - detail: json!({ - "kind": "grounding", - "claim_text": claim.text, - "segment_id": claim.segment_id, - "stakes": format!("{:?}", claim.stakes).to_lowercase(), - "retrievable": true, - }), + detail: { + let mut d = json!({ + "kind": "grounding", + "claim_text": claim.text, + "segment_id": claim.segment_id, + "stakes": format!("{:?}", claim.stakes).to_lowercase(), + "retrievable": true, + }); + // Only a panel can split on a pair, and only then is this + // said: a single model's report keeps its bytes. + if contested { + d["models_disagree"] = json!(true); + } + d + }, }); } } -- 2.54.0 (Apple Git-157)