diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-compose/src/verified.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-compose/src/verified.rs index 29a585f22..f8fd3072d 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-compose/src/verified.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-compose/src/verified.rs @@ -25,9 +25,9 @@ use std::collections::HashMap; use uuid::Uuid; use jouleclaw_schema::{ - Answer, AnswerSegment, AnswerStatus, AtomicClaim, ClaimAttribution, EpistemicMode, - Invariant, KnowledgeAxes, OriginalQuery, Provenance, QueryPlan, RecommendedAction, - Refusal, RetrievedItem, VerificationAction, VerificationReport, + Answer, AnswerSegment, AnswerStatus, AtomicClaim, ClaimAttribution, EntailmentLabel, + EpistemicMode, Invariant, KnowledgeAxes, OriginalQuery, Provenance, QueryPlan, + RecommendedAction, Refusal, RetrievedItem, VerificationAction, VerificationReport, }; use crate::draft::DraftSegment; @@ -208,9 +208,51 @@ fn build_answer(inputs: &ComposeInputs<'_>) -> Answer { ds.segment_id )); } - let cited_item = resolved - .into_iter() - .min_by_key(|it| (it.knowledge_axes.source_authority_tier, it.item_id)); + // And the most authoritative cited item *that supports the claim*, + // where the verifier judged the segment's claims against what it + // cites. Choosing among all cited items credited the claim to a + // source that does not say it whenever a more authoritative item was + // cited beside the one that does: an answer can cite a peer-reviewed + // paper and a forum post, be supported only by the post, and be + // attributed to the paper. The verifier flags that + // (`citation_violations`); this keeps the attribution from repeating + // it. Where no cited item was judged against the claim, there is + // nothing to choose by, and the most authoritative cited item stands. + let entails = |claim_id: Uuid, item_id: Uuid| { + let mut judged = inputs + .report + .entailments_consulted + .iter() + .filter(|e| e.claim_id == claim_id && e.premise_item_ids == [item_id]) + .peekable(); + judged.peek().is_some() && judged.all(|e| matches!(e.label, EntailmentLabel::Entails)) + }; + let judged = resolved.iter().any(|it| { + seg_claims.iter().any(|c| { + inputs + .report + .entailments_consulted + .iter() + .any(|e| e.claim_id == c.claim_id && e.premise_item_ids == [it.item_id]) + }) + }); + let by_authority = |it: &&RetrievedItem| (it.knowledge_axes.source_authority_tier, it.item_id); + let cited_item = if judged { + let supporting = resolved + .iter() + .copied() + .filter(|it| seg_claims.iter().any(|c| entails(c.claim_id, it.item_id))) + .min_by_key(by_authority); + if supporting.is_none() { + caveats.push(format!( + "Segment {} cites no item that supports it, so it is attributed to none", + ds.segment_id + )); + } + supporting + } else { + resolved.into_iter().min_by_key(by_authority) + }; let attribution = ClaimAttribution { schema_version: "6.0".into(), epistemic_mode: EpistemicMode::FromRetrieval, @@ -657,6 +699,78 @@ mod tests { } } + fn judged(claim_id: Uuid, item_id: Uuid, label: EntailmentLabel) -> EntailmentResult { + EntailmentResult { + schema_version: "2.0".into(), + result_id: Uuid::new_v4(), + claim_id, + premise_item_ids: vec![item_id], + label, + label_probabilities: EntailmentProbabilities { entails: 0.8, neutral: 0.1, contradicts: 0.1 }, + running_e_value: None, + model_id: "fixture".into(), + joules_spent: 1.0, + metadata: Default::default(), + } + } + + /// Where the verifier judged the claim against what its segment cites, + /// the attribution goes to the most authoritative cited item **that + /// supports it**. Choosing among all cited items credited the claim to a + /// peer-reviewed source that does not say it whenever it was cited beside + /// the forum post that does, which is the laundering the verifier refuses. + /// With no cited item supporting it, the segment is attributed to none, + /// and says so. + #[test] + fn attribution_goes_to_a_cited_item_that_supports_the_claim() { + let p = plan(); + let mut community = item("Paris is the capital of France."); + community.knowledge_axes = axes(4); + community.source_url = Some("https://forum.example/thread/1".into()); + let mut primary = item("A survey of unrelated results."); + primary.knowledge_axes = axes(1); + primary.source_url = Some("https://journal.example/paper".into()); + let seg = DraftSegment { + segment_id: "s0".into(), + text: "Paris is the capital of France.".into(), + cited_item_ids: vec![primary.item_id, community.item_id], + }; + let claim = claim_for("s0", "Paris is the capital of France."); + let items = [community.clone(), primary.clone()]; + let compose = |community_says: EntailmentLabel| { + let mut report = report_proceed(p.plan_id); + report.entailments_consulted = vec![ + judged(claim.claim_id, primary.item_id, EntailmentLabel::Neutral), + judged(claim.claim_id, community.item_id, community_says), + ]; + let inputs = ComposeInputs { + plan: &p, + draft_segments: std::slice::from_ref(&seg), + claims: std::slice::from_ref(&claim), + items: &items, + report: &report, + joules_spent_total: 100.0, + latency_ms: 1500, + }; + compose_verified_answer(&inputs).unwrap().as_answer().expect("Answer").clone() + }; + + let ans = compose(EntailmentLabel::Entails); + assert_eq!( + ans.segments[0].attribution.retrieval_source_uri.as_deref(), + Some("https://forum.example/thread/1"), + "the source that says it, not the more authoritative one beside it that does not" + ); + + let ans = compose(EntailmentLabel::Neutral); + assert_eq!(ans.segments[0].attribution.retrieval_source_uri, None, "no cited item supports it"); + assert!( + ans.caveats.iter().any(|c| c.contains("cites no item that supports it")), + "{:?}", + ans.caveats + ); + } + /// Two cited items of **equal** authority must resolve the same way /// whichever order they are cited in, or the attribution of an answer /// depends on citation order — which is not a property of the sources. diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/lib.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/lib.rs index ea1f61a00..af308de95 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/lib.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/lib.rs @@ -30,7 +30,7 @@ pub use entailer::{DebertaEntailer, EntailError, Entailer, FixtureEntailer, Pane pub use recovery::recovery_actions_for_violations; pub use report::{verify, VerifyError, VerifyInputs}; pub use valid_answer::{ - authority_violations, consistency_violations, coverage_violations, + authority_violations, citation_violations, consistency_violations, coverage_violations, freshness_violations, grounding_violations, }; pub use verdict::determine_verdict; diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/recovery.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/recovery.rs index 640586bdb..284380cfb 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/recovery.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/recovery.rs @@ -46,6 +46,7 @@ fn action_for(v: &Violation) -> RecoveryAction { RecommendedAction::Hedge, "best available source is below the required authority tier".to_string(), ), + "attribution" => attribution_action(v), "freshness" => ( RecommendedAction::Hedge, "best available source exceeds its freshness budget".to_string(), @@ -64,6 +65,29 @@ fn action_for(v: &Violation) -> RecoveryAction { } } +/// A claim supported by sources other than the ones it cites. Laundering, a +/// trusted source credited with what a weaker uncited one says, is dropped; +/// a claim merely cited wrongly, or not cited, is hedged, since its support +/// is real but not the source it names. +fn attribution_action(v: &Violation) -> (RecommendedAction, String) { + match v.detail.get("finding").and_then(|x| x.as_str()) { + Some("laundering") => ( + RecommendedAction::Drop, + "credited to a source that does not support it, while a less authoritative one it does not cite does: \ + drop" + .into(), + ), + Some("uncited") => ( + RecommendedAction::Hedge, + "supported, but the segment cites no source for it".into(), + ), + _ => ( + RecommendedAction::Hedge, + "supported only by sources the segment does not cite".into(), + ), + } +} + fn grounding_action(v: &Violation) -> (RecommendedAction, String) { let stakes = v.detail.get("stakes").and_then(|x| x.as_str()).unwrap_or(""); match (v.severity, stakes) { @@ -137,6 +161,21 @@ mod tests { assert!(matches!(action.action_type, RecommendedAction::Hedge)); } + /// A claim credited to a source that does not support it, while a less + /// authoritative uncited one does, is dropped; one merely cited wrongly, + /// or not cited, is hedged. An unrecognised kind would have been kept. + #[test] + fn attribution_drops_laundering_and_hedges_the_rest() { + let finding = |f: &str, severity| { + let mut x = v("attribution:x", severity, "attribution", "high"); + x.detail["finding"] = json!(f); + action_for(&x).action_type + }; + assert!(matches!(finding("laundering", ViolationSeverity::Critical), RecommendedAction::Drop)); + assert!(matches!(finding("misattributed", ViolationSeverity::Major), RecommendedAction::Hedge)); + assert!(matches!(finding("uncited", ViolationSeverity::Major), RecommendedAction::Hedge)); + } + #[test] fn coverage_keeps_for_orchestrator() { let action = action_for(&v("cov1", ViolationSeverity::Critical, "coverage", "")); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/report.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/report.rs index 480e2f80f..dd8711648 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/report.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/report.rs @@ -14,6 +14,8 @@ //! entailment on the at-risk subset. For a clean answer the entire //! pillar runs in low joules. +use std::collections::BTreeMap; + use chrono::Utc; use uuid::Uuid; @@ -25,7 +27,7 @@ use jouleclaw_schema::{ use crate::entailer::{entail_batch, EntailError, Entailer}; use crate::recovery::recovery_actions_for_violations; use crate::valid_answer::{ - authority_violations, consistency_violations, coverage_violations, + authority_violations, citation_violations, consistency_violations, coverage_violations, freshness_violations, grounding_violations, }; use crate::verdict::determine_verdict; @@ -59,6 +61,13 @@ pub struct VerifyInputs<'a> { pub authority: &'a [AuthorityRecord], pub claims: &'a [AtomicClaim], pub reroute_count: u32, + /// What each segment of the draft cites, by segment id + /// ([`VerifyInputs::with_citations`]). With it, each claim is judged on + /// its own citations ([`citation_violations`]), and the report may say + /// that every claim has provenance (I1) and that the authority of what is + /// cited meets the query (I4). Without it neither is checked, so neither + /// is claimed. + pub citations: Option<&'a BTreeMap>>, } /// Run the conflict-directed verification pipeline. @@ -99,6 +108,16 @@ pub fn verify( // 3. Entailment-driven checks. violations.extend(grounding_violations(inputs.claims, &entailments)); + if let Some(citations) = inputs.citations { + violations.extend(citation_violations( + inputs.plan, + inputs.claims, + &entailments, + citations, + inputs.items, + inputs.authority, + )); + } violations.extend(consistency_violations(inputs.claims, &entailments)); // 4. Verdict + recovery. @@ -107,12 +126,18 @@ pub fn verify( // 5. Build the report. let mut invariants_verified: Vec = vec![ - Invariant::I1EveryClaimHasProvenance.id().into(), - Invariant::I4AuthorityTierRespected.id().into(), Invariant::I5FreshnessRespected.id().into(), Invariant::I6ConflictsSurfaced.id().into(), Invariant::I9RerouteBounded.id().into(), ]; + // I1 (every claim has provenance) and I4 (the authority of *cited* + // sources meets the query) are about what the answer cites, so they are + // verified only when the citations were judged. They were listed always, + // while nothing here read a citation. + if inputs.citations.is_some() { + invariants_verified.insert(0, Invariant::I4AuthorityTierRespected.id().into()); + invariants_verified.insert(0, Invariant::I1EveryClaimHasProvenance.id().into()); + } // I2 only verifies when we ran entailment. if !entailments.is_empty() { invariants_verified.push(Invariant::I2AssertedImpliesEntailed.id().into()); @@ -163,6 +188,7 @@ impl<'a> VerifyInputs<'a> { authority, claims, reroute_count: 0, + citations: None, } } @@ -170,6 +196,13 @@ impl<'a> VerifyInputs<'a> { self.reroute_count = n; self } + + /// Judge each claim on what its segment cites (segment id → cited item + /// ids): [`citation_violations`]. + pub fn with_citations(mut self, citations: &'a BTreeMap>) -> Self { + self.citations = Some(citations); + self + } } /// Helper: post-verdict, decide whether the orchestrator can @@ -317,6 +350,39 @@ mod tests { .contains(&"I2".to_string()), "I2 should be verified when entailment ran"); } + /// End to end: a draft segment citing a primary source that says + /// nothing of the claim, beside a retrieved forum post that does. Judged + /// on any retrieved item, as `verify` did, it proceeds and lists I1 and + /// I4 as verified. Judged on its own citations it is refused, and only + /// then are I1 and I4 claimed, since only then were they checked. + #[test] + fn a_claim_is_judged_on_its_own_citations() { + let mut plan = simple_plan(); + plan.constraints.minimum_authority_tier = 2; + let primary = item_for("q0", "journal", 1, "A survey of unrelated results."); + let forum = item_for("q0", "forum", 4, "Paris is the capital of France."); + let items = vec![primary.clone(), forum]; + let segs = vec![("s0".into(), "Paris is the capital of France.".into())]; + let claims = atomize_sentences(&segs, &axes_with_tier(1)).unwrap(); + let entailer = FixtureEntailer::new(); + entailer.set_entails("Paris is the capital of France.", "Paris is the capital of France."); + + let blind = verify(&VerifyInputs::new(&plan, &items, &[], &claims), &entailer).unwrap(); + assert!(blind.violations.is_empty(), "{:?}", blind.violations); + for i in ["I1", "I4"] { + assert!(!blind.invariants_verified.contains(&i.to_string()), "{i} claimed unchecked"); + } + + let citations = BTreeMap::from([("s0".to_string(), vec![primary.item_id])]); + let judged = verify(&VerifyInputs::new(&plan, &items, &[], &claims).with_citations(&citations), &entailer).unwrap(); + assert!(matches!(judged.verdict, VerificationAction::Refuse), "{:?}", judged.violations); + assert_eq!(judged.violations.len(), 1); + assert_eq!(judged.violations[0].detail["finding"], "laundering"); + for i in ["I1", "I4"] { + assert!(judged.invariants_verified.contains(&i.to_string()), "{i} checked and not claimed"); + } + } + /// The same clean report as above — no violations, verdict /// `ProceedToComposition`, invariant I2 verified — is nonetheless refused /// as *grounded*, because a fixture produced its entailment labels. diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/valid_answer.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/valid_answer.rs index b545a7cc8..2be668afa 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/valid_answer.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-diagnose/src/valid_answer.rs @@ -16,8 +16,11 @@ //! retrieval could fix. The verdict logic in §6.6 reads this to //! decide RE_ROUTE vs REFUSE. +use std::collections::{BTreeMap, BTreeSet}; + use chrono::Utc; use serde_json::json; +use uuid::Uuid; use jouleclaw_schema::{ AtomicClaim, AuthorityRecord, EntailmentLabel, EntailmentResult, QueryPlan, RetrievedItem, @@ -112,8 +115,148 @@ pub fn grounding_violations( out } -/// **Authority** — cited sources meet the query's minimum tier -/// requirement (spec §6.1). +/// **Attribution** — every claim that some retrieved item supports is +/// supported by an item its own segment cites, and the most authoritative of +/// those cited supporting items meets the query's minimum tier. +/// +/// [`grounding_violations`] asks whether *any* retrieved item entails a claim, +/// and [`authority_violations`] whether *any* retrieved item of a sub-query +/// meets the tier. Neither reads what the answer cites. So an answer that +/// cites a primary source which does not support a claim, while the only +/// support is an uncited forum post, passed both: grounded by the post, +/// authoritative by the source. That is citation laundering: one weak source +/// makes the claim, and a trusted one is credited with it. This check judges +/// the claim on its own citations (`citations`, segment id → cited item ids): +/// +/// - a supported claim whose segment cites nothing is `uncited` (`Major`); +/// - one supported only by items it does not cite is `misattributed` +/// (`Major`), and `laundering` (`Critical`, not retrievable) when it also +/// cites an item more authoritative than any that supports it, which is +/// the signature of the attack; +/// - one supported by its own citations is held to the tier through the most +/// authoritative of them: an `authority` violation scoped to the claim +/// (`Critical`, retrievable) when even that one falls short. +/// +/// An item supports a claim as [`grounding_violations`] says one does: every +/// model that judged the pair entailed it. A premise of several items is +/// cited when all of them are. A claim nothing supports is left to +/// [`grounding_violations`], which already reports it. +pub fn citation_violations( + plan: &QueryPlan, + claims: &[AtomicClaim], + entailments: &[EntailmentResult], + citations: &BTreeMap>, + items: &[RetrievedItem], + authority: &[AuthorityRecord], +) -> Vec { + let min_tier = plan.constraints.minimum_authority_tier; + let tier_of = |id: &Uuid| items.iter().find(|it| it.item_id == *id).map(|it| effective_tier(it, authority)); + // A premise's tier is its weakest item's: it is only as authoritative as + // the least authoritative source it rests on. + let premise_tier = |p: &[Uuid]| p.iter().map(|id| tier_of(id).unwrap_or(u8::MAX)).max().unwrap_or(u8::MAX); + let mut out = Vec::new(); + for claim in claims { + let mut by_premise: BTreeMap<&[Uuid], Vec<&EntailmentLabel>> = BTreeMap::new(); + for e in entailments.iter().filter(|e| e.claim_id == claim.claim_id) { + by_premise.entry(e.premise_item_ids.as_slice()).or_default().push(&e.label); + } + let supporting: Vec<&[Uuid]> = by_premise + .iter() + .filter(|(_, labels)| labels.iter().all(|l| matches!(l, EntailmentLabel::Entails))) + .map(|(p, _)| *p) + .collect(); + if supporting.is_empty() { + continue; + } + let cited: BTreeSet = + citations.get(&claim.segment_id).map(|c| c.iter().copied().collect()).unwrap_or_default(); + let cited_support: Vec<&[Uuid]> = + supporting.iter().copied().filter(|p| p.iter().all(|id| cited.contains(id))).collect(); + let base = |finding: &str| { + json!({ + "kind": "attribution", + "finding": finding, + "claim_text": claim.text, + "segment_id": claim.segment_id, + "stakes": format!("{:?}", claim.stakes).to_lowercase(), + "cited_item_ids": cited.iter().map(Uuid::to_string).collect::>(), + "supporting_item_ids": supporting.iter().flat_map(|p| p.iter()).map(Uuid::to_string).collect::>(), + // Retrieval does not fix what an answer cites; composition does. + "retrievable": false, + }) + }; + if cited.is_empty() { + out.push(Violation { + violation_id: format!("attribution:{}", claim.claim_id), + message: format!("claim {:?} (segment {:?}) is supported but cites no source", claim.text, claim.segment_id), + severity: ViolationSeverity::Major, + claim_id: Some(claim.claim_id), + item_id: None, + detail: base("uncited"), + }); + continue; + } + if cited_support.is_empty() { + let best_support = supporting.iter().map(|p| premise_tier(p)).min().unwrap_or(u8::MAX); + let best_cited = cited.iter().filter_map(tier_of).min(); + let laundering = best_cited.is_some_and(|t| t < best_support); + let (severity, finding) = if laundering { + (ViolationSeverity::Critical, "laundering") + } else { + (ViolationSeverity::Major, "misattributed") + }; + let mut detail = base(finding); + detail["supporting_tier"] = json!(best_support); + if let Some(t) = best_cited { + detail["cited_tier"] = json!(t); + } + out.push(Violation { + violation_id: format!("attribution:{}", claim.claim_id), + message: if laundering { + format!( + "claim {:?} is supported only by an uncited tier-{best_support} source while it cites a \ + more authoritative one that does not support it", + claim.text + ) + } else { + format!("claim {:?} is supported only by sources it does not cite", claim.text) + }, + severity, + claim_id: Some(claim.claim_id), + item_id: None, + detail, + }); + continue; + } + let best = cited_support.iter().map(|p| premise_tier(p)).min().unwrap_or(u8::MAX); + if best > min_tier { + out.push(Violation { + violation_id: format!("authority:{}", claim.claim_id), + message: format!( + "claim {:?} rests on a cited tier-{best} source; the query requires tier ≤{min_tier}", + claim.text + ), + severity: ViolationSeverity::Critical, + claim_id: Some(claim.claim_id), + item_id: None, + detail: json!({ + "kind": "authority", + "scope": "cited", + "claim_text": claim.text, + "segment_id": claim.segment_id, + "required_tier": min_tier, + "found_tier": best, + "retrievable": true, + }), + }); + } + } + out +} + +/// **Authority** — retrieved sources meet the query's minimum tier +/// requirement (spec §6.1). The tier of what an answer *cites* is +/// [`citation_violations`]'s to judge. /// /// We check that at least one retrieved item per sub-query meets /// `plan.constraints.minimum_authority_tier`. The check inspects @@ -520,6 +663,133 @@ mod tests { assert!(v.is_empty()); } + fn cites(ids: &[Uuid]) -> BTreeMap> { + BTreeMap::from([("s0".to_string(), ids.to_vec())]) + } + + /// **The attack.** A claim cites a primary source that does not support + /// it, and the only support is a forum post it does not cite. Grounding + /// passes (the post entails it) and authority passes (a tier-1 item was + /// retrieved for the sub-query), so the old checks passed it by + /// construction. Judged on its own citations it is laundering: + /// `Critical`, not retrievable, so the verdict refuses. + #[test] + fn citation_laundering_is_caught() { + let plan = plan_with(&["q0"], 2); + let primary = item_for("q0", "journal", 1); + let forum = item_for("q0", "forum", 4); + let claim = claim_with("X cures Y", ClaimStakes::High); + let es = [ + entail(claim.claim_id, primary.item_id, EntailmentLabel::Neutral), + entail(claim.claim_id, forum.item_id, EntailmentLabel::Entails), + ]; + let items = [primary.clone(), forum.clone()]; + assert!(grounding_violations(std::slice::from_ref(&claim), &es).is_empty(), "grounded by the post"); + assert!(authority_violations(&plan, &items, &[]).is_empty(), "a tier-1 item was retrieved"); + + let v = citation_violations(&plan, std::slice::from_ref(&claim), &es, &cites(&[primary.item_id]), &items, &[]); + assert_eq!(v.len(), 1, "{v:?}"); + assert_eq!(v[0].detail["finding"], "laundering"); + assert!(matches!(v[0].severity, ViolationSeverity::Critical)); + assert!(!is_retrievable(&v[0])); + assert_eq!((v[0].detail["cited_tier"].as_u64(), v[0].detail["supporting_tier"].as_u64()), (Some(1), Some(4))); + assert!(matches!( + crate::verdict::determine_verdict(&v, &plan, 0), + jouleclaw_schema::VerificationAction::Refuse + )); + } + + /// Supported by an item it does not cite, while citing nothing more + /// authoritative: misattributed, `Major`, so the answer degrades rather + /// than refuses. + #[test] + fn a_claim_supported_only_by_an_uncited_item_is_misattributed() { + let plan = plan_with(&["q0"], 4); + let cited = item_for("q0", "blog", 4); + let uncited = item_for("q0", "journal", 1); + let claim = claim_with("X cures Y", ClaimStakes::Medium); + let es = [ + entail(claim.claim_id, cited.item_id, EntailmentLabel::Neutral), + entail(claim.claim_id, uncited.item_id, EntailmentLabel::Entails), + ]; + let v = citation_violations(&plan, &[claim], &es, &cites(&[cited.item_id]), &[cited, uncited], &[]); + assert_eq!(v.len(), 1, "{v:?}"); + assert_eq!(v[0].detail["finding"], "misattributed"); + assert!(matches!(v[0].severity, ViolationSeverity::Major)); + } + + /// A supported claim whose segment cites nothing has no provenance. + #[test] + fn a_supported_claim_that_cites_nothing_is_uncited() { + let plan = plan_with(&["q0"], 4); + let item = item_for("q0", "journal", 1); + let claim = claim_with("X cures Y", ClaimStakes::Medium); + let es = [entail(claim.claim_id, item.item_id, EntailmentLabel::Entails)]; + for citations in [cites(&[]), BTreeMap::new()] { + let v = citation_violations(&plan, std::slice::from_ref(&claim), &es, &citations, std::slice::from_ref(&item), &[]); + assert_eq!(v.len(), 1, "{v:?}"); + assert_eq!(v[0].detail["finding"], "uncited"); + } + } + + /// Authority is judged on what supports the claim among its citations, + /// not on the best item retrieved, and not on the best item cited: citing + /// a primary source beside the post that actually supports it does not + /// lend the claim the source's tier. + #[test] + fn authority_is_judged_on_the_cited_item_that_supports_the_claim() { + let plan = plan_with(&["q0"], 2); + let primary = item_for("q0", "journal", 1); + let forum = item_for("q0", "forum", 4); + let claim = claim_with("X cures Y", ClaimStakes::High); + let es = [ + entail(claim.claim_id, primary.item_id, EntailmentLabel::Neutral), + entail(claim.claim_id, forum.item_id, EntailmentLabel::Entails), + ]; + let both = cites(&[primary.item_id, forum.item_id]); + let v = citation_violations(&plan, std::slice::from_ref(&claim), &es, &both, &[primary.clone(), forum.clone()], &[]); + assert_eq!(v.len(), 1, "{v:?}"); + assert_eq!(v[0].detail["kind"], "authority"); + assert_eq!(v[0].detail["scope"], "cited"); + assert_eq!(v[0].detail["found_tier"], 4); + assert_eq!(v[0].claim_id, Some(claim.claim_id)); + assert!(is_retrievable(&v[0]), "a more authoritative source may yet be found"); + + // Supported by the primary source it cites: nothing to report. + let es = [ + entail(claim.claim_id, primary.item_id, EntailmentLabel::Entails), + entail(claim.claim_id, forum.item_id, EntailmentLabel::Entails), + ]; + assert!(citation_violations(&plan, &[claim], &es, &both, &[primary, forum], &[]).is_empty()); + } + + /// A cited item supports a claim only when every model that judged the + /// pair entailed it, as grounding requires; and a claim nothing supports + /// is grounding's to report, not this check's. + #[test] + fn support_needs_every_model_and_an_unsupported_claim_is_left_to_grounding() { + let plan = plan_with(&["q0"], 4); + let cited = item_for("q0", "journal", 1); + let claim = claim_with("X cures Y", ClaimStakes::Medium); + let split = [ + entail(claim.claim_id, cited.item_id, EntailmentLabel::Entails), + entail(claim.claim_id, cited.item_id, EntailmentLabel::Neutral), + ]; + let c = cites(&[cited.item_id]); + assert!(citation_violations(&plan, std::slice::from_ref(&claim), &split, &c, std::slice::from_ref(&cited), &[]).is_empty()); + assert_eq!(grounding_violations(std::slice::from_ref(&claim), &split).len(), 1); + + // The models split on the cited item while every model entails an + // uncited one: the claim is supported, but not by what it cites. One + // model's "entails" on the citation does not make it the support. + let uncited = item_for("q0", "forum", 4); + let mut es = split.to_vec(); + es.push(entail(claim.claim_id, uncited.item_id, EntailmentLabel::Entails)); + let v = citation_violations(&plan, &[claim], &es, &c, &[cited, uncited], &[]); + assert_eq!(v.len(), 1, "{v:?}"); + assert_eq!(v[0].detail["finding"], "laundering", "cites tier 1, supported only by an uncited tier 4"); + } + #[test] fn consistency_violation_when_both_entail_and_contradict() { let claim = claim_with("X is true", ClaimStakes::Medium); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-edge-cli/src/pipeline.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-edge-cli/src/pipeline.rs index 81ee9afb0..cab5d1a18 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-edge-cli/src/pipeline.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-edge-cli/src/pipeline.rs @@ -301,9 +301,13 @@ async fn run_inner<'e>( // Verify (with current reroute_passes count so the verdict // logic knows when the budget is exhausted). let stage = Instant::now(); + // Each claim is judged on what its own segment cites, not on any + // retrieved item. + let citations = citations_of(&draft); let report = verify( &VerifyInputs::new(&plan, &exec.items, &[], &claims) - .with_reroute_count(reroute_passes), + .with_reroute_count(reroute_passes) + .with_citations(&citations), entailer_ref, ) .map_err(|e| PipelineError::Verify(e.to_string()))?; @@ -373,9 +377,11 @@ async fn run_inner<'e>( VerificationAction::ReRouteWithRefinement ) { let stage = Instant::now(); + let citations = citations_of(&outcome.draft); let r = verify( &VerifyInputs::new(&outcome.plan, &outcome.items, &[], &outcome.claims) - .with_reroute_count(max_reroutes), + .with_reroute_count(max_reroutes) + .with_citations(&citations), entailer_ref, ) .map_err(|e| PipelineError::Verify(e.to_string()))?; @@ -449,6 +455,12 @@ const CACHE_VERSION_TAG: &str = "jouleclaw-edge-cli/0.1"; /// Most fields are derived from the AnswerOrRefusal itself; the /// pipeline-specific bookkeeping (plan, draft, claims, items, /// stages timings) we reconstruct cheaply from the cached Answer. +/// What each draft segment cites, by segment id: the citations each claim +/// is judged on. +fn citations_of(draft: &[DraftSegment]) -> BTreeMap> { + draft.iter().map(|s| (s.segment_id.clone(), s.cited_item_ids.clone())).collect() +} + fn rebuild_output_from_cache( query: &str, env: CachedAnswer,