diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-compose/src/verified.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-compose/src/verified.rs index 139bd6a8c..746a20a74 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-compose/src/verified.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-compose/src/verified.rs @@ -36,6 +36,10 @@ use crate::provenance::cache_key_for_subquery; /// Either an [`Answer`] (Verified / Degraded / Partial) or a /// structured [`Refusal`]. Returned by [`compose_verified_answer`] /// so the caller doesn't need to peek at the verdict. +// One per query, so the size gap between an answer and a refusal costs +// nothing measurable; boxing the answer would only add an allocation and +// change a public enum that callers match on. +#[allow(clippy::large_enum_variant)] #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub enum AnswerOrRefusal { Answer(Answer), @@ -319,6 +323,9 @@ fn build_provenance(inputs: &ComposeInputs<'_>) -> Provenance { items: inputs.items.to_vec(), claims: inputs.claims.to_vec(), entailments: inputs.report.entailments_consulted.clone(), + // The verifier's declaration travels with its results, so a stored + // answer can be re-verified against the same claim. + entailment_backing: inputs.report.grounding_backing.clone(), cache_key: combined, metadata: Default::default(), } @@ -564,6 +571,42 @@ mod tests { .contains(&"I13".to_string()), "I13 should be added"); } + /// The answer's provenance carries the backing the verifier declared, + /// beside the entailment results it covers, so an answer read back from + /// a cache can be held to the same claim. Any backing, the floor + /// included, travels unchanged. + #[test] + fn the_answer_carries_the_verifiers_backing() { + for backing in [ + Some(Backing::Unconsulted), + Some(Backing::SingleModel { model_id: "nli".into() }), + None, + ] { + let p = plan(); + let it = item("Paris is the capital of France."); + let seg = DraftSegment { + segment_id: "s0".into(), + text: "Wikidata: Paris is the capital of France.".into(), + cited_item_ids: vec![it.item_id], + }; + let claim = claim_for("s0", "Paris is the capital of France."); + let mut report = report_proceed(p.plan_id); + report.grounding_backing = backing.clone(); + let inputs = ComposeInputs { + plan: &p, + draft_segments: &[seg], + claims: &[claim], + items: &[it], + report: &report, + joules_spent_total: 1.0, + latency_ms: 1, + }; + let result = compose_verified_answer(&inputs).unwrap(); + let ans = result.as_answer().expect("Answer"); + assert_eq!(ans.provenance.entailment_backing, backing); + } + } + #[test] fn drop_action_removes_segment_and_records_caveat() { let p = plan(); diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-edge-cli/src/pipeline.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-edge-cli/src/pipeline.rs index db6f54081..81ee9afb0 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-edge-cli/src/pipeline.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-edge-cli/src/pipeline.rs @@ -477,24 +477,23 @@ fn rebuild_output_from_cache( entailments_consulted: a.provenance.entailments.clone(), invariants_verified: a.invariants_verified.clone(), joules_spent: 0.0, - // The cache preserves the entailment *results* but not the - // backing the original producer declared, so this rebuild - // cannot re-make that claim — it declares the floor. + // The backing the original verifier declared, stored with the + // entailment results in the answer's provenance, so the + // rebuilt report makes the same claim and + // `require_substantive_grounding` checks it against the same + // results. An answer cached before the backing was stored has + // none, and declares the floor. // - // Deriving one from the cached `model_id`s would be the - // laundering path: a run backed by `FixtureEntailer` records - // model_id "fixture-entailer", and reconstructing - // `SingleModel` from it would promote a stand-in into a - // substantive backing on the way out of the cache. A backing - // is an assertion the producer made; evidence that a model ran - // is not the same as a claim about what its labels are worth. - // - // Consequence, stated rather than hidden: a rehydrated report - // is refused by `require_substantive_grounding`. The fix is to - // persist the backing alongside the entailments in - // `Provenance`, which is a cache-format change and not this - // change. - grounding_backing: Some(Backing::Unstated), + // Never derived from the cached `model_id`s: a run backed by + // `FixtureEntailer` records model_id "fixture-entailer", and + // reconstructing `SingleModel` from it would promote a + // stand-in into a substantive backing on the way out of the + // cache. A backing is an assertion the producer made; evidence + // that a model ran is not a claim about what its labels are + // worth. + grounding_backing: Some( + a.provenance.entailment_backing.clone().unwrap_or(Backing::Unstated), + ), metadata: Default::default(), }, ), @@ -819,3 +818,91 @@ fn axes_for_template_claim() -> KnowledgeAxes { metadata: Default::default(), } } + +#[cfg(test)] +mod tests { + use super::*; + use jouleclaw_schema::{ + Answer, AnswerStatus, Backing, EntailmentLabel, EntailmentProbabilities, EntailmentResult, + GroundingError, Provenance, + }; + + /// A cached answer whose one entailment came from `model_id`, its + /// provenance carrying `backing`. + fn cached(model_id: &str, backing: Option) -> CachedAnswer { + let entailment = EntailmentResult { + schema_version: "2.0".into(), + result_id: uuid::Uuid::new_v4(), + claim_id: uuid::Uuid::new_v4(), + premise_item_ids: vec![], + label: EntailmentLabel::Entails, + label_probabilities: EntailmentProbabilities { entails: 0.9, neutral: 0.08, contradicts: 0.02 }, + running_e_value: None, + model_id: model_id.into(), + joules_spent: 0.0, + metadata: Default::default(), + }; + let plan_id = uuid::Uuid::new_v4(); + CachedAnswer { + schema_version: "1.0".into(), + cache_key: "k".into(), + cached_at: Utc::now(), + query_text: "q".into(), + result: AnswerOrRefusal::Answer(Answer { + schema_version: "2.0".into(), + answer_id: uuid::Uuid::new_v4(), + plan_id, + status: AnswerStatus::Verified, + segments: vec![], + provenance: Provenance { + schema_version: "2.0".into(), + plan_id, + items: vec![], + claims: vec![], + entailments: vec![entailment], + entailment_backing: backing, + cache_key: "k".into(), + metadata: Default::default(), + }, + invariants_verified: vec![], + joules_spent_total: 0.0, + latency_ms: 0, + caveats: vec![], + emitted_at: Utc::now(), + metadata: Default::default(), + }), + } + } + + /// An answer cached with its backing comes back out of the cache with the + /// same backing, held to the same results: a single model's labels are + /// grounded again, where every cached answer used to be refused. + #[test] + fn a_cached_answer_keeps_the_backing_its_verifier_declared() { + let backing = Backing::SingleModel { model_id: "nli-model".into() }; + let out = rebuild_output_from_cache("q", cached("nli-model", Some(backing.clone())), StageTimings::default()); + assert_eq!(out.report.require_substantive_grounding(), Ok(&backing)); + } + + /// The stored backing is a claim, still checked against the results it + /// came with: a fixture stays a fixture, a backing naming another model + /// than the one whose results are cached is refused, and an answer cached + /// before backings were stored declares the floor. + #[test] + fn a_cached_backing_is_never_promoted_or_trusted_past_its_results() { + let fixture = Backing::Fixture { model_ids: vec!["fixture-entailer".into()] }; + let out = rebuild_output_from_cache("q", cached("fixture-entailer", Some(fixture)), StageTimings::default()); + assert!(matches!(out.report.require_substantive_grounding(), Err(GroundingError::Fixture { .. }))); + + let claims_other = Backing::SingleModel { model_id: "a-better-model".into() }; + let out = rebuild_output_from_cache("q", cached("nli-model", Some(claims_other)), StageTimings::default()); + assert!(matches!( + out.report.require_substantive_grounding(), + Err(GroundingError::BackingDisagreesWithEvidence { .. }) + )); + + let out = rebuild_output_from_cache("q", cached("nli-model", None), StageTimings::default()); + assert_eq!(out.report.require_substantive_grounding(), Err(GroundingError::Unstated)); + } +} + diff --git a/jouleclaw/jouleclaw-rs/crates/jouleclaw-schema/src/answer.rs b/jouleclaw/jouleclaw-rs/crates/jouleclaw-schema/src/answer.rs index 0a1cd17b5..388f53d06 100644 --- a/jouleclaw/jouleclaw-rs/crates/jouleclaw-schema/src/answer.rs +++ b/jouleclaw/jouleclaw-rs/crates/jouleclaw-schema/src/answer.rs @@ -12,7 +12,7 @@ use uuid::Uuid; use crate::atomic_claim::AtomicClaim; use crate::common::Metadata; -use crate::entailment::EntailmentResult; +use crate::entailment::{Backing, EntailmentResult}; use crate::epistemic::ClaimAttribution; use crate::knowledge_axes::KnowledgeAxes; use crate::retrieved_item::RetrievedItem; @@ -69,6 +69,19 @@ pub struct Provenance { pub claims: Vec, /// Every entailment result consulted (focused subset). pub entailments: Vec, + /// What the labels in [`Self::entailments`] rest on: the backing the + /// verifier declared for them + /// ([`crate::VerificationReport::grounding_backing`]), carried with the + /// results so that a report rebuilt from a stored answer states the same + /// backing, and is held to it by + /// [`crate::VerificationReport::require_substantive_grounding`]. Without + /// it, a rebuilt report could only declare the floor, and every cached + /// answer was refused as ungrounded. + /// + /// Omitted from the wire when absent, so answers written before it + /// serialize byte-identically; absent reads as unstated. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub entailment_backing: Option, /// Cache key produced from the sub-queries (§7.3). pub cache_key: String, #[serde(default)] @@ -174,6 +187,7 @@ mod tests { items: vec![], claims: vec![], entailments: vec![], + entailment_backing: None, cache_key: "ab".into(), metadata: Default::default(), }, @@ -185,6 +199,17 @@ mod tests { metadata: Default::default(), }; let json = serde_json::to_string(&a).unwrap(); + // Absent, the backing is not on the wire: answers written before it + // existed keep their bytes. + assert!(!json.contains("entailment_backing"), "{json}"); + let back: Answer = serde_json::from_str(&json).unwrap(); + assert_eq!(a, back); + + // Present, it round-trips. + let mut a = a; + a.provenance.entailment_backing = Some(Backing::SingleModel { model_id: "nli".into() }); + let json = serde_json::to_string(&a).unwrap(); + assert!(json.contains("entailment_backing"), "{json}"); let back: Answer = serde_json::from_str(&json).unwrap(); assert_eq!(a, back); }