jouleclaw: an answer carries its verifier's backing, so a cached answer is not refused as ungrounded #276

Merged
dcharlot merged 1 commit from fix/answer-carries-entailment-backing into main 2026-10-16 09:41:29 -04:00
3 changed files with 173 additions and 18 deletions

View file

@ -36,6 +36,10 @@ use crate::provenance::cache_key_for_subquery;
/// Either an [`Answer`] (Verified / Degraded / Partial) or a
/// structured [`Refusal`]. Returned by [`compose_verified_answer`]
/// so the caller doesn't need to peek at the verdict.
// One per query, so the size gap between an answer and a refusal costs
// nothing measurable; boxing the answer would only add an allocation and
// change a public enum that callers match on.
#[allow(clippy::large_enum_variant)]
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub enum AnswerOrRefusal {
Answer(Answer),
@ -319,6 +323,9 @@ fn build_provenance(inputs: &ComposeInputs<'_>) -> Provenance {
items: inputs.items.to_vec(),
claims: inputs.claims.to_vec(),
entailments: inputs.report.entailments_consulted.clone(),
// The verifier's declaration travels with its results, so a stored
// answer can be re-verified against the same claim.
entailment_backing: inputs.report.grounding_backing.clone(),
cache_key: combined,
metadata: Default::default(),
}
@ -564,6 +571,42 @@ mod tests {
.contains(&"I13".to_string()), "I13 should be added");
}
/// The answer's provenance carries the backing the verifier declared,
/// beside the entailment results it covers, so an answer read back from
/// a cache can be held to the same claim. Any backing, the floor
/// included, travels unchanged.
#[test]
fn the_answer_carries_the_verifiers_backing() {
for backing in [
Some(Backing::Unconsulted),
Some(Backing::SingleModel { model_id: "nli".into() }),
None,
] {
let p = plan();
let it = item("Paris is the capital of France.");
let seg = DraftSegment {
segment_id: "s0".into(),
text: "Wikidata: Paris is the capital of France.".into(),
cited_item_ids: vec![it.item_id],
};
let claim = claim_for("s0", "Paris is the capital of France.");
let mut report = report_proceed(p.plan_id);
report.grounding_backing = backing.clone();
let inputs = ComposeInputs {
plan: &p,
draft_segments: &[seg],
claims: &[claim],
items: &[it],
report: &report,
joules_spent_total: 1.0,
latency_ms: 1,
};
let result = compose_verified_answer(&inputs).unwrap();
let ans = result.as_answer().expect("Answer");
assert_eq!(ans.provenance.entailment_backing, backing);
}
}
#[test]
fn drop_action_removes_segment_and_records_caveat() {
let p = plan();

View file

@ -477,24 +477,23 @@ fn rebuild_output_from_cache(
entailments_consulted: a.provenance.entailments.clone(),
invariants_verified: a.invariants_verified.clone(),
joules_spent: 0.0,
// The cache preserves the entailment *results* but not the
// backing the original producer declared, so this rebuild
// cannot re-make that claim — it declares the floor.
// The backing the original verifier declared, stored with the
// entailment results in the answer's provenance, so the
// rebuilt report makes the same claim and
// `require_substantive_grounding` checks it against the same
// results. An answer cached before the backing was stored has
// none, and declares the floor.
//
// Deriving one from the cached `model_id`s would be the
// laundering path: a run backed by `FixtureEntailer` records
// model_id "fixture-entailer", and reconstructing
// `SingleModel` from it would promote a stand-in into a
// substantive backing on the way out of the cache. A backing
// is an assertion the producer made; evidence that a model ran
// is not the same as a claim about what its labels are worth.
//
// Consequence, stated rather than hidden: a rehydrated report
// is refused by `require_substantive_grounding`. The fix is to
// persist the backing alongside the entailments in
// `Provenance`, which is a cache-format change and not this
// change.
grounding_backing: Some(Backing::Unstated),
// Never derived from the cached `model_id`s: a run backed by
// `FixtureEntailer` records model_id "fixture-entailer", and
// reconstructing `SingleModel` from it would promote a
// stand-in into a substantive backing on the way out of the
// cache. A backing is an assertion the producer made; evidence
// that a model ran is not a claim about what its labels are
// worth.
grounding_backing: Some(
a.provenance.entailment_backing.clone().unwrap_or(Backing::Unstated),
),
metadata: Default::default(),
},
),
@ -819,3 +818,91 @@ fn axes_for_template_claim() -> KnowledgeAxes {
metadata: Default::default(),
}
}
#[cfg(test)]
mod tests {
use super::*;
use jouleclaw_schema::{
Answer, AnswerStatus, Backing, EntailmentLabel, EntailmentProbabilities, EntailmentResult,
GroundingError, Provenance,
};
/// A cached answer whose one entailment came from `model_id`, its
/// provenance carrying `backing`.
fn cached(model_id: &str, backing: Option<Backing>) -> CachedAnswer {
let entailment = EntailmentResult {
schema_version: "2.0".into(),
result_id: uuid::Uuid::new_v4(),
claim_id: uuid::Uuid::new_v4(),
premise_item_ids: vec![],
label: EntailmentLabel::Entails,
label_probabilities: EntailmentProbabilities { entails: 0.9, neutral: 0.08, contradicts: 0.02 },
running_e_value: None,
model_id: model_id.into(),
joules_spent: 0.0,
metadata: Default::default(),
};
let plan_id = uuid::Uuid::new_v4();
CachedAnswer {
schema_version: "1.0".into(),
cache_key: "k".into(),
cached_at: Utc::now(),
query_text: "q".into(),
result: AnswerOrRefusal::Answer(Answer {
schema_version: "2.0".into(),
answer_id: uuid::Uuid::new_v4(),
plan_id,
status: AnswerStatus::Verified,
segments: vec![],
provenance: Provenance {
schema_version: "2.0".into(),
plan_id,
items: vec![],
claims: vec![],
entailments: vec![entailment],
entailment_backing: backing,
cache_key: "k".into(),
metadata: Default::default(),
},
invariants_verified: vec![],
joules_spent_total: 0.0,
latency_ms: 0,
caveats: vec![],
emitted_at: Utc::now(),
metadata: Default::default(),
}),
}
}
/// An answer cached with its backing comes back out of the cache with the
/// same backing, held to the same results: a single model's labels are
/// grounded again, where every cached answer used to be refused.
#[test]
fn a_cached_answer_keeps_the_backing_its_verifier_declared() {
let backing = Backing::SingleModel { model_id: "nli-model".into() };
let out = rebuild_output_from_cache("q", cached("nli-model", Some(backing.clone())), StageTimings::default());
assert_eq!(out.report.require_substantive_grounding(), Ok(&backing));
}
/// The stored backing is a claim, still checked against the results it
/// came with: a fixture stays a fixture, a backing naming another model
/// than the one whose results are cached is refused, and an answer cached
/// before backings were stored declares the floor.
#[test]
fn a_cached_backing_is_never_promoted_or_trusted_past_its_results() {
let fixture = Backing::Fixture { model_ids: vec!["fixture-entailer".into()] };
let out = rebuild_output_from_cache("q", cached("fixture-entailer", Some(fixture)), StageTimings::default());
assert!(matches!(out.report.require_substantive_grounding(), Err(GroundingError::Fixture { .. })));
let claims_other = Backing::SingleModel { model_id: "a-better-model".into() };
let out = rebuild_output_from_cache("q", cached("nli-model", Some(claims_other)), StageTimings::default());
assert!(matches!(
out.report.require_substantive_grounding(),
Err(GroundingError::BackingDisagreesWithEvidence { .. })
));
let out = rebuild_output_from_cache("q", cached("nli-model", None), StageTimings::default());
assert_eq!(out.report.require_substantive_grounding(), Err(GroundingError::Unstated));
}
}

View file

@ -12,7 +12,7 @@ use uuid::Uuid;
use crate::atomic_claim::AtomicClaim;
use crate::common::Metadata;
use crate::entailment::EntailmentResult;
use crate::entailment::{Backing, EntailmentResult};
use crate::epistemic::ClaimAttribution;
use crate::knowledge_axes::KnowledgeAxes;
use crate::retrieved_item::RetrievedItem;
@ -69,6 +69,19 @@ pub struct Provenance {
pub claims: Vec<AtomicClaim>,
/// Every entailment result consulted (focused subset).
pub entailments: Vec<EntailmentResult>,
/// What the labels in [`Self::entailments`] rest on: the backing the
/// verifier declared for them
/// ([`crate::VerificationReport::grounding_backing`]), carried with the
/// results so that a report rebuilt from a stored answer states the same
/// backing, and is held to it by
/// [`crate::VerificationReport::require_substantive_grounding`]. Without
/// it, a rebuilt report could only declare the floor, and every cached
/// answer was refused as ungrounded.
///
/// Omitted from the wire when absent, so answers written before it
/// serialize byte-identically; absent reads as unstated.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub entailment_backing: Option<Backing>,
/// Cache key produced from the sub-queries (§7.3).
pub cache_key: String,
#[serde(default)]
@ -174,6 +187,7 @@ mod tests {
items: vec![],
claims: vec![],
entailments: vec![],
entailment_backing: None,
cache_key: "ab".into(),
metadata: Default::default(),
},
@ -185,6 +199,17 @@ mod tests {
metadata: Default::default(),
};
let json = serde_json::to_string(&a).unwrap();
// Absent, the backing is not on the wire: answers written before it
// existed keep their bytes.
assert!(!json.contains("entailment_backing"), "{json}");
let back: Answer = serde_json::from_str(&json).unwrap();
assert_eq!(a, back);
// Present, it round-trips.
let mut a = a;
a.provenance.entailment_backing = Some(Backing::SingleModel { model_id: "nli".into() });
let json = serde_json::to_string(&a).unwrap();
assert!(json.contains("entailment_backing"), "{json}");
let back: Answer = serde_json::from_str(&json).unwrap();
assert_eq!(a, back);
}