jouleclaw: an answer carries its verifier's backing, so a cached answer is not refused as ungrounded #276
3 changed files with 173 additions and 18 deletions
|
|
@ -36,6 +36,10 @@ use crate::provenance::cache_key_for_subquery;
|
|||
/// Either an [`Answer`] (Verified / Degraded / Partial) or a
|
||||
/// structured [`Refusal`]. Returned by [`compose_verified_answer`]
|
||||
/// so the caller doesn't need to peek at the verdict.
|
||||
// One per query, so the size gap between an answer and a refusal costs
|
||||
// nothing measurable; boxing the answer would only add an allocation and
|
||||
// change a public enum that callers match on.
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
||||
pub enum AnswerOrRefusal {
|
||||
Answer(Answer),
|
||||
|
|
@ -319,6 +323,9 @@ fn build_provenance(inputs: &ComposeInputs<'_>) -> Provenance {
|
|||
items: inputs.items.to_vec(),
|
||||
claims: inputs.claims.to_vec(),
|
||||
entailments: inputs.report.entailments_consulted.clone(),
|
||||
// The verifier's declaration travels with its results, so a stored
|
||||
// answer can be re-verified against the same claim.
|
||||
entailment_backing: inputs.report.grounding_backing.clone(),
|
||||
cache_key: combined,
|
||||
metadata: Default::default(),
|
||||
}
|
||||
|
|
@ -564,6 +571,42 @@ mod tests {
|
|||
.contains(&"I13".to_string()), "I13 should be added");
|
||||
}
|
||||
|
||||
/// The answer's provenance carries the backing the verifier declared,
|
||||
/// beside the entailment results it covers, so an answer read back from
|
||||
/// a cache can be held to the same claim. Any backing, the floor
|
||||
/// included, travels unchanged.
|
||||
#[test]
|
||||
fn the_answer_carries_the_verifiers_backing() {
|
||||
for backing in [
|
||||
Some(Backing::Unconsulted),
|
||||
Some(Backing::SingleModel { model_id: "nli".into() }),
|
||||
None,
|
||||
] {
|
||||
let p = plan();
|
||||
let it = item("Paris is the capital of France.");
|
||||
let seg = DraftSegment {
|
||||
segment_id: "s0".into(),
|
||||
text: "Wikidata: Paris is the capital of France.".into(),
|
||||
cited_item_ids: vec![it.item_id],
|
||||
};
|
||||
let claim = claim_for("s0", "Paris is the capital of France.");
|
||||
let mut report = report_proceed(p.plan_id);
|
||||
report.grounding_backing = backing.clone();
|
||||
let inputs = ComposeInputs {
|
||||
plan: &p,
|
||||
draft_segments: &[seg],
|
||||
claims: &[claim],
|
||||
items: &[it],
|
||||
report: &report,
|
||||
joules_spent_total: 1.0,
|
||||
latency_ms: 1,
|
||||
};
|
||||
let result = compose_verified_answer(&inputs).unwrap();
|
||||
let ans = result.as_answer().expect("Answer");
|
||||
assert_eq!(ans.provenance.entailment_backing, backing);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_action_removes_segment_and_records_caveat() {
|
||||
let p = plan();
|
||||
|
|
|
|||
|
|
@ -477,24 +477,23 @@ fn rebuild_output_from_cache(
|
|||
entailments_consulted: a.provenance.entailments.clone(),
|
||||
invariants_verified: a.invariants_verified.clone(),
|
||||
joules_spent: 0.0,
|
||||
// The cache preserves the entailment *results* but not the
|
||||
// backing the original producer declared, so this rebuild
|
||||
// cannot re-make that claim — it declares the floor.
|
||||
// The backing the original verifier declared, stored with the
|
||||
// entailment results in the answer's provenance, so the
|
||||
// rebuilt report makes the same claim and
|
||||
// `require_substantive_grounding` checks it against the same
|
||||
// results. An answer cached before the backing was stored has
|
||||
// none, and declares the floor.
|
||||
//
|
||||
// Deriving one from the cached `model_id`s would be the
|
||||
// laundering path: a run backed by `FixtureEntailer` records
|
||||
// model_id "fixture-entailer", and reconstructing
|
||||
// `SingleModel` from it would promote a stand-in into a
|
||||
// substantive backing on the way out of the cache. A backing
|
||||
// is an assertion the producer made; evidence that a model ran
|
||||
// is not the same as a claim about what its labels are worth.
|
||||
//
|
||||
// Consequence, stated rather than hidden: a rehydrated report
|
||||
// is refused by `require_substantive_grounding`. The fix is to
|
||||
// persist the backing alongside the entailments in
|
||||
// `Provenance`, which is a cache-format change and not this
|
||||
// change.
|
||||
grounding_backing: Some(Backing::Unstated),
|
||||
// Never derived from the cached `model_id`s: a run backed by
|
||||
// `FixtureEntailer` records model_id "fixture-entailer", and
|
||||
// reconstructing `SingleModel` from it would promote a
|
||||
// stand-in into a substantive backing on the way out of the
|
||||
// cache. A backing is an assertion the producer made; evidence
|
||||
// that a model ran is not a claim about what its labels are
|
||||
// worth.
|
||||
grounding_backing: Some(
|
||||
a.provenance.entailment_backing.clone().unwrap_or(Backing::Unstated),
|
||||
),
|
||||
metadata: Default::default(),
|
||||
},
|
||||
),
|
||||
|
|
@ -819,3 +818,91 @@ fn axes_for_template_claim() -> KnowledgeAxes {
|
|||
metadata: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use jouleclaw_schema::{
|
||||
Answer, AnswerStatus, Backing, EntailmentLabel, EntailmentProbabilities, EntailmentResult,
|
||||
GroundingError, Provenance,
|
||||
};
|
||||
|
||||
/// A cached answer whose one entailment came from `model_id`, its
|
||||
/// provenance carrying `backing`.
|
||||
fn cached(model_id: &str, backing: Option<Backing>) -> CachedAnswer {
|
||||
let entailment = EntailmentResult {
|
||||
schema_version: "2.0".into(),
|
||||
result_id: uuid::Uuid::new_v4(),
|
||||
claim_id: uuid::Uuid::new_v4(),
|
||||
premise_item_ids: vec![],
|
||||
label: EntailmentLabel::Entails,
|
||||
label_probabilities: EntailmentProbabilities { entails: 0.9, neutral: 0.08, contradicts: 0.02 },
|
||||
running_e_value: None,
|
||||
model_id: model_id.into(),
|
||||
joules_spent: 0.0,
|
||||
metadata: Default::default(),
|
||||
};
|
||||
let plan_id = uuid::Uuid::new_v4();
|
||||
CachedAnswer {
|
||||
schema_version: "1.0".into(),
|
||||
cache_key: "k".into(),
|
||||
cached_at: Utc::now(),
|
||||
query_text: "q".into(),
|
||||
result: AnswerOrRefusal::Answer(Answer {
|
||||
schema_version: "2.0".into(),
|
||||
answer_id: uuid::Uuid::new_v4(),
|
||||
plan_id,
|
||||
status: AnswerStatus::Verified,
|
||||
segments: vec![],
|
||||
provenance: Provenance {
|
||||
schema_version: "2.0".into(),
|
||||
plan_id,
|
||||
items: vec![],
|
||||
claims: vec![],
|
||||
entailments: vec![entailment],
|
||||
entailment_backing: backing,
|
||||
cache_key: "k".into(),
|
||||
metadata: Default::default(),
|
||||
},
|
||||
invariants_verified: vec![],
|
||||
joules_spent_total: 0.0,
|
||||
latency_ms: 0,
|
||||
caveats: vec![],
|
||||
emitted_at: Utc::now(),
|
||||
metadata: Default::default(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// An answer cached with its backing comes back out of the cache with the
|
||||
/// same backing, held to the same results: a single model's labels are
|
||||
/// grounded again, where every cached answer used to be refused.
|
||||
#[test]
|
||||
fn a_cached_answer_keeps_the_backing_its_verifier_declared() {
|
||||
let backing = Backing::SingleModel { model_id: "nli-model".into() };
|
||||
let out = rebuild_output_from_cache("q", cached("nli-model", Some(backing.clone())), StageTimings::default());
|
||||
assert_eq!(out.report.require_substantive_grounding(), Ok(&backing));
|
||||
}
|
||||
|
||||
/// The stored backing is a claim, still checked against the results it
|
||||
/// came with: a fixture stays a fixture, a backing naming another model
|
||||
/// than the one whose results are cached is refused, and an answer cached
|
||||
/// before backings were stored declares the floor.
|
||||
#[test]
|
||||
fn a_cached_backing_is_never_promoted_or_trusted_past_its_results() {
|
||||
let fixture = Backing::Fixture { model_ids: vec!["fixture-entailer".into()] };
|
||||
let out = rebuild_output_from_cache("q", cached("fixture-entailer", Some(fixture)), StageTimings::default());
|
||||
assert!(matches!(out.report.require_substantive_grounding(), Err(GroundingError::Fixture { .. })));
|
||||
|
||||
let claims_other = Backing::SingleModel { model_id: "a-better-model".into() };
|
||||
let out = rebuild_output_from_cache("q", cached("nli-model", Some(claims_other)), StageTimings::default());
|
||||
assert!(matches!(
|
||||
out.report.require_substantive_grounding(),
|
||||
Err(GroundingError::BackingDisagreesWithEvidence { .. })
|
||||
));
|
||||
|
||||
let out = rebuild_output_from_cache("q", cached("nli-model", None), StageTimings::default());
|
||||
assert_eq!(out.report.require_substantive_grounding(), Err(GroundingError::Unstated));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ use uuid::Uuid;
|
|||
|
||||
use crate::atomic_claim::AtomicClaim;
|
||||
use crate::common::Metadata;
|
||||
use crate::entailment::EntailmentResult;
|
||||
use crate::entailment::{Backing, EntailmentResult};
|
||||
use crate::epistemic::ClaimAttribution;
|
||||
use crate::knowledge_axes::KnowledgeAxes;
|
||||
use crate::retrieved_item::RetrievedItem;
|
||||
|
|
@ -69,6 +69,19 @@ pub struct Provenance {
|
|||
pub claims: Vec<AtomicClaim>,
|
||||
/// Every entailment result consulted (focused subset).
|
||||
pub entailments: Vec<EntailmentResult>,
|
||||
/// What the labels in [`Self::entailments`] rest on: the backing the
|
||||
/// verifier declared for them
|
||||
/// ([`crate::VerificationReport::grounding_backing`]), carried with the
|
||||
/// results so that a report rebuilt from a stored answer states the same
|
||||
/// backing, and is held to it by
|
||||
/// [`crate::VerificationReport::require_substantive_grounding`]. Without
|
||||
/// it, a rebuilt report could only declare the floor, and every cached
|
||||
/// answer was refused as ungrounded.
|
||||
///
|
||||
/// Omitted from the wire when absent, so answers written before it
|
||||
/// serialize byte-identically; absent reads as unstated.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub entailment_backing: Option<Backing>,
|
||||
/// Cache key produced from the sub-queries (§7.3).
|
||||
pub cache_key: String,
|
||||
#[serde(default)]
|
||||
|
|
@ -174,6 +187,7 @@ mod tests {
|
|||
items: vec![],
|
||||
claims: vec![],
|
||||
entailments: vec![],
|
||||
entailment_backing: None,
|
||||
cache_key: "ab".into(),
|
||||
metadata: Default::default(),
|
||||
},
|
||||
|
|
@ -185,6 +199,17 @@ mod tests {
|
|||
metadata: Default::default(),
|
||||
};
|
||||
let json = serde_json::to_string(&a).unwrap();
|
||||
// Absent, the backing is not on the wire: answers written before it
|
||||
// existed keep their bytes.
|
||||
assert!(!json.contains("entailment_backing"), "{json}");
|
||||
let back: Answer = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(a, back);
|
||||
|
||||
// Present, it round-trips.
|
||||
let mut a = a;
|
||||
a.provenance.entailment_backing = Some(Backing::SingleModel { model_id: "nli".into() });
|
||||
let json = serde_json::to_string(&a).unwrap();
|
||||
assert!(json.contains("entailment_backing"), "{json}");
|
||||
let back: Answer = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(a, back);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue