diagnose: a verification verdict must say which instrument produced it #91

Merged
dcharlot merged 1 commit from diagnose/grounding-backing into main 2026-09-10 21:15:24 -04:00
7 changed files with 497 additions and 7 deletions

View file

@ -526,6 +526,10 @@ mod tests {
entailments_consulted: vec![],
invariants_verified: vec!["I1".into(), "I2".into()],
joules_spent: 0.0,
// A test fixture, and it says so: these compose tests are about
// composition, not grounding, and a helper that quietly declared a
// real backing would make every one of them look grounded.
grounding_backing: Some(Backing::Unconsulted),
metadata: Default::default(),
}
}

View file

@ -24,7 +24,8 @@ use std::time::Instant;
use uuid::Uuid;
use jouleclaw_schema::{
AtomicClaim, EntailmentLabel, EntailmentProbabilities, EntailmentResult, RetrievedItem,
AtomicClaim, Backing, EntailmentLabel, EntailmentProbabilities, EntailmentResult,
RetrievedItem,
};
use jouleclaw_deberta::{NliEngine, NliInference, NliInferenceError};
@ -68,6 +69,17 @@ pub trait Entailer: Send + Sync {
/// Stable model id surfaced in `EntailmentResult.model_id`.
fn model_id(&self) -> &str;
/// What this backend's labels can be relied on as.
///
/// Defaults to [`Backing::Unstated`] — the **floor**, deliberately. A
/// backend that does not declare is not assumed substantive, so an
/// implementation that forgets under-claims rather than over-claims. This
/// is the same discipline as `Vmm::attained_tier` defaulting to S0: the
/// cost of silence should fall on the claim, not on the reader.
fn backing(&self) -> Backing {
Backing::Unstated
}
/// Wrap a raw entailment for (claim, item) in a schema-shaped
/// [`EntailmentResult`] with attribution. Implementations
/// usually don't override this; the default builds the result
@ -157,6 +169,12 @@ impl Entailer for DebertaEntailer {
})
}
fn backing(&self) -> Backing {
// One learned model's judgement. Substantive, and explicitly not
// corroborated — nothing here cross-checks a second instrument.
Backing::SingleModel { model_id: self.engine.model_id().to_string() }
}
fn model_id(&self) -> &str {
self.engine.model_id()
}
@ -268,6 +286,16 @@ impl Entailer for FixtureEntailer {
Ok(self.default.lock().unwrap().clone())
}
fn backing(&self) -> Backing {
// Says what it is. A report built on this backend is refused by
// `require_substantive_grounding`, which is the entire point: a
// fixture answers whatever it was told to, so a verdict resting on
// one establishes nothing about the world. Naming it in the sealed
// report is what keeps a test-shaped artifact from reading like a
// verified one downstream.
Backing::Fixture { model_ids: vec![self.model_id.clone()] }
}
fn model_id(&self) -> &str {
&self.model_id
}

View file

@ -18,8 +18,8 @@ use chrono::Utc;
use uuid::Uuid;
use jouleclaw_schema::{
AtomicClaim, AuthorityRecord, Invariant, QueryPlan, RetrievedItem, VerificationAction,
VerificationReport,
AtomicClaim, AuthorityRecord, Backing, Invariant, QueryPlan, RetrievedItem,
VerificationAction, VerificationReport,
};
use crate::entailer::{entail_batch, EntailError, Entailer};
@ -120,6 +120,17 @@ pub fn verify<E: Entailer + ?Sized>(
// I10 always holds — refusals are structured by construction.
invariants_verified.push(Invariant::I10RefusalIsStructured.id().into());
// Declare what the grounding rests on, from the backend that actually
// produced the labels — not from an assumption at the call site. When
// nothing was consulted the verdict is *silent* about grounding, which is
// a different statement from "ran and found support", and the two must not
// collapse into one categorical verdict.
let grounding_backing = if entailments.is_empty() {
Backing::Unconsulted
} else {
entailer.backing()
};
let report = VerificationReport {
schema_version: "2.0".into(),
report_id: Uuid::new_v4(),
@ -132,6 +143,7 @@ pub fn verify<E: Entailer + ?Sized>(
invariants_verified,
joules_spent,
metadata: Default::default(),
grounding_backing: Some(grounding_backing),
};
Ok(report)
}
@ -305,6 +317,69 @@ mod tests {
.contains(&"I2".to_string()), "I2 should be verified when entailment ran");
}
/// The same clean report as above — no violations, verdict
/// `ProceedToComposition`, invariant I2 verified — is nonetheless refused
/// as *grounded*, because a fixture produced its entailment labels.
///
/// This is the gap the backing closes. Every signal a consumer previously
/// had says this answer is verified, and all of them are true: the checks
/// ran, found nothing wrong, and I2 genuinely holds over the labels it was
/// given. What none of them said is that the labels came from a stand-in
/// that answers whatever it was told to. The verdict is about the labels;
/// the backing is about whether the labels are worth anything.
#[test]
fn a_clean_verdict_from_a_fixture_is_still_not_grounded() {
let plan = simple_plan();
let item = item_for("q0", "wikidata:Q90", 1, "Paris is the capital of France.");
let items = vec![item];
let segs = vec![("s0".into(), "Paris is the capital of France.".into())];
let claims = atomize_sentences(&segs, &axes_with_tier(1)).unwrap();
let entailer = FixtureEntailer::new();
entailer.set_entails(
"Paris is the capital of France.",
"Paris is the capital of France.",
);
let report =
verify(&VerifyInputs::new(&plan, &items, &[], &claims), &entailer).unwrap();
// Everything that was checkable before still says "verified".
assert!(report.violations.is_empty());
assert!(matches!(report.verdict, VerificationAction::ProceedToComposition));
assert!(report.invariants_verified.contains(&"I2".to_string()));
// And the report names the stand-in rather than letting it pass.
assert_eq!(
report.grounding_backing,
Some(Backing::Fixture { model_ids: vec!["fixture-entailer".into()] })
);
assert!(matches!(
report.require_substantive_grounding(),
Err(jouleclaw_schema::GroundingError::Fixture { .. })
));
}
/// When no entailment runs, the report says the verdict is *silent* about
/// grounding rather than supportive of it.
#[test]
fn a_report_with_no_entailment_declares_itself_unconsulted() {
let plan = simple_plan();
let item = item_for("q0", "wikidata:Q90", 1, "Paris is the capital of France.");
let entailer = FixtureEntailer::new();
// No claims => entailment never runs.
let report =
verify(&VerifyInputs::new(&plan, &[item], &[], &[]), &entailer).unwrap();
assert_eq!(report.grounding_backing, Some(Backing::Unconsulted));
assert!(
!report.invariants_verified.contains(&"I2".to_string()),
"I2 must not be claimed when nothing was entailed"
);
assert!(matches!(
report.require_substantive_grounding(),
Err(jouleclaw_schema::GroundingError::Unconsulted)
));
}
#[test]
fn unsupported_claim_drives_grounding_violation_and_degrade() {
let plan = simple_plan();

View file

@ -477,6 +477,24 @@ fn rebuild_output_from_cache(
entailments_consulted: a.provenance.entailments.clone(),
invariants_verified: a.invariants_verified.clone(),
joules_spent: 0.0,
// The cache preserves the entailment *results* but not the
// backing the original producer declared, so this rebuild
// cannot re-make that claim — it declares the floor.
//
// Deriving one from the cached `model_id`s would be the
// laundering path: a run backed by `FixtureEntailer` records
// model_id "fixture-entailer", and reconstructing
// `SingleModel` from it would promote a stand-in into a
// substantive backing on the way out of the cache. A backing
// is an assertion the producer made; evidence that a model ran
// is not the same as a claim about what its labels are worth.
//
// Consequence, stated rather than hidden: a rehydrated report
// is refused by `require_substantive_grounding`. The fix is to
// persist the backing alongside the entailments in
// `Provenance`, which is a cache-format change and not this
// change.
grounding_backing: Some(Backing::Unstated),
metadata: Default::default(),
},
),
@ -494,6 +512,10 @@ fn rebuild_output_from_cache(
entailments_consulted: vec![],
invariants_verified: vec![],
joules_spent: 0.0,
// A refusal consulted nothing, so this is not the floor but
// the accurate statement: the verdict is silent about
// grounding because no entailment ran.
grounding_backing: Some(Backing::Unconsulted),
metadata: Default::default(),
},
),

View file

@ -128,3 +128,85 @@ mod tests {
assert_eq!(r, back);
}
}
/// What a verification verdict's grounding actually rests on.
///
/// # Why a verdict needs to say this
///
/// [`crate::VerificationReport::verdict`] is a categorical outcome, and a
/// consumer that reads it without walking `entailments_consulted` sees a
/// decision with no indication of what produced it. The labels underneath are
/// produced by a **learned model** — a DeBERTa-v3 NLI head, per §6.4 — and a
/// learned model's output is a judgement, not a derivation. Swap the checkpoint
/// and a claim can flip while every downstream artifact reports the same shape.
///
/// That is not a reason to stop using NLI: deciding whether prose entails a
/// claim is not a decidable problem, and a calibrated judgement is the honest
/// instrument for it. It is a reason for the verdict to **say which instrument
/// it used**, so a reader can tell a model's opinion from a checkable fact.
///
/// # The floor, not the ceiling, is the default
///
/// [`Unstated`](Self::Unstated) exists so that a producer which says nothing
/// under-claims rather than over-claims — the same discipline as
/// `Vmm::attained_tier` returning the floor when a backend forgets to report.
/// A report that does not name its backing is refused by
/// [`crate::VerificationReport::require_substantive_grounding`], because the
/// absence of a claim is not a claim of substance.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Backing {
/// The producer did not say. Treated as the floor: no grounding claim.
Unstated,
/// No entailment ran at all. The verdict is silent about grounding rather
/// than supportive of it — an important distinction from "ran and found
/// nothing", which is a grounding violation.
Unconsulted,
/// A programmable stand-in supplied the labels. Legitimate in tests; never
/// a basis for presenting an answer as verified. This is the case the
/// discipline check exists to catch, and it is the same shape as a proof
/// cascade closed by a stub.
Fixture { model_ids: Vec<String> },
/// One learned model's labels. Substantive, and explicitly a single
/// instrument's judgement — not corroborated.
SingleModel { model_id: String },
/// Independent models that agreed on every consulted claim.
Concurring { model_ids: Vec<String> },
/// Independent models that disagreed on at least one claim. Substantive and
/// *informative*: a contested claim is a real finding, not a failure, and
/// collapsing it to one model's answer is how disagreement gets hidden.
Contested { model_ids: Vec<String> },
}
impl Backing {
/// Whether this backing can support presenting an answer as grounded.
///
/// `Unstated` and `Unconsulted` cannot, because nothing was established.
/// `Fixture` cannot, because a stand-in establishes only what it was told
/// to say.
pub fn is_substantive(&self) -> bool {
matches!(
self,
Backing::SingleModel { .. } | Backing::Concurring { .. } | Backing::Contested { .. }
)
}
/// The model ids behind this backing, for a reader that wants to pin them.
pub fn model_ids(&self) -> Vec<&str> {
match self {
Backing::Unstated | Backing::Unconsulted => Vec::new(),
Backing::SingleModel { model_id } => vec![model_id.as_str()],
Backing::Fixture { model_ids }
| Backing::Concurring { model_ids }
| Backing::Contested { model_ids } => {
model_ids.iter().map(String::as_str).collect()
}
}
}
}
impl Default for Backing {
fn default() -> Self {
Backing::Unstated
}
}

View file

@ -32,7 +32,7 @@ pub use answer::{Answer, AnswerSegment, AnswerStatus, Provenance, Refusal};
pub use atomic_claim::{AtomicClaim, ClaimStakes};
pub use authority::{AuthorityRecord, AuthorityTier};
pub use common::{Metadata, SchemaVersion};
pub use entailment::{EntailmentLabel, EntailmentProbabilities, EntailmentResult};
pub use entailment::{Backing, EntailmentLabel, EntailmentProbabilities, EntailmentResult};
pub use epistemic::{ClaimAttribution, EpistemicMode, PriorClaimClass};
pub use invariants::{Invariant, InvariantsVerified};
pub use knowledge_axes::{GranularityClass, KnowledgeAxes, ScopeClass, TemporalStabilityClass};
@ -49,6 +49,6 @@ pub use system_capabilities::{
CapabilityStatus, ReasonerCapability, RetrieverCapability, SystemCapabilities,
};
pub use verification::{
RecommendedAction, RecoveryAction, VerificationAction, VerificationReport, Violation,
ViolationSeverity,
GroundingError, RecommendedAction, RecoveryAction, VerificationAction, VerificationReport,
Violation, ViolationSeverity,
};

View file

@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::common::Metadata;
use crate::entailment::EntailmentResult;
use crate::entailment::{Backing, EntailmentResult};
/// Per-violation severity. Drives the verdict logic in §6.6:
/// any `Critical` violation triggers RE_ROUTE or REFUSE depending
@ -108,10 +108,128 @@ pub struct VerificationReport {
pub invariants_verified: Vec<String>,
/// Joules spent during verification, contributing to §8.5.
pub joules_spent: f64,
/// What the grounding verdict rests on — which instrument produced the
/// entailment labels underneath [`Self::verdict`].
///
/// Optional and **omitted from the wire when absent**, so reports and
/// vectors written before this field existed serialize byte-identically
/// and keep verifying. Absent is not "fine": it reads as
/// [`Backing::Unstated`], which [`Self::require_substantive_grounding`]
/// refuses.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub grounding_backing: Option<Backing>,
#[serde(default)]
pub metadata: Metadata,
}
/// Why a report cannot be presented as grounded.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum GroundingError {
/// The report does not say what produced its entailment labels.
Unstated,
/// No entailment ran, so the verdict is silent about grounding.
Unconsulted,
/// A programmable stand-in supplied the labels.
Fixture { model_ids: Vec<String> },
/// The report claims a backing but consulted no entailments, or consulted
/// entailments the backing does not account for.
BackingDisagreesWithEvidence { claimed: Vec<String>, consulted: Vec<String> },
/// The caller pinned an expected model and the report used another.
UnexpectedModel { expected: String, actual: Vec<String> },
}
impl core::fmt::Display for GroundingError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
GroundingError::Unstated => write!(
f,
"report does not name the instrument behind its entailment \
labels; an unstated backing is not a claim of substance"
),
GroundingError::Unconsulted => {
write!(f, "no entailment ran: the verdict is silent about grounding")
}
GroundingError::Fixture { model_ids } => write!(
f,
"grounding rests on a programmable stand-in ({}); a fixture \
establishes only what it was told to say",
model_ids.join(", ")
),
GroundingError::BackingDisagreesWithEvidence { claimed, consulted } => write!(
f,
"declared backing ({}) is not the set of models that actually \
produced the consulted entailments ({})",
claimed.join(", "),
consulted.join(", ")
),
GroundingError::UnexpectedModel { expected, actual } => write!(
f,
"expected grounding from {expected}, got {}",
actual.join(", ")
),
}
}
}
impl std::error::Error for GroundingError {}
impl VerificationReport {
/// Refuse to treat this report as grounded unless something substantive
/// produced its entailment labels.
///
/// The sibling of `proof-cascade`'s `require_substantive_proof`,
/// `sandbox-attest`'s `require_mechanism_evidence` and
/// `jcp-action-receipt`'s `check_discipline`: the artifact carries its
/// provenance either way, and this is the opt-in call that turns carried
/// provenance into an enforced precondition. Existing consumers are
/// unaffected until they ask.
///
/// It also cross-checks the declared backing against
/// [`Self::entailments_consulted`], so the declaration cannot simply assert
/// a better instrument than the one whose results are attached.
pub fn require_substantive_grounding(&self) -> Result<&Backing, GroundingError> {
let backing = self.grounding_backing.as_ref().unwrap_or(&Backing::Unstated);
match backing {
Backing::Unstated => return Err(GroundingError::Unstated),
Backing::Unconsulted => return Err(GroundingError::Unconsulted),
Backing::Fixture { model_ids } => {
return Err(GroundingError::Fixture { model_ids: model_ids.clone() })
}
_ => {}
}
// A declaration that outruns its evidence is worse than none: it reads
// as corroboration a reader cannot obtain from the report.
let mut consulted: Vec<String> =
self.entailments_consulted.iter().map(|e| e.model_id.clone()).collect();
consulted.sort();
consulted.dedup();
let mut claimed: Vec<String> =
backing.model_ids().into_iter().map(str::to_string).collect();
claimed.sort();
claimed.dedup();
if consulted != claimed {
return Err(GroundingError::BackingDisagreesWithEvidence { claimed, consulted });
}
Ok(backing)
}
/// As [`Self::require_substantive_grounding`], additionally pinning the
/// model. For a caller who has decided *which* instrument it trusts, rather
/// than only that one was used.
pub fn require_grounding_from(&self, expected_model: &str) -> Result<&Backing, GroundingError> {
let backing = self.require_substantive_grounding()?;
let actual = backing.model_ids();
if !actual.iter().any(|m| *m == expected_model) {
return Err(GroundingError::UnexpectedModel {
expected: expected_model.to_string(),
actual: actual.into_iter().map(str::to_string).collect(),
});
}
Ok(backing)
}
}
#[cfg(test)]
mod tests {
use super::*;
@ -129,6 +247,7 @@ mod tests {
entailments_consulted: vec![],
invariants_verified: vec!["I1".into(), "I2".into(), "I13".into()],
joules_spent: 0.0,
grounding_backing: None,
metadata: Default::default(),
};
let json = serde_json::to_string(&r).unwrap();
@ -149,4 +268,164 @@ mod tests {
let json = serde_json::to_string(&v).unwrap();
assert!(json.contains("\"severity\":\"critical\""));
}
#[cfg(test)]
mod grounding_tests {
use super::*;
use crate::entailment::{EntailmentLabel, EntailmentProbabilities};
fn report(backing: Option<Backing>, models: &[&str]) -> VerificationReport {
let plan_id = Uuid::new_v4();
VerificationReport {
schema_version: "2.0".into(),
report_id: Uuid::new_v4(),
plan_id,
generated_at: Utc::now(),
verdict: VerificationAction::ProceedToComposition,
violations: vec![],
recovery_actions: vec![],
entailments_consulted: models
.iter()
.map(|m| EntailmentResult {
schema_version: "2.0".into(),
result_id: Uuid::new_v4(),
claim_id: Uuid::new_v4(),
premise_item_ids: vec![Uuid::new_v4()],
label: EntailmentLabel::Entails,
label_probabilities: EntailmentProbabilities {
entails: 0.9,
neutral: 0.07,
contradicts: 0.03,
},
running_e_value: None,
model_id: (*m).to_string(),
joules_spent: 1.0,
metadata: Default::default(),
})
.collect(),
invariants_verified: vec![],
joules_spent: 0.0,
grounding_backing: backing,
metadata: Default::default(),
}
}
/// The discipline this family runs on: a new optional field must be absent
/// from the wire when unset, so reports and conformance vectors written
/// before it existed serialize byte-identically and keep verifying.
#[test]
fn an_absent_backing_does_not_reach_the_wire() {
let r = report(None, &[]);
let json = serde_json::to_string(&r).unwrap();
assert!(
!json.contains("grounding_backing"),
"an unset backing must not appear in the serialized report"
);
// And it round-trips to the same absence rather than to a default.
let back: VerificationReport = serde_json::from_str(&json).unwrap();
assert_eq!(back.grounding_backing, None);
}
/// Silence is not a claim of substance. A producer that says nothing gets
/// the floor, so forgetting under-claims instead of over-claiming.
#[test]
fn an_unstated_backing_is_refused() {
assert_eq!(
report(None, &[]).require_substantive_grounding(),
Err(GroundingError::Unstated)
);
assert_eq!(
report(Some(Backing::Unstated), &[]).require_substantive_grounding(),
Err(GroundingError::Unstated)
);
}
/// The core rule: a stand-in answers whatever it was told to, so a verdict
/// resting on one establishes nothing about the world — the same shape as a
/// proof cascade closed by a stub.
#[test]
fn a_fixture_backed_verdict_is_refused() {
let r = report(
Some(Backing::Fixture { model_ids: vec!["fixture-entailer".into()] }),
&["fixture-entailer"],
);
match r.require_substantive_grounding() {
Err(GroundingError::Fixture { model_ids }) => {
assert_eq!(model_ids, vec!["fixture-entailer".to_string()]);
}
other => panic!("a fixture must never read as grounded: {other:?}"),
}
}
/// "Nothing ran" and "ran and found no support" are different statements.
/// The first is silence about grounding; the second is a violation. A
/// verdict must not let them collapse into one categorical outcome.
#[test]
fn unconsulted_is_distinct_from_a_grounding_violation() {
assert_eq!(
report(Some(Backing::Unconsulted), &[]).require_substantive_grounding(),
Err(GroundingError::Unconsulted)
);
}
/// A declaration that outruns its evidence is worse than none: it reads as
/// corroboration the reader cannot obtain from the report itself.
#[test]
fn a_backing_may_not_claim_models_the_evidence_does_not_show() {
// Claims two concurring models; only one actually produced results.
let r = report(
Some(Backing::Concurring {
model_ids: vec!["deberta-v3".into(), "some-second-model".into()],
}),
&["deberta-v3"],
);
match r.require_substantive_grounding() {
Err(GroundingError::BackingDisagreesWithEvidence { claimed, consulted }) => {
assert_eq!(consulted, vec!["deberta-v3".to_string()]);
assert_eq!(claimed.len(), 2);
}
other => panic!("an unsupported corroboration claim must be refused: {other:?}"),
}
}
/// A real single-model verdict passes, and says plainly that it is one
/// instrument's judgement rather than a corroborated one.
#[test]
fn a_single_learned_model_is_substantive_but_uncorroborated() {
let r = report(
Some(Backing::SingleModel { model_id: "deberta-v3".into() }),
&["deberta-v3"],
);
let backing = r.require_substantive_grounding().expect("substantive");
assert!(backing.is_substantive());
assert_eq!(backing.model_ids(), vec!["deberta-v3"]);
assert!(
!matches!(backing, Backing::Concurring { .. }),
"one model is not corroboration"
);
// A caller that has decided which instrument it trusts can pin it.
assert!(r.require_grounding_from("deberta-v3").is_ok());
match r.require_grounding_from("some-other-model") {
Err(GroundingError::UnexpectedModel { expected, actual }) => {
assert_eq!(expected, "some-other-model");
assert_eq!(actual, vec!["deberta-v3".to_string()]);
}
other => panic!("pinning must reject a different model: {other:?}"),
}
}
/// Disagreement between instruments is a real finding, not a failure. It
/// stays substantive — collapsing it to one model's answer is how a
/// contested claim gets quietly resolved in favour of whichever ran last.
#[test]
fn contested_is_substantive_and_says_so() {
let r = report(
Some(Backing::Contested { model_ids: vec!["a".into(), "b".into()] }),
&["a", "b"],
);
let backing = r.require_substantive_grounding().expect("contested is substantive");
assert!(matches!(backing, Backing::Contested { .. }));
}
}
}