diagnose: a verification verdict must say which instrument produced it #91
7 changed files with 497 additions and 7 deletions
|
|
@ -526,6 +526,10 @@ mod tests {
|
|||
entailments_consulted: vec![],
|
||||
invariants_verified: vec!["I1".into(), "I2".into()],
|
||||
joules_spent: 0.0,
|
||||
// A test fixture, and it says so: these compose tests are about
|
||||
// composition, not grounding, and a helper that quietly declared a
|
||||
// real backing would make every one of them look grounded.
|
||||
grounding_backing: Some(Backing::Unconsulted),
|
||||
metadata: Default::default(),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@ use std::time::Instant;
|
|||
use uuid::Uuid;
|
||||
|
||||
use jouleclaw_schema::{
|
||||
AtomicClaim, EntailmentLabel, EntailmentProbabilities, EntailmentResult, RetrievedItem,
|
||||
AtomicClaim, Backing, EntailmentLabel, EntailmentProbabilities, EntailmentResult,
|
||||
RetrievedItem,
|
||||
};
|
||||
|
||||
use jouleclaw_deberta::{NliEngine, NliInference, NliInferenceError};
|
||||
|
|
@ -68,6 +69,17 @@ pub trait Entailer: Send + Sync {
|
|||
/// Stable model id surfaced in `EntailmentResult.model_id`.
|
||||
fn model_id(&self) -> &str;
|
||||
|
||||
/// What this backend's labels can be relied on as.
|
||||
///
|
||||
/// Defaults to [`Backing::Unstated`] — the **floor**, deliberately. A
|
||||
/// backend that does not declare is not assumed substantive, so an
|
||||
/// implementation that forgets under-claims rather than over-claims. This
|
||||
/// is the same discipline as `Vmm::attained_tier` defaulting to S0: the
|
||||
/// cost of silence should fall on the claim, not on the reader.
|
||||
fn backing(&self) -> Backing {
|
||||
Backing::Unstated
|
||||
}
|
||||
|
||||
/// Wrap a raw entailment for (claim, item) in a schema-shaped
|
||||
/// [`EntailmentResult`] with attribution. Implementations
|
||||
/// usually don't override this; the default builds the result
|
||||
|
|
@ -157,6 +169,12 @@ impl Entailer for DebertaEntailer {
|
|||
})
|
||||
}
|
||||
|
||||
fn backing(&self) -> Backing {
|
||||
// One learned model's judgement. Substantive, and explicitly not
|
||||
// corroborated — nothing here cross-checks a second instrument.
|
||||
Backing::SingleModel { model_id: self.engine.model_id().to_string() }
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
self.engine.model_id()
|
||||
}
|
||||
|
|
@ -268,6 +286,16 @@ impl Entailer for FixtureEntailer {
|
|||
Ok(self.default.lock().unwrap().clone())
|
||||
}
|
||||
|
||||
fn backing(&self) -> Backing {
|
||||
// Says what it is. A report built on this backend is refused by
|
||||
// `require_substantive_grounding`, which is the entire point: a
|
||||
// fixture answers whatever it was told to, so a verdict resting on
|
||||
// one establishes nothing about the world. Naming it in the sealed
|
||||
// report is what keeps a test-shaped artifact from reading like a
|
||||
// verified one downstream.
|
||||
Backing::Fixture { model_ids: vec![self.model_id.clone()] }
|
||||
}
|
||||
|
||||
fn model_id(&self) -> &str {
|
||||
&self.model_id
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ use chrono::Utc;
|
|||
use uuid::Uuid;
|
||||
|
||||
use jouleclaw_schema::{
|
||||
AtomicClaim, AuthorityRecord, Invariant, QueryPlan, RetrievedItem, VerificationAction,
|
||||
VerificationReport,
|
||||
AtomicClaim, AuthorityRecord, Backing, Invariant, QueryPlan, RetrievedItem,
|
||||
VerificationAction, VerificationReport,
|
||||
};
|
||||
|
||||
use crate::entailer::{entail_batch, EntailError, Entailer};
|
||||
|
|
@ -120,6 +120,17 @@ pub fn verify<E: Entailer + ?Sized>(
|
|||
// I10 always holds — refusals are structured by construction.
|
||||
invariants_verified.push(Invariant::I10RefusalIsStructured.id().into());
|
||||
|
||||
// Declare what the grounding rests on, from the backend that actually
|
||||
// produced the labels — not from an assumption at the call site. When
|
||||
// nothing was consulted the verdict is *silent* about grounding, which is
|
||||
// a different statement from "ran and found support", and the two must not
|
||||
// collapse into one categorical verdict.
|
||||
let grounding_backing = if entailments.is_empty() {
|
||||
Backing::Unconsulted
|
||||
} else {
|
||||
entailer.backing()
|
||||
};
|
||||
|
||||
let report = VerificationReport {
|
||||
schema_version: "2.0".into(),
|
||||
report_id: Uuid::new_v4(),
|
||||
|
|
@ -132,6 +143,7 @@ pub fn verify<E: Entailer + ?Sized>(
|
|||
invariants_verified,
|
||||
joules_spent,
|
||||
metadata: Default::default(),
|
||||
grounding_backing: Some(grounding_backing),
|
||||
};
|
||||
Ok(report)
|
||||
}
|
||||
|
|
@ -305,6 +317,69 @@ mod tests {
|
|||
.contains(&"I2".to_string()), "I2 should be verified when entailment ran");
|
||||
}
|
||||
|
||||
/// The same clean report as above — no violations, verdict
|
||||
/// `ProceedToComposition`, invariant I2 verified — is nonetheless refused
|
||||
/// as *grounded*, because a fixture produced its entailment labels.
|
||||
///
|
||||
/// This is the gap the backing closes. Every signal a consumer previously
|
||||
/// had says this answer is verified, and all of them are true: the checks
|
||||
/// ran, found nothing wrong, and I2 genuinely holds over the labels it was
|
||||
/// given. What none of them said is that the labels came from a stand-in
|
||||
/// that answers whatever it was told to. The verdict is about the labels;
|
||||
/// the backing is about whether the labels are worth anything.
|
||||
#[test]
|
||||
fn a_clean_verdict_from_a_fixture_is_still_not_grounded() {
|
||||
let plan = simple_plan();
|
||||
let item = item_for("q0", "wikidata:Q90", 1, "Paris is the capital of France.");
|
||||
let items = vec![item];
|
||||
let segs = vec![("s0".into(), "Paris is the capital of France.".into())];
|
||||
let claims = atomize_sentences(&segs, &axes_with_tier(1)).unwrap();
|
||||
let entailer = FixtureEntailer::new();
|
||||
entailer.set_entails(
|
||||
"Paris is the capital of France.",
|
||||
"Paris is the capital of France.",
|
||||
);
|
||||
let report =
|
||||
verify(&VerifyInputs::new(&plan, &items, &[], &claims), &entailer).unwrap();
|
||||
|
||||
// Everything that was checkable before still says "verified".
|
||||
assert!(report.violations.is_empty());
|
||||
assert!(matches!(report.verdict, VerificationAction::ProceedToComposition));
|
||||
assert!(report.invariants_verified.contains(&"I2".to_string()));
|
||||
|
||||
// And the report names the stand-in rather than letting it pass.
|
||||
assert_eq!(
|
||||
report.grounding_backing,
|
||||
Some(Backing::Fixture { model_ids: vec!["fixture-entailer".into()] })
|
||||
);
|
||||
assert!(matches!(
|
||||
report.require_substantive_grounding(),
|
||||
Err(jouleclaw_schema::GroundingError::Fixture { .. })
|
||||
));
|
||||
}
|
||||
|
||||
/// When no entailment runs, the report says the verdict is *silent* about
|
||||
/// grounding rather than supportive of it.
|
||||
#[test]
|
||||
fn a_report_with_no_entailment_declares_itself_unconsulted() {
|
||||
let plan = simple_plan();
|
||||
let item = item_for("q0", "wikidata:Q90", 1, "Paris is the capital of France.");
|
||||
let entailer = FixtureEntailer::new();
|
||||
// No claims => entailment never runs.
|
||||
let report =
|
||||
verify(&VerifyInputs::new(&plan, &[item], &[], &[]), &entailer).unwrap();
|
||||
|
||||
assert_eq!(report.grounding_backing, Some(Backing::Unconsulted));
|
||||
assert!(
|
||||
!report.invariants_verified.contains(&"I2".to_string()),
|
||||
"I2 must not be claimed when nothing was entailed"
|
||||
);
|
||||
assert!(matches!(
|
||||
report.require_substantive_grounding(),
|
||||
Err(jouleclaw_schema::GroundingError::Unconsulted)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_claim_drives_grounding_violation_and_degrade() {
|
||||
let plan = simple_plan();
|
||||
|
|
|
|||
|
|
@ -477,6 +477,24 @@ fn rebuild_output_from_cache(
|
|||
entailments_consulted: a.provenance.entailments.clone(),
|
||||
invariants_verified: a.invariants_verified.clone(),
|
||||
joules_spent: 0.0,
|
||||
// The cache preserves the entailment *results* but not the
|
||||
// backing the original producer declared, so this rebuild
|
||||
// cannot re-make that claim — it declares the floor.
|
||||
//
|
||||
// Deriving one from the cached `model_id`s would be the
|
||||
// laundering path: a run backed by `FixtureEntailer` records
|
||||
// model_id "fixture-entailer", and reconstructing
|
||||
// `SingleModel` from it would promote a stand-in into a
|
||||
// substantive backing on the way out of the cache. A backing
|
||||
// is an assertion the producer made; evidence that a model ran
|
||||
// is not the same as a claim about what its labels are worth.
|
||||
//
|
||||
// Consequence, stated rather than hidden: a rehydrated report
|
||||
// is refused by `require_substantive_grounding`. The fix is to
|
||||
// persist the backing alongside the entailments in
|
||||
// `Provenance`, which is a cache-format change and not this
|
||||
// change.
|
||||
grounding_backing: Some(Backing::Unstated),
|
||||
metadata: Default::default(),
|
||||
},
|
||||
),
|
||||
|
|
@ -494,6 +512,10 @@ fn rebuild_output_from_cache(
|
|||
entailments_consulted: vec![],
|
||||
invariants_verified: vec![],
|
||||
joules_spent: 0.0,
|
||||
// A refusal consulted nothing, so this is not the floor but
|
||||
// the accurate statement: the verdict is silent about
|
||||
// grounding because no entailment ran.
|
||||
grounding_backing: Some(Backing::Unconsulted),
|
||||
metadata: Default::default(),
|
||||
},
|
||||
),
|
||||
|
|
|
|||
|
|
@ -128,3 +128,85 @@ mod tests {
|
|||
assert_eq!(r, back);
|
||||
}
|
||||
}
|
||||
|
||||
/// What a verification verdict's grounding actually rests on.
|
||||
///
|
||||
/// # Why a verdict needs to say this
|
||||
///
|
||||
/// [`crate::VerificationReport::verdict`] is a categorical outcome, and a
|
||||
/// consumer that reads it without walking `entailments_consulted` sees a
|
||||
/// decision with no indication of what produced it. The labels underneath are
|
||||
/// produced by a **learned model** — a DeBERTa-v3 NLI head, per §6.4 — and a
|
||||
/// learned model's output is a judgement, not a derivation. Swap the checkpoint
|
||||
/// and a claim can flip while every downstream artifact reports the same shape.
|
||||
///
|
||||
/// That is not a reason to stop using NLI: deciding whether prose entails a
|
||||
/// claim is not a decidable problem, and a calibrated judgement is the honest
|
||||
/// instrument for it. It is a reason for the verdict to **say which instrument
|
||||
/// it used**, so a reader can tell a model's opinion from a checkable fact.
|
||||
///
|
||||
/// # The floor, not the ceiling, is the default
|
||||
///
|
||||
/// [`Unstated`](Self::Unstated) exists so that a producer which says nothing
|
||||
/// under-claims rather than over-claims — the same discipline as
|
||||
/// `Vmm::attained_tier` returning the floor when a backend forgets to report.
|
||||
/// A report that does not name its backing is refused by
|
||||
/// [`crate::VerificationReport::require_substantive_grounding`], because the
|
||||
/// absence of a claim is not a claim of substance.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Backing {
|
||||
/// The producer did not say. Treated as the floor: no grounding claim.
|
||||
Unstated,
|
||||
/// No entailment ran at all. The verdict is silent about grounding rather
|
||||
/// than supportive of it — an important distinction from "ran and found
|
||||
/// nothing", which is a grounding violation.
|
||||
Unconsulted,
|
||||
/// A programmable stand-in supplied the labels. Legitimate in tests; never
|
||||
/// a basis for presenting an answer as verified. This is the case the
|
||||
/// discipline check exists to catch, and it is the same shape as a proof
|
||||
/// cascade closed by a stub.
|
||||
Fixture { model_ids: Vec<String> },
|
||||
/// One learned model's labels. Substantive, and explicitly a single
|
||||
/// instrument's judgement — not corroborated.
|
||||
SingleModel { model_id: String },
|
||||
/// Independent models that agreed on every consulted claim.
|
||||
Concurring { model_ids: Vec<String> },
|
||||
/// Independent models that disagreed on at least one claim. Substantive and
|
||||
/// *informative*: a contested claim is a real finding, not a failure, and
|
||||
/// collapsing it to one model's answer is how disagreement gets hidden.
|
||||
Contested { model_ids: Vec<String> },
|
||||
}
|
||||
|
||||
impl Backing {
|
||||
/// Whether this backing can support presenting an answer as grounded.
|
||||
///
|
||||
/// `Unstated` and `Unconsulted` cannot, because nothing was established.
|
||||
/// `Fixture` cannot, because a stand-in establishes only what it was told
|
||||
/// to say.
|
||||
pub fn is_substantive(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Backing::SingleModel { .. } | Backing::Concurring { .. } | Backing::Contested { .. }
|
||||
)
|
||||
}
|
||||
|
||||
/// The model ids behind this backing, for a reader that wants to pin them.
|
||||
pub fn model_ids(&self) -> Vec<&str> {
|
||||
match self {
|
||||
Backing::Unstated | Backing::Unconsulted => Vec::new(),
|
||||
Backing::SingleModel { model_id } => vec![model_id.as_str()],
|
||||
Backing::Fixture { model_ids }
|
||||
| Backing::Concurring { model_ids }
|
||||
| Backing::Contested { model_ids } => {
|
||||
model_ids.iter().map(String::as_str).collect()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Backing {
|
||||
fn default() -> Self {
|
||||
Backing::Unstated
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ pub use answer::{Answer, AnswerSegment, AnswerStatus, Provenance, Refusal};
|
|||
pub use atomic_claim::{AtomicClaim, ClaimStakes};
|
||||
pub use authority::{AuthorityRecord, AuthorityTier};
|
||||
pub use common::{Metadata, SchemaVersion};
|
||||
pub use entailment::{EntailmentLabel, EntailmentProbabilities, EntailmentResult};
|
||||
pub use entailment::{Backing, EntailmentLabel, EntailmentProbabilities, EntailmentResult};
|
||||
pub use epistemic::{ClaimAttribution, EpistemicMode, PriorClaimClass};
|
||||
pub use invariants::{Invariant, InvariantsVerified};
|
||||
pub use knowledge_axes::{GranularityClass, KnowledgeAxes, ScopeClass, TemporalStabilityClass};
|
||||
|
|
@ -49,6 +49,6 @@ pub use system_capabilities::{
|
|||
CapabilityStatus, ReasonerCapability, RetrieverCapability, SystemCapabilities,
|
||||
};
|
||||
pub use verification::{
|
||||
RecommendedAction, RecoveryAction, VerificationAction, VerificationReport, Violation,
|
||||
ViolationSeverity,
|
||||
GroundingError, RecommendedAction, RecoveryAction, VerificationAction, VerificationReport,
|
||||
Violation, ViolationSeverity,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize};
|
|||
use uuid::Uuid;
|
||||
|
||||
use crate::common::Metadata;
|
||||
use crate::entailment::EntailmentResult;
|
||||
use crate::entailment::{Backing, EntailmentResult};
|
||||
|
||||
/// Per-violation severity. Drives the verdict logic in §6.6:
|
||||
/// any `Critical` violation triggers RE_ROUTE or REFUSE depending
|
||||
|
|
@ -108,10 +108,128 @@ pub struct VerificationReport {
|
|||
pub invariants_verified: Vec<String>,
|
||||
/// Joules spent during verification, contributing to §8.5.
|
||||
pub joules_spent: f64,
|
||||
/// What the grounding verdict rests on — which instrument produced the
|
||||
/// entailment labels underneath [`Self::verdict`].
|
||||
///
|
||||
/// Optional and **omitted from the wire when absent**, so reports and
|
||||
/// vectors written before this field existed serialize byte-identically
|
||||
/// and keep verifying. Absent is not "fine": it reads as
|
||||
/// [`Backing::Unstated`], which [`Self::require_substantive_grounding`]
|
||||
/// refuses.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub grounding_backing: Option<Backing>,
|
||||
#[serde(default)]
|
||||
pub metadata: Metadata,
|
||||
}
|
||||
|
||||
/// Why a report cannot be presented as grounded.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum GroundingError {
|
||||
/// The report does not say what produced its entailment labels.
|
||||
Unstated,
|
||||
/// No entailment ran, so the verdict is silent about grounding.
|
||||
Unconsulted,
|
||||
/// A programmable stand-in supplied the labels.
|
||||
Fixture { model_ids: Vec<String> },
|
||||
/// The report claims a backing but consulted no entailments, or consulted
|
||||
/// entailments the backing does not account for.
|
||||
BackingDisagreesWithEvidence { claimed: Vec<String>, consulted: Vec<String> },
|
||||
/// The caller pinned an expected model and the report used another.
|
||||
UnexpectedModel { expected: String, actual: Vec<String> },
|
||||
}
|
||||
|
||||
impl core::fmt::Display for GroundingError {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
match self {
|
||||
GroundingError::Unstated => write!(
|
||||
f,
|
||||
"report does not name the instrument behind its entailment \
|
||||
labels; an unstated backing is not a claim of substance"
|
||||
),
|
||||
GroundingError::Unconsulted => {
|
||||
write!(f, "no entailment ran: the verdict is silent about grounding")
|
||||
}
|
||||
GroundingError::Fixture { model_ids } => write!(
|
||||
f,
|
||||
"grounding rests on a programmable stand-in ({}); a fixture \
|
||||
establishes only what it was told to say",
|
||||
model_ids.join(", ")
|
||||
),
|
||||
GroundingError::BackingDisagreesWithEvidence { claimed, consulted } => write!(
|
||||
f,
|
||||
"declared backing ({}) is not the set of models that actually \
|
||||
produced the consulted entailments ({})",
|
||||
claimed.join(", "),
|
||||
consulted.join(", ")
|
||||
),
|
||||
GroundingError::UnexpectedModel { expected, actual } => write!(
|
||||
f,
|
||||
"expected grounding from {expected}, got {}",
|
||||
actual.join(", ")
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for GroundingError {}
|
||||
|
||||
impl VerificationReport {
|
||||
/// Refuse to treat this report as grounded unless something substantive
|
||||
/// produced its entailment labels.
|
||||
///
|
||||
/// The sibling of `proof-cascade`'s `require_substantive_proof`,
|
||||
/// `sandbox-attest`'s `require_mechanism_evidence` and
|
||||
/// `jcp-action-receipt`'s `check_discipline`: the artifact carries its
|
||||
/// provenance either way, and this is the opt-in call that turns carried
|
||||
/// provenance into an enforced precondition. Existing consumers are
|
||||
/// unaffected until they ask.
|
||||
///
|
||||
/// It also cross-checks the declared backing against
|
||||
/// [`Self::entailments_consulted`], so the declaration cannot simply assert
|
||||
/// a better instrument than the one whose results are attached.
|
||||
pub fn require_substantive_grounding(&self) -> Result<&Backing, GroundingError> {
|
||||
let backing = self.grounding_backing.as_ref().unwrap_or(&Backing::Unstated);
|
||||
match backing {
|
||||
Backing::Unstated => return Err(GroundingError::Unstated),
|
||||
Backing::Unconsulted => return Err(GroundingError::Unconsulted),
|
||||
Backing::Fixture { model_ids } => {
|
||||
return Err(GroundingError::Fixture { model_ids: model_ids.clone() })
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// A declaration that outruns its evidence is worse than none: it reads
|
||||
// as corroboration a reader cannot obtain from the report.
|
||||
let mut consulted: Vec<String> =
|
||||
self.entailments_consulted.iter().map(|e| e.model_id.clone()).collect();
|
||||
consulted.sort();
|
||||
consulted.dedup();
|
||||
let mut claimed: Vec<String> =
|
||||
backing.model_ids().into_iter().map(str::to_string).collect();
|
||||
claimed.sort();
|
||||
claimed.dedup();
|
||||
if consulted != claimed {
|
||||
return Err(GroundingError::BackingDisagreesWithEvidence { claimed, consulted });
|
||||
}
|
||||
Ok(backing)
|
||||
}
|
||||
|
||||
/// As [`Self::require_substantive_grounding`], additionally pinning the
|
||||
/// model. For a caller who has decided *which* instrument it trusts, rather
|
||||
/// than only that one was used.
|
||||
pub fn require_grounding_from(&self, expected_model: &str) -> Result<&Backing, GroundingError> {
|
||||
let backing = self.require_substantive_grounding()?;
|
||||
let actual = backing.model_ids();
|
||||
if !actual.iter().any(|m| *m == expected_model) {
|
||||
return Err(GroundingError::UnexpectedModel {
|
||||
expected: expected_model.to_string(),
|
||||
actual: actual.into_iter().map(str::to_string).collect(),
|
||||
});
|
||||
}
|
||||
Ok(backing)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -129,6 +247,7 @@ mod tests {
|
|||
entailments_consulted: vec![],
|
||||
invariants_verified: vec!["I1".into(), "I2".into(), "I13".into()],
|
||||
joules_spent: 0.0,
|
||||
grounding_backing: None,
|
||||
metadata: Default::default(),
|
||||
};
|
||||
let json = serde_json::to_string(&r).unwrap();
|
||||
|
|
@ -149,4 +268,164 @@ mod tests {
|
|||
let json = serde_json::to_string(&v).unwrap();
|
||||
assert!(json.contains("\"severity\":\"critical\""));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod grounding_tests {
|
||||
use super::*;
|
||||
use crate::entailment::{EntailmentLabel, EntailmentProbabilities};
|
||||
|
||||
fn report(backing: Option<Backing>, models: &[&str]) -> VerificationReport {
|
||||
let plan_id = Uuid::new_v4();
|
||||
VerificationReport {
|
||||
schema_version: "2.0".into(),
|
||||
report_id: Uuid::new_v4(),
|
||||
plan_id,
|
||||
generated_at: Utc::now(),
|
||||
verdict: VerificationAction::ProceedToComposition,
|
||||
violations: vec![],
|
||||
recovery_actions: vec![],
|
||||
entailments_consulted: models
|
||||
.iter()
|
||||
.map(|m| EntailmentResult {
|
||||
schema_version: "2.0".into(),
|
||||
result_id: Uuid::new_v4(),
|
||||
claim_id: Uuid::new_v4(),
|
||||
premise_item_ids: vec![Uuid::new_v4()],
|
||||
label: EntailmentLabel::Entails,
|
||||
label_probabilities: EntailmentProbabilities {
|
||||
entails: 0.9,
|
||||
neutral: 0.07,
|
||||
contradicts: 0.03,
|
||||
},
|
||||
running_e_value: None,
|
||||
model_id: (*m).to_string(),
|
||||
joules_spent: 1.0,
|
||||
metadata: Default::default(),
|
||||
})
|
||||
.collect(),
|
||||
invariants_verified: vec![],
|
||||
joules_spent: 0.0,
|
||||
grounding_backing: backing,
|
||||
metadata: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The discipline this family runs on: a new optional field must be absent
|
||||
/// from the wire when unset, so reports and conformance vectors written
|
||||
/// before it existed serialize byte-identically and keep verifying.
|
||||
#[test]
|
||||
fn an_absent_backing_does_not_reach_the_wire() {
|
||||
let r = report(None, &[]);
|
||||
let json = serde_json::to_string(&r).unwrap();
|
||||
assert!(
|
||||
!json.contains("grounding_backing"),
|
||||
"an unset backing must not appear in the serialized report"
|
||||
);
|
||||
// And it round-trips to the same absence rather than to a default.
|
||||
let back: VerificationReport = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(back.grounding_backing, None);
|
||||
}
|
||||
|
||||
/// Silence is not a claim of substance. A producer that says nothing gets
|
||||
/// the floor, so forgetting under-claims instead of over-claiming.
|
||||
#[test]
|
||||
fn an_unstated_backing_is_refused() {
|
||||
assert_eq!(
|
||||
report(None, &[]).require_substantive_grounding(),
|
||||
Err(GroundingError::Unstated)
|
||||
);
|
||||
assert_eq!(
|
||||
report(Some(Backing::Unstated), &[]).require_substantive_grounding(),
|
||||
Err(GroundingError::Unstated)
|
||||
);
|
||||
}
|
||||
|
||||
/// The core rule: a stand-in answers whatever it was told to, so a verdict
|
||||
/// resting on one establishes nothing about the world — the same shape as a
|
||||
/// proof cascade closed by a stub.
|
||||
#[test]
|
||||
fn a_fixture_backed_verdict_is_refused() {
|
||||
let r = report(
|
||||
Some(Backing::Fixture { model_ids: vec!["fixture-entailer".into()] }),
|
||||
&["fixture-entailer"],
|
||||
);
|
||||
match r.require_substantive_grounding() {
|
||||
Err(GroundingError::Fixture { model_ids }) => {
|
||||
assert_eq!(model_ids, vec!["fixture-entailer".to_string()]);
|
||||
}
|
||||
other => panic!("a fixture must never read as grounded: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// "Nothing ran" and "ran and found no support" are different statements.
|
||||
/// The first is silence about grounding; the second is a violation. A
|
||||
/// verdict must not let them collapse into one categorical outcome.
|
||||
#[test]
|
||||
fn unconsulted_is_distinct_from_a_grounding_violation() {
|
||||
assert_eq!(
|
||||
report(Some(Backing::Unconsulted), &[]).require_substantive_grounding(),
|
||||
Err(GroundingError::Unconsulted)
|
||||
);
|
||||
}
|
||||
|
||||
/// A declaration that outruns its evidence is worse than none: it reads as
|
||||
/// corroboration the reader cannot obtain from the report itself.
|
||||
#[test]
|
||||
fn a_backing_may_not_claim_models_the_evidence_does_not_show() {
|
||||
// Claims two concurring models; only one actually produced results.
|
||||
let r = report(
|
||||
Some(Backing::Concurring {
|
||||
model_ids: vec!["deberta-v3".into(), "some-second-model".into()],
|
||||
}),
|
||||
&["deberta-v3"],
|
||||
);
|
||||
match r.require_substantive_grounding() {
|
||||
Err(GroundingError::BackingDisagreesWithEvidence { claimed, consulted }) => {
|
||||
assert_eq!(consulted, vec!["deberta-v3".to_string()]);
|
||||
assert_eq!(claimed.len(), 2);
|
||||
}
|
||||
other => panic!("an unsupported corroboration claim must be refused: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// A real single-model verdict passes, and says plainly that it is one
|
||||
/// instrument's judgement rather than a corroborated one.
|
||||
#[test]
|
||||
fn a_single_learned_model_is_substantive_but_uncorroborated() {
|
||||
let r = report(
|
||||
Some(Backing::SingleModel { model_id: "deberta-v3".into() }),
|
||||
&["deberta-v3"],
|
||||
);
|
||||
let backing = r.require_substantive_grounding().expect("substantive");
|
||||
assert!(backing.is_substantive());
|
||||
assert_eq!(backing.model_ids(), vec!["deberta-v3"]);
|
||||
assert!(
|
||||
!matches!(backing, Backing::Concurring { .. }),
|
||||
"one model is not corroboration"
|
||||
);
|
||||
|
||||
// A caller that has decided which instrument it trusts can pin it.
|
||||
assert!(r.require_grounding_from("deberta-v3").is_ok());
|
||||
match r.require_grounding_from("some-other-model") {
|
||||
Err(GroundingError::UnexpectedModel { expected, actual }) => {
|
||||
assert_eq!(expected, "some-other-model");
|
||||
assert_eq!(actual, vec!["deberta-v3".to_string()]);
|
||||
}
|
||||
other => panic!("pinning must reject a different model: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Disagreement between instruments is a real finding, not a failure. It
|
||||
/// stays substantive — collapsing it to one model's answer is how a
|
||||
/// contested claim gets quietly resolved in favour of whichever ran last.
|
||||
#[test]
|
||||
fn contested_is_substantive_and_says_so() {
|
||||
let r = report(
|
||||
Some(Backing::Contested { model_ids: vec!["a".into(), "b".into()] }),
|
||||
&["a", "b"],
|
||||
);
|
||||
let backing = r.require_substantive_grounding().expect("contested is substantive");
|
||||
assert!(matches!(backing, Backing::Contested { .. }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue