0055f59b3c
JCR-1 ships ten conformance vectors with three independent verifier
implementations byte-equal. The triangle primitives (conservation,
cone, oracle, elicitor) had no equivalent — implementers in other
languages had to read the Rust source to know what "compliant" means.
This commit closes that gap. Each primitive gets:
conformance/vectors.json — the canonical fixture set
conformance/README.md — the conformance contract spec
tests/conformance.rs — Rust verifier wired into cargo test
Vectors:
conservation/1 — 5 vectors: conforming, below_spec, hallucination,
missing_delta, plus a multi-finding case
cone/1 — 6 vectors: every datum_default band entry,
jouleclaw_cascade_default L0, escalate-lut-to-
formulaic, escalate-from-apex (refused)
oracle/1 — 5 vectors: always_pass, always_fail, composite
all-pass with chain dedup, composite short-
circuit on first fail, empty-composite trivial-
pass
elicitor/1 — 5 vectors: empty-walker-done, single-required-
ask, two-required-fill-first-ask-second, fully-
filled-done, optional-slot-never-asked
Each verifier asserts byte-equality between the reference impl's
serde_json output and the vector's `expected_*`. Adding a vector
extends the conformance contract for every implementer.
The Python and Go reference verifiers (the JCR-1 multi-language
pattern) are roadmap, not part of this commit — the vectors are
language-neutral so other-language verifiers can be added without
changing the contract.
4 new tests, one per primitive — all green.
Co-Authored-By: Claude Opus 4.7 <[email protected]>
80 lines
3 KiB
Rust
80 lines
3 KiB
Rust
//! Oracle conformance — the reference Oracle implementations
|
|
//! (`AlwaysPassOracle`, `AlwaysFailOracle`, `CompositeOracle`) must
|
|
//! produce the verdict listed in `conformance/vectors.json` for every
|
|
//! (oracle_program, candidate) pair, byte-equal under serde_json.
|
|
|
|
use jouleclaw_oracle::{
|
|
AlwaysFailOracle, AlwaysPassOracle, Candidate, CompositeOracle, Oracle, Verdict,
|
|
};
|
|
use serde_json::Value;
|
|
|
|
const VECTORS: &str = include_str!("../conformance/vectors.json");
|
|
|
|
fn parse_candidate(v: &Value) -> Candidate {
|
|
Candidate {
|
|
id: v["id"].as_str().unwrap_or("").to_string(),
|
|
content: v["content"].as_str().unwrap_or("").to_string(),
|
|
language: v["language"].as_str().unwrap_or("").to_string(),
|
|
source_tier: v["source_tier"].as_str().unwrap_or("").to_string(),
|
|
provenance: v["provenance"].as_str().unwrap_or("").to_string(),
|
|
}
|
|
}
|
|
|
|
fn build_oracle(program: &[Value]) -> Box<dyn Oracle> {
|
|
// The first node is either a leaf (always_pass / always_fail) or
|
|
// a composite header (kind="composite", id=…) followed by its
|
|
// members.
|
|
let head = &program[0];
|
|
match head["kind"].as_str().expect("kind") {
|
|
"always_pass" => Box::new(AlwaysPassOracle),
|
|
"always_fail" => Box::new(AlwaysFailOracle::new(
|
|
head["reason"].as_str().unwrap_or(""),
|
|
)),
|
|
"composite" => {
|
|
let id = head["id"].as_str().expect("composite id");
|
|
let mut comp = CompositeOracle::new(id);
|
|
for member in &program[1..] {
|
|
comp = comp.add(build_oracle(std::slice::from_ref(member)));
|
|
}
|
|
Box::new(comp)
|
|
}
|
|
other => panic!("unknown oracle kind: {other}"),
|
|
}
|
|
}
|
|
|
|
fn check_vector(v: &Value) {
|
|
let n = v["n"].as_i64().expect("n");
|
|
let desc = v["description"].as_str().expect("description");
|
|
let input = &v["input"];
|
|
let expected = &v["expected_verdict"];
|
|
|
|
let candidate = parse_candidate(&input["candidate"]);
|
|
let program = input["oracle_program"]
|
|
.as_array()
|
|
.expect("oracle_program array");
|
|
let oracle = build_oracle(program);
|
|
|
|
// Drive the async verify via a current-thread runtime.
|
|
let rt = tokio::runtime::Builder::new_current_thread()
|
|
.enable_all()
|
|
.build()
|
|
.expect("tokio runtime");
|
|
let verdict: Verdict = rt.block_on(oracle.verify(&candidate)).expect("verify");
|
|
|
|
let got = serde_json::to_value(&verdict).expect("verdict serialises");
|
|
assert_eq!(
|
|
&got, expected,
|
|
"vector {n} ({desc}): verdict mismatch.\n got = {got}\n expected = {expected}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn every_oracle_vector_matches_the_reference() {
|
|
let suite: Value = serde_json::from_str(VECTORS).expect("vectors.json parses");
|
|
let count = suite["vector_count"].as_u64().expect("vector_count") as usize;
|
|
let vectors = suite["vectors"].as_array().expect("vectors array");
|
|
assert_eq!(vectors.len(), count, "vector_count mismatch");
|
|
for v in vectors {
|
|
check_vector(v);
|
|
}
|
|
}
|