From 27fa4e5e599030b1ac09a8a159111402d9e8d8cd Mon Sep 17 00:00:00 2001 From: David Charlot Date: Tue, 15 Sep 2026 11:13:11 -0400 Subject: [PATCH 1/2] joulecontract: the published negative vectors were never actually run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pack had two tests: every file's content address against `MANIFEST.json`, and the positive DSSE envelope. Neither *executed* a negative vector. All eleven `negative-*.json` were published, digest-pinned, and never read by a verifier — so the vectors written to catch a verifier that stops demoting were exactly the ones that could not catch it. This replays each through `validate_sealed` under the context they were generated in, and compares all four recorded fields: the demotion list, the maximum profile, the rendered §15 claim string, and whether the claimed profile is permitted. **Equality, not "it demoted".** These are degradation vectors, not accept/reject ones. Each records the verdict a conformant verifier must produce — `metered_unattested` demotes Energy to `ModelBased`, `generator_unattested` caps Governance at `declared`. Asserting merely that something was demoted would pass a verifier that demoted the wrong axis, or by the wrong amount, or that refused the receipt outright — and refusing is not the specified behaviour either, so the replay also asserts each vector still validates structurally. **A wrong assumption the vectors corrected.** My first version asserted every negative vector must leave its claimed profile impermissible. Two do not: `recourse_undeclared` and `static_only_frame` record a *disclosure* demotion rather than a cap, and `static_only` is prohibited only at JR-4 while these vectors claim JR-3. The honest verdict there is "permitted, and here is what you are trusting". Asserting `!permits` would have pressured two published vectors toward a stricter reading than the spec takes, so the invariant is the demotion list instead. **Falsified, not assumed.** Removing the `EngagementUndisclosed` demotion rule from the verifier leaves the *entire* pre-existing workspace suite green, and fails this replay naming the vector and the axis. `StaticOnly`, `EngagementUndisclosed` and `IndependentKeyOnly` appear in no unit test in `joulecontract-receipt` at all — the published pack was their only coverage, and it was not running. (A probe against `MeteredUnattested` is *not* silent: two unit tests catch that one. The gap is uneven, which is itself worth knowing.) The replay asserts it saw at least eleven vectors, so a wrong path, a renamed prefix or an empty glob fails rather than passing having checked nothing — the same vacuity that let the original gap hide. It also pins the three context scalars it replays under, so a regenerated pack under a different context is a failure rather than a silent comparison between two different questions. `joulecontract-rs` is already in the CI matrix, so these vectors now run on every change. Verified: 15 test binaries green across the workspace, 1 new. The verifier crate is byte-identical to `main` — the falsification probes were reverted and the revert checked with `git diff --quiet`. Co-Authored-By: Claude Opus 5 --- .../joulecontract-conformance/src/lib.rs | 132 ++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/joulecontract/joulecontract-rs/crates/joulecontract-conformance/src/lib.rs b/joulecontract/joulecontract-rs/crates/joulecontract-conformance/src/lib.rs index 65643c875..9f208c9b9 100644 --- a/joulecontract/joulecontract-rs/crates/joulecontract-conformance/src/lib.rs +++ b/joulecontract/joulecontract-rs/crates/joulecontract-conformance/src/lib.rs @@ -416,3 +416,135 @@ mod vector_pack_tests { assert_eq!(statement.subject.len(), 2); } } + +#[cfg(test)] +mod negative_vector_replay { + use super::fixtures::appendix_a_sealed; + use joulecontract_receipt::{validate_sealed, Receipt, SealedContext}; + + /// Every published `negative-*` vector is replayed through the real + /// verifier and must reproduce its recorded verdict exactly. + /// + /// # What was missing + /// + /// The pack already had two tests: one checking every file's content + /// address against `MANIFEST.json`, and one verifying the positive DSSE + /// envelope. Neither *ran* a negative vector. They were published, + /// digest-pinned, and never executed — so a verifier that stopped demoting + /// would have kept every test green while the eleven vectors written to + /// catch exactly that sat on disk unread. + /// + /// # Why equality, not just "it demoted" + /// + /// These are degradation vectors, not accept/reject ones. Each records the + /// composed verdict a conformant verifier must produce: `metered_unattested` + /// demotes Energy to `ModelBased`, `generator_unattested` caps Governance at + /// `declared`, and so on. Asserting merely that *something* was demoted + /// would pass a verifier that demoted the wrong axis, or by the wrong + /// amount, or that refused the receipt outright — and refusing is not the + /// specified behaviour either. The whole content of these vectors is the + /// **exact** resulting claim, so the test compares all four recorded + /// fields: the demotion list, the maximum profile, the rendered §15 claim + /// string, and whether the claimed profile is permitted. + #[test] + fn every_negative_vector_reproduces_its_recorded_verdict() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../conformance/v1"); + if !dir.exists() { + return; // crate vendored without the pack + } + + // Rebuilt exactly as `gen_vectors` builds it: the negative cases mutate + // the receipt and share this one context. + let (contract, _r, bundle, log, interval) = appendix_a_sealed(); + let ctx = SealedContext { + bundle: &bundle, + key_log: &log, + interval: &interval, + transition_end_at: 150, + max_anchor_latency_secs: 3600, + open_breach_unamended: false, + }; + + let mut replayed = 0usize; + let mut entries: Vec<_> = std::fs::read_dir(&dir) + .expect("read conformance dir") + .filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|n| n.starts_with("negative-") && n.ends_with(".json")) + .collect(); + entries.sort(); + + for name in &entries { + let body = std::fs::read(dir.join(name)).unwrap_or_else(|e| panic!("{name}: {e}")); + let v: serde_json::Value = serde_json::from_slice(&body) + .unwrap_or_else(|e| panic!("{name}: parse: {e}")); + + // The context the vector was generated under must be the context we + // are replaying under, or the comparison is between two different + // questions. Checked rather than assumed. + let sc = &v["sealed_context"]; + assert_eq!(sc["transition_end_at"], 150, "{name}: context drifted"); + assert_eq!(sc["max_anchor_latency_secs"], 3600, "{name}: context drifted"); + assert_eq!(sc["open_breach_unamended"], false, "{name}: context drifted"); + + let receipt: Receipt = serde_json::from_value(v["receipt"].clone()) + .unwrap_or_else(|e| panic!("{name}: receipt: {e}")); + + // A negative vector is demoted, never malformed: the receipt must + // still validate structurally. A verifier that rejected it outright + // would be wrong in the other direction. + let verdict = validate_sealed(&contract, &receipt, &ctx) + .unwrap_or_else(|e| panic!("{name}: must demote, not refuse: {e}")); + + let expected = &v["expected_composed"]; + assert_eq!( + serde_json::to_value(&verdict.demotions).unwrap(), + expected["demotions"], + "{name}: demotion list" + ); + assert_eq!( + serde_json::to_value(&verdict.max_profile).unwrap(), + expected["max_profile"], + "{name}: maximum permitted profile" + ); + assert_eq!( + serde_json::Value::String(verdict.claim()), + expected["claim"], + "{name}: rendered claim" + ); + assert_eq!( + serde_json::Value::Bool(verdict.permits(&receipt.claimed_profile)), + expected["permits_claimed_profile"], + "{name}: whether the claimed profile is permitted" + ); + + // What makes a vector negative is that the verifier *flagged* + // something — not that it capped the profile. Two of the published + // eleven deliberately still permit the claimed profile: + // `recourse_undeclared` and `static_only_frame` record a + // **disclosure** demotion rather than a cap. `static_only` is + // prohibited only at JR-4 and these vectors claim JR-3, so the + // honest verdict is "permitted, and here is what you are trusting". + // + // So the invariant is the demotion list, not the permission bit. + // Asserting `!permits` here would have been wrong, and would have + // pressured those two vectors toward a stricter reading than the + // spec takes. + assert!( + !verdict.demotions.is_empty(), + "{name}: a negative vector that produces no demotion is not \ + testing anything" + ); + + replayed += 1; + } + + // Anti-vacuity. A wrong path, a renamed prefix or an empty glob would + // otherwise let this test pass having checked nothing — the same shape + // as the gap it was written to close. + assert!( + replayed >= 11, + "expected at least the 11 published negative vectors, replayed {replayed}" + ); + } +} -- 2.54.0 (Apple Git-157) From 7eee267a06f2cbee2995fefbb600105361a8dbf6 Mon Sep 17 00:00:00 2001 From: David Charlot Date: Thu, 17 Sep 2026 06:54:01 -0400 Subject: [PATCH 2/2] joulecontract: the negative replay could pass with the pack missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The replay test opened with the same `if !dir.exists() { return; }` the other two vector-pack tests had — and that early return fires *before* the `replayed >= 11` guard at the end of the function. So a missing or moved pack reported green, which is precisely the case that guard was written to catch. I had described the test as anti-vacuous; with that return in place it was not. `main` has since replaced the same pattern in `vector_pack_tests` with an assertion, on the grounds that this crate is not published and so only ever runs in-repo where the pack is always present. That reasoning applies unchanged here, so the replay now asserts too, and the three pack tests in this file agree. Verified in both directions rather than one: - pack present: 10 passed - pack moved aside: the replay FAILS, naming the path it could not find Before this change the second case passed. The pack was restored and confirmed byte-identical with `git status`, not by assumption. Co-Authored-By: Claude Opus 5 --- .../crates/joulecontract-conformance/src/lib.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/joulecontract/joulecontract-rs/crates/joulecontract-conformance/src/lib.rs b/joulecontract/joulecontract-rs/crates/joulecontract-conformance/src/lib.rs index 9f208c9b9..8780003ac 100644 --- a/joulecontract/joulecontract-rs/crates/joulecontract-conformance/src/lib.rs +++ b/joulecontract/joulecontract-rs/crates/joulecontract-conformance/src/lib.rs @@ -449,9 +449,16 @@ mod negative_vector_replay { #[test] fn every_negative_vector_reproduces_its_recorded_verdict() { let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../../conformance/v1"); - if !dir.exists() { - return; // crate vendored without the pack - } + // Assert, not skip — matching `vector_pack_tests`. An early `return` here + // would fire *before* the `replayed >= 11` guard below and let a missing + // pack pass vacuously, which is precisely the failure that guard exists to + // catch. The crate is not published, so it only runs in-repo, where the + // pack is always present. + assert!( + dir.exists(), + "conformance pack missing at {} — it is what this test replays", + dir.display() + ); // Rebuilt exactly as `gen_vectors` builds it: the negative cases mutate // the receipt and share this one context. -- 2.54.0 (Apple Git-157)