open-standards/wai/wai-rs/tests/robustness.rs

//! Robustness evidence for soft bindings (extensions/robustness-evidence.md) and its corpus
//! `live-conformance/robustness/`: every file regenerates byte for byte from its fixed seeds —
//! wai-mark:tent64's records among them, which runs the harness over the measurement sets of
//! synthesis §7.5.8 — and no other file is there; every case gets what it states; §12 lists
//! exactly the reference's codes, and the corpus refuses a case for each; §4 registers exactly
//! the reference's transforms; the tent64 records' tables are the synthesis corpus's
//! `battery.json`, measured by another program; and the tent64 claim is synthesis §7.5.8's
//! list. Run it in release.
#![cfg(feature = "robustness")]

use serde_json::Value;
use std::collections::{BTreeMap, BTreeSet};
use std::path::PathBuf;
use std::sync::OnceLock;
use wai::robustness::corpus::{self, World, TENT64_CLAIM_CLIPS, TENT64_CLAIM_PICTURES};
use wai::robustness::{Reason, REASONS, TRANSFORMS};

fn wai_dir() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..")
}

fn tree(root: &PathBuf) -> BTreeMap<String, Vec<u8>> {
    let mut out = BTreeMap::new();
    let mut stack = vec![root.clone()];
    while let Some(d) = stack.pop() {
        for e in std::fs::read_dir(&d).expect("a committed corpus").flatten() {
            let p = e.path();
            if p.is_dir() {
                stack.push(p);
                continue;
            }
            let rel = p.strip_prefix(root).expect("under the corpus").to_string_lossy().replace('\\', "/");
            if rel != "README.md" {
                out.insert(rel, std::fs::read(&p).expect("a file"));
            }
        }
    }
    out
}

fn committed() -> BTreeMap<String, Vec<u8>> {
    tree(&wai_dir().join("live-conformance/robustness"))
}

/// The world, computed once for every test of this file.
fn world() -> &'static World {
    static W: OnceLock<World> = OnceLock::new();
    W.get_or_init(corpus::world)
}

fn text(rel: &str) -> String {
    std::fs::read_to_string(wai_dir().join(rel)).expect("a text")
}

/// The text of the section that starts at `head`, up to the next heading of its level or above.
fn section(text: &str, head: &str) -> String {
    let at = text.find(&format!("\n{head}")).unwrap_or_else(|| panic!("a section {head}"));
    let level = head.chars().take_while(|c| *c == '#').count();
    let end = text[at + 1..]
        .match_indices("\n#")
        .find(|(i, _)| text[at + 1 + i + 1..].chars().take_while(|c| *c == '#').count() <= level)
        .map_or(text.len(), |(i, _)| at + 1 + i);
    text[at..end].to_string()
}

fn fold(s: &str) -> String {
    s.split_whitespace().collect::<Vec<_>>().join(" ")
}

/// The first column of a section's table rows, between backticks.
fn first_column(s: &str) -> Vec<String> {
    s.lines()
        .filter(|l| l.starts_with("| `"))
        .filter_map(|l| l.split('`').nth(1).map(String::from))
        .collect()
}

#[test]
fn the_committed_corpus_regenerates_from_its_seeds() {
    let generated: BTreeMap<String, Vec<u8>> = corpus::generate_from(world()).into_iter().collect();
    let files = committed();
    for k in generated.keys() {
        assert!(files.get(k) == generated.get(k), "{k} differs from its regeneration, or is missing");
    }
    for k in files.keys() {
        assert!(generated.contains_key(k), "{k} is not generated");
    }
}

#[test]
fn every_case_gets_what_it_states() {
    let fails = corpus::verify(&committed());
    assert!(fails.is_empty(), "{fails:#?}");
}

#[test]
fn the_extension_lists_exactly_the_references_reasons_and_the_corpus_refuses_each() {
    let ext = text("extensions/robustness-evidence.md");
    let listed = first_column(&section(&ext, "## 12. Reasons"));
    let ours: Vec<String> = REASONS.iter().map(|r| r.label().to_string()).collect();
    assert_eq!(listed, ours, "§12 lists exactly the reference's codes, in its order");
    let doc: Value = serde_json::from_slice(&committed()["vectors.json"]).expect("vectors.json");
    let refused: BTreeSet<String> = doc["cases"]
        .as_array()
        .expect("cases")
        .iter()
        .filter_map(|c| c["expected"]["reason"].as_str().map(String::from))
        .collect();
    for r in REASONS {
        assert!(refused.contains(r.label()), "no case is refused {}", r.label());
    }
    assert!(REASONS.contains(&Reason::TableIrreproducible));
}

#[test]
fn the_extension_registers_exactly_the_references_transforms() {
    let ext = text("extensions/robustness-evidence.md");
    let listed = first_column(&section(&ext, "## 4. The transforms"));
    assert_eq!(listed, TRANSFORMS.iter().map(|t| t.to_string()).collect::<Vec<_>>());
}

#[test]
fn the_tent64_records_are_the_synthesis_measurement() {
    let battery: Value = serde_json::from_slice(&std::fs::read(wai_dir().join("live-conformance/synthesis-mark/battery.json")).expect("battery.json")).expect("JSON");
    let payload: BTreeMap<&str, &str> = battery["pictures"]
        .as_array()
        .expect("pictures")
        .iter()
        .map(|p| (p["name"].as_str().expect("a name"), p["payload"].as_str().expect("a payload")))
        .collect();
    for (name, record) in &world().tent64 {
        let set = name.strip_prefix("tent64-").expect("a set");
        let doc = &world().tent64_docs.iter().find(|(n, _)| n == name).expect("its corpus").1;
        for (item, value) in doc.items.iter().zip(record["values"].as_array().expect("values")) {
            assert_eq!(Some(&value.as_str().expect("hex")), payload.get(item.name.as_str()), "{name}: {}'s payload", item.name);
        }
        for row in record["table"].as_array().expect("a table") {
            let attack = row["attack"].as_str().expect("a name");
            let find = |list: &str| {
                battery[list].as_array().expect(list).iter().find(|r| r["attack"] == attack && r["set"] == set).unwrap_or_else(|| panic!("{set} {attack} in battery.json")).clone()
            };
            let (t, u) = (find("table"), find("unmarked"));
            assert_eq!(row["trials"], t["trials"], "{set} {attack}");
            assert_eq!(row["survived"], t["detected"], "{set} {attack}");
            assert_eq!(row["wrong"], t["wrong"], "{set} {attack}");
            assert_eq!(row["negatives"], u["trials"], "{set} {attack}");
            assert_eq!(row["false_positives"], u["detections"], "{set} {attack}");
        }
    }
}

#[test]
fn the_tent64_claim_is_synthesis_758s_list() {
    let ext = text("extensions/synthesis.md");
    let s758 = fold(&section(&ext, "#### 7.5.8 "));
    let at = s758.find("For pictures, these attacks, each with the parameter measured: ").expect("§7.5.8's claims");
    let rest = &s758[at..];
    let clips_at = rest.find(" For clips: ").expect("the clips' claims");
    let names = |s: &str| -> Vec<String> { s.split('`').skip(1).step_by(2).map(String::from).collect() };
    let pictures = names(&rest[..clips_at]);
    let clips = names(&rest[clips_at..rest[clips_at..].find(". ").map_or(rest.len(), |e| clips_at + e)]);
    assert_eq!(pictures, TENT64_CLAIM_PICTURES.to_vec());
    assert_eq!(clips, TENT64_CLAIM_CLIPS.to_vec());
    // And the claim record holds exactly those, each measured.
    let claim = &world().tent64_claim;
    assert_eq!(claim["survives"].as_array().expect("survives").len(), pictures.len() + clips.len());
}

#[test]
fn jwp_receipts_lists_the_span_reasons() {
    let jwp = fold(&text("extensions/jwp-receipts.md"));
    for (step, r) in [(4, "span_malformed"), (4, "span_inverted"), (6, "recovery_outside_span")] {
        assert!(jwp.contains(&format!("| {step} | `{r}` |")), "jwp-receipts §3.1 lists {r} at step {step}");
    }
}