open-standards/wai/wai-rs/tests/cli_conform_sinks.rs

//! `conform-result/1` run against WAI's native sink (conformance-results §4.1), through its Rust
//! module and through the C interface its platform integrations link, against its browser sink,
//! against the score decoder and the integer-synthesis learned decoder, and against a cache in
//! front of the reference origin, each run through `wai conform` as another implementation would
//! be. Each run is conformant, `wai verify` accepts each record, `wai verify --conformant` admits
//! each against the corpus of this checkout, and gives each area its verdict. Planted faults: a
//! native sink that presents an object whose digest does not match, a browser sink that shows a
//! frame whose object was refused as verified, a decoder that changes one byte of its output and a
//! cache that applies a content coding are each refused (`not_conformant`); the native sink run
//! against a copy of the corpus changed to suit it is conformant there, and refused against this
//! checkout's (`corpus_mismatch`); a case the corpus marks informative that does not pass leaves
//! the verdict conformant and a required one does not; a record that marks a required case
//! informative, signed again, is refused (`corpus_mismatch`); a record of a suite this build does
//! not run is refused (`suite_unknown`).
#![cfg(all(feature = "quickstart", feature = "score", feature = "native_sink", feature = "browser_sink", feature = "neural_int"))]

use serde_json::Value;
use std::path::{Path, PathBuf};
use std::process::{Command, Output};

fn tmp(name: &str) -> PathBuf {
    let d = std::env::temp_dir().join(format!("wai-conform-sinks-{name}-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&d);
    std::fs::create_dir_all(&d).unwrap();
    d
}

fn wai_dir() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf()
}

fn wai(dir: &Path, args: &[&str]) -> Output {
    Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(dir).args(args).output().expect("wai runs")
}

fn text(o: &Output) -> String {
    format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))
}

fn q(p: &str) -> String {
    format!("'{}'", p.replace('\'', "'\\''"))
}

/// `wai conform <kind> --suite <suite> --impl <imp> --root <root> --json`, its record written
/// to `out`: the exit status and the JSON envelope.
fn conform(dir: &Path, root: &Path, kind: &str, suite: &str, imp: &str, out: &Path) -> (i32, Value) {
    let o = wai(dir, &["conform", kind, "--suite", suite, "--impl", imp, "--root", root.to_str().unwrap(), "--out", out.to_str().unwrap(), "--yes", "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("JSON: {}", text(&o)));
    (o.status.code().unwrap_or(-1), v)
}

/// The reference native sink, through its module or its C interface.
fn native(c_abi: bool) -> String {
    format!("{} answers {}{{case}}", q(env!("CARGO_BIN_EXE_wai_native_sink_vectors")), if c_abi { "--c-abi " } else { "" })
}

/// A sink that presents an object whose digest does not match the SHA-256 its name states: the
/// reference sink's answers with that refusal's `present` turned true.
fn planted() -> String {
    format!("{} | sed '/object_digest_mismatch/s/\"present\":false/\"present\":true/'", native(false))
}

/// The reference browser sink, its answer to one call.
fn browser() -> String {
    format!("{} answers {{case}}", q(env!("CARGO_BIN_EXE_wai_browser_sink_vectors")))
}

/// The reference integer-synthesis decoder.
fn int_synth() -> String {
    format!("{} decode {{model}} {{in}} {{out}}", q(env!("CARGO_BIN_EXE_wai_int_synth_vectors")))
}

#[test]
fn the_sinks_and_the_decoders_conform_and_each_record_is_admitted() {
    let d = tmp("own");
    let root = wai_dir();
    let w = q(env!("CARGO_BIN_EXE_wai"));
    let runs = [
        ("sink", "native-sink", native(false), "native-sink"),
        ("sink", "native-sink", native(true), "native-sink-c-abi"),
        ("sink", "browser", browser(), "browser"),
        ("decoder", "score", format!("{w} score render {{in}} --pcm {{out}}"), "score"),
        ("decoder", "neural", int_synth(), "neural"),
    ];
    for (kind, suite, imp, name) in &runs {
        let (code, v) = conform(&d, &root, kind, suite, imp, &d.join(format!("{name}.wai-conform.json")));
        assert_eq!(code, 0, "{name}: {v:#}");
        assert_eq!(v["data"]["conformant"], true, "{name}");
        assert_eq!(v["data"]["failed"], 0, "{name}");
    }
    let rec: Value = serde_json::from_slice(&std::fs::read(d.join("native-sink.wai-conform.json")).unwrap()).unwrap();
    assert_eq!((rec["kind"].as_str(), rec["suite"].as_str()), (Some("sink"), Some("native-sink")));
    // Every session of the corpus and every configuration it refuses is a case.
    let corpus: Value = serde_json::from_slice(&std::fs::read(root.join("live-conformance/native-sink/vectors.json")).unwrap()).unwrap();
    let n = corpus["cases"].as_array().unwrap().len() + corpus["configs"].as_array().unwrap().len();
    assert_eq!(rec["summary"]["cases"].as_u64(), Some(n as u64), "{}", rec["summary"]);

    // Every call of the browser corpus, its chain aside, and every vector of the learned decoder's.
    let rec: Value = serde_json::from_slice(&std::fs::read(d.join("browser.wai-conform.json")).unwrap()).unwrap();
    assert_eq!(rec["summary"]["cases"].as_u64(), Some(110), "{}", rec["summary"]);
    let rec: Value = serde_json::from_slice(&std::fs::read(d.join("neural.wai-conform.json")).unwrap()).unwrap();
    let names: Vec<&str> = rec["cases"].as_array().unwrap().iter().map(|c| c["name"].as_str().unwrap()).collect();
    assert_eq!(names, vec!["gradient", "bloom", "facade", "photo"]);
    // No suite of these marks a case informative, and none takes --level.
    for (kind, suite, imp, _) in &runs[2..] {
        let o = wai(&d, &["conform", kind, "--suite", suite, "--impl", imp, "--root", root.to_str().unwrap(), "--level", "all", "--out", d.join("x.wai-conform.json").to_str().unwrap()]);
        assert_eq!(o.status.code(), Some(2), "{suite}: {}", text(&o));
        assert!(text(&o).contains("marks no case informative"), "{}", text(&o));
    }
    // The decoder names the parameter set it decodes against.
    let o = wai(&d, &["conform", "decoder", "--suite", "neural", "--impl", &format!("{} decode {{in}} {{out}}", q(env!("CARGO_BIN_EXE_wai_int_synth_vectors"))), "--root", root.to_str().unwrap(), "--out", d.join("x.wai-conform.json").to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(2), "{}", text(&o));
    assert!(text(&o).contains("names {model}"), "{}", text(&o));

    let o = wai(&d, &["verify", d.to_str().unwrap()]);
    assert!(o.status.success() && text(&o).contains("5 conformance results verified"), "{}", text(&o));
    let o = wai(&d, &["verify", "--conformant", "--root", root.to_str().unwrap(), d.to_str().unwrap()]);
    assert!(o.status.success(), "{}", text(&o));
    assert_eq!(text(&o).matches("ok   admitted").count(), 5, "{}", text(&o));
    assert!(text(&o).contains("every conformance result admits its implementation"), "{}", text(&o));
    // An area verdict for each area the records are in: media's two suites run here, its floor
    // suites named as run by no record, and live's two sinks, its relay suite named as run by no
    // record.
    assert!(text(&o).contains("area media: conformant over its 2 records, every one admitted (decoder neural, decoder score); not run, no record names it: decoder floor, decoder video-floor\n"), "{}", text(&o));
    assert!(text(&o).contains("area live: conformant over its 3 records, every one admitted (sink browser, sink native-sink); not run, no record names it: relay moqt\n"), "{}", text(&o));
    assert!(!text(&o).contains("area trust") && !text(&o).contains("area delivery"), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

#[test]
fn a_planted_failing_browser_sink_and_a_decoder_that_changes_one_byte_are_refused() {
    let d = tmp("planted-browser");
    let root = wai_dir();
    // A browser sink that shows a frame whose object, chain or provenance was refused as verified.
    let shown = format!("{} | sed 's/\"state\":\"refused\"/\"state\":\"verified\"/'", browser());
    let (code, v) = conform(&d, &root, "sink", "browser", &shown, &d.join("browser.wai-conform.json"));
    assert_eq!(code, 1, "{v:#}");
    assert_eq!(v["data"]["conformant"], false);
    let failed: Vec<&str> = v["data"]["failures"].as_array().unwrap().iter().map(|f| f["name"].as_str().unwrap()).collect();
    assert_eq!(failed, vec!["frames/cmaf_provenance_refused", "frames/chain_refused", "frames/object_refused", "frames/refused_over_pending"], "exactly the frames it shows wrongly");
    // A decoder that changes one byte of what it decodes, the hundredth.
    let flip = format!("{} >/dev/null && dd if={{out}} bs=1 skip=100 count=1 2>/dev/null | LC_ALL=C tr '\\000-\\377' '\\001-\\377\\000' | dd of={{out}} bs=1 seek=100 count=1 conv=notrunc 2>/dev/null", int_synth());
    let (code, v) = conform(&d, &root, "decoder", "neural", &flip, &d.join("neural.wai-conform.json"));
    assert_eq!(code, 1, "{v:#}");
    assert_eq!((v["data"]["failed"].as_u64(), v["data"]["conformant"].as_bool()), (Some(4), Some(false)), "one byte changed fails every vector: {v:#}");
    // Each record is a genuine record of a run that did not pass: it verifies, and admits nothing.
    let o = wai(&d, &["verify", d.to_str().unwrap()]);
    assert!(o.status.success() && text(&o).matches("not conformant").count() == 2, "{}", text(&o));
    let o = wai(&d, &["verify", "--conformant", "--root", root.to_str().unwrap(), d.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert_eq!(text(&o).matches("FAIL admission rule 2 not_conformant").count(), 2, "{}", text(&o));
    assert!(text(&o).contains("area media: not conformant: refused decoder neural") && text(&o).contains("area live: not conformant: refused sink browser"), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

/// A cache in front of the reference origin, started by `mutators.py`: `relay` changes nothing,
/// the others each break http-delivery §9 one way.
fn cache(mode: &str, root: &Path) -> String {
    format!("python3 -I {} --mode {mode} --listen {{port}} --upstream {{origin}}", q(root.join("cache-conformance/mutators.py").to_str().unwrap()))
}

/// What a cache run reads, copied under `dir`: the corpus's tests (`tests` written over them where
/// given), its driver and proxies, the BLAKE3 the driver checks with, and the objects and the
/// parameter set it stages.
fn cache_corpus_copy(dir: &Path, tests: Option<&Value>) -> PathBuf {
    let root = wai_dir();
    let copy = dir.join("wai");
    fn copy_tree(from: &Path, to: &Path) {
        std::fs::create_dir_all(to).unwrap();
        for e in std::fs::read_dir(from).unwrap().flatten() {
            let t = to.join(e.file_name());
            if e.path().is_dir() { copy_tree(&e.path(), &t) } else { std::fs::copy(e.path(), t).unwrap(); }
        }
    }
    for d in ["registry", "claim-conformance", "cache-conformance", "http-conformance", "wai-rs/tests/vectors/int_mlicpp"] {
        std::fs::create_dir_all(copy.join(d)).unwrap();
    }
    for f in ["SPEC.md", "CHANGELOG.md", "registry/capabilities.json", "claim-conformance/_blake3.py", "cache-conformance/driver.py", "cache-conformance/mutators.py", "cache-conformance/tests.json", "wai-rs/tests/vectors/int_mlicpp/model.wmb"] {
        std::fs::copy(root.join(f), copy.join(f)).unwrap();
    }
    copy_tree(&root.join("http-conformance"), &copy.join("http-conformance"));
    if let Some(t) = tests {
        std::fs::write(copy.join("cache-conformance/tests.json"), serde_json::to_vec_pretty(t).unwrap()).unwrap();
    }
    copy
}

#[test]
fn a_cache_conforms_on_its_required_tests_and_an_informative_one_is_not_counted() {
    let d = tmp("cache");
    let root = wai_dir();
    let r = root.to_str().unwrap();
    // A relay, which §9 rule 7 allows: every MUST test passes, the informative ones say it stored
    // nothing, and the record marks each of them informative.
    let (code, v) = conform(&d, &root, "cache", "cache", &cache("relay", &root), &d.join("relay.wai-conform.json"));
    assert_eq!(code, 0, "{v:#}");
    assert_eq!((v["data"]["conformant"].as_bool(), v["data"]["informative"].as_u64()), (Some(true), Some(4)), "{v:#}");
    let rec: Value = serde_json::from_slice(&std::fs::read(d.join("relay.wai-conform.json")).unwrap()).unwrap();
    let informative: Vec<&str> = rec["cases"].as_array().unwrap().iter().filter(|c| c["level"] == "informative").map(|c| c["name"].as_str().unwrap()).collect();
    assert_eq!(informative, vec!["S1 stored", "S2 range_stored", "S3 stored_after_pause", "S4 origin_share"]);
    // --level all holds every case required: no case is marked, and the stricter record is admitted too.
    let o = wai(&d, &["conform", "cache", "--suite", "cache", "--impl", &cache("relay", &root), "--root", r, "--level", "all", "--out", d.join("relay-all.wai-conform.json").to_str().unwrap(), "--json"]);
    assert_eq!(o.status.code(), Some(0), "{}", text(&o));
    let all = std::fs::read_to_string(d.join("relay-all.wai-conform.json")).unwrap();
    assert!(!all.contains("\"level\""), "{all}");
    assert_eq!(wai(&d, &["conform", "cache", "--suite", "cache", "--impl", &cache("relay", &root), "--root", r, "--level", "most", "--out", d.join("x.wai-conform.json").to_str().unwrap()]).status.code(), Some(2));
    // A cache command that names no port is a usage error, before anything starts; one that ends
    // before it listens is an implementation that does not answer, said as soon as it ends.
    assert_eq!(wai(&d, &["conform", "cache", "--suite", "cache", "--impl", "python3 -I x.py {origin}", "--root", r, "--out", d.join("x.wai-conform.json").to_str().unwrap()]).status.code(), Some(2));
    let t0 = std::time::Instant::now();
    let o = wai(&d, &["conform", "cache", "--suite", "cache", "--impl", "false {port} {origin}", "--root", r, "--out", d.join("x.wai-conform.json").to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(6), "{}", text(&o));
    assert!(text(&o).contains("ended (exit Some(1)) before it answered") && t0.elapsed() < std::time::Duration::from_secs(30), "{:?} {}", t0.elapsed(), text(&o));
    assert!(!d.join("x.wai-conform.json").exists(), "no record of a run that did not start");
    let o = wai(&d, &["verify", "--conformant", "--root", r, d.to_str().unwrap()]);
    assert!(o.status.success(), "{}", text(&o));
    assert_eq!(text(&o).matches("ok   admitted").count(), 2, "{}", text(&o));
    assert!(text(&o).contains("area delivery: conformant over its 2 records, every one admitted (cache cache); not run, no record names it: packager policy, origin http"), "{}", text(&o));

    // A cache that applies a content coding fails C7, a MUST test: not conformant, refused at rule 2.
    let p = d.join("gzip");
    std::fs::create_dir_all(&p).unwrap();
    let (code, v) = conform(&d, &root, "cache", "cache", &cache("gzip", &root), &p.join("gzip.wai-conform.json"));
    assert_eq!(code, 1, "{v:#}");
    let failed: Vec<(&str, bool)> = v["data"]["failures"].as_array().unwrap().iter().map(|f| (f["name"].as_str().unwrap(), f["informative"] == true)).collect();
    assert_eq!(failed, vec![("C7 no_coding", false)], "{v:#}");
    let o = wai(&d, &["verify", "--conformant", "--root", r, p.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("FAIL admission rule 2 not_conformant"), "{}", text(&o));

    // The same cache against a copy of the corpus that marks C7 informative: its failure is
    // reported and not counted, so the run is conformant there and admitted by a party holding
    // that copy; against this checkout's corpus, where C7 is required, it is refused.
    let mut tests: Value = serde_json::from_slice(&std::fs::read(root.join("cache-conformance/tests.json")).unwrap()).unwrap();
    for t in tests["tests"].as_array_mut().unwrap() {
        if t["id"] == "C7" {
            t["level"] = Value::from("informative");
        }
    }
    let copy = cache_corpus_copy(&d.join("copy"), Some(&tests));
    let out = d.join("informative.wai-conform.json");
    let (code, v) = conform(&d, &copy, "cache", "cache", &cache("gzip", &root), &out);
    assert_eq!(code, 0, "an informative case that did not pass leaves the run conformant: {v:#}");
    assert_eq!((v["data"]["failed"].as_u64(), v["data"]["informative_failed"].as_u64()), (Some(1), Some(1)), "{v:#}");
    assert!(wai(&d, &["verify", "--conformant", "--root", copy.to_str().unwrap(), out.to_str().unwrap()]).status.success());
    let o = wai(&d, &["verify", "--conformant", "--root", r, out.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("FAIL admission rule 4 corpus_mismatch"), "{}", text(&o));

    // The record of the run against this checkout's corpus, C7 marked informative, the verdict
    // made to agree and signed again: it verifies, and its digest is this corpus's, but the corpus
    // requires C7, so it admits nothing.
    let rec: Value = serde_json::from_slice(&std::fs::read(p.join("gzip.wai-conform.json")).unwrap()).unwrap();
    let mut m = rec.as_object().unwrap().clone();
    m.remove("sig");
    m.remove("signer_pubkey");
    for c in m["cases"].as_array_mut().unwrap() {
        if c["name"] == "C7 no_coding" {
            c["level"] = Value::from("informative");
        }
    }
    m.insert("conformant".into(), Value::Bool(true));
    let m = wai::conform_result::sign(m, &ed25519_dalek::SigningKey::from_bytes(&[0x46; 32])).unwrap();
    let relevel = d.join("relevelled.wai-conform.json");
    std::fs::write(&relevel, serde_json::to_vec_pretty(&Value::Object(m)).unwrap()).unwrap();
    let o = wai(&d, &["verify", relevel.to_str().unwrap()]);
    assert!(o.status.success() && text(&o).contains("conformant"), "signed again, it verifies: {}", text(&o));
    let o = wai(&d, &["verify", "--conformant", "--root", r, relevel.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("FAIL admission rule 4 corpus_mismatch: it marks C7 no_coding informative"), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

#[test]
fn a_planted_failing_sink_is_refused() {
    let d = tmp("planted");
    let root = wai_dir();
    let out = d.join("planted.wai-conform.json");
    let (code, v) = conform(&d, &root, "sink", "native-sink", &planted(), &out);
    assert_eq!(code, 1, "{v:#}");
    assert_eq!(v["data"]["conformant"], false);
    let failed: Vec<&str> = v["data"]["failures"].as_array().unwrap().iter().map(|f| f["name"].as_str().unwrap()).collect();
    assert!(failed.contains(&"tampered-fragment"), "{failed:?}");
    assert!(!failed.contains(&"signed-vod"), "a session with no refused object still passes: {failed:?}");
    // The record is a genuine record of a run that did not pass: it verifies, and admits nothing.
    let o = wai(&d, &["verify", out.to_str().unwrap()]);
    assert!(o.status.success() && text(&o).contains("not conformant"), "{}", text(&o));
    let o = wai(&d, &["verify", "--conformant", "--root", root.to_str().unwrap(), out.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("FAIL admission rule 2 not_conformant"), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

#[test]
fn a_sink_run_against_a_corpus_changed_to_suit_it_is_refused() {
    let d = tmp("changed");
    let root = wai_dir();
    // A copy of what a run reads, its answers changed to the planted sink's.
    let copy = d.join("wai");
    std::fs::create_dir_all(copy.join("registry")).unwrap();
    std::fs::create_dir_all(copy.join("live-conformance/native-sink")).unwrap();
    for f in ["SPEC.md", "CHANGELOG.md", "registry/capabilities.json"] {
        std::fs::copy(root.join(f), copy.join(f)).unwrap();
    }
    let mut corpus: Value = serde_json::from_slice(&std::fs::read(root.join("live-conformance/native-sink/vectors.json")).unwrap()).unwrap();
    for c in corpus["cases"].as_array_mut().unwrap() {
        for call in c["calls"].as_array_mut().unwrap() {
            if call["answer"]["refused"] == "object_digest_mismatch" {
                call["answer"]["present"] = Value::Bool(true);
            }
        }
    }
    std::fs::write(copy.join("live-conformance/native-sink/vectors.json"), serde_json::to_vec_pretty(&corpus).unwrap()).unwrap();
    let out = d.join("suited.wai-conform.json");
    let (code, v) = conform(&d, &copy, "sink", "native-sink", &planted(), &out);
    assert_eq!(code, 0, "against the changed corpus the planted sink passes: {v:#}");
    let o = wai(&d, &["verify", "--conformant", "--root", copy.to_str().unwrap(), out.to_str().unwrap()]);
    assert!(o.status.success(), "a relying party holding the changed corpus admits it: {}", text(&o));
    let o = wai(&d, &["verify", "--conformant", "--root", root.to_str().unwrap(), out.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("FAIL admission rule 4 corpus_mismatch"), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

#[test]
fn a_record_of_a_suite_this_build_does_not_run_is_refused() {
    let d = tmp("unknown");
    let root = wai_dir();
    let out = d.join("native-sink.wai-conform.json");
    let (code, _) = conform(&d, &root, "sink", "native-sink", &native(false), &out);
    assert_eq!(code, 0);
    // --level is for a suite whose corpus marks a case informative; the native sink's marks none.
    let o = wai(&d, &["conform", "sink", "--suite", "native-sink", "--impl", &native(false), "--root", root.to_str().unwrap(), "--level", "required", "--out", d.join("x.wai-conform.json").to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(2), "{}", text(&o));
    let rec: Value = serde_json::from_slice(&std::fs::read(&out).unwrap()).unwrap();
    let mut r = rec.as_object().unwrap().clone();
    r.remove("sig");
    r.remove("signer_pubkey");
    r.insert("suite".into(), Value::from("nonesuch"));
    let key = ed25519_dalek::SigningKey::from_bytes(&[0x45; 32]);
    let r = wai::conform_result::sign(r, &key).unwrap();
    let other = d.join("nonesuch.wai-conform.json");
    std::fs::write(&other, serde_json::to_vec_pretty(&Value::Object(r)).unwrap()).unwrap();
    assert!(wai(&d, &["verify", other.to_str().unwrap()]).status.success(), "signed again, it verifies");
    let o = wai(&d, &["verify", "--conformant", "--root", root.to_str().unwrap(), other.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("FAIL admission rule 3 suite_unknown"), "{}", text(&o));
    // --root alone is a usage error: it names the corpus --conformant compares with.
    assert_eq!(wai(&d, &["verify", "--root", root.to_str().unwrap(), out.to_str().unwrap()]).status.code(), Some(2));
    let _ = std::fs::remove_dir_all(&d);
}