open-standards/wai/wai-rs/tests/cli_conform_sinks.rs

//! `conform-result/1` run against WAI's native sink (conformance-results §4.1), through its Rust
//! module and through the C interface its platform integrations link, and against the score
//! decoder, each run through `wai conform` as another implementation would be. Each run
//! is conformant, `wai verify` accepts each record, and `wai verify --conformant` admits each
//! against the corpus of this checkout. A planted sink that presents an object whose digest does
//! not match is refused (`not_conformant`); the same sink run against a copy of the corpus changed
//! to suit it is conformant there, and refused against this checkout's (`corpus_mismatch`); a
//! record of a suite this build does not run is refused (`suite_unknown`).
#![cfg(all(feature = "quickstart", feature = "score", feature = "native_sink"))]

use serde_json::Value;
use std::path::{Path, PathBuf};
use std::process::{Command, Output};

fn tmp(name: &str) -> PathBuf {
    let d = std::env::temp_dir().join(format!("wai-conform-sinks-{name}-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&d);
    std::fs::create_dir_all(&d).unwrap();
    d
}

fn wai_dir() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf()
}

fn wai(dir: &Path, args: &[&str]) -> Output {
    Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(dir).args(args).output().expect("wai runs")
}

fn text(o: &Output) -> String {
    format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))
}

fn q(p: &str) -> String {
    format!("'{}'", p.replace('\'', "'\\''"))
}

/// `wai conform <kind> --suite <suite> --impl <imp> --root <root> --json`, its record written
/// to `out`: the exit status and the JSON envelope.
fn conform(dir: &Path, root: &Path, kind: &str, suite: &str, imp: &str, out: &Path) -> (i32, Value) {
    let o = wai(dir, &["conform", kind, "--suite", suite, "--impl", imp, "--root", root.to_str().unwrap(), "--out", out.to_str().unwrap(), "--yes", "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("JSON: {}", text(&o)));
    (o.status.code().unwrap_or(-1), v)
}

/// The reference native sink, through its module or its C interface.
fn native(c_abi: bool) -> String {
    format!("{} answers {}{{case}}", q(env!("CARGO_BIN_EXE_wai_native_sink_vectors")), if c_abi { "--c-abi " } else { "" })
}

/// A sink that presents an object whose digest does not match the SHA-256 its name states: the
/// reference sink's answers with that refusal's `present` turned true.
fn planted() -> String {
    format!("{} | sed '/object_digest_mismatch/s/\"present\":false/\"present\":true/'", native(false))
}

#[test]
fn the_native_sink_and_the_score_decoder_conform_and_each_record_is_admitted() {
    let d = tmp("own");
    let root = wai_dir();
    let w = q(env!("CARGO_BIN_EXE_wai"));
    let runs = [
        ("sink", "native-sink", native(false), "native-sink"),
        ("sink", "native-sink", native(true), "native-sink-c-abi"),
        ("decoder", "score", format!("{w} score render {{in}} --pcm {{out}}"), "score"),
    ];
    for (kind, suite, imp, name) in &runs {
        let (code, v) = conform(&d, &root, kind, suite, imp, &d.join(format!("{name}.wai-conform.json")));
        assert_eq!(code, 0, "{name}: {v:#}");
        assert_eq!(v["data"]["conformant"], true, "{name}");
        assert_eq!(v["data"]["failed"], 0, "{name}");
    }
    let rec: Value = serde_json::from_slice(&std::fs::read(d.join("native-sink.wai-conform.json")).unwrap()).unwrap();
    assert_eq!((rec["kind"].as_str(), rec["suite"].as_str()), (Some("sink"), Some("native-sink")));
    // Every session of the corpus and every configuration it refuses is a case.
    let corpus: Value = serde_json::from_slice(&std::fs::read(root.join("live-conformance/native-sink/vectors.json")).unwrap()).unwrap();
    let n = corpus["cases"].as_array().unwrap().len() + corpus["configs"].as_array().unwrap().len();
    assert_eq!(rec["summary"]["cases"].as_u64(), Some(n as u64), "{}", rec["summary"]);

    let o = wai(&d, &["verify", d.to_str().unwrap()]);
    assert!(o.status.success() && text(&o).contains("3 conformance results verified"), "{}", text(&o));
    let o = wai(&d, &["verify", "--conformant", "--root", root.to_str().unwrap(), d.to_str().unwrap()]);
    assert!(o.status.success(), "{}", text(&o));
    assert_eq!(text(&o).matches("ok   admitted").count(), 3, "{}", text(&o));
    assert!(text(&o).contains("every conformance result admits its implementation"), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

#[test]
fn a_planted_failing_sink_is_refused() {
    let d = tmp("planted");
    let root = wai_dir();
    let out = d.join("planted.wai-conform.json");
    let (code, v) = conform(&d, &root, "sink", "native-sink", &planted(), &out);
    assert_eq!(code, 1, "{v:#}");
    assert_eq!(v["data"]["conformant"], false);
    let failed: Vec<&str> = v["data"]["failures"].as_array().unwrap().iter().map(|f| f["name"].as_str().unwrap()).collect();
    assert!(failed.contains(&"tampered-fragment"), "{failed:?}");
    assert!(!failed.contains(&"signed-vod"), "a session with no refused object still passes: {failed:?}");
    // The record is a genuine record of a run that did not pass: it verifies, and admits nothing.
    let o = wai(&d, &["verify", out.to_str().unwrap()]);
    assert!(o.status.success() && text(&o).contains("not conformant"), "{}", text(&o));
    let o = wai(&d, &["verify", "--conformant", "--root", root.to_str().unwrap(), out.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("FAIL admission rule 2 not_conformant"), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

#[test]
fn a_sink_run_against_a_corpus_changed_to_suit_it_is_refused() {
    let d = tmp("changed");
    let root = wai_dir();
    // A copy of what a run reads, its answers changed to the planted sink's.
    let copy = d.join("wai");
    std::fs::create_dir_all(copy.join("registry")).unwrap();
    std::fs::create_dir_all(copy.join("live-conformance/native-sink")).unwrap();
    for f in ["SPEC.md", "CHANGELOG.md", "registry/capabilities.json"] {
        std::fs::copy(root.join(f), copy.join(f)).unwrap();
    }
    let mut corpus: Value = serde_json::from_slice(&std::fs::read(root.join("live-conformance/native-sink/vectors.json")).unwrap()).unwrap();
    for c in corpus["cases"].as_array_mut().unwrap() {
        for call in c["calls"].as_array_mut().unwrap() {
            if call["answer"]["refused"] == "object_digest_mismatch" {
                call["answer"]["present"] = Value::Bool(true);
            }
        }
    }
    std::fs::write(copy.join("live-conformance/native-sink/vectors.json"), serde_json::to_vec_pretty(&corpus).unwrap()).unwrap();
    let out = d.join("suited.wai-conform.json");
    let (code, v) = conform(&d, &copy, "sink", "native-sink", &planted(), &out);
    assert_eq!(code, 0, "against the changed corpus the planted sink passes: {v:#}");
    let o = wai(&d, &["verify", "--conformant", "--root", copy.to_str().unwrap(), out.to_str().unwrap()]);
    assert!(o.status.success(), "a relying party holding the changed corpus admits it: {}", text(&o));
    let o = wai(&d, &["verify", "--conformant", "--root", root.to_str().unwrap(), out.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("FAIL admission rule 4 corpus_mismatch"), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

#[test]
fn a_record_of_a_suite_this_build_does_not_run_is_refused() {
    let d = tmp("unknown");
    let root = wai_dir();
    let out = d.join("native-sink.wai-conform.json");
    let (code, _) = conform(&d, &root, "sink", "native-sink", &native(false), &out);
    assert_eq!(code, 0);
    let rec: Value = serde_json::from_slice(&std::fs::read(&out).unwrap()).unwrap();
    let mut r = rec.as_object().unwrap().clone();
    r.remove("sig");
    r.remove("signer_pubkey");
    r.insert("suite".into(), Value::from("nonesuch"));
    let key = ed25519_dalek::SigningKey::from_bytes(&[0x45; 32]);
    let r = wai::conform_result::sign(r, &key).unwrap();
    let other = d.join("nonesuch.wai-conform.json");
    std::fs::write(&other, serde_json::to_vec_pretty(&Value::Object(r)).unwrap()).unwrap();
    assert!(wai(&d, &["verify", other.to_str().unwrap()]).status.success(), "signed again, it verifies");
    let o = wai(&d, &["verify", "--conformant", "--root", root.to_str().unwrap(), other.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("FAIL admission rule 3 suite_unknown"), "{}", text(&o));
    // --root alone is a usage error: it names the corpus --conformant compares with.
    assert_eq!(wai(&d, &["verify", "--root", root.to_str().unwrap(), out.to_str().unwrap()]).status.code(), Some(2));
    let _ = std::fs::remove_dir_all(&d);
}