open-standards/wai/wai-rs/tests/cli_conform_relay.rs

//! `wai conform relay --suite moqt`, the hybrid-signature verifier suite and `wai conformance gen`,
//! through the `wai` binary:
//!
//! - a relay that passes every case conforms: the interop client (here a stand-in that answers
//!   TAP as `joule-moq-interop` does) runs once a case with the relay's URL and the draft, the
//!   record names its command line and its SHA-256, `wai verify --conformant` admits it against
//!   this checkout's corpus and gives the live area its verdict;
//! - planted: a relay failing one case fails the record at that case alone, its run exits 1, the
//!   record still verifies and admission refuses it at rule 2; a test the client skips fails
//!   where the case is a test and passes at the draft that has none; a client not found, a URL
//!   that reaches no relay and a draft the client does not speak each have their exit status;
//! - the hybrid-signature suite (with `pq_signatures`): the reference verifier conforms case by
//!   case and over the whole corpus, one that changes a verdict fails at those cases, and the
//!   corpus's independent verifier, `pq_verify.py`, agrees on every case;
//! - `wai conformance gen`: each corpus's tool run in its modes' order, in `wai-rs`, given the
//!   corpus's directory; a tool that fails fails the run; a corpus whose tool writes nothing is
//!   refused unless only verified; a dry run names each command and runs none.
#![cfg(feature = "quickstart")]
// The stand-ins below are shell scripts: the tests run where a shell does.
#![cfg_attr(not(unix), allow(dead_code, unused_imports))]

use serde_json::Value;
use std::path::{Path, PathBuf};
use std::process::{Command, Output};

// A program a test runs is written by a process of its own (ETXTBSY: support/program.rs).
#[path = "support/program.rs"]
mod program;

fn tmp(name: &str) -> PathBuf {
    let d = std::env::temp_dir().join(format!("wai-conform-relay-{name}-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&d);
    std::fs::create_dir_all(&d).unwrap();
    d
}

fn wai_dir() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf()
}

/// `wai` in `dir`, the relay's client `client` (its environment variable unset for `None`).
fn wai(dir: &Path, args: &[&str], client: Option<&Path>) -> Output {
    let mut c = Command::new(env!("CARGO_BIN_EXE_wai"));
    c.current_dir(dir).args(args);
    match client {
        Some(p) => c.env("WAI_MOQ_INTEROP_BIN", p),
        None => c.env_remove("WAI_MOQ_INTEROP_BIN"),
    };
    c.output().expect("wai runs")
}

fn text(o: &Output) -> String {
    format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))
}

fn json(o: &Output) -> Value {
    serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("JSON: {}", text(o)))
}

fn sha256_hex(b: &[u8]) -> String {
    use sha2::Digest;
    sha2::Sha256::digest(b).iter().map(|x| format!("{x:02x}")).collect()
}

/// A stand-in for the interop client, `dir/joule-moq-interop`: it records its arguments in
/// `dir/args.log` and answers its `--test` as TAP version 14, as the relay it stands for would:
/// `not ok` for a test in `fail`, skipped for one in `skip`, `ok` for the rest.
#[cfg(unix)]
fn client(dir: &Path, fail: &[&str], skip: &[&str]) -> PathBuf {
    let p = dir.join("joule-moq-interop");
    let log = dir.join("args.log");
    program::write_program(
        &p,
        &format!(
            r##"#!/bin/sh
echo "$@" >> '{}'
test="" draft=18
while [ $# -gt 0 ]; do case "$1" in --test) test="$2"; shift 2 ;; --draft) draft="$2"; shift 2 ;; *) shift ;; esac; done
echo "TAP version 14"
echo "# a stand-in for joule-moq-interop"
echo "1..1"
case " {} " in *" $test "*) echo "not ok 1 - $test"; echo "  ---"; echo '  message: "the relay answered nothing"'; echo "  ..."; exit 1 ;; esac
case " {} " in *" $test "*) echo "ok 1 - $test # SKIP draft-$draft has no such message"; exit 0 ;; esac
echo "ok 1 - $test"
echo "  ---"
echo "  duration_ms: 3"
echo "  ..."
"##,
            log.display(),
            fail.join(" "),
            skip.join(" ")
        ),
    );
    p
}

/// `wai conform relay --suite moqt` against `url` with `client`, its record `out`.
fn relay(dir: &Path, client: &Path, url: &str, extra: &[&str], out: &Path) -> Output {
    let root = wai_dir();
    let mut a = vec!["conform", "relay", "--suite", "moqt", "--impl", url, "--root", root.to_str().unwrap(), "--out", out.to_str().unwrap(), "--yes", "--json"];
    a.extend_from_slice(extra);
    wai(dir, &a, Some(client))
}

const CASES: [&str; 9] = [
    "setup-only",
    "announce-only",
    "publish-namespace-done",
    "subscribe-namespace-lifecycle",
    "subscribe-error",
    "rendezvous-timeout",
    "announce-subscribe",
    "subscribe-before-announce",
    "wai-receipts",
];

#[cfg(unix)]
#[test]
fn a_relay_that_passes_every_case_conforms_and_is_admitted() {
    let d = tmp("pass");
    let c = client(&d, &[], &[]);
    let rec_dir = d.join("records");
    std::fs::create_dir_all(&rec_dir).unwrap();
    let out = rec_dir.join("moqt.wai-conform.json");
    let o = relay(&d, &c, "moqt://127.0.0.1:4443/moq", &["--tls-disable-verify", "--name", "wai-relay", "--version", "dev"], &out);
    let v = json(&o);
    assert_eq!(o.status.code(), Some(0), "{v:#}");
    assert_eq!((v["data"]["conformant"].as_bool(), v["data"]["cases"].as_u64(), v["data"]["failed"].as_u64()), (Some(true), Some(9), Some(0)), "{v:#}");
    // One run of the client a case, in the corpus's order, at the default draft.
    let log = std::fs::read_to_string(d.join("args.log")).unwrap();
    let runs: Vec<&str> = log.lines().collect();
    assert_eq!(runs.len(), 9);
    for (r, name) in runs.iter().zip(CASES) {
        assert_eq!(*r, format!("--relay moqt://127.0.0.1:4443/moq --draft 18 --tls-disable-verify --test {name}"));
    }
    // The record: a relay of the moqt suite, run through the client's command line, the relay's
    // URL in it, the client hashed.
    let rec: Value = serde_json::from_slice(&std::fs::read(&out).unwrap()).unwrap();
    assert_eq!((rec["kind"].as_str(), rec["suite"].as_str(), rec["corpus"]["dir"].as_str()), (Some("relay"), Some("moqt"), Some("moqt-conformance")));
    let im = &rec["implementation"];
    assert_eq!((im["adapter"].as_str(), im["name"].as_str(), im["version"].as_str()), (Some("command"), Some("wai-relay"), Some("dev")));
    assert!(im["impl"].as_str().unwrap().ends_with(" --relay 'moqt://127.0.0.1:4443/moq' --draft 18 --tls-disable-verify --test {case}"), "{im}");
    assert_eq!(im["files"][0]["path"].as_str(), Some(c.to_str().unwrap()));
    assert_eq!(im["files"][0]["sha256"].as_str(), Some(sha256_hex(&std::fs::read(&c).unwrap()).as_str()));
    let names: Vec<&str> = rec["cases"].as_array().unwrap().iter().map(|x| x["name"].as_str().unwrap()).collect();
    assert_eq!(names, CASES);
    assert!(rec["cases"][0]["expected"].as_str().unwrap().starts_with("ok: "), "{}", rec["cases"][0]);
    // Verified, and admitted against this checkout's corpus, the live area's verdict given.
    let o = wai(&d, &["verify", "--conformant", "--root", wai_dir().to_str().unwrap(), rec_dir.to_str().unwrap()], None);
    assert!(o.status.success(), "{}", text(&o));
    assert!(text(&o).contains("area live: conformant"), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

#[cfg(unix)]
#[test]
fn a_relay_failing_one_case_fails_the_record() {
    let d = tmp("fail");
    let c = client(&d, &["subscribe-error"], &[]);
    let rec_dir = d.join("records");
    std::fs::create_dir_all(&rec_dir).unwrap();
    let out = rec_dir.join("moqt.wai-conform.json");
    let o = relay(&d, &c, "moqt://127.0.0.1:4443/moq", &[], &out);
    let v = json(&o);
    assert_eq!(o.status.code(), Some(1), "{v:#}");
    assert_eq!((v["data"]["conformant"].as_bool(), v["data"]["passed"].as_u64(), v["data"]["failed"].as_u64()), (Some(false), Some(8), Some(1)), "{v:#}");
    let f = &v["data"]["failures"][0];
    assert_eq!((f["name"].as_str(), f["got"].as_str()), (Some("subscribe-error"), Some("not ok: the relay answered nothing")), "{f}");
    // The record of a failed run is a genuine record of it: verified, and refused at admission.
    let o = wai(&d, &["verify", rec_dir.to_str().unwrap()], None);
    assert!(o.status.success(), "{}", text(&o));
    let o = wai(&d, &["verify", "--conformant", "--root", wai_dir().to_str().unwrap(), rec_dir.to_str().unwrap()], None);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("FAIL admission rule 2 not_conformant"), "{}", text(&o));
    // In text, the case and what the client said.
    let o = wai(&d, &["conform", "relay", "--suite", "moqt", "--impl", "moqt://127.0.0.1:4443/moq", "--root", wai_dir().to_str().unwrap(), "--out", out.to_str().unwrap(), "--yes"], Some(&c));
    assert_eq!(o.status.code(), Some(1));
    assert!(text(&o).contains("FAIL subscribe-error") && text(&o).contains("8 of 9 cases passed: not conformant"), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

#[cfg(unix)]
#[test]
fn a_skipped_test_passes_only_at_a_draft_where_it_is_no_test() {
    let d = tmp("skip");
    let c = client(&d, &[], &["rendezvous-timeout"]);
    let out = d.join("moqt.wai-conform.json");
    // At the runner's draft rendezvous-timeout is a test: a skip fails it.
    let o = relay(&d, &c, "https://127.0.0.1:4443/moq", &[], &out);
    let v = json(&o);
    assert_eq!(o.status.code(), Some(1), "{v:#}");
    assert_eq!(v["data"]["failures"][0]["name"], "rendezvous-timeout", "{v:#}");
    assert!(v["data"]["failures"][0]["got"].as_str().unwrap().starts_with("skipped at draft 18, where it is a test"), "{v:#}");
    // Draft 16 has no RENDEZVOUS_TIMEOUT: the client skips it there, and the case passes.
    let o = relay(&d, &c, "https://127.0.0.1:4443/moq", &["--draft", "16"], &out);
    let v = json(&o);
    assert_eq!(o.status.code(), Some(0), "{v:#}");
    let rec: Value = serde_json::from_slice(&std::fs::read(&out).unwrap()).unwrap();
    assert!(rec["implementation"]["impl"].as_str().unwrap().contains(" --draft 16 --test {case}"), "{rec:#}");
    let rt = rec["cases"].as_array().unwrap().iter().find(|x| x["name"] == "rendezvous-timeout").unwrap().clone();
    assert_eq!(rt["pass"], true);
    assert!(rt["expected"].as_str().unwrap().contains("no test at draft 16"), "{rt}");
    // A skip of a case that is a test at every draft fails at draft 16 too.
    let c = client(&d, &[], &["setup-only"]);
    let o = relay(&d, &c, "https://127.0.0.1:4443/moq", &["--draft", "16"], &out);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

#[cfg(unix)]
#[test]
fn each_failure_to_run_a_relay_has_its_exit_status() {
    let d = tmp("exits");
    let c = client(&d, &[], &[]);
    let out = d.join("moqt.wai-conform.json");
    // A URL that reaches no relay, a draft the client does not speak, a relay's flags on another kind.
    let root = wai_dir();
    for (args, code) in [
        (vec!["--impl", "http://127.0.0.1:4443/moq"], 2),
        (vec!["--impl", "moqt://127.0.0.1:4443/moq", "--draft", "17"], 2),
    ] {
        let mut a = vec!["conform", "relay", "--suite", "moqt", "--root", root.to_str().unwrap(), "--out", out.to_str().unwrap()];
        a.extend(args);
        let o = wai(&d, &a, Some(&c));
        assert_eq!(o.status.code(), Some(code), "{a:?}: {}", text(&o));
    }
    let o = wai(&d, &["conform", "verifier", "--suite", "jwp", "--impl", "true", "--draft", "18", "--root", wai_dir().to_str().unwrap()], Some(&c));
    assert_eq!(o.status.code(), Some(2), "{}", text(&o));
    // A client that is not where its variable says: exit 5, and nothing run.
    let o = wai(&d, &["conform", "relay", "--suite", "moqt", "--impl", "moqt://127.0.0.1:4443/moq", "--root", wai_dir().to_str().unwrap(), "--out", out.to_str().unwrap()], Some(&d.join("no-such-client")));
    assert_eq!(o.status.code(), Some(5), "{}", text(&o));
    assert!(text(&o).contains("WAI_MOQ_INTEROP_BIN"), "{}", text(&o));
    assert!(!out.exists());
    // A dry run names the client's command line and runs nothing.
    let o = wai(&d, &["conform", "relay", "--suite", "moqt", "--impl", "moqt://127.0.0.1:4443/moq", "--root", wai_dir().to_str().unwrap(), "--out", out.to_str().unwrap(), "--dry-run"], Some(&c));
    assert!(o.status.success(), "{}", text(&o));
    assert!(text(&o).contains("--relay 'moqt://127.0.0.1:4443/moq' --draft 18 --test {case}"), "{}", text(&o));
    assert!(!out.exists() && !d.join("args.log").exists());
    // The list names the relay kind, its suite in the live area, and its client.
    let o = wai(&d, &["conformance", "list", "--root", wai_dir().to_str().unwrap(), "--json"], None);
    let v = json(&o);
    let s = v["data"]["suites"].as_array().unwrap().iter().find(|s| s["suite"] == "moqt").unwrap().clone();
    assert_eq!((s["kind"].as_str(), s["area"].as_str(), s["cases"].as_u64()), (Some("relay"), Some("live"), Some(9)), "{s}");
    assert_eq!(s["adapters"], serde_json::json!(["joule-moq-interop"]));
    let _ = std::fs::remove_dir_all(&d);
}

/// The hybrid-signature verifier suite: the reference, case by case and whole; a changed verdict;
/// and its independent verifier beside it (pq_verify.py, which needs `cryptography`).
#[cfg(feature = "pq_signatures")]
#[cfg(unix)]
#[test]
fn the_hybrid_signature_verifiers_conform_and_one_changed_verdict_does_not() {
    let d = tmp("pq");
    let root = wai_dir();
    let r = root.to_str().unwrap();
    let pq = format!("'{}'", env!("CARGO_BIN_EXE_wai_pq_vectors"));
    let run = |imp: &str, extra: &[&str]| {
        let mut a = vec!["conform", "verifier", "--suite", "pq", "--impl", imp, "--root", r, "--yes", "--json"];
        a.extend_from_slice(extra);
        wai(&d, &a, None)
    };
    // Case by case, the verdict alone (the reference prints `name<TAB>verdict`), and whole.
    for imp in [format!("{pq} verdicts {{case}} | cut -f2-"), format!("{pq} verdicts {{corpus}}")] {
        let o = run(&imp, &[]);
        let v = json(&o);
        assert_eq!(o.status.code(), Some(0), "{imp}: {v:#}");
        assert!(v["data"]["cases"].as_u64().unwrap() >= 40, "{v:#}");
    }
    // A verifier that reads every hybrid record as classical fails at those records alone.
    let o = run(&format!("{pq} verdicts {{case}} | cut -f2- | sed 's/accept form=hybrid/accept form=classical/'"), &[]);
    let v = json(&o);
    assert_eq!(o.status.code(), Some(1), "{v:#}");
    let failed: Vec<&str> = v["data"]["failures"].as_array().unwrap().iter().map(|f| f["name"].as_str().unwrap()).collect();
    assert_eq!(failed, ["record_synthesis_hybrid", "record_exploration_hybrid"], "{v:#}");
    // The corpus's independent verifier, case by case beside the reference: they agree.
    let o = run(&format!("{pq} verdicts {{corpus}}"), &["--independent"]);
    let v = json(&o);
    assert_eq!(o.status.code(), Some(0), "{v:#}");
    let ind = &v["data"]["independent"];
    assert_eq!((ind["verifier"].as_str(), ind["conformant"].as_bool()), (Some("pq"), Some(true)), "{ind:#}");
    assert!(ind["command"].as_str().unwrap().ends_with("pq-conformance/pq_verify.py verdicts pq-conformance/vectors.json"), "{ind}");
    assert_eq!(ind["disagreements"].as_array().unwrap().len(), 0, "{ind:#}");
    let _ = std::fs::remove_dir_all(&d);
}

/// A checkout's `wai/` for `wai conformance gen` to write into: the files that make it one, the
/// directories of the corpora named, and `wai-rs/` the tools run in.
fn gen_root(d: &Path, corpora: &[&str]) -> PathBuf {
    let root = d.join("wai");
    std::fs::create_dir_all(root.join("registry")).unwrap();
    std::fs::create_dir_all(root.join("wai-rs")).unwrap();
    std::fs::write(root.join("SPEC.md"), "").unwrap();
    std::fs::write(root.join("registry/capabilities.json"), "{}").unwrap();
    for c in corpora {
        std::fs::create_dir_all(root.join(c)).unwrap();
    }
    root
}

/// A stand-in for a corpus's tool, `bin_dir/<name>`: it writes, in the corpus directory it is
/// given (from the directory it runs in), a file named for its mode, and exits `code`.
#[cfg(unix)]
fn tool(bin_dir: &Path, name: &str, code: i32) {
    std::fs::create_dir_all(bin_dir).unwrap();
    program::write_program(&bin_dir.join(name), &format!("#!/bin/sh\n[ \"$(basename \"$PWD\")\" = wai-rs ] || exit 9\necho \"$1\" >> \"$2/modes\"\nexit {code}\n"));
}

#[cfg(unix)]
#[test]
fn a_corpus_is_written_by_its_generator_in_the_order_of_its_modes() {
    let d = tmp("gen");
    let root = gen_root(&d, &["score-conformance", "staged-measure-conformance", "oer-conformance"]);
    let bins = d.join("bin");
    tool(&bins, "wai_score_vectors", 0);
    tool(&bins, "wai_stage_measure", 0);
    tool(&bins, "wai_oer_vectors", 0);
    let r = root.to_str().unwrap();
    let b = bins.to_str().unwrap();
    // A dry run names each command and runs none.
    let o = wai(&d, &["conformance", "gen", "--suite", "score,staged-measure", "--root", r, "--bin-dir", b, "--dry-run", "--json"], None);
    let v = json(&o);
    assert!(o.status.success(), "{v:#}");
    assert_eq!(v["dry_run"], true, "{v:#}");
    let cmds: Vec<String> = v["data"]["corpora"].as_array().unwrap().iter().flat_map(|c| c["commands"].as_array().unwrap().iter().map(|x| x.as_str().unwrap().to_string())).collect();
    assert_eq!(cmds, [format!("{b}/wai_score_vectors gen ../score-conformance"), format!("{b}/wai_stage_measure gen-fixtures ../staged-measure-conformance"), format!("{b}/wai_stage_measure generate-goldens ../staged-measure-conformance")]);
    assert!(!root.join("score-conformance/modes").exists());
    // Run: each tool in wai-rs, given its corpus's directory, its modes in order.
    let o = wai(&d, &["conformance", "gen", "--suite", "staged-measure,score", "--root", r, "--bin-dir", b], None);
    assert!(o.status.success(), "{}", text(&o));
    assert_eq!(std::fs::read_to_string(root.join("score-conformance/modes")).unwrap(), "gen\n");
    assert_eq!(std::fs::read_to_string(root.join("staged-measure-conformance/modes")).unwrap(), "gen-fixtures\ngenerate-goldens\n");
    assert!(text(&o).contains("2 of 2 corpora written"), "{}", text(&o));
    // --verify runs the verifier alone.
    let o = wai(&d, &["conformance", "gen", "--suite", "score", "--verify", "--root", r, "--bin-dir", b], None);
    assert!(o.status.success(), "{}", text(&o));
    assert_eq!(std::fs::read_to_string(root.join("score-conformance/modes")).unwrap(), "gen\nverify\n");
    // A corpus whose tool writes nothing is refused to be written, and may be verified.
    let o = wai(&d, &["conformance", "gen", "--suite", "oer", "--root", r, "--bin-dir", b], None);
    assert_eq!(o.status.code(), Some(5), "{}", text(&o));
    assert!(text(&o).contains("gen: none"), "{}", text(&o));
    assert!(wai(&d, &["conformance", "gen", "--suite", "oer", "--verify", "--root", r, "--bin-dir", b], None).status.success());
    // A tool that fails fails the run, and names the corpus; one not built there is exit 5; an
    // unknown corpus is exit 5; no --suite is a usage error.
    tool(&bins, "wai_score_vectors", 3);
    let o = wai(&d, &["conformance", "gen", "--suite", "score", "--root", r, "--bin-dir", b, "--json"], None);
    let v = json(&o);
    assert_eq!(o.status.code(), Some(1), "{v:#}");
    assert_eq!((v["data"]["failed"].as_u64(), v["data"]["corpora"][0]["runs"][0]["exit_code"].as_i64()), (Some(1), Some(3)), "{v:#}");
    assert_eq!(wai(&d, &["conformance", "gen", "--suite", "pin", "--root", r, "--bin-dir", b], None).status.code(), Some(5));
    assert_eq!(wai(&d, &["conformance", "gen", "--suite", "no-such", "--root", r, "--bin-dir", b], None).status.code(), Some(5));
    assert_eq!(wai(&d, &["conformance", "gen", "--root", r, "--bin-dir", b], None).status.code(), Some(2));
    // The list names every corpus and its generator.
    let o = wai(&d, &["conformance", "list", "--json"], None);
    let v = json(&o);
    let gens = v["data"]["generators"].as_array().unwrap();
    assert!(gens.iter().any(|g| g["corpus"] == "pin" && g["modes"] == serde_json::json!(["generate"])), "{gens:?}");
    assert!(gens.len() >= 20, "{}", gens.len());
    let _ = std::fs::remove_dir_all(&d);
}