open-standards/wai/wai-rs/tests/cli_conform.rs

//! `wai conform` run against implementations of each kind, through both adapters, and its
//! records checked by `wai verify`:
//!
//! - the reference verifiers of the claim, JWP and key-discovery corpora (`wai_*_vectors
//!   verdicts`, whole-corpus runs) conform, and so does the key-discovery reference behind
//!   an HTTP adapter; one that changes one verdict does not, at that case alone;
//! - the score reference renderer (`wai score render`) conforms, and one that changes one
//!   byte of its output fails every case;
//! - the reference origin (`wai http origin`) serving the HTTP corpus's objects conforms,
//!   and one serving an object with one byte changed fails that object;
//! - a record changed after signing, in a case's outcome or in its summary, is refused by
//!   `wai verify`; a dry run writes nothing; a record is not replaced without `--yes`;
//!   each failure to run has its exit status;
//! - `--format junit` and `--format html` are the records' report: a test case a case, a failure
//!   exactly where a case failed, the independent verifier's record told apart from the
//!   implementation's; `--format json` is the envelope `--json` prints;
//! - `--independent` beside an implementation runs the corpus's independent verifier on the same
//!   cases, its own signed record beside the implementation's: the run passes when both conform
//!   and agree, and a case the two disagree on fails it and is named, two passing answers that
//!   differ on a case the corpus states in part among them; with the real Python verifier too;
//!   beside the origin, the HTTP corpus's verifier over the whole corpus, which failing fails it;
//! - `--independent` alone runs the independent verifiers `--suite` names (each with its own
//!   command line, in `wai/`), a record a corpus in `--out <dir>` (a file there is a usage
//!   error), a failed one failing the run and naming the Python package it needs if it is missing;
//!   each record is written as its verifier ends, a verifier that cannot start or a corpus that
//!   cannot be read is that verifier's failed case and the rest still run, a relative `--python`
//!   is the caller's wherever `--root` is, and `wai verify --conformant` admits nothing on such a
//!   record, whose one case is the whole corpus;
//! - the floor corpus's two suites under the class rule (`--presents`): a sink that presents video
//!   and decodes every case passes the mandatory floor and the video floor, a record each, both
//!   admitted; one with no AV1 decoder (it answers `unsupported wai.video.av1`) has the video case
//!   not run, never passed, and does not pass the video floor; one that presents audio alone is
//!   held to the mandatory floor, its video floor reported not run with the rule and no record
//!   written; one changed output byte fails its case; the independent verifier's checks it did
//!   not run are said not run.
#![cfg(all(feature = "quickstart", feature = "score", feature = "key_discovery"))]

use serde_json::Value;
use std::io::{BufRead, Read, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio};

// A program a test runs is written by a process of its own (ETXTBSY: support/program.rs).
#[path = "support/program.rs"]
mod program;

// The origin a test starts is killed and reaped however the test ends.
#[path = "support/running.rs"]
mod running;

use running::Running;

fn tmp(name: &str) -> PathBuf {
    let d = std::env::temp_dir().join(format!("wai-conform-{name}-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&d);
    std::fs::create_dir_all(&d).unwrap();
    d
}

fn wai_dir() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf()
}

fn wai(dir: &Path, args: &[&str]) -> Output {
    Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(dir).args(args).output().expect("wai runs")
}

fn text(o: &Output) -> String {
    format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))
}

/// `wai conform <kind> --suite <suite> --impl <imp> --json` writing its record to `out`.
fn conform(dir: &Path, kind: &str, suite: &str, imp: &str, out: &Path) -> (i32, Value) {
    let root = wai_dir();
    let o = wai(dir, &["conform", kind, "--suite", suite, "--impl", imp, "--root", root.to_str().unwrap(), "--out", out.to_str().unwrap(), "--yes", "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("JSON: {}", text(&o)));
    (o.status.code().unwrap_or(-1), v)
}

fn q(p: &str) -> String {
    format!("'{}'", p.replace('\'', "'\\''"))
}

#[test]
fn the_reference_verifiers_conform_and_one_changed_verdict_does_not() {
    let d = tmp("verifiers");
    for (suite, bin) in [("claim", env!("CARGO_BIN_EXE_wai_claim_vectors")), ("jwp", env!("CARGO_BIN_EXE_wai_jwp_vectors")), ("signer", env!("CARGO_BIN_EXE_wai_signer_vectors"))] {
        let out = d.join(format!("{suite}.wai-conform.json"));
        let (code, v) = conform(&d, "verifier", suite, &format!("{} verdicts {{corpus}}", q(bin)), &out);
        assert_eq!(code, 0, "{suite}: {v:#}");
        assert_eq!(v["data"]["conformant"], true);
        assert!(v["data"]["cases"].as_u64().unwrap() > 100, "{v}");
        let rec: Value = serde_json::from_slice(&std::fs::read(&out).unwrap()).unwrap();
        assert_eq!(rec["wai"], "conform-result/1");
        assert!(rec["corpus"]["digest"].as_str().unwrap().starts_with("sha256:"));
        assert!(rec["standard"].as_str().unwrap().starts_with('v'));
        assert!(rec["implementation"]["files"].as_array().unwrap().iter().any(|f| f["path"].as_str().unwrap().ends_with(&format!("wai_{}_vectors", if suite == "signer" { "signer" } else { suite }))), "{}", rec["implementation"]);
    }
    let o = wai(&d, &["verify", d.to_str().unwrap()]);
    assert!(o.status.success(), "{}", text(&o));
    assert!(text(&o).contains("3 conformance results verified"), "{}", text(&o));

    // One verdict changed: its case fails, and no other.
    let bad = format!("{} verdicts {{corpus}} | awk 'NR == 1 {{ sub(/\\t/, \"\\t~\") }} 1'", q(env!("CARGO_BIN_EXE_wai_signer_vectors")));
    let (code, v) = conform(&d, "verifier", "signer", &bad, &d.join("bad.wai-conform.json"));
    assert_eq!(code, 1, "{v:#}");
    assert_eq!(v["data"]["conformant"], false);
    assert_eq!(v["data"]["failed"], 1, "{v:#}");
    // A record that says "not conformant" is still a genuine record.
    let o = wai(&d, &["verify", d.join("bad.wai-conform.json").to_str().unwrap()]);
    assert!(o.status.success() && text(&o).contains("not conformant"), "{}", text(&o));
}

#[test]
fn a_tampered_record_is_refused() {
    let d = tmp("tamper");
    let out = d.join("signer.wai-conform.json");
    let bad = format!("{} verdicts {{corpus}} | awk 'NR == 1 {{ sub(/\\t/, \"\\t~\") }} 1'", q(env!("CARGO_BIN_EXE_wai_signer_vectors")));
    let (code, _) = conform(&d, "verifier", "signer", &bad, &out);
    assert_eq!(code, 1);
    let good = std::fs::read(&out).unwrap();
    let mut rec: Value = serde_json::from_slice(&good).unwrap();
    // The failed case turned into a pass, the summary and verdict made to agree with it: the
    // signature no longer covers what the record says.
    let cases = rec["cases"].as_array_mut().unwrap();
    let i = cases.iter().position(|c| c["pass"] == false).unwrap();
    cases[i]["pass"] = Value::Bool(true);
    rec["summary"]["passed"] = Value::from(rec["summary"]["passed"].as_u64().unwrap() + 1);
    rec["summary"]["failed"] = Value::from(0);
    rec["conformant"] = Value::Bool(true);
    std::fs::write(&out, serde_json::to_vec_pretty(&rec).unwrap()).unwrap();
    let o = wai(&d, &["verify", out.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("the signature does not verify"), "{}", text(&o));

    // One byte of the corpus digest changed.
    let s = String::from_utf8(good.clone()).unwrap();
    let at = s.find("\"sha256:").unwrap() + 8;
    let mut b = good.clone();
    b[at] = if b[at] == b'0' { b'1' } else { b'0' };
    std::fs::write(&out, &b).unwrap();
    assert_eq!(wai(&d, &["verify", out.to_str().unwrap()]).status.code(), Some(1));

    // The summary alone changed, the signature left: refused at the signature
    // (conformance-results §4 step 4, before the summary is counted).
    let mut rec: Value = serde_json::from_slice(&good).unwrap();
    rec["summary"]["failed"] = Value::from(0);
    std::fs::write(&out, serde_json::to_vec(&rec).unwrap()).unwrap();
    let o = wai(&d, &["verify", out.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1));
    assert!(text(&o).contains("step 4 signature_invalid"), "{}", text(&o));

    std::fs::write(&out, &good).unwrap();
    assert!(wai(&d, &["verify", out.to_str().unwrap()]).status.success());
}

#[test]
fn the_score_renderer_conforms_and_one_flipped_output_byte_fails() {
    let d = tmp("score");
    let w = q(env!("CARGO_BIN_EXE_wai"));
    let (code, v) = conform(&d, "decoder", "score", &format!("{w} score render {{in}} --pcm {{out}}"), &d.join("score.wai-conform.json"));
    assert_eq!(code, 0, "{v:#}");
    // The corpus's eight vectors (score-conformance/vectors/): three at 22 050 Hz, five at 48 000 Hz.
    assert_eq!(v["data"]["passed"], 8);
    // One byte of the output changed (byte 100 of the PCM, plus one).
    let flip = format!("{w} score render {{in}} --pcm {{out}} >/dev/null && dd if={{out}} bs=1 skip=100 count=1 2>/dev/null | LC_ALL=C tr '\\000-\\377' '\\001-\\377\\000' | dd of={{out}} bs=1 seek=100 count=1 conv=notrunc 2>/dev/null");
    let (code, v) = conform(&d, "decoder", "score", &flip, &d.join("flip.wai-conform.json"));
    assert_eq!(code, 1, "{v:#}");
    assert_eq!(v["data"]["failed"], 8, "{v:#}");
    assert_eq!(v["data"]["passed"], 0);
}

/// `wai http origin --root <root>` on a port the system picks: the child and its URL, from
/// the line it prints once it listens. It is held as it is spawned, so one that never says
/// where it listens is killed too.
fn origin(root: &Path) -> (Running, String) {
    let mut c = Running::new(Command::new(env!("CARGO_BIN_EXE_wai")).args(["http", "origin", "--root", root.to_str().unwrap(), "--port", "0"]).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().unwrap());
    let mut line = String::new();
    std::io::BufReader::new(c.stdout.as_mut().unwrap()).read_line(&mut line).unwrap();
    let url = line.split_whitespace().find(|w| w.starts_with("http://127.0.0.1:")).unwrap_or_else(|| panic!("{line}")).trim_end_matches('/').to_string();
    (c, url)
}

#[test]
fn the_reference_origin_conforms_and_one_changed_object_fails() {
    let d = tmp("origin");
    let objects = wai_dir().join("http-conformance").join("objects");
    let (c, url) = origin(&objects);
    let (code, v) = conform(&d, "origin", "http", &url, &d.join("http.wai-conform.json"));
    // Beside it, the HTTP corpus's independent verifier over the whole corpus: one that fails
    // fails the run, though the origin conforms.
    #[cfg(unix)]
    let beside = {
        let fails = fake_python(&d, "fails", "echo 'FAIL an object'; exit 1");
        let root = wai_dir();
        let o = wai(&d, &["conform", "origin", "--suite", "http", "--impl", &url, "--root", root.to_str().unwrap(), "--out", d.join("hb.wai-conform.json").to_str().unwrap(), "--yes", "--independent", "--python", fails.to_str().unwrap(), "--json"]);
        (o.status.code(), serde_json::from_slice::<Value>(&o.stdout).unwrap_or_else(|_| panic!("JSON: {}", text(&o))))
    };
    drop(c);
    assert_eq!(code, 0, "{v:#}");
    #[cfg(unix)]
    {
        let (code, v) = beside;
        assert_eq!(code, Some(1), "{v:#}");
        assert_eq!((v["ok"].as_bool(), v["data"]["conformant"].as_bool()), (Some(false), Some(true)), "the origin conforms, and the run fails: {v:#}");
        let ind = &v["data"]["independent"];
        assert_eq!((ind["verifier"].as_str(), ind["conformant"].as_bool()), (Some("http"), Some(false)), "{ind:#}");
        assert!(ind["command"].as_str().unwrap().ends_with("http-conformance/http_verify.py http-conformance"), "{ind}");
        assert!(d.join("hb.independent.wai-conform.json").is_file());
    }

    // A copy of the objects with one byte of one object changed.
    let copy = d.join("objects");
    std::fs::create_dir_all(&copy).unwrap();
    let mut changed = None;
    for e in std::fs::read_dir(&objects).unwrap().flatten() {
        if e.path().is_file() {
            let mut b = std::fs::read(e.path()).unwrap();
            let name = e.file_name().to_string_lossy().into_owned();
            if changed.is_none() && name.ends_with(".wai") && b.len() > 2000 {
                let n = b.len();
                b[n - 1] ^= 1;
                changed = Some(name);
            }
            std::fs::write(copy.join(e.file_name()), b).unwrap();
        }
    }
    let (c, url) = origin(&copy);
    let (code, v) = conform(&d, "origin", "http", &url, &d.join("bad.wai-conform.json"));
    drop(c);
    assert_eq!(code, 1, "{v:#}");
    assert_eq!(v["data"]["failed"], 1, "{v:#}");
    assert!(v["data"]["failures"][0]["got"].as_str().unwrap().contains("the body is not the object"), "{v:#}");
}

/// A verifier behind HTTP: the reference's verdicts, served one case a request.
#[test]
fn a_verifier_behind_http_conforms() {
    let d = tmp("http-verifier");
    let corpus = wai_dir().join("signer-conformance").join("vectors.json");
    let o = Command::new(env!("CARGO_BIN_EXE_wai_signer_vectors")).arg("verdicts").arg(&corpus).output().unwrap();
    let verdicts: std::collections::HashMap<String, String> =
        String::from_utf8_lossy(&o.stdout).lines().filter_map(|l| l.split_once('\t')).map(|(a, b)| (a.to_string(), b.to_string())).collect();
    let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
    let url = format!("http://{}/verify", listener.local_addr().unwrap());
    std::thread::spawn(move || {
        for s in listener.incoming().flatten() {
            let mut s = s;
            let mut buf = Vec::new();
            let mut chunk = [0u8; 65536];
            let body = loop {
                let n = s.read(&mut chunk).unwrap_or(0);
                if n == 0 {
                    break None;
                }
                buf.extend_from_slice(&chunk[..n]);
                if let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
                    let head = String::from_utf8_lossy(&buf[..end]).to_ascii_lowercase();
                    let len: usize = head.lines().find_map(|l| l.strip_prefix("content-length:")).and_then(|v| v.trim().parse().ok()).unwrap_or(0);
                    if buf.len() >= end + 4 + len {
                        break Some(buf[end + 4..end + 4 + len].to_vec());
                    }
                }
            };
            let v: Value = body.and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or(Value::Null);
            let verdict = verdicts.get(v["name"].as_str().unwrap_or("")).cloned().unwrap_or_default();
            let reply = serde_json::json!({"verdict": verdict}).to_string();
            let _ = s.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{reply}", reply.len()).as_bytes());
        }
    });
    let (code, v) = conform(&d, "verifier", "signer", &url, &d.join("signer.wai-conform.json"));
    assert_eq!(code, 0, "{v:#}");
    let rec: Value = serde_json::from_slice(&std::fs::read(d.join("signer.wai-conform.json")).unwrap()).unwrap();
    assert_eq!(rec["implementation"]["adapter"], "http");
}

#[test]
fn a_dry_run_writes_nothing_and_each_failure_has_its_exit_status() {
    let d = tmp("statuses");
    let root = wai_dir();
    let r = root.to_str().unwrap();
    let w = q(env!("CARGO_BIN_EXE_wai"));
    let imp = format!("{w} score render {{in}} --pcm {{out}}");
    let o = wai(&d, &["conform", "decoder", "--suite", "score", "--impl", &imp, "--root", r, "--dry-run", "--json"]);
    assert!(o.status.success(), "{}", text(&o));
    let v: Value = serde_json::from_slice(&o.stdout).unwrap();
    assert_eq!(v["dry_run"], true);
    assert!(std::fs::read_dir(&d).unwrap().next().is_none(), "a dry run writes nothing");

    // The record exists: not replaced without --yes (4), the proposal and the confirming command
    // in the envelope.
    std::fs::write(d.join("score.wai-conform.json"), "{}").unwrap();
    let o = wai(&d, &["conform", "decoder", "--suite", "score", "--impl", &imp, "--root", r, "--json"]);
    assert_eq!(o.status.code(), Some(4), "{}", text(&o));
    let v: Value = serde_json::from_slice(&o.stdout).unwrap();
    assert_eq!((v["error"]["code"].as_str(), v["proposed"][0]["replaces"].as_bool()), (Some("confirmation-required"), Some(true)), "{v:#}");
    assert!(v["confirm"].as_str().unwrap().ends_with(" --yes"), "{v:#}");
    assert_eq!(std::fs::read(d.join("score.wai-conform.json")).unwrap(), b"{}");

    let code = |args: &[&str]| wai(&d, args).status.code();
    assert_eq!(code(&["conform", "decoder", "--suite", "score", "--root", r]), Some(2), "no --impl");
    assert_eq!(code(&["conform", "transcoder", "--impl", "x", "--root", r]), Some(5), "no such kind");
    // A relay is reached at a moqt:// or https:// URL, not by a command.
    assert_eq!(code(&["conform", "relay", "--impl", "x", "--root", r]), Some(2), "a relay's --impl is its URL");
    assert_eq!(code(&["conform", "decoder", "--suite", "nope", "--impl", &imp, "--root", r]), Some(5), "no such suite");
    assert_eq!(code(&["conform", "decoder", "--suite", "score", "--impl", "no-such-program-here {in}", "--root", r]), Some(5), "no such program");
    let free = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
    assert_eq!(code(&["conform", "origin", "--impl", &format!("http://127.0.0.1:{free}"), "--root", r, "--out", d.join("o.wai-conform.json").to_str().unwrap()]), Some(6), "nothing listening");
    let o = wai(&d, &["conform", "--list", "--root", r, "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap();
    assert!(v["data"]["suites"].as_array().unwrap().iter().all(|s| s["cases"].as_u64().unwrap() > 0), "{v:#}");
}

/// A stand-in for `python3`, a shell script at `dir/name`: `body` runs with the verifier's
/// arguments (`$1` the script, then its arguments). A process of its own writes it
/// (support/program.rs): one this process wrote could still be open for writing, in the child of
/// a sibling test's spawn, when `wai` ran it, and the verifier would read as one that could not
/// start.
#[cfg(unix)]
fn fake_python(dir: &Path, name: &str, body: &str) -> PathBuf {
    let p = dir.join(name);
    program::write_program(&p, &format!("#!/bin/sh\n{body}\n"));
    p
}

#[test]
fn junit_and_html_are_the_records_report() {
    let d = tmp("formats");
    let root = wai_dir();
    let r = root.to_str().unwrap();
    let good = format!("{} verdicts {{corpus}}", q(env!("CARGO_BIN_EXE_wai_signer_vectors")));
    let o = wai(&d, &["conform", "verifier", "--suite", "signer", "--impl", &good, "--root", r, "--out", "s.wai-conform.json", "--format", "junit"]);
    assert!(o.status.success(), "{}", text(&o));
    let x = String::from_utf8(o.stdout.clone()).unwrap();
    let rec: Value = serde_json::from_slice(&std::fs::read(d.join("s.wai-conform.json")).unwrap()).unwrap();
    let n = rec["cases"].as_array().unwrap().len();
    assert!(x.starts_with("<?xml"), "only the report is on standard output: {x}");
    assert!(x.contains(&format!("<testsuites name=\"wai conform\" tests=\"{n}\" failures=\"0\"")), "{x}");
    assert_eq!(x.matches("<testcase ").count(), n);
    assert!(x.contains(&format!("<property name=\"corpus_digest\" value=\"{}\"/>", rec["corpus"]["digest"].as_str().unwrap())), "the report names its record");
    assert!(String::from_utf8_lossy(&o.stderr).contains("cases passed: conformant"), "the summary goes to standard error");

    // One verdict changed: one failure, at that case.
    let bad = format!("{good} | awk 'NR == 1 {{ sub(/\\t/, \"\\t~\") }} 1'");
    let o = wai(&d, &["conform", "verifier", "--suite", "signer", "--impl", &bad, "--root", r, "--out", "b.wai-conform.json", "--format", "junit"]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    let x = String::from_utf8(o.stdout.clone()).unwrap();
    assert!(x.contains("failures=\"1\""), "{x}");
    assert_eq!(x.matches("<failure ").count(), 1);
    let rec: Value = serde_json::from_slice(&std::fs::read(d.join("b.wai-conform.json")).unwrap()).unwrap();
    let failed = rec["cases"].as_array().unwrap().iter().find(|c| c["pass"] == false).unwrap();
    let name = failed["name"].as_str().unwrap();
    let at = x.find(&format!("name=\"{name}\">")).unwrap_or_else(|| panic!("{name} in {x}"));
    assert!(x[at..].trim_start_matches(|c| c != '\n').trim_start().starts_with("<failure "), "the failure is the failed case's");

    let o = wai(&d, &["conform", "verifier", "--suite", "signer", "--impl", &bad, "--root", r, "--out", "b.wai-conform.json", "--yes", "--format", "html"]);
    assert_eq!(o.status.code(), Some(1));
    let h = String::from_utf8(o.stdout).unwrap();
    assert!(h.starts_with("<!doctype html>") && h.contains("not conformant"), "{h}");
    assert_eq!(h.matches("<tr class=\"fail\">").count(), 1);
    assert_eq!(h.matches("<tr class=\"pass\">").count(), n - 1);

    // With the independent verifier beside it: two test suites, which the report tells apart.
    #[cfg(unix)]
    {
        let same = fake_python(&d, "same", &format!("shift; {} \"$@\"", q(env!("CARGO_BIN_EXE_wai_signer_vectors"))));
        let o = wai(&d, &["conform", "verifier", "--suite", "signer", "--impl", &good, "--root", r, "--out", "s.wai-conform.json", "--yes", "--independent", "--python", same.to_str().unwrap(), "--format", "junit"]);
        assert!(o.status.success(), "{}", text(&o));
        let x = String::from_utf8(o.stdout).unwrap();
        let suites: Vec<&str> = x.lines().filter_map(|l| l.trim().strip_prefix("<testsuite name=\"")).map(|l| &l[..l.find('"').unwrap()]).collect();
        assert_eq!(suites, vec!["verifier signer", "verifier signer (independent verifier)"], "{x}");
        let classes: std::collections::BTreeSet<&str> = x.split("classname=\"").skip(1).map(|l| &l[..l.find('"').unwrap()]).collect();
        assert_eq!(classes.into_iter().collect::<Vec<_>>(), vec!["verifier.signer", "verifier.signer.independent_verifier"], "{x}");
        let rec: Value = serde_json::from_slice(&std::fs::read(d.join("s.independent.wai-conform.json")).unwrap()).unwrap();
        assert_eq!(rec["implementation"]["name"], "independent verifier");
    }

    // Two outputs at once, or one that is not there, are a usage error.
    let code = |args: &[&str]| wai(&d, args).status.code();
    assert_eq!(code(&["conform", "verifier", "--suite", "signer", "--impl", &good, "--root", r, "--format", "xml"]), Some(2));
    assert_eq!(code(&["conform", "verifier", "--suite", "signer", "--impl", &good, "--root", r, "--format", "html", "--json"]), Some(2));
}

#[test]
fn format_json_is_the_json_envelope() {
    let d = tmp("format-json");
    let root = wai_dir();
    let r = root.to_str().unwrap();
    let imp = format!("{} score render {{in}} --pcm {{out}}", q(env!("CARGO_BIN_EXE_wai")));
    let call = |a: &[&str]| {
        let o = wai(&d, a);
        (o.status.code(), serde_json::from_slice::<Value>(&o.stdout).unwrap_or_else(|_| panic!("JSON on standard output: {}", text(&o))))
    };
    let run = |extra: &[&str]| call(&[&["conform", "decoder", "--suite", "score", "--impl", imp.as_str(), "--root", r][..], extra].concat());
    // A dry run: the same envelope, word for word.
    let (a, b) = (run(&["--dry-run", "--json"]), run(&["--dry-run", "--format", "json"]));
    assert_eq!(a, b);
    assert_eq!((a.0, a.1["dry_run"].as_bool()), (Some(0), Some(true)), "{:#}", a.1);
    // A run: the same envelope, but for what the run itself says (its time and key).
    let (a, b) = (run(&["--out", "a.wai-conform.json", "--json"]), run(&["--out", "b.wai-conform.json", "--format", "json"]));
    assert_eq!((a.0, b.0), (Some(0), Some(0)), "{:#} {:#}", a.1, b.1);
    let keys = |v: &Value| v["data"].as_object().unwrap().keys().cloned().collect::<Vec<_>>();
    assert_eq!((a.1["op"].as_str(), keys(&a.1)), (b.1["op"].as_str(), keys(&b.1)));
    assert_eq!((a.1["data"]["passed"].as_u64(), a.1["data"]["corpus_digest"].as_str()), (b.1["data"]["passed"].as_u64(), b.1["data"]["corpus_digest"].as_str()));
    // A failure: the error envelope, for each spelling.
    let nope = |f: &[&str]| call(&[&["conform", "decoder", "--suite", "nope", "--impl", imp.as_str(), "--root", r][..], f].concat());
    let (a, b) = (nope(&["--json"]), nope(&["--format", "json"]));
    assert_eq!(a, b);
    assert_eq!((a.0, a.1["ok"].as_bool()), (Some(5), Some(false)), "{:#}", a.1);
}

#[cfg(unix)]
#[test]
fn beside_an_implementation_the_independent_verifier_must_agree() {
    let d = tmp("beside");
    let root = wai_dir();
    let r = root.to_str().unwrap();
    let reference = q(env!("CARGO_BIN_EXE_wai_signer_vectors"));
    let imp = format!("{reference} verdicts {{corpus}}");
    // A stand-in for the independent verifier: the reference's verdicts ($1 is the script).
    let same = fake_python(&d, "same", &format!("shift; {reference} \"$@\""));
    let o = wai(&d, &["conform", "verifier", "--suite", "signer", "--impl", &imp, "--root", r, "--independent", "--python", same.to_str().unwrap(), "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("{}", text(&o)));
    assert_eq!(o.status.code(), Some(0), "{v:#}");
    let ind = &v["data"]["independent"];
    assert_eq!((ind["verifier"].as_str(), ind["conformant"].as_bool()), (Some("signer"), Some(true)), "{v:#}");
    assert_eq!(ind["disagreements"].as_array().unwrap().len(), 0);
    assert!(ind["command"].as_str().unwrap().ends_with("signer-conformance/signer_verify.py verdicts signer-conformance/vectors.json"), "{ind}");
    // Two records, both signed, both verified.
    assert!(d.join("signer.independent.wai-conform.json").is_file());
    let o = wai(&d, &["verify", d.to_str().unwrap()]);
    assert!(o.status.success() && text(&o).contains("2 conformance results verified"), "{}", text(&o));

    // The independent verifier changes one verdict: the run fails, and names the case.
    let flipped = fake_python(&d, "flipped", &format!("shift; {reference} \"$@\" | awk 'NR == 1 {{ sub(/\\t/, \"\\t~\") }} 1'"));
    let o = wai(&d, &["conform", "verifier", "--suite", "signer", "--impl", &imp, "--root", r, "--independent", "--python", flipped.to_str().unwrap(), "--yes", "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("{}", text(&o)));
    assert_eq!(o.status.code(), Some(1), "{v:#}");
    assert_eq!(v["ok"], false);
    assert_eq!(v["data"]["conformant"], true, "the implementation itself conforms");
    let ind = &v["data"]["independent"];
    assert_eq!((ind["conformant"].as_bool(), ind["failed"].as_u64()), (Some(false), Some(1)), "{ind:#}");
    let dis = ind["disagreements"].as_array().unwrap();
    assert_eq!(dis.len(), 1, "{ind:#}");
    assert!(dis[0]["independent"].as_str().unwrap().contains('~'), "{dis:?}");
    // In text, the disagreement is said.
    let o = wai(&d, &["conform", "verifier", "--suite", "signer", "--impl", &imp, "--root", r, "--independent", "--python", flipped.to_str().unwrap(), "--yes"]);
    assert_eq!(o.status.code(), Some(1));
    assert!(text(&o).contains("disagree on 1 case"), "{}", text(&o));

    // The real independent verifier (signer_verify.py, which needs the `cryptography` package).
    let o = wai(&d, &["conform", "verifier", "--suite", "signer", "--impl", &imp, "--root", r, "--independent", "--yes", "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("{}", text(&o)));
    assert_eq!(o.status.code(), Some(0), "{v:#}");
    assert_eq!(v["data"]["independent"]["conformant"], true);

    // A suite whose corpus has no independent verifier says so, and the run stands on its own.
    let w = q(env!("CARGO_BIN_EXE_wai"));
    let o = wai(&d, &["conform", "decoder", "--suite", "score", "--impl", &format!("{w} score render {{in}} --pcm {{out}}"), "--root", r, "--independent", "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("{}", text(&o)));
    assert_eq!(o.status.code(), Some(0), "{v:#}");
    assert!(v["data"]["independent"].is_null(), "{v:#}");
}

#[cfg(unix)]
#[test]
fn alone_the_independent_verifiers_run_as_the_table_says_each_a_record() {
    let d = tmp("alone");
    let root = wai_dir();
    let r = root.to_str().unwrap();
    // A stand-in that passes every verifier but the live rooms', and prints what it was given.
    let fake = fake_python(&d, "fake", "echo \"ran $*\"; case \"$*\" in *live_rooms_verify.py*) echo 'FAIL a room'; exit 1;; esac; exit 0");
    let out = d.join("records");
    let o = wai(&d, &["conform", "--independent", "--suite", "live-rooms,pin", "--root", r, "--python", fake.to_str().unwrap(), "--out", out.to_str().unwrap(), "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("{}", text(&o)));
    assert_eq!(o.status.code(), Some(1), "{v:#}");
    let rows = v["data"]["independent"].as_array().unwrap();
    assert_eq!(rows.iter().map(|x| x["name"].as_str().unwrap()).collect::<Vec<_>>(), vec!["live-rooms", "pin"]);
    assert_eq!((rows[0]["pass"].as_bool(), rows[1]["pass"].as_bool()), (Some(false), Some(true)), "{v:#}");
    assert_eq!(rows[0]["command"].as_str().unwrap(), format!("{} -I live-conformance/live_rooms_verify.py live-conformance/rooms", fake.display()));
    assert!(rows[0]["got"].as_str().unwrap().starts_with("exit 1: FAIL a room"), "{v:#}");
    assert!(rows[1]["command"].as_str().unwrap().ends_with(" pin-conformance/pin_verify.py pin-conformance"), "not isolated, as the README runs it");
    assert_eq!((v["data"]["passed"].as_u64(), v["data"]["failed"].as_u64(), v["data"]["conformant"].as_bool()), (Some(1), Some(1), Some(false)));
    // A record a corpus, each verified: the failed one a genuine record of a failed run.
    let rec: Value = serde_json::from_slice(&std::fs::read(out.join("live-rooms.wai-conform.json")).unwrap()).unwrap();
    assert_eq!((rec["kind"].as_str(), rec["suite"].as_str(), rec["conformant"].as_bool()), (Some("verifier"), Some("live-rooms"), Some(false)));
    assert_eq!(rec["corpus"]["dir"], "live-conformance/rooms");
    assert!(rec["implementation"]["files"].as_array().unwrap().iter().any(|f| f["path"] == "live-conformance/live_rooms_verify.py"), "{}", rec["implementation"]);
    let o = wai(&d, &["verify", out.to_str().unwrap()]);
    assert!(o.status.success() && text(&o).contains("2 conformance results verified"), "{}", text(&o));
    // The records are not replaced without --yes.
    let o = wai(&d, &["conform", "--independent", "--suite", "pin", "--root", r, "--python", fake.to_str().unwrap(), "--out", out.to_str().unwrap(), "--json"]);
    assert_eq!(o.status.code(), Some(4), "{}", text(&o));

    // JUnit: a test suite a verifier, the failure at the live rooms'.
    let o = wai(&d, &["conform", "--independent", "--suite", "live-rooms,pin", "--root", r, "--python", fake.to_str().unwrap(), "--format", "junit"]);
    assert_eq!(o.status.code(), Some(1));
    let x = String::from_utf8(o.stdout).unwrap();
    assert!(x.contains("tests=\"2\" failures=\"1\""), "{x}");
    assert!(x.contains("<testsuite name=\"verifier live-rooms\" tests=\"1\" failures=\"1\""), "{x}");
    assert!(x.contains("<testsuite name=\"verifier pin\" tests=\"1\" failures=\"0\""), "{x}");
    assert!(String::from_utf8_lossy(&o.stderr).contains("| FAIL a room"), "the failed verifier's output is shown: {}", String::from_utf8_lossy(&o.stderr));

    // The real verifier: the pin corpus's, standard library only.
    let o = wai(&d, &["conform", "--independent", "--suite", "pin", "--root", r]);
    assert!(o.status.success(), "{}", text(&o));
    assert!(text(&o).contains("1 of 1 independent verifiers passed"), "{}", text(&o));

    // A dry run runs and writes nothing.
    let dry = d.join("dry");
    let o = wai(&d, &["conform", "--independent", "--suite", "trust", "--root", r, "--python", fake.to_str().unwrap(), "--out", dry.to_str().unwrap(), "--dry-run", "--json"]);
    assert!(o.status.success(), "{}", text(&o));
    let v: Value = serde_json::from_slice(&o.stdout).unwrap();
    assert_eq!(v["dry_run"], true);
    assert!(v["data"]["independent"].as_array().unwrap().iter().any(|x| x["name"] == "claim"));
    assert!(!dry.exists());

    let code = |args: &[&str]| wai(&d, args).status.code();
    assert_eq!(code(&["conform", "--independent", "--suite", "nope", "--root", r]), Some(5), "no such area or verifier");
    assert_eq!(code(&["conform", "--independent", "--impl", "x", "--root", r]), Some(2), "an implementation needs a kind");
    assert_eq!(code(&["conform", "--independent", "--suite", "pin", "--root", r, "--python", "/no/such/python"]), Some(5), "no such interpreter");
    assert_eq!(code(&["conform", "--root", r]), Some(2), "neither a kind nor --independent");
}

/// A checkout of the standard that holds what `--root` needs and one claim vector, `x`, which
/// declares an accepted claim and pins no link hash: any `accept link_hash=` passes it.
#[cfg(unix)]
fn one_claim_root(d: &Path) -> PathBuf {
    let root = d.join("wai");
    std::fs::create_dir_all(root.join("registry")).unwrap();
    std::fs::create_dir_all(root.join("claim-conformance")).unwrap();
    std::fs::write(root.join("SPEC.md"), "# WAI\n").unwrap();
    std::fs::write(root.join("registry/capabilities.json"), "{}").unwrap();
    std::fs::write(root.join("claim-conformance/vectors.json"), r#"{"vectors": [{"name": "x", "expected": {"outcome": "accept"}}]}"#).unwrap();
    root
}

#[cfg(unix)]
#[test]
fn two_answers_that_pass_a_case_stated_in_part_but_differ_fail_the_run() {
    let d = tmp("both-pass");
    let root = one_claim_root(&d);
    let r = root.to_str().unwrap();
    let imp = "printf 'x\\taccept link_hash=aa\\n'; : {corpus}";
    // The independent verifier accepts it too, with another link hash: each passes, and they disagree.
    let other = fake_python(&d, "other", "printf 'x\\taccept link_hash=bb\\n'");
    let o = wai(&d, &["conform", "verifier", "--suite", "claim", "--impl", imp, "--root", r, "--independent", "--python", other.to_str().unwrap(), "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("{}", text(&o)));
    assert_eq!(o.status.code(), Some(1), "{v:#}");
    assert_eq!(v["ok"], false);
    let ind = &v["data"]["independent"];
    assert_eq!((v["data"]["conformant"].as_bool(), ind["conformant"].as_bool()), (Some(true), Some(true)), "each passes the case: {v:#}");
    let dis = ind["disagreements"].as_array().unwrap();
    assert_eq!(dis.len(), 1, "{ind:#}");
    assert_eq!((dis[0]["name"].as_str(), dis[0]["implementation"].as_str(), dis[0]["independent"].as_str()), (Some("x"), Some("accept link_hash=aa"), Some("accept link_hash=bb")), "{dis:?}");
    // The same answer from both: they agree, and the run passes.
    let same = fake_python(&d, "same", "printf 'x\\taccept link_hash=aa\\n'");
    let o = wai(&d, &["conform", "verifier", "--suite", "claim", "--impl", imp, "--root", r, "--independent", "--python", same.to_str().unwrap(), "--yes", "--json"]);
    assert_eq!(o.status.code(), Some(0), "{}", text(&o));
}

#[cfg(unix)]
#[test]
fn each_record_is_written_as_its_verifier_ends() {
    let d = tmp("as-made");
    let root = wai_dir();
    let out = d.join("records");
    let seen = d.join("seen");
    // Each run counts the records already there: the second sees the first's.
    let count = fake_python(&d, "count", &format!("ls {} | wc -l >> {}", q(out.to_str().unwrap()), q(seen.to_str().unwrap())));
    let o = wai(&d, &["conform", "--independent", "--suite", "pin,loc", "--root", root.to_str().unwrap(), "--python", count.to_str().unwrap(), "--out", out.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(0), "{}", text(&o));
    let counts: Vec<String> = std::fs::read_to_string(&seen).unwrap().split_whitespace().map(String::from).collect();
    assert_eq!(counts, ["0", "1"], "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

/// `one_claim_root` with the pin corpus's directory, a link in it that points at nothing.
#[cfg(unix)]
fn root_with_an_unreadable_corpus(d: &Path) -> PathBuf {
    let root = one_claim_root(d);
    std::fs::create_dir_all(root.join("pin-conformance")).unwrap();
    std::os::unix::fs::symlink("nowhere", root.join("pin-conformance/gone.json")).unwrap();
    root
}

#[cfg(unix)]
#[test]
fn a_corpus_that_cannot_be_read_is_its_verifiers_failed_case_and_the_rest_still_run() {
    let d = tmp("unread");
    let root = root_with_an_unreadable_corpus(&d);
    let r = root.to_str().unwrap();
    let yes = fake_python(&d, "yes", "exit 0");
    let out = d.join("records");
    let o = wai(&d, &["conform", "--independent", "--suite", "pin,claim", "--root", r, "--python", yes.to_str().unwrap(), "--out", out.to_str().unwrap(), "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("{}", text(&o)));
    assert_eq!(o.status.code(), Some(1), "{v:#}");
    let row = |n: &str| v["data"]["independent"].as_array().unwrap().iter().find(|x| x["name"] == n).unwrap_or_else(|| panic!("{n}: {v:#}")).clone();
    assert_eq!(row("pin")["pass"], false, "{v:#}");
    assert!(row("pin")["got"].as_str().unwrap().starts_with("the corpus was not read: "), "{v:#}");
    assert_eq!(row("claim")["pass"], true, "{v:#}");
    // Each has its record, the failed one's a run that did not pass, and both verify.
    let rec: Value = serde_json::from_slice(&std::fs::read(out.join("pin.wai-conform.json")).unwrap()).unwrap();
    assert_eq!((rec["conformant"].as_bool(), rec["corpus"]["files"].as_u64()), (Some(false), Some(0)), "{rec:#}");
    assert!(out.join("claim.wai-conform.json").is_file());
    let o = wai(&d, &["verify", out.to_str().unwrap()]);
    assert!(o.status.success(), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

#[cfg(unix)]
#[test]
fn a_verifier_that_cannot_start_is_a_failed_case_with_its_record() {
    let d = tmp("not-started");
    let root = one_claim_root(&d);
    // A file `--python` finds, which cannot be run.
    let not_run = d.join("not-run");
    std::fs::write(&not_run, "").unwrap();
    let out = d.join("records");
    let o = wai(&d, &["conform", "--independent", "--suite", "claim", "--root", root.to_str().unwrap(), "--python", not_run.to_str().unwrap(), "--out", out.to_str().unwrap(), "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("{}", text(&o)));
    assert_eq!(o.status.code(), Some(1), "{v:#}");
    let got = v["data"]["independent"][0]["got"].as_str().unwrap();
    assert!(got.starts_with("did not start: ") && !got.contains("pip install"), "{got}");
    assert!(out.join("claim.wai-conform.json").is_file(), "{v:#}");
    let _ = std::fs::remove_dir_all(&d);
}

#[cfg(unix)]
#[test]
fn a_relative_python_is_the_callers_wherever_the_root_is() {
    let d = tmp("relative-python");
    let root = one_claim_root(&d);
    let r = root.to_str().unwrap();
    std::fs::create_dir_all(d.join("venv")).unwrap();
    fake_python(&d.join("venv"), "python3", "exit 0");
    // Found from the caller's directory, and run from there too, though the run is in --root.
    let o = wai(&d, &["conform", "--independent", "--suite", "claim", "--root", r, "--python", "./venv/python3", "--out", "records", "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("{}", text(&o)));
    assert_eq!(o.status.code(), Some(0), "{v:#}");
    let rec: Value = serde_json::from_slice(&std::fs::read(d.join("records/claim.wai-conform.json")).unwrap()).unwrap();
    let hashed = rec["implementation"]["files"][0]["path"].as_str().unwrap();
    assert!(Path::new(hashed).is_absolute() && hashed.ends_with("venv/python3"), "the file hashed is the one run: {rec:#}");
    let _ = std::fs::remove_dir_all(&d);
}

#[cfg(unix)]
#[test]
fn a_whole_corpus_record_admits_nothing_and_a_run_of_the_suites_cases_does() {
    let d = tmp("whole-corpus");
    let root = one_claim_root(&d);
    let r = root.to_str().unwrap();
    // A program that exits 0 stands for the verifier: its one case is the corpus, the file it
    // digests is the suite's, and its kind and name are the suite's.
    let yes = fake_python(&d, "yes", "exit 0");
    let out = d.join("records");
    let o = wai(&d, &["conform", "--independent", "--suite", "claim", "--root", r, "--python", yes.to_str().unwrap(), "--out", out.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(0), "{}", text(&o));
    let whole = out.join("claim.wai-conform.json");
    let rec: Value = serde_json::from_slice(&std::fs::read(&whole).unwrap()).unwrap();
    assert_eq!((rec["kind"].as_str(), rec["suite"].as_str(), rec["conformant"].as_bool()), (Some("verifier"), Some("claim"), Some(true)), "{rec:#}");
    let o = wai(&d, &["verify", whole.to_str().unwrap()]);
    assert!(o.status.success(), "a genuine record of what ran: {}", text(&o));
    let o = wai(&d, &["verify", "--conformant", "--root", r, whole.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("FAIL admission rule 4 corpus_mismatch"), "{}", text(&o));
    // The same corpus and suite run case by case, one case here too, is admitted.
    let by_case = d.join("by-case.wai-conform.json");
    let o = wai(&d, &["conform", "verifier", "--suite", "claim", "--impl", "printf 'x\\taccept link_hash=aa\\n'; : {corpus}", "--root", r, "--out", by_case.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(0), "{}", text(&o));
    let o = wai(&d, &["verify", "--conformant", "--root", r, by_case.to_str().unwrap()]);
    assert!(o.status.success() && text(&o).contains("ok   admitted"), "{}", text(&o));
    let _ = std::fs::remove_dir_all(&d);
}

#[cfg(unix)]
#[test]
fn a_verifier_that_fails_without_its_package_names_it() {
    let d = tmp("needs");
    let root = wai_dir();
    let r = root.to_str().unwrap();
    // A Python without the package: `-c 'import cryptography'` fails, and so does the verifier.
    let bare = fake_python(&d, "bare", "case \"$1\" in -c) exit 1;; esac; echo 'FAIL a claim'; exit 1");
    let o = wai(&d, &["conform", "--independent", "--suite", "claim", "--root", r, "--python", bare.to_str().unwrap(), "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("{}", text(&o)));
    assert_eq!(o.status.code(), Some(1), "{v:#}");
    let got = v["data"]["independent"][0]["got"].as_str().unwrap();
    assert!(got.starts_with("exit 1: FAIL a claim"), "{got}");
    assert!(got.contains(&format!("it needs the Python package cryptography: {} -m pip install cryptography", bare.display())), "{got}");
    // A verifier that needs no package is given no such hint.
    let o = wai(&d, &["conform", "--independent", "--suite", "pin", "--root", r, "--python", bare.to_str().unwrap(), "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("{}", text(&o)));
    assert!(!v["data"]["independent"][0]["got"].as_str().unwrap().contains("pip install"), "{v:#}");

    // --out with --independent alone names a directory: a file there is a usage error.
    let file = d.join("a-file");
    std::fs::write(&file, "").unwrap();
    let o = wai(&d, &["conform", "--independent", "--suite", "pin", "--root", r, "--python", bare.to_str().unwrap(), "--out", file.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(2), "{}", text(&o));
    assert!(text(&o).contains("names a directory"), "{}", text(&o));
}

/// The decoded outputs of the floor corpus's cases, each `<case>.out` in `dir`, as a sink writes
/// them: the mandatory floor's by the reference decoders (RGB8, s16le interleaved, the bytes), the
/// video case's the committed reference decode of its stream (I420 frames in output order).
fn floor_outputs(dir: &Path) -> PathBuf {
    let corpus = wai_dir().join("floor-conformance");
    let doc: Value = serde_json::from_slice(&std::fs::read(corpus.join("cases.json")).unwrap()).unwrap();
    let outs = dir.join("outs");
    std::fs::create_dir_all(&outs).unwrap();
    for c in doc["cases"].as_array().unwrap() {
        let name = c["name"].as_str().unwrap();
        let env = std::fs::read(corpus.join(c["envelope"].as_str().unwrap())).unwrap();
        let payload = wai::container::Wai::from_bytes(&env).unwrap().payload;
        let out = match c["capability"].as_str().unwrap() {
            "wai.image.png" => wai::codecs::image::png_decode(&payload).unwrap().0,
            "wai.audio.flac" => wai::codecs::audio::flac_decode_i16(&payload, 1 << 27).unwrap().0.iter().flat_map(|x| x.to_le_bytes()).collect(),
            "wai.text.zstd" => wai::codecs::text::zstd_decode(&payload).unwrap(),
            "wai.video.av1" => std::fs::read(wai_dir().join("platform-decode-conformance/grain-free-av1/base.i420")).unwrap(),
            other => panic!("{other}"),
        };
        std::fs::write(outs.join(format!("{name}.out")), out).unwrap();
    }
    outs
}

/// `wai conform decoder --suite <suite> --impl <imp> [--presents <p>] --json`, the record to `out`.
fn floor_run(dir: &Path, suite: &str, imp: &str, presents: Option<&str>, out: &Path, more: &[&str]) -> (i32, Value, String) {
    let root = wai_dir();
    let mut args = vec!["conform", "decoder", "--suite", suite, "--impl", imp, "--root", root.to_str().unwrap(), "--out", out.to_str().unwrap(), "--yes"];
    if let Some(p) = presents {
        args.extend(["--presents", p]);
    }
    args.extend(more);
    let o = wai(dir, &[args.as_slice(), &["--json"]].concat());
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("JSON: {}", text(&o)));
    let t = wai(dir, &args);
    (o.status.code().unwrap_or(-1), v, text(&t))
}

#[cfg(unix)]
#[test]
fn the_floor_suites_hold_a_sink_to_the_class_rule() {
    let d = tmp("floor");
    let r = wai_dir();
    let outs = floor_outputs(&d);
    // A sink's decoder, given the envelope ($1) and where its output goes ($2): the case's output,
    // by the envelope's name.
    let copy = format!("cp {}/\"$(basename \"$1\" .wai)\".out \"$2\"", q(outs.to_str().unwrap()));
    let script = |name: &str, body: &str| format!("{} {{in}} {{out}}", q(fake_python(&d, name, body).to_str().unwrap()));
    let decodes = script("decodes", &copy);
    // One with no AV1 decoder, which says so.
    let no_av1 = script("no-av1", &format!("case \"$(basename \"$1\")\" in av1-*) echo unsupported wai.video.av1; exit 3;; esac; {copy}"));

    // (1) It presents video and decodes every case: both floors pass, a record each, both admitted.
    let out = d.join("sink.wai-conform.json");
    let (code, v, t) = floor_run(&d, "floor", &decodes, Some("audio,video"), &out, &[]);
    assert_eq!(code, 0, "{v:#}\n{t}");
    let data = &v["data"];
    assert_eq!((data["conformant"].as_bool(), data["cases"].as_u64(), data["not_run"].as_u64()), (Some(true), Some(4), Some(0)), "{data:#}");
    let runs = data["runs"].as_array().unwrap();
    assert_eq!(runs.iter().map(|r| r["suite"].as_str().unwrap()).collect::<Vec<_>>(), vec!["floor", "video-floor"]);
    assert_eq!((runs[0]["cases"].as_u64(), runs[1]["cases"].as_u64()), (Some(3), Some(1)));
    assert!(data["suites_not_run"].as_array().unwrap().is_empty());
    let video_rec = d.join("sink.video-floor.wai-conform.json");
    assert!(out.is_file() && video_rec.is_file());
    assert!(t.contains("the mandatory floor (decoder floor): 3 of 3 cases passed: conformant"), "{t}");
    assert!(t.contains("the video floor (decoder video-floor): 1 of 1 cases passed: conformant"), "{t}");
    assert!(t.contains("conformant to the mandatory floor and the video floor"), "{t}");
    let o = wai(&d, &["verify", "--conformant", "--root", r.to_str().unwrap(), out.to_str().unwrap(), video_rec.to_str().unwrap()]);
    assert!(o.status.success(), "{}", text(&o));
    assert_eq!(text(&o).matches("ok   admitted").count(), 2, "{}", text(&o));

    // (2) It presents video and has no AV1 decoder: the video case is not run, never passed, and
    // the sink does not pass the video floor; its mandatory floor stands.
    let out = d.join("no-av1.wai-conform.json");
    let (code, v, t) = floor_run(&d, "floor", &no_av1, Some("video"), &out, &[]);
    assert_eq!(code, 1, "{v:#}\n{t}");
    let data = &v["data"];
    assert_eq!((data["conformant"].as_bool(), data["not_run"].as_u64(), data["passed"].as_u64()), (Some(false), Some(1), Some(3)), "{data:#}");
    let video = &data["runs"][1];
    assert_eq!((video["conformant"].as_bool(), video["not_run"].as_u64(), video["passed"].as_u64()), (Some(false), Some(1), Some(0)), "{video:#}");
    let o = &video["outcomes"][0];
    assert_eq!((o["pass"].as_bool(), o["not_run"].as_bool()), (Some(false), Some(true)));
    assert!(o["got"].as_str().unwrap().starts_with("not run: ") && o["got"].as_str().unwrap().contains("unsupported wai.video.av1"), "{o}");
    assert!(t.contains("NOT RUN av1-video-floor") && !t.contains("PASS av1-video-floor") && !t.contains("FAIL av1-video-floor"), "{t}");
    assert!(t.contains("the video floor (decoder video-floor): 0 of 1 cases passed, 1 not run: not conformant"), "{t}");
    assert!(t.contains("not conformant: it does not pass the video floor"), "{t}");
    let rec: Value = serde_json::from_slice(&std::fs::read(d.join("no-av1.video-floor.wai-conform.json")).unwrap()).unwrap();
    assert_eq!((rec["conformant"].as_bool(), rec["cases"][0]["pass"].as_bool()), (Some(false), Some(false)), "{rec:#}");
    let o = wai(&d, &["verify", "--conformant", "--root", r.to_str().unwrap(), d.join("no-av1.video-floor.wai-conform.json").to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("not_conformant"), "{}", text(&o));

    // (3) It presents audio alone: held to the mandatory floor, its video floor not run, with the
    // rule, and no record of it written.
    let out = d.join("audio.wai-conform.json");
    let (code, v, t) = floor_run(&d, "floor", &no_av1, Some("audio"), &out, &[]);
    assert_eq!(code, 0, "{v:#}\n{t}");
    let data = &v["data"];
    assert_eq!((data["conformant"].as_bool(), data["cases"].as_u64(), data["runs"].as_array().unwrap().len()), (Some(true), Some(3), 1), "{data:#}");
    let nr = &data["suites_not_run"][0];
    assert_eq!(nr["suite"], "video-floor");
    assert!(nr["why"].as_str().unwrap().contains("presents audio and no video"), "{nr}");
    assert!(out.is_file() && !d.join("audio.video-floor.wai-conform.json").exists());
    assert!(t.contains("the video floor (decoder video-floor): not run: the sink presents audio and no video"), "{t}");
    assert!(t.contains("conformant to the mandatory floor") && !t.contains("PASS av1"), "{t}");

    // Without --presents: the mandatory floor alone, the video floor not run for want of the classes.
    let (code, v, _) = floor_run(&d, "floor", &decodes, None, &d.join("unsaid.wai-conform.json"), &[]);
    assert_eq!(code, 0, "{v:#}");
    assert!(v["data"]["suites_not_run"][0]["why"].as_str().unwrap().contains("--presents"), "{v:#}");
    // The video floor asked for alone runs alone; for a sink that presents no video, it is not run
    // and nothing is written.
    let (code, v, _) = floor_run(&d, "video-floor", &decodes, None, &d.join("v.wai-conform.json"), &[]);
    assert_eq!((code, v["data"]["cases"].as_u64(), v["data"]["conformant"].as_bool()), (0, Some(1), Some(true)), "{v:#}");
    let (code, v, _) = floor_run(&d, "video-floor", &decodes, Some("none"), &d.join("n.wai-conform.json"), &[]);
    assert_eq!((code, v["data"]["cases"].as_u64()), (0, Some(0)), "{v:#}");
    assert!(v["data"]["conformant"].is_null() && !d.join("n.wai-conform.json").exists(), "{v:#}");

    // One byte of the picture changed: that case fails, and is not said not run.
    let other = std::fs::read(outs.join("png-rgb8.out")).unwrap()[7] ^ 1;
    let flip = script("flip", &format!("{copy} && case \"$(basename \"$1\")\" in png-*) printf '\\{other:03o}' | dd of=\"$2\" bs=1 seek=7 count=1 conv=notrunc 2>/dev/null;; esac; exit 0"));
    let (code, v, t) = floor_run(&d, "floor", &flip, Some("image"), &d.join("flip.wai-conform.json"), &[]);
    assert_eq!(code, 1, "{v:#}");
    assert_eq!((v["data"]["failed"].as_u64(), v["data"]["not_run"].as_u64()), (Some(1), Some(0)), "{v:#}");
    assert!(t.contains("FAIL png-rgb8"), "{t}");

    // A dry run writes nothing; --presents is the floor's alone, and names media classes.
    let dry = tmp("floor-dry");
    let o = wai(&dry, &["conform", "decoder", "--suite", "floor", "--impl", &decodes, "--presents", "video", "--root", r.to_str().unwrap(), "--dry-run", "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("{}", text(&o)));
    assert!(o.status.success() && v["data"]["runs"].as_array().unwrap().len() == 2, "{v:#}");
    assert!(std::fs::read_dir(&dry).unwrap().next().is_none(), "a dry run writes nothing");
    let code = |args: &[&str]| wai(&d, args).status.code();
    assert_eq!(code(&["conform", "decoder", "--suite", "floor", "--impl", &decodes, "--presents", "colour", "--root", r.to_str().unwrap()]), Some(2));
    let w = q(env!("CARGO_BIN_EXE_wai"));
    assert_eq!(code(&["conform", "decoder", "--suite", "score", "--impl", &format!("{w} score render {{in}} --pcm {{out}}"), "--presents", "audio", "--root", r.to_str().unwrap()]), Some(2));

    // The independent verifier beside it: a check it says it did not run is said not run.
    let said = fake_python(&d, "said", "echo 'av1-video-floor: decode not run (no AV1 decoder: pass --dav1d PATH)'; exit 0");
    let (code, v, t) = floor_run(&d, "floor", &decodes, Some("video"), &d.join("ind.wai-conform.json"), &["--independent", "--python", said.to_str().unwrap()]);
    assert_eq!(code, 0, "{v:#}\n{t}");
    let ind = &v["data"]["independent"];
    assert_eq!((ind["verifier"].as_str(), ind["conformant"].as_bool()), (Some("floor"), Some(true)), "{ind:#}");
    assert!(ind["not_run"][0].as_str().unwrap().contains("decode not run (no AV1 decoder"), "{ind:#}");
    assert!(ind["got"].as_str().unwrap().contains("not run:"), "{ind:#}");
    assert!(t.contains("NOT RUN (independent) av1-video-floor: decode not run"), "{t}");
    assert!(d.join("ind.independent.wai-conform.json").is_file());
    let _ = std::fs::remove_dir_all(&d);
    let _ = std::fs::remove_dir_all(&dry);
}