open-standards/wai/wai-rs/tests/cli_conform.rs

//! `wai conform` run against implementations of each kind, through both adapters, and its
//! records checked by `wai verify`:
//!
//! - the reference verifiers of the claim, JWP and key-discovery corpora (`wai_*_vectors
//!   verdicts`, whole-corpus runs) conform, and so does the key-discovery reference behind
//!   an HTTP adapter; one that changes one verdict does not, at that case alone;
//! - the score reference renderer (`wai score render`) conforms, and one that changes one
//!   byte of its output fails every case;
//! - the reference origin (`wai http origin`) serving the HTTP corpus's objects conforms,
//!   and one serving an object with one byte changed fails that object;
//! - a record changed after signing, in a case's outcome or in its summary, is refused by
//!   `wai verify`; a dry run writes nothing; a record is not replaced without `--yes`;
//!   each failure to run has its exit status.
#![cfg(all(feature = "quickstart", feature = "score", feature = "key_discovery"))]

use serde_json::Value;
use std::io::{BufRead, Read, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Output, Stdio};

fn tmp(name: &str) -> PathBuf {
    let d = std::env::temp_dir().join(format!("wai-conform-{name}-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&d);
    std::fs::create_dir_all(&d).unwrap();
    d
}

fn wai_dir() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf()
}

fn wai(dir: &Path, args: &[&str]) -> Output {
    Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(dir).args(args).output().expect("wai runs")
}

fn text(o: &Output) -> String {
    format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))
}

/// `wai conform <kind> --suite <suite> --impl <imp> --json` writing its record to `out`.
fn conform(dir: &Path, kind: &str, suite: &str, imp: &str, out: &Path) -> (i32, Value) {
    let root = wai_dir();
    let o = wai(dir, &["conform", kind, "--suite", suite, "--impl", imp, "--root", root.to_str().unwrap(), "--out", out.to_str().unwrap(), "--yes", "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("JSON: {}", text(&o)));
    (o.status.code().unwrap_or(-1), v)
}

fn q(p: &str) -> String {
    format!("'{}'", p.replace('\'', "'\\''"))
}

#[test]
fn the_reference_verifiers_conform_and_one_changed_verdict_does_not() {
    let d = tmp("verifiers");
    for (suite, bin) in [("claim", env!("CARGO_BIN_EXE_wai_claim_vectors")), ("jwp", env!("CARGO_BIN_EXE_wai_jwp_vectors")), ("signer", env!("CARGO_BIN_EXE_wai_signer_vectors"))] {
        let out = d.join(format!("{suite}.wai-conform.json"));
        let (code, v) = conform(&d, "verifier", suite, &format!("{} verdicts {{corpus}}", q(bin)), &out);
        assert_eq!(code, 0, "{suite}: {v:#}");
        assert_eq!(v["data"]["conformant"], true);
        assert!(v["data"]["cases"].as_u64().unwrap() > 100, "{v}");
        let rec: Value = serde_json::from_slice(&std::fs::read(&out).unwrap()).unwrap();
        assert_eq!(rec["wai"], "conform-result/1");
        assert!(rec["corpus"]["digest"].as_str().unwrap().starts_with("sha256:"));
        assert!(rec["standard"].as_str().unwrap().starts_with('v'));
        assert!(rec["implementation"]["files"].as_array().unwrap().iter().any(|f| f["path"].as_str().unwrap().ends_with(&format!("wai_{}_vectors", if suite == "signer" { "signer" } else { suite }))), "{}", rec["implementation"]);
    }
    let o = wai(&d, &["verify", d.to_str().unwrap()]);
    assert!(o.status.success(), "{}", text(&o));
    assert!(text(&o).contains("3 conformance results verified"), "{}", text(&o));

    // One verdict changed: its case fails, and no other.
    let bad = format!("{} verdicts {{corpus}} | awk 'NR == 1 {{ sub(/\\t/, \"\\t~\") }} 1'", q(env!("CARGO_BIN_EXE_wai_signer_vectors")));
    let (code, v) = conform(&d, "verifier", "signer", &bad, &d.join("bad.wai-conform.json"));
    assert_eq!(code, 1, "{v:#}");
    assert_eq!(v["data"]["conformant"], false);
    assert_eq!(v["data"]["failed"], 1, "{v:#}");
    // A record that says "not conformant" is still a genuine record.
    let o = wai(&d, &["verify", d.join("bad.wai-conform.json").to_str().unwrap()]);
    assert!(o.status.success() && text(&o).contains("not conformant"), "{}", text(&o));
}

#[test]
fn a_tampered_record_is_refused() {
    let d = tmp("tamper");
    let out = d.join("signer.wai-conform.json");
    let bad = format!("{} verdicts {{corpus}} | awk 'NR == 1 {{ sub(/\\t/, \"\\t~\") }} 1'", q(env!("CARGO_BIN_EXE_wai_signer_vectors")));
    let (code, _) = conform(&d, "verifier", "signer", &bad, &out);
    assert_eq!(code, 1);
    let good = std::fs::read(&out).unwrap();
    let mut rec: Value = serde_json::from_slice(&good).unwrap();
    // The failed case turned into a pass, the summary and verdict made to agree with it: the
    // signature no longer covers what the record says.
    let cases = rec["cases"].as_array_mut().unwrap();
    let i = cases.iter().position(|c| c["pass"] == false).unwrap();
    cases[i]["pass"] = Value::Bool(true);
    rec["summary"]["passed"] = Value::from(rec["summary"]["passed"].as_u64().unwrap() + 1);
    rec["summary"]["failed"] = Value::from(0);
    rec["conformant"] = Value::Bool(true);
    std::fs::write(&out, serde_json::to_vec_pretty(&rec).unwrap()).unwrap();
    let o = wai(&d, &["verify", out.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1), "{}", text(&o));
    assert!(text(&o).contains("the signature does not verify"), "{}", text(&o));

    // One byte of the corpus digest changed.
    let s = String::from_utf8(good.clone()).unwrap();
    let at = s.find("\"sha256:").unwrap() + 8;
    let mut b = good.clone();
    b[at] = if b[at] == b'0' { b'1' } else { b'0' };
    std::fs::write(&out, &b).unwrap();
    assert_eq!(wai(&d, &["verify", out.to_str().unwrap()]).status.code(), Some(1));

    // The summary alone changed, the signature left: refused at the signature
    // (conformance-results §4 step 4, before the summary is counted).
    let mut rec: Value = serde_json::from_slice(&good).unwrap();
    rec["summary"]["failed"] = Value::from(0);
    std::fs::write(&out, serde_json::to_vec(&rec).unwrap()).unwrap();
    let o = wai(&d, &["verify", out.to_str().unwrap()]);
    assert_eq!(o.status.code(), Some(1));
    assert!(text(&o).contains("step 4 signature_invalid"), "{}", text(&o));

    std::fs::write(&out, &good).unwrap();
    assert!(wai(&d, &["verify", out.to_str().unwrap()]).status.success());
}

#[test]
fn the_score_renderer_conforms_and_one_flipped_output_byte_fails() {
    let d = tmp("score");
    let w = q(env!("CARGO_BIN_EXE_wai"));
    let (code, v) = conform(&d, "decoder", "score", &format!("{w} score render {{in}} --pcm {{out}}"), &d.join("score.wai-conform.json"));
    assert_eq!(code, 0, "{v:#}");
    assert_eq!(v["data"]["passed"], 3);
    // One byte of the output changed (byte 100 of the PCM, plus one).
    let flip = format!("{w} score render {{in}} --pcm {{out}} >/dev/null && dd if={{out}} bs=1 skip=100 count=1 2>/dev/null | LC_ALL=C tr '\\000-\\377' '\\001-\\377\\000' | dd of={{out}} bs=1 seek=100 count=1 conv=notrunc 2>/dev/null");
    let (code, v) = conform(&d, "decoder", "score", &flip, &d.join("flip.wai-conform.json"));
    assert_eq!(code, 1, "{v:#}");
    assert_eq!(v["data"]["failed"], 3, "{v:#}");
    assert_eq!(v["data"]["passed"], 0);
}

/// `wai http origin --root <root>` on a port the system picks: the child and its URL, from
/// the line it prints once it listens.
fn origin(root: &Path) -> (Child, String) {
    let mut c = Command::new(env!("CARGO_BIN_EXE_wai")).args(["http", "origin", "--root", root.to_str().unwrap(), "--port", "0"]).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().unwrap();
    let mut line = String::new();
    std::io::BufReader::new(c.stdout.as_mut().unwrap()).read_line(&mut line).unwrap();
    let url = line.split_whitespace().find(|w| w.starts_with("http://127.0.0.1:")).unwrap_or_else(|| panic!("{line}")).trim_end_matches('/').to_string();
    (c, url)
}

#[test]
fn the_reference_origin_conforms_and_one_changed_object_fails() {
    let d = tmp("origin");
    let objects = wai_dir().join("http-conformance").join("objects");
    let (mut c, url) = origin(&objects);
    let (code, v) = conform(&d, "origin", "http", &url, &d.join("http.wai-conform.json"));
    let _ = c.kill();
    assert_eq!(code, 0, "{v:#}");

    // A copy of the objects with one byte of one object changed.
    let copy = d.join("objects");
    std::fs::create_dir_all(&copy).unwrap();
    let mut changed = None;
    for e in std::fs::read_dir(&objects).unwrap().flatten() {
        if e.path().is_file() {
            let mut b = std::fs::read(e.path()).unwrap();
            let name = e.file_name().to_string_lossy().into_owned();
            if changed.is_none() && name.ends_with(".wai") && b.len() > 2000 {
                let n = b.len();
                b[n - 1] ^= 1;
                changed = Some(name);
            }
            std::fs::write(copy.join(e.file_name()), b).unwrap();
        }
    }
    let (mut c, url) = origin(&copy);
    let (code, v) = conform(&d, "origin", "http", &url, &d.join("bad.wai-conform.json"));
    let _ = c.kill();
    assert_eq!(code, 1, "{v:#}");
    assert_eq!(v["data"]["failed"], 1, "{v:#}");
    assert!(v["data"]["failures"][0]["got"].as_str().unwrap().contains("the body is not the object"), "{v:#}");
}

/// A verifier behind HTTP: the reference's verdicts, served one case a request.
#[test]
fn a_verifier_behind_http_conforms() {
    let d = tmp("http-verifier");
    let corpus = wai_dir().join("signer-conformance").join("vectors.json");
    let o = Command::new(env!("CARGO_BIN_EXE_wai_signer_vectors")).arg("verdicts").arg(&corpus).output().unwrap();
    let verdicts: std::collections::HashMap<String, String> =
        String::from_utf8_lossy(&o.stdout).lines().filter_map(|l| l.split_once('\t')).map(|(a, b)| (a.to_string(), b.to_string())).collect();
    let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
    let url = format!("http://{}/verify", listener.local_addr().unwrap());
    std::thread::spawn(move || {
        for s in listener.incoming().flatten() {
            let mut s = s;
            let mut buf = Vec::new();
            let mut chunk = [0u8; 65536];
            let body = loop {
                let n = s.read(&mut chunk).unwrap_or(0);
                if n == 0 {
                    break None;
                }
                buf.extend_from_slice(&chunk[..n]);
                if let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") {
                    let head = String::from_utf8_lossy(&buf[..end]).to_ascii_lowercase();
                    let len: usize = head.lines().find_map(|l| l.strip_prefix("content-length:")).and_then(|v| v.trim().parse().ok()).unwrap_or(0);
                    if buf.len() >= end + 4 + len {
                        break Some(buf[end + 4..end + 4 + len].to_vec());
                    }
                }
            };
            let v: Value = body.and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or(Value::Null);
            let verdict = verdicts.get(v["name"].as_str().unwrap_or("")).cloned().unwrap_or_default();
            let reply = serde_json::json!({"verdict": verdict}).to_string();
            let _ = s.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{reply}", reply.len()).as_bytes());
        }
    });
    let (code, v) = conform(&d, "verifier", "signer", &url, &d.join("signer.wai-conform.json"));
    assert_eq!(code, 0, "{v:#}");
    let rec: Value = serde_json::from_slice(&std::fs::read(d.join("signer.wai-conform.json")).unwrap()).unwrap();
    assert_eq!(rec["implementation"]["adapter"], "http");
}

#[test]
fn a_dry_run_writes_nothing_and_each_failure_has_its_exit_status() {
    let d = tmp("statuses");
    let root = wai_dir();
    let r = root.to_str().unwrap();
    let w = q(env!("CARGO_BIN_EXE_wai"));
    let imp = format!("{w} score render {{in}} --pcm {{out}}");
    let o = wai(&d, &["conform", "decoder", "--suite", "score", "--impl", &imp, "--root", r, "--dry-run", "--json"]);
    assert!(o.status.success(), "{}", text(&o));
    let v: Value = serde_json::from_slice(&o.stdout).unwrap();
    assert_eq!(v["dry_run"], true);
    assert!(std::fs::read_dir(&d).unwrap().next().is_none(), "a dry run writes nothing");

    // The record exists: not replaced without --yes (4), the proposal and the confirming command
    // in the envelope.
    std::fs::write(d.join("score.wai-conform.json"), "{}").unwrap();
    let o = wai(&d, &["conform", "decoder", "--suite", "score", "--impl", &imp, "--root", r, "--json"]);
    assert_eq!(o.status.code(), Some(4), "{}", text(&o));
    let v: Value = serde_json::from_slice(&o.stdout).unwrap();
    assert_eq!((v["error"]["code"].as_str(), v["proposed"][0]["replaces"].as_bool()), (Some("confirmation-required"), Some(true)), "{v:#}");
    assert!(v["confirm"].as_str().unwrap().ends_with(" --yes"), "{v:#}");
    assert_eq!(std::fs::read(d.join("score.wai-conform.json")).unwrap(), b"{}");

    let code = |args: &[&str]| wai(&d, args).status.code();
    assert_eq!(code(&["conform", "decoder", "--suite", "score", "--root", r]), Some(2), "no --impl");
    assert_eq!(code(&["conform", "relay", "--impl", "x", "--root", r]), Some(5), "no such kind");
    assert_eq!(code(&["conform", "decoder", "--suite", "nope", "--impl", &imp, "--root", r]), Some(5), "no such suite");
    assert_eq!(code(&["conform", "decoder", "--suite", "score", "--impl", "no-such-program-here {in}", "--root", r]), Some(5), "no such program");
    let free = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
    assert_eq!(code(&["conform", "origin", "--impl", &format!("http://127.0.0.1:{free}"), "--root", r, "--out", d.join("o.wai-conform.json").to_str().unwrap()]), Some(6), "nothing listening");
    let o = wai(&d, &["conform", "--list", "--root", r, "--json"]);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap();
    assert!(v["data"]["suites"].as_array().unwrap().iter().all(|s| s["cases"].as_u64().unwrap() > 0), "{v:#}");
}