open-standards/wai/wai-rs/tests/cli_dx.rs

//! The developer's path through the `wai` binary, as a person runs it: `wai hello` writes an
//! object, its receipt index and its group's receipt file, and `wai verify` checks them
//! offline and refuses each when it is altered; `wai create` writes each example of
//! `wai/examples/` as a project, file for file and byte for byte, and never into a directory
//! that holds anything. `wai verify` also reads the envelopes under a path: a planted envelope
//! that pins nothing where its capability decodes against a parameter set is reported
//! `prior_unpinned`, and still verifies; one whose pin breaks SPEC §3.1 does not. Piped to a
//! reader that stops reading (`wai ... | head`), `wai` ends as a Unix tool does, without a word;
//! any other failure to write its output is still an error.
#![cfg(feature = "quickstart")]

use std::path::{Path, PathBuf};
use std::process::{Command, Output};

fn tmp(name: &str) -> PathBuf {
    let d = std::env::temp_dir().join(format!("wai-dx-{name}-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&d);
    std::fs::create_dir_all(&d).unwrap();
    d
}

fn wai(dir: &Path, args: &[&str]) -> Output {
    Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(dir).args(args).output().expect("wai runs")
}

fn text(o: &Output) -> String {
    format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))
}

/// The one file under `dir` whose name ends with `suffix`.
fn one(dir: &Path, suffix: &str) -> PathBuf {
    let mut v: Vec<PathBuf> = std::fs::read_dir(dir).unwrap().flatten().map(|e| e.path()).filter(|p| p.to_string_lossy().ends_with(suffix)).collect();
    assert_eq!(v.len(), 1, "one {suffix} in {}", dir.display());
    v.remove(0)
}

#[test]
fn hello_runs_the_four_steps_and_verify_checks_what_it_wrote() {
    let d = tmp("hello");
    let o = wai(&d, &["hello", "h"]);
    assert!(o.status.success(), "{}", text(&o));
    let t = text(&o);
    for step in ["1 Name", "2 Dispatch", "3 Rebuild", "4 Receipt", "byte for byte", "delivered-verified"] {
        assert!(t.contains(step), "hello says {step}: {t}");
    }
    let h = d.join("h");
    let v = wai(&d, &["verify", "h"]);
    assert!(v.status.success(), "{}", text(&v));
    // The object is an envelope there too, and its pins are read.
    assert!(text(&v).contains("delivered-verified") && text(&v).contains("1 receipt index and 1 envelope verified"), "{}", text(&v));

    // A byte of the object changed: its content hash is not the receipt's.
    let obj = one(&h, ".wai");
    let good = std::fs::read(&obj).unwrap();
    let mut bad = good.clone();
    let n = bad.len();
    bad[n - 1] ^= 1;
    std::fs::write(&obj, &bad).unwrap();
    let v = wai(&d, &["verify", "h"]);
    assert_eq!(v.status.code(), Some(1), "{}", text(&v));
    assert!(text(&v).contains("FAIL"), "{}", text(&v));
    std::fs::write(&obj, &good).unwrap();

    // A byte of the group's receipt file changed: its name is no longer its link hash, or its
    // signature no longer verifies.
    let rcpt = one(&h.join("rcpt"), ".json");
    let g = std::fs::read_to_string(&rcpt).unwrap();
    let i = g.find("\"sig\":\"").expect("a signature") + 7;
    let mut b = g.into_bytes();
    b[i] = if b[i] == b'0' { b'1' } else { b'0' };
    std::fs::write(&rcpt, &b).unwrap();
    let v = wai(&d, &["verify", "h"]);
    assert_eq!(v.status.code(), Some(1), "{}", text(&v));

    // Nothing to verify is an error, not a pass.
    std::fs::create_dir_all(d.join("empty")).unwrap();
    let v = wai(&d, &["verify", "empty"]);
    assert_eq!(v.status.code(), Some(2), "{}", text(&v));
    // And hello never writes into a directory of someone else's files.
    std::fs::write(d.join("empty/mine.txt"), "mine").unwrap();
    assert_eq!(wai(&d, &["hello", "empty"]).status.code(), Some(2));
    let _ = std::fs::remove_dir_all(&d);
}

/// SPEC §3.1, *Pin by default*: `wai verify` reports `prior_unpinned` on a planted envelope
/// that pins nothing over a capability that decodes against a parameter set, and the envelope
/// still verifies; it reports nothing on one over a standard codec or on a pinned one; and it
/// refuses one whose pin a verifier can tell is wrong without holding a set.
#[test]
fn verify_reports_an_unpinned_envelope() {
    use wai::container::{Conditioning, Manifest, ModelRequirement, Prior, Wai};
    let d = tmp("unpinned");
    let p = d.join("planted");
    std::fs::create_dir_all(&p).unwrap();
    std::fs::write(d.join("payload.bin"), b"opaque test bytes").unwrap();
    let wrap = |cap: &str, out: &str| {
        let o = wai(&d, &["wrap", cap, "payload.bin", "-o", out, "--intent", "improve"]);
        assert!(o.status.success(), "{}", text(&o));
        o
    };
    let w = wrap("wai.neural.bmshj2018", "planted/unpinned.wai");
    assert!(text(&w).contains("note prior_unpinned: wai.neural.bmshj2018"), "wrap says it pinned nothing: {}", text(&w));
    wrap("wai.image.png", "planted/png.wai");
    let pin = Prior::new("int_mlicpp/test", "7ef44f3b1fc4a2df6c3e3b68460f595cae3d31cb7f9018699e29e738340272a4", "int-rans/p16+int-mlicpp/i64");
    let pinned = Wai::new(Manifest::new("image", "improve", ModelRequirement::new("wai.neural.int_mlicpp"),
                                        Conditioning::new("int_mlicpp").with_prior(pin.clone())), b"M".to_vec());
    std::fs::write(p.join("pinned.wai"), pinned.to_bytes().unwrap()).unwrap();

    let v = wai(&d, &["verify", "planted"]);
    let t = text(&v);
    assert!(v.status.success(), "a report is not a refusal: {t}");
    assert!(t.contains("note prior_unpinned: wai.neural.bmshj2018 decodes against a parameter set"), "{t}");
    assert_eq!(t.matches("prior_unpinned").count(), 1, "the PNG and the pinned envelope are not reported: {t}");
    // The two `wai wrap` wrote leave their receipts beside them, which `wai verify` reads too.
    assert!(t.contains("3 envelopes and 2 operation receipts verified"), "{t}");
    let v = wai(&d, &["verify", "planted/unpinned.wai", "--json"]);
    assert!(v.status.success() && text(&v).contains("prior_unpinned"), "{}", text(&v));

    // A pin over a capability that reads no set: the envelope does not conform.
    let bad = Wai::new(Manifest::new("image", "replicate", ModelRequirement::new("wai.image.png"),
                                     Conditioning::new("png").with_prior(pin)), b"PNG".to_vec());
    std::fs::write(p.join("bad.wai"), bad.to_bytes().unwrap()).unwrap();
    let v = wai(&d, &["verify", "planted"]);
    assert_eq!(v.status.code(), Some(1), "{}", text(&v));
    assert!(text(&v).contains("FAIL WAI1 wai.image.png: prior_not_applicable"), "{}", text(&v));
    let _ = std::fs::remove_dir_all(&d);
}

/// Every file of an example but its `test/` directory, relative to it, sorted; of a project, but the
/// toolchain's own `.wai/`, where `wai create` keeps its receipt.
fn example_files(dir: &Path) -> Vec<String> {
    fn walk(root: &Path, p: &Path, out: &mut Vec<String>) {
        for e in std::fs::read_dir(p).unwrap().flatten() {
            let q = e.path();
            let rel = q.strip_prefix(root).unwrap().to_string_lossy().replace('\\', "/");
            if rel == "test" || rel.starts_with("test/") || rel.ends_with(".DS_Store") || rel == "target" || rel.starts_with("target/") || rel == ".wai" || rel.starts_with(".wai/") {
                continue;
            }
            if q.is_dir() {
                walk(root, &q, out);
            } else {
                out.push(rel);
            }
        }
    }
    let mut v = Vec::new();
    walk(dir, dir, &mut v);
    v.sort();
    v
}

#[test]
fn create_writes_every_example_as_it_is_in_the_repository() {
    let d = tmp("create");
    let examples = Path::new(env!("CARGO_MANIFEST_DIR")).join("../examples");
    let o = wai(&d, &["create", "--list", "--json"]);
    assert!(o.status.success(), "{}", text(&o));
    // The JSON envelope (envelope/1): the list is its data.
    let env: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap();
    assert_eq!((env["ok"].as_bool(), env["op"].as_str()), (Some(true), Some("project.create")), "{env}");
    let list = env["data"].as_array().unwrap();
    let names: Vec<&str> = list.iter().map(|t| t["template"].as_str().unwrap()).collect();
    // Every example directory is a template, and every template an example.
    let mut dirs: Vec<String> = std::fs::read_dir(&examples).unwrap().flatten().filter(|e| e.path().is_dir()).map(|e| e.file_name().to_string_lossy().into_owned()).collect();
    dirs.sort();
    let mut sorted = names.clone();
    sorted.sort();
    assert_eq!(sorted, dirs, "the templates are the examples");
    for t in list {
        let name = t["template"].as_str().unwrap();
        let generated: Vec<&str> = t["files"].as_array().unwrap().iter().map(|f| f.as_str().unwrap()).filter(|f| !examples.join(name).join(f).exists()).collect();
        let o = wai(&d, &["create", name, "--template", name]);
        assert!(o.status.success(), "{name}: {}", text(&o));
        let made = example_files(&d.join(name));
        let mut want = example_files(&examples.join(name));
        want.extend(generated.iter().map(|g| g.to_string()));
        want.sort();
        assert_eq!(made, want, "{name}: the project holds the example's files, and those made for it");
        // And the receipt of what `wai create` wrote, in the toolchain's own directory.
        assert!(d.join(name).join(".wai/receipts/wai.json.wai-receipt.json").is_file(), "{name}: its receipt");
        for f in example_files(&examples.join(name)) {
            assert_eq!(std::fs::read(d.join(name).join(&f)).unwrap(), std::fs::read(examples.join(name).join(&f)).unwrap(), "{name}/{f}: byte for byte");
        }
    }
    // Never into a directory that holds anything.
    let o = wai(&d, &["create", "video-room", "--template", "video-room"]);
    assert_eq!(o.status.code(), Some(2), "{}", text(&o));
    assert!(text(&o).contains("not empty"));
    let o = wai(&d, &["create", "x", "--template", "no-such"]);
    assert_eq!(o.status.code(), Some(2));
    let _ = std::fs::remove_dir_all(&d);
}

/// The lineup of the corpus's sample channel (`live-conformance/channels/`), written to `dir`
/// beside a copy of its receipt: what `wai channel verify` reads.
fn sample_channel(dir: &Path) -> (PathBuf, PathBuf) {
    let corpus = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../live-conformance/channels");
    let v: serde_json::Value = serde_json::from_slice(&std::fs::read(corpus.join("vectors.json")).unwrap()).unwrap();
    let hex = v["imports"][0]["outcome"]["lineup"].as_str().expect("the sample's lineup");
    let lineup: Vec<u8> = (0..hex.len()).step_by(2).map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap()).collect();
    let (r, l) = (dir.join("sample.channel.json"), dir.join("lineup.wlu"));
    std::fs::copy(corpus.join("sample.channel.json"), &r).unwrap();
    std::fs::write(&l, lineup).unwrap();
    (r, l)
}

/// A reader that stops reading (`wai channel verify ... | head`) ends `wai` as it ends a Unix
/// tool, with nothing on standard error: on Unix by SIGPIPE, which a shell reports as 141, and on
/// Windows with status 141. The pipe's reader is closed before `wai` starts, so its first write
/// finds the reader gone: an operation's text, its JSON envelope, and the help the door prints.
#[test]
fn a_reader_that_stops_reading_ends_wai_without_a_word() {
    let d = tmp("pipe");
    let (r, l) = sample_channel(&d);
    let (r, l) = (r.to_str().unwrap(), l.to_str().unwrap());
    // Read whole, the sample verifies.
    let o = wai(&d, &["channel", "verify", "--receipt", r, "--lineup", l]);
    assert!(o.status.success() && !o.stdout.is_empty(), "{}", text(&o));
    for args in [&["channel", "verify", "--receipt", r, "--lineup", l][..], &["ops", "list", "--json"], &["--help"]] {
        let (reader, writer) = std::io::pipe().unwrap();
        drop(reader);
        let o = Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(&d).args(args).stdout(writer).output().expect("wai runs");
        assert!(o.stderr.is_empty(), "{args:?}: {}", String::from_utf8_lossy(&o.stderr));
        #[cfg(unix)]
        assert_eq!(std::os::unix::process::ExitStatusExt::signal(&o.status), Some(13), "{args:?}: ended by SIGPIPE, not {:?}", o.status);
        #[cfg(windows)]
        assert_eq!(o.status.code(), Some(141), "{args:?}");
    }
    let _ = std::fs::remove_dir_all(&d);
}

/// Any other failure to write standard output still fails `wai`, as the internal error it was
/// (exit 7), saying what failed: a full device on Linux, a handle open for reading alone on
/// Windows. (With `--json` the error's envelope cannot be written either, and that second
/// failure ends the run with the panic's status, 101, as before.) (macOS has neither to hand: a descriptor open for reading alone gives EBADF there,
/// which the standard library takes for a closed output and writes nothing to.)
#[cfg(any(target_os = "linux", windows))]
#[test]
fn a_write_that_fails_otherwise_still_fails_wai() {
    let d = tmp("write-fails");
    let (r, l) = sample_channel(&d);
    #[cfg(target_os = "linux")]
    let out = || std::fs::OpenOptions::new().write(true).open("/dev/full").unwrap();
    #[cfg(windows)]
    let out = || {
        let p = d.join("read-only");
        std::fs::write(&p, b"").unwrap();
        std::fs::File::open(&p).unwrap()
    };
    for args in [&["channel", "verify", "--receipt", r.to_str().unwrap(), "--lineup", l.to_str().unwrap()][..], &["ops", "list"]] {
        let o = Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(&d).args(args).stdout(out()).output().expect("wai runs");
        let err = String::from_utf8_lossy(&o.stderr);
        assert_eq!(o.status.code(), Some(7), "{args:?}: {err}");
        assert!(err.contains("failed printing to stdout"), "{args:?}: {err}");
    }
    let _ = std::fs::remove_dir_all(&d);
}