open-standards/wai/wai-rs/tests/cli_dx.rs

//! The developer's path through the `wai` binary, as a person runs it: `wai hello` writes an
//! object, its receipt index and its group's receipt file, and `wai verify` checks them
//! offline and refuses each when it is altered; `wai create` writes each example of
//! `wai/examples/` as a project, file for file and byte for byte, and never into a directory
//! that holds anything. `wai verify` also reads the envelopes under a path: a planted envelope
//! that pins nothing where its capability decodes against a parameter set is reported
//! `prior_unpinned`, and still verifies; one whose pin breaks SPEC §3.1 does not.
#![cfg(feature = "quickstart")]

use std::path::{Path, PathBuf};
use std::process::{Command, Output};

fn tmp(name: &str) -> PathBuf {
    let d = std::env::temp_dir().join(format!("wai-dx-{name}-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&d);
    std::fs::create_dir_all(&d).unwrap();
    d
}

fn wai(dir: &Path, args: &[&str]) -> Output {
    Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(dir).args(args).output().expect("wai runs")
}

fn text(o: &Output) -> String {
    format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))
}

/// The one file under `dir` whose name ends with `suffix`.
fn one(dir: &Path, suffix: &str) -> PathBuf {
    let mut v: Vec<PathBuf> = std::fs::read_dir(dir).unwrap().flatten().map(|e| e.path()).filter(|p| p.to_string_lossy().ends_with(suffix)).collect();
    assert_eq!(v.len(), 1, "one {suffix} in {}", dir.display());
    v.remove(0)
}

#[test]
fn hello_runs_the_four_steps_and_verify_checks_what_it_wrote() {
    let d = tmp("hello");
    let o = wai(&d, &["hello", "h"]);
    assert!(o.status.success(), "{}", text(&o));
    let t = text(&o);
    for step in ["1 Name", "2 Dispatch", "3 Rebuild", "4 Receipt", "byte for byte", "delivered-verified"] {
        assert!(t.contains(step), "hello says {step}: {t}");
    }
    let h = d.join("h");
    let v = wai(&d, &["verify", "h"]);
    assert!(v.status.success(), "{}", text(&v));
    // The object is an envelope there too, and its pins are read.
    assert!(text(&v).contains("delivered-verified") && text(&v).contains("1 receipt index and 1 envelope verified"), "{}", text(&v));

    // A byte of the object changed: its content hash is not the receipt's.
    let obj = one(&h, ".wai");
    let good = std::fs::read(&obj).unwrap();
    let mut bad = good.clone();
    let n = bad.len();
    bad[n - 1] ^= 1;
    std::fs::write(&obj, &bad).unwrap();
    let v = wai(&d, &["verify", "h"]);
    assert_eq!(v.status.code(), Some(1), "{}", text(&v));
    assert!(text(&v).contains("FAIL"), "{}", text(&v));
    std::fs::write(&obj, &good).unwrap();

    // A byte of the group's receipt file changed: its name is no longer its link hash, or its
    // signature no longer verifies.
    let rcpt = one(&h.join("rcpt"), ".json");
    let g = std::fs::read_to_string(&rcpt).unwrap();
    let i = g.find("\"sig\":\"").expect("a signature") + 7;
    let mut b = g.into_bytes();
    b[i] = if b[i] == b'0' { b'1' } else { b'0' };
    std::fs::write(&rcpt, &b).unwrap();
    let v = wai(&d, &["verify", "h"]);
    assert_eq!(v.status.code(), Some(1), "{}", text(&v));

    // Nothing to verify is an error, not a pass.
    std::fs::create_dir_all(d.join("empty")).unwrap();
    let v = wai(&d, &["verify", "empty"]);
    assert_eq!(v.status.code(), Some(2), "{}", text(&v));
    // And hello never writes into a directory of someone else's files.
    std::fs::write(d.join("empty/mine.txt"), "mine").unwrap();
    assert_eq!(wai(&d, &["hello", "empty"]).status.code(), Some(2));
    let _ = std::fs::remove_dir_all(&d);
}

/// SPEC §3.1, *Pin by default*: `wai verify` reports `prior_unpinned` on a planted envelope
/// that pins nothing over a capability that decodes against a parameter set, and the envelope
/// still verifies; it reports nothing on one over a standard codec or on a pinned one; and it
/// refuses one whose pin a verifier can tell is wrong without holding a set.
#[test]
fn verify_reports_an_unpinned_envelope() {
    use wai::container::{Conditioning, Manifest, ModelRequirement, Prior, Wai};
    let d = tmp("unpinned");
    let p = d.join("planted");
    std::fs::create_dir_all(&p).unwrap();
    std::fs::write(d.join("payload.bin"), b"opaque test bytes").unwrap();
    let wrap = |cap: &str, out: &str| {
        let o = wai(&d, &["wrap", cap, "payload.bin", "-o", out, "--intent", "improve"]);
        assert!(o.status.success(), "{}", text(&o));
        o
    };
    let w = wrap("wai.neural.bmshj2018", "planted/unpinned.wai");
    assert!(text(&w).contains("note prior_unpinned: wai.neural.bmshj2018"), "wrap says it pinned nothing: {}", text(&w));
    wrap("wai.image.png", "planted/png.wai");
    let pin = Prior::new("int_mlicpp/test", "7ef44f3b1fc4a2df6c3e3b68460f595cae3d31cb7f9018699e29e738340272a4", "int-rans/p16+int-mlicpp/i64");
    let pinned = Wai::new(Manifest::new("image", "improve", ModelRequirement::new("wai.neural.int_mlicpp"),
                                        Conditioning::new("int_mlicpp").with_prior(pin.clone())), b"M".to_vec());
    std::fs::write(p.join("pinned.wai"), pinned.to_bytes().unwrap()).unwrap();

    let v = wai(&d, &["verify", "planted"]);
    let t = text(&v);
    assert!(v.status.success(), "a report is not a refusal: {t}");
    assert!(t.contains("note prior_unpinned: wai.neural.bmshj2018 decodes against a parameter set"), "{t}");
    assert_eq!(t.matches("prior_unpinned").count(), 1, "the PNG and the pinned envelope are not reported: {t}");
    assert!(t.contains("3 envelopes verified"), "{t}");
    let v = wai(&d, &["verify", "planted/unpinned.wai", "--json"]);
    assert!(v.status.success() && text(&v).contains("prior_unpinned"), "{}", text(&v));

    // A pin over a capability that reads no set: the envelope does not conform.
    let bad = Wai::new(Manifest::new("image", "replicate", ModelRequirement::new("wai.image.png"),
                                     Conditioning::new("png").with_prior(pin)), b"PNG".to_vec());
    std::fs::write(p.join("bad.wai"), bad.to_bytes().unwrap()).unwrap();
    let v = wai(&d, &["verify", "planted"]);
    assert_eq!(v.status.code(), Some(1), "{}", text(&v));
    assert!(text(&v).contains("FAIL WAI1 wai.image.png: prior_not_applicable"), "{}", text(&v));
    let _ = std::fs::remove_dir_all(&d);
}

/// Every file of an example but its `test/` directory, relative to it, sorted.
fn example_files(dir: &Path) -> Vec<String> {
    fn walk(root: &Path, p: &Path, out: &mut Vec<String>) {
        for e in std::fs::read_dir(p).unwrap().flatten() {
            let q = e.path();
            let rel = q.strip_prefix(root).unwrap().to_string_lossy().replace('\\', "/");
            if rel == "test" || rel.starts_with("test/") || rel.ends_with(".DS_Store") || rel == "target" || rel.starts_with("target/") {
                continue;
            }
            if q.is_dir() {
                walk(root, &q, out);
            } else {
                out.push(rel);
            }
        }
    }
    let mut v = Vec::new();
    walk(dir, dir, &mut v);
    v.sort();
    v
}

#[test]
fn create_writes_every_example_as_it_is_in_the_repository() {
    let d = tmp("create");
    let examples = Path::new(env!("CARGO_MANIFEST_DIR")).join("../examples");
    let o = wai(&d, &["create", "--list", "--json"]);
    assert!(o.status.success(), "{}", text(&o));
    // The JSON envelope (envelope/1): the list is its data.
    let env: serde_json::Value = serde_json::from_slice(&o.stdout).unwrap();
    assert_eq!((env["ok"].as_bool(), env["op"].as_str()), (Some(true), Some("project.create")), "{env}");
    let list = env["data"].as_array().unwrap();
    let names: Vec<&str> = list.iter().map(|t| t["template"].as_str().unwrap()).collect();
    // Every example directory is a template, and every template an example.
    let mut dirs: Vec<String> = std::fs::read_dir(&examples).unwrap().flatten().filter(|e| e.path().is_dir()).map(|e| e.file_name().to_string_lossy().into_owned()).collect();
    dirs.sort();
    let mut sorted = names.clone();
    sorted.sort();
    assert_eq!(sorted, dirs, "the templates are the examples");
    for t in list {
        let name = t["template"].as_str().unwrap();
        let generated: Vec<&str> = t["files"].as_array().unwrap().iter().map(|f| f.as_str().unwrap()).filter(|f| !examples.join(name).join(f).exists()).collect();
        let o = wai(&d, &["create", name, "--template", name]);
        assert!(o.status.success(), "{name}: {}", text(&o));
        let made = example_files(&d.join(name));
        let mut want = example_files(&examples.join(name));
        want.extend(generated.iter().map(|g| g.to_string()));
        want.sort();
        assert_eq!(made, want, "{name}: the project holds the example's files, and those made for it");
        for f in example_files(&examples.join(name)) {
            assert_eq!(std::fs::read(d.join(name).join(&f)).unwrap(), std::fs::read(examples.join(name).join(&f)).unwrap(), "{name}/{f}: byte for byte");
        }
    }
    // Never into a directory that holds anything.
    let o = wai(&d, &["create", "video-room", "--template", "video-room"]);
    assert_eq!(o.status.code(), Some(2), "{}", text(&o));
    assert!(text(&o).contains("not empty"));
    let o = wai(&d, &["create", "x", "--template", "no-such"]);
    assert_eq!(o.status.code(), Some(2));
    let _ = std::fs::remove_dir_all(&d);
}