open-standards/wai/wai-rs/tests/cli_receipts.rs

//! Operation receipts through the `wai` binary (`--features quickstart,generate,synthesis --test
//! cli_receipts`): every developer-facing write leaves a receipt that `wai verify` reads
//! (`src/bin/wai_cli/dx/receipt.rs`), and a generation an origin claim beside it
//! (extensions/synthesis.md §3.5).
//!
//! - `wai generate` signs an origin claim (step `generate`, the model by its SHA-256, type
//!   `algorithmic`, operation `wai.generate.spectral`, its record the receipt) and the receipt;
//!   `wai verify` reads both, and so does the claim's second implementation,
//!   `live-conformance/synthesis_verify.py claim`, which gives it the statement wai-rs gives it;
//! - `wai create`, `wai lock`, `wai build`, `wai decode`, `wai wrap`, `wai unwrap` and `wai key
//!   create|rotate|revoke|export` each write one, in a project's `.wai/receipts/` or beside the file
//!   written, each naming the digest of the `wai.json` it ran under;
//! - planted faults: a changed output (a picture, a decoded file, a file of a built tree, a key's
//!   record) fails `wai verify`; a receipt with another project's digest fails it; a claim beside
//!   another picture's receipt fails it; and `--key` that names no key refuses with nothing written.
#![cfg(all(feature = "quickstart", feature = "generate", feature = "synthesis"))]

use serde_json::Value;
use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio};

fn root() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
}

fn model() -> String {
    root().join("tests/vectors/generate/generate_spectral_model.json").to_string_lossy().into_owned()
}

fn scratch(name: &str) -> PathBuf {
    let d = Path::new(env!("CARGO_TARGET_TMPDIR")).join("cli_receipts").join(name);
    let _ = std::fs::remove_dir_all(&d);
    std::fs::create_dir_all(&d).unwrap();
    std::fs::canonicalize(&d).unwrap()
}

/// `wai` in `d`, its keystore `<d>/ks` (`$WAI_KEYSTORE`), so no run reads or writes another.
fn wai_in(d: &Path, ks: &Path, args: &[&str]) -> Output {
    Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(d).args(args).env("WAI_KEYSTORE", ks).env("NO_COLOR", "1").stdin(Stdio::null()).output().expect("wai runs")
}

fn text(o: &Output) -> String {
    format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))
}

fn ok(o: &Output) -> String {
    assert!(o.status.success(), "wai failed: {}", text(o));
    String::from_utf8_lossy(&o.stdout).into_owned()
}

/// The first JSON value printed on a line of its own in `s`, however many lines it takes.
fn json_in(s: &str) -> Value {
    let at = if s.starts_with('{') { 0 } else { s.find("\n{").map(|i| i + 1).unwrap_or_else(|| panic!("no JSON in {s}")) };
    serde_json::Deserializer::from_str(&s[at..]).into_iter::<Value>().next().expect("a value").unwrap_or_else(|e| panic!("{e}: {s}"))
}

/// The `data` of the JSON envelope a `--json` run prints.
fn data(o: &Output) -> Value {
    json_in(&ok(o))["data"].clone()
}

fn jload(p: &Path) -> Value {
    serde_json::from_slice(&std::fs::read(p).unwrap_or_else(|e| panic!("{}: {e}", p.display()))).unwrap()
}

fn sha256_hex(b: &[u8]) -> String {
    wai::pin::sha256(b).iter().map(|x| format!("{x:02x}")).collect()
}

fn flip_last(p: &Path) {
    let mut b = std::fs::read(p).unwrap();
    let n = b.len() - 1;
    b[n] ^= 1;
    std::fs::write(p, b).unwrap();
}

/// `wai verify <paths>` in `d`: its exit status and its output.
fn verify(d: &Path, ks: &Path, paths: &[&str]) -> (Option<i32>, String) {
    let mut a = vec!["verify"];
    a.extend_from_slice(paths);
    let o = wai_in(d, ks, &a);
    (o.status.code(), text(&o))
}

/// The claim's second implementation (live-conformance/synthesis_verify.py), on one claim and the
/// samples it is about: its exit status and its outcome.
fn second(claim: &Path, samples: &Path) -> (Option<i32>, Value) {
    let script = root().join("../live-conformance/synthesis_verify.py");
    let o = Command::new(std::env::var_os("PYTHON").unwrap_or_else(|| "python3".into()))
        .arg("-I")
        .arg(script)
        .args(["claim", claim.to_str().unwrap(), samples.to_str().unwrap()])
        .stdin(Stdio::null())
        .output()
        .expect("python3 runs");
    let out = String::from_utf8_lossy(&o.stdout).into_owned();
    let v = out.lines().rev().find(|l| l.starts_with('{')).and_then(|l| serde_json::from_str(l).ok()).unwrap_or(Value::Null);
    (o.status.code(), v)
}

const GEN: &[&str] = &["generate", "--prompt", "a quiet harbour at dusk", "--seed", "7", "--steps", "2", "--size", "16x12"];

#[test]
fn a_generation_signs_its_origin_claim_and_receipt_and_verify_reads_both() {
    let d = scratch("generate");
    let ks = d.join("ks");
    let m = model();
    let mut a = GEN.to_vec();
    a.extend(["--model", &m, "-o", "pic.rgb", "--json"]);
    let v = data(&wai_in(&d, &ks, &a));
    let pic = d.join("pic.rgb");
    let rgb = std::fs::read(&pic).unwrap();
    assert_eq!(v["sha256"].as_str(), Some(sha256_hex(&rgb).as_str()));
    // Outside a project: both beside the picture, signed by a key made for this run.
    let receipt = d.join("pic.rgb.wai-receipt.json");
    let claim_file = d.join("pic.rgb.wai-origin.json");
    assert_eq!(v["receipt"]["file"].as_str(), Some(receipt.to_str().unwrap()), "{v}");
    assert_eq!(v["origin"]["file"].as_str(), Some(claim_file.to_str().unwrap()), "{v}");
    assert_eq!((v["receipt"]["run_key"].as_bool(), v["receipt"]["project_sha256"].as_str()), (Some(true), None), "{v}");
    // The claim: what §3.5 says a generation's claim is.
    let c = jload(&claim_file);
    let r = jload(&receipt);
    let model_sha = sha256_hex(&std::fs::read(&m).unwrap());
    assert_eq!(c["kind"], "wai-origin");
    assert_eq!(c["subject"], serde_json::json!({"type": "samples", "samples": sha256_hex(&rgb)}));
    assert_eq!((c["media"].as_str(), c["class"].as_str(), c["step"].as_str()), (Some("image"), Some("synthesized"), Some("generate")));
    assert_eq!(c["operation"], "wai.generate.spectral");
    assert_eq!(c["models"], serde_json::json!([{"digest": model_sha, "type": "algorithmic"}]));
    assert_eq!((c["source"].as_str(), c["oversight"].as_str()), (Some("algorithmicMedia"), Some("prompt_guided")));
    assert_eq!(c["record"], r["receipt_hash"], "the claim's record is its receipt");
    assert_eq!(c["signer_pubkey"], r["signer_pubkey"], "one key signs both");
    // The receipt: the model and the program read, the picture written.
    let ops: Vec<&str> = r["steps"].as_array().unwrap().iter().map(|s| s["op"].as_str().unwrap()).collect();
    assert_eq!(ops, ["read", "read", "media.generate pic.rgb"]);
    assert_eq!((r["capability"].as_str(), r["media"].as_str()), (Some("wai.generate.spectral"), Some("image")));
    let (code, out) = verify(&d, &ks, &["."]);
    assert_eq!(code, Some(0), "{out}");
    assert!(out.contains("1 operation receipt and 1 origin claim verified"), "{out}");
    assert!(out.contains("synthesized image, step generate by wai.generate.spectral"), "{out}");
    // The second implementation reads the claim as wai-rs does.
    let (code, s) = second(&claim_file, &pic);
    assert_eq!(code, Some(0), "synthesis_verify.py claim: {s}");
    assert_eq!((s["outcome"].as_str(), s["statement"]["source"].as_str(), s["statement"]["class"].as_str()), (Some("accept"), Some("algorithmicMedia"), Some("synthesized")), "{s}");
    assert_eq!(s["statement"]["link"], v["origin"]["link"], "both name the claim by one link hash");

    // Planted fault: a picture changed after it was made fails both.
    let kept = std::fs::read(&pic).unwrap();
    flip_last(&pic);
    let (code, out) = verify(&d, &ks, &["."]);
    assert_eq!(code, Some(1), "{out}");
    assert!(out.contains("FAIL wrote pic.rgb, which has changed since"), "{out}");
    assert!(out.contains("FAIL its subject is samples of sha256"), "{out}");
    assert_eq!(second(&claim_file, &pic).0, Some(1), "the second implementation refuses it too");
    std::fs::write(&pic, &kept).unwrap();

    // Planted fault: a claim beside another picture's receipt.
    let mut b = GEN.to_vec();
    b.extend(["--model", &m, "-o", "other.rgb", "--seed", "8"]);
    ok(&wai_in(&d, &ks, &b));
    std::fs::copy(d.join("other.rgb.wai-origin.json"), &claim_file).unwrap();
    let (code, out) = verify(&d, &ks, &["pic.rgb.wai-origin.json"]);
    assert_eq!(code, Some(1), "{out}");
    assert!(out.contains("FAIL its record is"), "{out}");
    // An edited claim no longer verifies.
    let mut edited = jload(&d.join("other.rgb.wai-origin.json"));
    edited["models"][0]["type"] = "trained".into();
    std::fs::write(&claim_file, serde_json::to_vec(&edited).unwrap()).unwrap();
    let (code, out) = verify(&d, &ks, &["pic.rgb.wai-origin.json"]);
    assert_eq!(code, Some(1), "{out}");
    assert!(out.contains("FAIL the claim: bad_signature") || out.contains("FAIL the claim: source_mismatch"), "{out}");
}

#[test]
fn a_named_key_signs_and_one_that_is_not_there_refuses_with_nothing_written() {
    let d = scratch("key");
    let ks = d.join("ks");
    let k = data(&wai_in(&d, &ks, &["key", "create", "maker", "--for", "dev", "--store", "file", "--json"]));
    let m = model();
    let mut a = GEN.to_vec();
    a.extend(["--model", &m, "-o", "pic.rgb", "--key", "maker", "--json"]);
    let v = data(&wai_in(&d, &ks, &a));
    assert_eq!(v["receipt"]["run_key"].as_bool(), Some(false));
    assert_eq!(v["receipt"]["signer"].as_str(), k["did_key"].as_str(), "{v}");
    let (code, out) = verify(&d, &ks, &["pic.rgb.wai-receipt.json"]);
    assert_eq!(code, Some(0), "{out}");
    assert!(out.contains("signed by the keystore's key maker (maker)"), "{out}");
    // The key's own receipt, in the keystore, is signed by the key it made.
    let kr = jload(&ks.join("maker.json.wai-receipt.json"));
    assert_eq!(kr["signer_pubkey"].as_str(), k["key"].as_str());
    for (op, extra) in [("generate", vec!["--model", m.as_str(), "--prompt", "p", "--seed", "1", "--steps", "1", "--size", "4x4", "-o", "x.rgb"]), ("lock", vec![])] {
        let mut a = vec![op];
        a.extend(extra);
        a.extend(["--key", "nobody"]);
        let o = wai_in(&d, &ks, &a);
        assert!(!o.status.success(), "{op}: {}", text(&o));
        assert!(text(&o).contains("nobody"), "{op}: {}", text(&o));
    }
    assert!(!d.join("x.rgb").exists() && !d.join("x.rgb.wai-receipt.json").exists(), "a key that does not sign stops the write");
}

/// A project file's canonical digest, as `wai check --json` states it.
fn digest_of(p: &Path, ks: &Path) -> String {
    let o = wai_in(p, ks, &["check", "--json"]);
    let env = json_in(&String::from_utf8_lossy(&o.stdout));
    env["data"]["digest"].as_str().unwrap_or_else(|| panic!("{env}")).to_string()
}

#[test]
fn a_projects_receipts_name_its_digest_and_another_projects_are_refused() {
    let d = scratch("project");
    let ks = d.join("ks");
    let made = wai_in(&d, &ks, &["create", "room", "--template", "video-room"]);
    assert!(text(&made).contains(".wai/receipts/wai.json.wai-receipt.json"), "{}", text(&made));
    let p = d.join("room");
    let digest = digest_of(&p, &ks);
    let created = jload(&p.join(".wai/receipts/wai.json.wai-receipt.json"));
    let steps = created["steps"].as_array().unwrap();
    assert_eq!((steps[0]["op"].as_str(), steps[0]["output_hash"].as_str()), (Some("wai.json"), Some(digest.as_str())), "S4: the project's digest");
    assert_eq!(steps.last().unwrap()["op"], "project.create wai.json", "the project file last");
    assert!(steps.iter().any(|s| s["op"] == "project.create wai.lock"));
    // lock, build, and a generation inside the project: each in .wai/receipts/, each naming it.
    let lock = data(&wai_in(&p, &ks, &["lock", "--json"]));
    assert_eq!(lock["receipt"]["project_sha256"].as_str(), Some(digest.as_str()), "{lock}");
    let built = data(&wai_in(&p, &ks, &["build", "--json"]));
    assert_eq!(built["receipt"]["file"].as_str(), Some(p.join(".wai/receipts/dist/wai-build.json.wai-receipt.json").to_str().unwrap()), "{built}");
    let m = model();
    let mut a = GEN.to_vec();
    a.extend(["--model", &m, "-o", "art/pic.rgb", "--json"]);
    std::fs::create_dir_all(p.join("art")).unwrap();
    let g = data(&wai_in(&p, &ks, &a));
    assert_eq!(g["receipt"]["file"].as_str(), Some(p.join(".wai/receipts/art/pic.rgb.wai-receipt.json").to_str().unwrap()), "{g}");
    assert_eq!(g["receipt"]["project_sha256"].as_str(), Some(digest.as_str()));
    let (code, out) = verify(&p, &ks, &["."]);
    assert_eq!(code, Some(0), "{out}");
    assert!(out.contains("4 operation receipts and 1 origin claim verified"), "{out}");
    assert!(out.contains("the tree of dist/wai-build.json"), "{out}");
    // `.wai/` is never built into dist/.
    assert!(!p.join("dist/.wai").exists());
    // A build that runs again replaces its receipt, and still verifies.
    data(&wai_in(&p, &ks, &["build", "--json"]));

    // Planted fault: a file of the built tree changed.
    let page = p.join("dist/index.html");
    let kept = std::fs::read(&page).unwrap();
    std::fs::write(&page, b"<p>changed</p>").unwrap();
    let (code, out) = verify(&p, &ks, &[".wai/receipts/dist/wai-build.json.wai-receipt.json"]);
    assert_eq!(code, Some(1), "{out}");
    assert!(out.contains("FAIL the tree of dist/wai-build.json: index.html: changed since the build"), "{out}");
    std::fs::write(&page, &kept).unwrap();
    // Planted fault: a file in the tree the record does not list.
    std::fs::write(p.join("dist/extra.txt"), b"x").unwrap();
    let (code, out) = verify(&p, &ks, &[".wai/receipts/dist/wai-build.json.wai-receipt.json"]);
    assert_eq!(code, Some(1), "{out}");
    assert!(out.contains("extra.txt: in the tree, and the build record does not list it"), "{out}");
    std::fs::remove_file(p.join("dist/extra.txt")).unwrap();

    // Planted fault: another project's receipt. A copy of the project with another title: the lock
    // is the same bytes, and its receipt names the first project's digest.
    let q = d.join("other");
    copy_dir(&p, &q);
    let wj = std::fs::read_to_string(q.join("wai.json")).unwrap();
    let mut v: Value = serde_json::from_str(&wj).unwrap();
    v["title"] = "Another room".into();
    std::fs::write(q.join("wai.json"), serde_json::to_string_pretty(&v).unwrap()).unwrap();
    assert_ne!(digest_of(&q, &ks), digest);
    let (code, out) = verify(&q, &ks, &[".wai/receipts/wai.lock.wai-receipt.json"]);
    assert_eq!(code, Some(1), "{out}");
    assert!(out.contains("FAIL written under another project file") && out.contains("ok   wrote wai.lock"), "the lock is as written; its project is not: {out}");
    // The lock written again in that project: its receipt names that project, and verifies.
    ok(&wai_in(&q, &ks, &["lock"]));
    let (code, out) = verify(&q, &ks, &[".wai/receipts/wai.lock.wai-receipt.json"]);
    assert_eq!(code, Some(0), "{out}");
    // And a project with no project file holds none of its receipts to anything.
    std::fs::remove_file(q.join("wai.json")).unwrap();
    let (code, out) = verify(&q, &ks, &[".wai/receipts/wai.lock.wai-receipt.json"]);
    assert_eq!(code, Some(1), "{out}");
    assert!(out.contains("is not there to hold its project"), "{out}");
}

fn copy_dir(from: &Path, to: &Path) {
    std::fs::create_dir_all(to).unwrap();
    for e in std::fs::read_dir(from).unwrap().flatten() {
        let t = to.join(e.file_name());
        if e.path().is_dir() {
            copy_dir(&e.path(), &t);
        } else {
            std::fs::copy(e.path(), t).unwrap();
        }
    }
}

#[test]
fn decode_wrap_and_unwrap_write_receipts_beside_what_they_write() {
    let d = scratch("media");
    let ks = d.join("ks");
    ok(&wai_in(&d, &ks, &["hello", "h"]));
    let object = std::fs::read_dir(d.join("h")).unwrap().flatten().map(|e| e.path()).find(|p| p.extension().is_some_and(|x| x == "wai")).unwrap();
    let obj = object.to_str().unwrap();
    let decoded = ok(&wai_in(&d, &ks, &["decode", obj, "-o", "out.rgb"]));
    assert!(decoded.contains("receipt → ") && decoded.contains("out.rgb.wai-receipt.json"), "{decoded}");
    let r = jload(&d.join("out.rgb.wai-receipt.json"));
    let ops: Vec<&str> = r["steps"].as_array().unwrap().iter().map(|s| s["op"].as_str().unwrap()).collect();
    let read = format!("read h/{}", object.file_name().unwrap().to_str().unwrap());
    assert_eq!(ops, [read.as_str(), "media.decode out.rgb.meta", "media.decode out.rgb"], "the envelope read, the shape and the samples written");
    assert_eq!(r["capability"], "wai.image.png");
    // A file read from outside the receipt's base is named by its content hash alone.
    std::fs::create_dir_all(d.join("out")).unwrap();
    ok(&wai_in(&d, &ks, &["decode", obj, "-o", "out/x.rgb"]));
    let x = jload(&d.join("out/x.rgb.wai-receipt.json"));
    assert_eq!(x["steps"][0]["op"], "read", "{x}");
    ok(&wai_in(&d, &ks, &["unwrap", obj, "-o", "pic.png"]));
    ok(&wai_in(&d, &ks, &["wrap", "wai.image.png", "pic.png", "-o", "again.wai"]));
    let w = jload(&d.join("again.wai.wai-receipt.json"));
    let ops: Vec<&str> = w["steps"].as_array().unwrap().iter().map(|s| s["op"].as_str().unwrap()).collect();
    assert_eq!(ops, ["read pic.png", "envelope.wrap again.wai"]);
    let (code, out) = verify(&d, &ks, &["."]);
    assert_eq!(code, Some(0), "{out}");
    assert!(out.contains("4 operation receipts verified"), "{out}");
    assert!(out.contains("ok   read pic.png, as it is now"), "{out}");
    // Planted fault: the decoded samples changed, and the payload the envelope was made from.
    flip_last(&d.join("out.rgb"));
    flip_last(&d.join("pic.png"));
    let (code, out) = verify(&d, &ks, &["."]);
    assert_eq!(code, Some(1), "{out}");
    assert!(out.contains("FAIL wrote out.rgb, which has changed since"), "{out}");
    assert!(out.contains("FAIL wrote pic.png, which has changed since"), "{out}");
    // What wrap read has changed since: that is noted, and refuses nothing of the envelope's.
    assert!(out.contains("note read pic.png, which has changed since"), "{out}");
    assert!(out.contains("ok   wrote again.wai"), "{out}");
    // A dry run writes no receipt, and names the one it would write.
    let o = wai_in(&d, &ks, &["decode", obj, "-o", "dry.rgb", "--dry-run", "--json"]);
    assert!(o.status.success(), "{}", text(&o));
    assert!(text(&o).contains("dry.rgb.wai-receipt.json"), "{}", text(&o));
    assert!(!d.join("dry.rgb").exists() && !d.join("dry.rgb.wai-receipt.json").exists());
}

#[test]
fn key_operations_write_receipts_in_the_keystore_that_verify_reads() {
    let d = scratch("keys");
    let ks = d.join("ks");
    let made = data(&wai_in(&d, &ks, &["key", "create", "rel", "--for", "release", "--store", "file", "--json"]));
    let rec = ks.join("rel.json.wai-receipt.json");
    assert_eq!(made["receipt"]["file"].as_str(), Some(rec.to_str().unwrap()), "{made}");
    assert_eq!(jload(&rec)["signer_pubkey"].as_str(), made["key"].as_str(), "the key signs its own record's receipt");
    let rot = data(&wai_in(&d, &ks, &["key", "rotate", "rel", "--yes", "--json"]));
    assert_eq!(jload(&rec)["signer_pubkey"].as_str(), rot["key"].as_str(), "the successor signs the rotation's");
    let rev = data(&wai_in(&d, &ks, &["key", "revoke", "rel", "--yes", "--json"]));
    assert_ne!(jload(&rec)["signer_pubkey"].as_str(), rev["key"].as_str(), "a revoked key signs nothing, its revocation among it");
    ok(&wai_in(&d, &ks, &["key", "create", "other", "--for", "dev", "--store", "file"]));
    ok(&wai_in(&d, &ks, &["key", "export", "other", "--out", "other.pub"]));
    assert!(d.join("other.pub.wai-receipt.json").is_file());
    // The receipts are not keys: the keystore lists its two.
    let list = data(&wai_in(&d, &ks, &["key", "list", "--json"]));
    assert_eq!(list["keys"].as_array().map(Vec::len), Some(2), "{list}");
    let ksp = ks.to_str().unwrap();
    let (code, out) = verify(&d, &ks, &[ksp, "other.pub.wai-receipt.json"]);
    assert_eq!(code, Some(0), "{out}");
    assert!(out.contains("3 operation receipts verified"), "{out}");
    // Planted fault: a key's record edited by hand.
    let path = ks.join("other.json");
    let t = std::fs::read_to_string(&path).unwrap().replace("\"dev\"", "\"audit\"");
    std::fs::write(&path, t).unwrap();
    let (code, out) = verify(&d, &ks, &[ksp]);
    assert_eq!(code, Some(1), "{out}");
    assert!(out.contains("FAIL wrote other.json, which has changed since"), "{out}");
}