//! Operation receipts through the `wai` binary (`--features quickstart,generate,synthesis --test //! cli_receipts`): every developer-facing write leaves a receipt that `wai verify` reads //! (`src/bin/wai_cli/dx/receipt.rs`), and a generation an origin claim beside it //! (extensions/synthesis.md §3.5). //! //! - `wai generate` signs an origin claim (step `generate`, the model by its SHA-256, type //! `algorithmic`, operation `wai.generate.spectral`, its record the receipt) and the receipt; //! `wai verify` reads both, and so does the claim's second implementation, //! `live-conformance/synthesis_verify.py claim`, which gives it the statement wai-rs gives it; //! - `wai create`, `wai lock`, `wai build`, `wai decode`, `wai wrap`, `wai unwrap` and `wai key //! create|rotate|revoke|export` each write one, in a project's `.wai/receipts/` or beside the file //! written, each naming the digest of the `wai.json` it ran under; //! - planted faults: a changed output (a picture, a decoded file, a file of a built tree, a key's //! record) fails `wai verify`; a receipt with another project's digest fails it; a claim beside //! another picture's receipt fails it; and `--key` that names no key refuses with nothing written. #![cfg(all(feature = "quickstart", feature = "generate", feature = "synthesis"))] use serde_json::Value; use std::path::{Path, PathBuf}; use std::process::{Command, Output, Stdio}; fn root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) } fn model() -> String { root().join("tests/vectors/generate/generate_spectral_model.json").to_string_lossy().into_owned() } fn scratch(name: &str) -> PathBuf { let d = Path::new(env!("CARGO_TARGET_TMPDIR")).join("cli_receipts").join(name); let _ = std::fs::remove_dir_all(&d); std::fs::create_dir_all(&d).unwrap(); std::fs::canonicalize(&d).unwrap() } /// `wai` in `d`, its keystore `/ks` (`$WAI_KEYSTORE`), so no run reads or writes another. fn wai_in(d: &Path, ks: &Path, args: &[&str]) -> Output { Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(d).args(args).env("WAI_KEYSTORE", ks).env("NO_COLOR", "1").stdin(Stdio::null()).output().expect("wai runs") } fn text(o: &Output) -> String { format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr)) } fn ok(o: &Output) -> String { assert!(o.status.success(), "wai failed: {}", text(o)); String::from_utf8_lossy(&o.stdout).into_owned() } /// The first JSON value printed on a line of its own in `s`, however many lines it takes. fn json_in(s: &str) -> Value { let at = if s.starts_with('{') { 0 } else { s.find("\n{").map(|i| i + 1).unwrap_or_else(|| panic!("no JSON in {s}")) }; serde_json::Deserializer::from_str(&s[at..]).into_iter::().next().expect("a value").unwrap_or_else(|e| panic!("{e}: {s}")) } /// The `data` of the JSON envelope a `--json` run prints. fn data(o: &Output) -> Value { json_in(&ok(o))["data"].clone() } fn jload(p: &Path) -> Value { serde_json::from_slice(&std::fs::read(p).unwrap_or_else(|e| panic!("{}: {e}", p.display()))).unwrap() } fn sha256_hex(b: &[u8]) -> String { wai::pin::sha256(b).iter().map(|x| format!("{x:02x}")).collect() } fn flip_last(p: &Path) { let mut b = std::fs::read(p).unwrap(); let n = b.len() - 1; b[n] ^= 1; std::fs::write(p, b).unwrap(); } /// `wai verify ` in `d`: its exit status and its output. fn verify(d: &Path, ks: &Path, paths: &[&str]) -> (Option, String) { let mut a = vec!["verify"]; a.extend_from_slice(paths); let o = wai_in(d, ks, &a); (o.status.code(), text(&o)) } /// The claim's second implementation (live-conformance/synthesis_verify.py), on one claim and the /// samples it is about: its exit status and its outcome. fn second(claim: &Path, samples: &Path) -> (Option, Value) { let script = root().join("../live-conformance/synthesis_verify.py"); let o = Command::new(std::env::var_os("PYTHON").unwrap_or_else(|| "python3".into())) .arg("-I") .arg(script) .args(["claim", claim.to_str().unwrap(), samples.to_str().unwrap()]) .stdin(Stdio::null()) .output() .expect("python3 runs"); let out = String::from_utf8_lossy(&o.stdout).into_owned(); let v = out.lines().rev().find(|l| l.starts_with('{')).and_then(|l| serde_json::from_str(l).ok()).unwrap_or(Value::Null); (o.status.code(), v) } const GEN: &[&str] = &["generate", "--prompt", "a quiet harbour at dusk", "--seed", "7", "--steps", "2", "--size", "16x12"]; #[test] fn a_generation_signs_its_origin_claim_and_receipt_and_verify_reads_both() { let d = scratch("generate"); let ks = d.join("ks"); let m = model(); let mut a = GEN.to_vec(); a.extend(["--model", &m, "-o", "pic.rgb", "--json"]); let v = data(&wai_in(&d, &ks, &a)); let pic = d.join("pic.rgb"); let rgb = std::fs::read(&pic).unwrap(); assert_eq!(v["sha256"].as_str(), Some(sha256_hex(&rgb).as_str())); // Outside a project: both beside the picture, signed by a key made for this run. let receipt = d.join("pic.rgb.wai-receipt.json"); let claim_file = d.join("pic.rgb.wai-origin.json"); assert_eq!(v["receipt"]["file"].as_str(), Some(receipt.to_str().unwrap()), "{v}"); assert_eq!(v["origin"]["file"].as_str(), Some(claim_file.to_str().unwrap()), "{v}"); assert_eq!((v["receipt"]["run_key"].as_bool(), v["receipt"]["project_sha256"].as_str()), (Some(true), None), "{v}"); // The claim: what §3.5 says a generation's claim is. let c = jload(&claim_file); let r = jload(&receipt); let model_sha = sha256_hex(&std::fs::read(&m).unwrap()); assert_eq!(c["kind"], "wai-origin"); assert_eq!(c["subject"], serde_json::json!({"type": "samples", "samples": sha256_hex(&rgb)})); assert_eq!((c["media"].as_str(), c["class"].as_str(), c["step"].as_str()), (Some("image"), Some("synthesized"), Some("generate"))); assert_eq!(c["operation"], "wai.generate.spectral"); assert_eq!(c["models"], serde_json::json!([{"digest": model_sha, "type": "algorithmic"}])); assert_eq!((c["source"].as_str(), c["oversight"].as_str()), (Some("algorithmicMedia"), Some("prompt_guided"))); assert_eq!(c["record"], r["receipt_hash"], "the claim's record is its receipt"); assert_eq!(c["signer_pubkey"], r["signer_pubkey"], "one key signs both"); // The receipt: the model and the program read, the picture written. let ops: Vec<&str> = r["steps"].as_array().unwrap().iter().map(|s| s["op"].as_str().unwrap()).collect(); assert_eq!(ops, ["read", "read", "media.generate pic.rgb"]); assert_eq!((r["capability"].as_str(), r["media"].as_str()), (Some("wai.generate.spectral"), Some("image"))); let (code, out) = verify(&d, &ks, &["."]); assert_eq!(code, Some(0), "{out}"); assert!(out.contains("1 operation receipt and 1 origin claim verified"), "{out}"); assert!(out.contains("synthesized image, step generate by wai.generate.spectral"), "{out}"); // The second implementation reads the claim as wai-rs does. let (code, s) = second(&claim_file, &pic); assert_eq!(code, Some(0), "synthesis_verify.py claim: {s}"); assert_eq!((s["outcome"].as_str(), s["statement"]["source"].as_str(), s["statement"]["class"].as_str()), (Some("accept"), Some("algorithmicMedia"), Some("synthesized")), "{s}"); assert_eq!(s["statement"]["link"], v["origin"]["link"], "both name the claim by one link hash"); // Planted fault: a picture changed after it was made fails both. let kept = std::fs::read(&pic).unwrap(); flip_last(&pic); let (code, out) = verify(&d, &ks, &["."]); assert_eq!(code, Some(1), "{out}"); assert!(out.contains("FAIL wrote pic.rgb, which has changed since"), "{out}"); assert!(out.contains("FAIL its subject is samples of sha256"), "{out}"); assert_eq!(second(&claim_file, &pic).0, Some(1), "the second implementation refuses it too"); std::fs::write(&pic, &kept).unwrap(); // Planted fault: a claim beside another picture's receipt. let mut b = GEN.to_vec(); b.extend(["--model", &m, "-o", "other.rgb", "--seed", "8"]); ok(&wai_in(&d, &ks, &b)); std::fs::copy(d.join("other.rgb.wai-origin.json"), &claim_file).unwrap(); let (code, out) = verify(&d, &ks, &["pic.rgb.wai-origin.json"]); assert_eq!(code, Some(1), "{out}"); assert!(out.contains("FAIL its record is"), "{out}"); // An edited claim no longer verifies. let mut edited = jload(&d.join("other.rgb.wai-origin.json")); edited["models"][0]["type"] = "trained".into(); std::fs::write(&claim_file, serde_json::to_vec(&edited).unwrap()).unwrap(); let (code, out) = verify(&d, &ks, &["pic.rgb.wai-origin.json"]); assert_eq!(code, Some(1), "{out}"); assert!(out.contains("FAIL the claim: bad_signature") || out.contains("FAIL the claim: source_mismatch"), "{out}"); } #[test] fn a_named_key_signs_and_one_that_is_not_there_refuses_with_nothing_written() { let d = scratch("key"); let ks = d.join("ks"); let k = data(&wai_in(&d, &ks, &["key", "create", "maker", "--for", "dev", "--store", "file", "--json"])); let m = model(); let mut a = GEN.to_vec(); a.extend(["--model", &m, "-o", "pic.rgb", "--key", "maker", "--json"]); let v = data(&wai_in(&d, &ks, &a)); assert_eq!(v["receipt"]["run_key"].as_bool(), Some(false)); assert_eq!(v["receipt"]["signer"].as_str(), k["did_key"].as_str(), "{v}"); let (code, out) = verify(&d, &ks, &["pic.rgb.wai-receipt.json"]); assert_eq!(code, Some(0), "{out}"); assert!(out.contains("signed by the keystore's key maker (maker)"), "{out}"); // The key's own receipt, in the keystore, is signed by the key it made. let kr = jload(&ks.join("maker.json.wai-receipt.json")); assert_eq!(kr["signer_pubkey"].as_str(), k["key"].as_str()); for (op, extra) in [("generate", vec!["--model", m.as_str(), "--prompt", "p", "--seed", "1", "--steps", "1", "--size", "4x4", "-o", "x.rgb"]), ("lock", vec![])] { let mut a = vec![op]; a.extend(extra); a.extend(["--key", "nobody"]); let o = wai_in(&d, &ks, &a); assert!(!o.status.success(), "{op}: {}", text(&o)); assert!(text(&o).contains("nobody"), "{op}: {}", text(&o)); } assert!(!d.join("x.rgb").exists() && !d.join("x.rgb.wai-receipt.json").exists(), "a key that does not sign stops the write"); } /// A project file's canonical digest, as `wai check --json` states it. fn digest_of(p: &Path, ks: &Path) -> String { let o = wai_in(p, ks, &["check", "--json"]); let env = json_in(&String::from_utf8_lossy(&o.stdout)); env["data"]["digest"].as_str().unwrap_or_else(|| panic!("{env}")).to_string() } #[test] fn a_projects_receipts_name_its_digest_and_another_projects_are_refused() { let d = scratch("project"); let ks = d.join("ks"); let made = wai_in(&d, &ks, &["create", "room", "--template", "video-room"]); assert!(text(&made).contains(".wai/receipts/wai.json.wai-receipt.json"), "{}", text(&made)); let p = d.join("room"); let digest = digest_of(&p, &ks); let created = jload(&p.join(".wai/receipts/wai.json.wai-receipt.json")); let steps = created["steps"].as_array().unwrap(); assert_eq!((steps[0]["op"].as_str(), steps[0]["output_hash"].as_str()), (Some("wai.json"), Some(digest.as_str())), "S4: the project's digest"); assert_eq!(steps.last().unwrap()["op"], "project.create wai.json", "the project file last"); assert!(steps.iter().any(|s| s["op"] == "project.create wai.lock")); // lock, build, and a generation inside the project: each in .wai/receipts/, each naming it. let lock = data(&wai_in(&p, &ks, &["lock", "--json"])); assert_eq!(lock["receipt"]["project_sha256"].as_str(), Some(digest.as_str()), "{lock}"); let built = data(&wai_in(&p, &ks, &["build", "--json"])); assert_eq!(built["receipt"]["file"].as_str(), Some(p.join(".wai/receipts/dist/wai-build.json.wai-receipt.json").to_str().unwrap()), "{built}"); let m = model(); let mut a = GEN.to_vec(); a.extend(["--model", &m, "-o", "art/pic.rgb", "--json"]); std::fs::create_dir_all(p.join("art")).unwrap(); let g = data(&wai_in(&p, &ks, &a)); assert_eq!(g["receipt"]["file"].as_str(), Some(p.join(".wai/receipts/art/pic.rgb.wai-receipt.json").to_str().unwrap()), "{g}"); assert_eq!(g["receipt"]["project_sha256"].as_str(), Some(digest.as_str())); let (code, out) = verify(&p, &ks, &["."]); assert_eq!(code, Some(0), "{out}"); assert!(out.contains("4 operation receipts and 1 origin claim verified"), "{out}"); assert!(out.contains("the tree of dist/wai-build.json"), "{out}"); // `.wai/` is never built into dist/. assert!(!p.join("dist/.wai").exists()); // A build that runs again replaces its receipt, and still verifies. data(&wai_in(&p, &ks, &["build", "--json"])); // Planted fault: a file of the built tree changed. let page = p.join("dist/index.html"); let kept = std::fs::read(&page).unwrap(); std::fs::write(&page, b"

changed

").unwrap(); let (code, out) = verify(&p, &ks, &[".wai/receipts/dist/wai-build.json.wai-receipt.json"]); assert_eq!(code, Some(1), "{out}"); assert!(out.contains("FAIL the tree of dist/wai-build.json: index.html: changed since the build"), "{out}"); std::fs::write(&page, &kept).unwrap(); // Planted fault: a file in the tree the record does not list. std::fs::write(p.join("dist/extra.txt"), b"x").unwrap(); let (code, out) = verify(&p, &ks, &[".wai/receipts/dist/wai-build.json.wai-receipt.json"]); assert_eq!(code, Some(1), "{out}"); assert!(out.contains("extra.txt: in the tree, and the build record does not list it"), "{out}"); std::fs::remove_file(p.join("dist/extra.txt")).unwrap(); // Planted fault: another project's receipt. A copy of the project with another title: the lock // is the same bytes, and its receipt names the first project's digest. let q = d.join("other"); copy_dir(&p, &q); let wj = std::fs::read_to_string(q.join("wai.json")).unwrap(); let mut v: Value = serde_json::from_str(&wj).unwrap(); v["title"] = "Another room".into(); std::fs::write(q.join("wai.json"), serde_json::to_string_pretty(&v).unwrap()).unwrap(); assert_ne!(digest_of(&q, &ks), digest); let (code, out) = verify(&q, &ks, &[".wai/receipts/wai.lock.wai-receipt.json"]); assert_eq!(code, Some(1), "{out}"); assert!(out.contains("FAIL written under another project file") && out.contains("ok wrote wai.lock"), "the lock is as written; its project is not: {out}"); // The lock written again in that project: its receipt names that project, and verifies. ok(&wai_in(&q, &ks, &["lock"])); let (code, out) = verify(&q, &ks, &[".wai/receipts/wai.lock.wai-receipt.json"]); assert_eq!(code, Some(0), "{out}"); // And a project with no project file holds none of its receipts to anything. std::fs::remove_file(q.join("wai.json")).unwrap(); let (code, out) = verify(&q, &ks, &[".wai/receipts/wai.lock.wai-receipt.json"]); assert_eq!(code, Some(1), "{out}"); assert!(out.contains("is not there to hold its project"), "{out}"); } fn copy_dir(from: &Path, to: &Path) { std::fs::create_dir_all(to).unwrap(); for e in std::fs::read_dir(from).unwrap().flatten() { let t = to.join(e.file_name()); if e.path().is_dir() { copy_dir(&e.path(), &t); } else { std::fs::copy(e.path(), t).unwrap(); } } } #[test] fn decode_wrap_and_unwrap_write_receipts_beside_what_they_write() { let d = scratch("media"); let ks = d.join("ks"); ok(&wai_in(&d, &ks, &["hello", "h"])); let object = std::fs::read_dir(d.join("h")).unwrap().flatten().map(|e| e.path()).find(|p| p.extension().is_some_and(|x| x == "wai")).unwrap(); let obj = object.to_str().unwrap(); let decoded = ok(&wai_in(&d, &ks, &["decode", obj, "-o", "out.rgb"])); assert!(decoded.contains("receipt → ") && decoded.contains("out.rgb.wai-receipt.json"), "{decoded}"); let r = jload(&d.join("out.rgb.wai-receipt.json")); let ops: Vec<&str> = r["steps"].as_array().unwrap().iter().map(|s| s["op"].as_str().unwrap()).collect(); let read = format!("read h/{}", object.file_name().unwrap().to_str().unwrap()); assert_eq!(ops, [read.as_str(), "media.decode out.rgb.meta", "media.decode out.rgb"], "the envelope read, the shape and the samples written"); assert_eq!(r["capability"], "wai.image.png"); // A file read from outside the receipt's base is named by its content hash alone. std::fs::create_dir_all(d.join("out")).unwrap(); ok(&wai_in(&d, &ks, &["decode", obj, "-o", "out/x.rgb"])); let x = jload(&d.join("out/x.rgb.wai-receipt.json")); assert_eq!(x["steps"][0]["op"], "read", "{x}"); ok(&wai_in(&d, &ks, &["unwrap", obj, "-o", "pic.png"])); ok(&wai_in(&d, &ks, &["wrap", "wai.image.png", "pic.png", "-o", "again.wai"])); let w = jload(&d.join("again.wai.wai-receipt.json")); let ops: Vec<&str> = w["steps"].as_array().unwrap().iter().map(|s| s["op"].as_str().unwrap()).collect(); assert_eq!(ops, ["read pic.png", "envelope.wrap again.wai"]); let (code, out) = verify(&d, &ks, &["."]); assert_eq!(code, Some(0), "{out}"); assert!(out.contains("4 operation receipts verified"), "{out}"); assert!(out.contains("ok read pic.png, as it is now"), "{out}"); // Planted fault: the decoded samples changed, and the payload the envelope was made from. flip_last(&d.join("out.rgb")); flip_last(&d.join("pic.png")); let (code, out) = verify(&d, &ks, &["."]); assert_eq!(code, Some(1), "{out}"); assert!(out.contains("FAIL wrote out.rgb, which has changed since"), "{out}"); assert!(out.contains("FAIL wrote pic.png, which has changed since"), "{out}"); // What wrap read has changed since: that is noted, and refuses nothing of the envelope's. assert!(out.contains("note read pic.png, which has changed since"), "{out}"); assert!(out.contains("ok wrote again.wai"), "{out}"); // A dry run writes no receipt, and names the one it would write. let o = wai_in(&d, &ks, &["decode", obj, "-o", "dry.rgb", "--dry-run", "--json"]); assert!(o.status.success(), "{}", text(&o)); assert!(text(&o).contains("dry.rgb.wai-receipt.json"), "{}", text(&o)); assert!(!d.join("dry.rgb").exists() && !d.join("dry.rgb.wai-receipt.json").exists()); } #[test] fn key_operations_write_receipts_in_the_keystore_that_verify_reads() { let d = scratch("keys"); let ks = d.join("ks"); let made = data(&wai_in(&d, &ks, &["key", "create", "rel", "--for", "release", "--store", "file", "--json"])); let rec = ks.join("rel.json.wai-receipt.json"); assert_eq!(made["receipt"]["file"].as_str(), Some(rec.to_str().unwrap()), "{made}"); assert_eq!(jload(&rec)["signer_pubkey"].as_str(), made["key"].as_str(), "the key signs its own record's receipt"); let rot = data(&wai_in(&d, &ks, &["key", "rotate", "rel", "--yes", "--json"])); assert_eq!(jload(&rec)["signer_pubkey"].as_str(), rot["key"].as_str(), "the successor signs the rotation's"); let rev = data(&wai_in(&d, &ks, &["key", "revoke", "rel", "--yes", "--json"])); assert_ne!(jload(&rec)["signer_pubkey"].as_str(), rev["key"].as_str(), "a revoked key signs nothing, its revocation among it"); ok(&wai_in(&d, &ks, &["key", "create", "other", "--for", "dev", "--store", "file"])); ok(&wai_in(&d, &ks, &["key", "export", "other", "--out", "other.pub"])); assert!(d.join("other.pub.wai-receipt.json").is_file()); // The receipts are not keys: the keystore lists its two. let list = data(&wai_in(&d, &ks, &["key", "list", "--json"])); assert_eq!(list["keys"].as_array().map(Vec::len), Some(2), "{list}"); let ksp = ks.to_str().unwrap(); let (code, out) = verify(&d, &ks, &[ksp, "other.pub.wai-receipt.json"]); assert_eq!(code, Some(0), "{out}"); assert!(out.contains("3 operation receipts verified"), "{out}"); // Planted fault: a key's record edited by hand. let path = ks.join("other.json"); let t = std::fs::read_to_string(&path).unwrap().replace("\"dev\"", "\"audit\""); std::fs::write(&path, t).unwrap(); let (code, out) = verify(&d, &ks, &[ksp]); assert_eq!(code, Some(1), "{out}"); assert!(out.contains("FAIL wrote other.json, which has changed since"), "{out}"); }