//! `wai conform` run against implementations of each kind, through both adapters, and its //! records checked by `wai verify`: //! //! - the reference verifiers of the claim, JWP and key-discovery corpora (`wai_*_vectors //! verdicts`, whole-corpus runs) conform, and so does the key-discovery reference behind //! an HTTP adapter; one that changes one verdict does not, at that case alone; //! - the score reference renderer (`wai score render`) conforms, and one that changes one //! byte of its output fails every case; //! - the reference origin (`wai http origin`) serving the HTTP corpus's objects conforms, //! and one serving an object with one byte changed fails that object; //! - a record changed after signing, in a case's outcome or in its summary, is refused by //! `wai verify`; a dry run writes nothing; a record is not replaced without `--yes`; //! each failure to run has its exit status. #![cfg(all(feature = "quickstart", feature = "score", feature = "key_discovery"))] use serde_json::Value; use std::io::{BufRead, Read, Write}; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Output, Stdio}; fn tmp(name: &str) -> PathBuf { let d = std::env::temp_dir().join(format!("wai-conform-{name}-{}", std::process::id())); let _ = std::fs::remove_dir_all(&d); std::fs::create_dir_all(&d).unwrap(); d } fn wai_dir() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf() } fn wai(dir: &Path, args: &[&str]) -> Output { Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(dir).args(args).output().expect("wai runs") } fn text(o: &Output) -> String { format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr)) } /// `wai conform --suite --impl --json` writing its record to `out`. fn conform(dir: &Path, kind: &str, suite: &str, imp: &str, out: &Path) -> (i32, Value) { let root = wai_dir(); let o = wai(dir, &["conform", kind, "--suite", suite, "--impl", imp, "--root", root.to_str().unwrap(), "--out", out.to_str().unwrap(), "--yes", "--json"]); let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("JSON: {}", text(&o))); (o.status.code().unwrap_or(-1), v) } fn q(p: &str) -> String { format!("'{}'", p.replace('\'', "'\\''")) } #[test] fn the_reference_verifiers_conform_and_one_changed_verdict_does_not() { let d = tmp("verifiers"); for (suite, bin) in [("claim", env!("CARGO_BIN_EXE_wai_claim_vectors")), ("jwp", env!("CARGO_BIN_EXE_wai_jwp_vectors")), ("signer", env!("CARGO_BIN_EXE_wai_signer_vectors"))] { let out = d.join(format!("{suite}.wai-conform.json")); let (code, v) = conform(&d, "verifier", suite, &format!("{} verdicts {{corpus}}", q(bin)), &out); assert_eq!(code, 0, "{suite}: {v:#}"); assert_eq!(v["data"]["conformant"], true); assert!(v["data"]["cases"].as_u64().unwrap() > 100, "{v}"); let rec: Value = serde_json::from_slice(&std::fs::read(&out).unwrap()).unwrap(); assert_eq!(rec["wai"], "conform-result/1"); assert!(rec["corpus"]["digest"].as_str().unwrap().starts_with("sha256:")); assert!(rec["standard"].as_str().unwrap().starts_with('v')); assert!(rec["implementation"]["files"].as_array().unwrap().iter().any(|f| f["path"].as_str().unwrap().ends_with(&format!("wai_{}_vectors", if suite == "signer" { "signer" } else { suite }))), "{}", rec["implementation"]); } let o = wai(&d, &["verify", d.to_str().unwrap()]); assert!(o.status.success(), "{}", text(&o)); assert!(text(&o).contains("3 conformance results verified"), "{}", text(&o)); // One verdict changed: its case fails, and no other. let bad = format!("{} verdicts {{corpus}} | awk 'NR == 1 {{ sub(/\\t/, \"\\t~\") }} 1'", q(env!("CARGO_BIN_EXE_wai_signer_vectors"))); let (code, v) = conform(&d, "verifier", "signer", &bad, &d.join("bad.wai-conform.json")); assert_eq!(code, 1, "{v:#}"); assert_eq!(v["data"]["conformant"], false); assert_eq!(v["data"]["failed"], 1, "{v:#}"); // A record that says "not conformant" is still a genuine record. let o = wai(&d, &["verify", d.join("bad.wai-conform.json").to_str().unwrap()]); assert!(o.status.success() && text(&o).contains("not conformant"), "{}", text(&o)); } #[test] fn a_tampered_record_is_refused() { let d = tmp("tamper"); let out = d.join("signer.wai-conform.json"); let bad = format!("{} verdicts {{corpus}} | awk 'NR == 1 {{ sub(/\\t/, \"\\t~\") }} 1'", q(env!("CARGO_BIN_EXE_wai_signer_vectors"))); let (code, _) = conform(&d, "verifier", "signer", &bad, &out); assert_eq!(code, 1); let good = std::fs::read(&out).unwrap(); let mut rec: Value = serde_json::from_slice(&good).unwrap(); // The failed case turned into a pass, the summary and verdict made to agree with it: the // signature no longer covers what the record says. let cases = rec["cases"].as_array_mut().unwrap(); let i = cases.iter().position(|c| c["pass"] == false).unwrap(); cases[i]["pass"] = Value::Bool(true); rec["summary"]["passed"] = Value::from(rec["summary"]["passed"].as_u64().unwrap() + 1); rec["summary"]["failed"] = Value::from(0); rec["conformant"] = Value::Bool(true); std::fs::write(&out, serde_json::to_vec_pretty(&rec).unwrap()).unwrap(); let o = wai(&d, &["verify", out.to_str().unwrap()]); assert_eq!(o.status.code(), Some(1), "{}", text(&o)); assert!(text(&o).contains("the signature does not verify"), "{}", text(&o)); // One byte of the corpus digest changed. let s = String::from_utf8(good.clone()).unwrap(); let at = s.find("\"sha256:").unwrap() + 8; let mut b = good.clone(); b[at] = if b[at] == b'0' { b'1' } else { b'0' }; std::fs::write(&out, &b).unwrap(); assert_eq!(wai(&d, &["verify", out.to_str().unwrap()]).status.code(), Some(1)); // The summary alone changed, the signature left: refused at the signature // (conformance-results ยง4 step 4, before the summary is counted). let mut rec: Value = serde_json::from_slice(&good).unwrap(); rec["summary"]["failed"] = Value::from(0); std::fs::write(&out, serde_json::to_vec(&rec).unwrap()).unwrap(); let o = wai(&d, &["verify", out.to_str().unwrap()]); assert_eq!(o.status.code(), Some(1)); assert!(text(&o).contains("step 4 signature_invalid"), "{}", text(&o)); std::fs::write(&out, &good).unwrap(); assert!(wai(&d, &["verify", out.to_str().unwrap()]).status.success()); } #[test] fn the_score_renderer_conforms_and_one_flipped_output_byte_fails() { let d = tmp("score"); let w = q(env!("CARGO_BIN_EXE_wai")); let (code, v) = conform(&d, "decoder", "score", &format!("{w} score render {{in}} --pcm {{out}}"), &d.join("score.wai-conform.json")); assert_eq!(code, 0, "{v:#}"); assert_eq!(v["data"]["passed"], 3); // One byte of the output changed (byte 100 of the PCM, plus one). let flip = format!("{w} score render {{in}} --pcm {{out}} >/dev/null && dd if={{out}} bs=1 skip=100 count=1 2>/dev/null | LC_ALL=C tr '\\000-\\377' '\\001-\\377\\000' | dd of={{out}} bs=1 seek=100 count=1 conv=notrunc 2>/dev/null"); let (code, v) = conform(&d, "decoder", "score", &flip, &d.join("flip.wai-conform.json")); assert_eq!(code, 1, "{v:#}"); assert_eq!(v["data"]["failed"], 3, "{v:#}"); assert_eq!(v["data"]["passed"], 0); } /// `wai http origin --root ` on a port the system picks: the child and its URL, from /// the line it prints once it listens. fn origin(root: &Path) -> (Child, String) { let mut c = Command::new(env!("CARGO_BIN_EXE_wai")).args(["http", "origin", "--root", root.to_str().unwrap(), "--port", "0"]).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().unwrap(); let mut line = String::new(); std::io::BufReader::new(c.stdout.as_mut().unwrap()).read_line(&mut line).unwrap(); let url = line.split_whitespace().find(|w| w.starts_with("http://127.0.0.1:")).unwrap_or_else(|| panic!("{line}")).trim_end_matches('/').to_string(); (c, url) } #[test] fn the_reference_origin_conforms_and_one_changed_object_fails() { let d = tmp("origin"); let objects = wai_dir().join("http-conformance").join("objects"); let (mut c, url) = origin(&objects); let (code, v) = conform(&d, "origin", "http", &url, &d.join("http.wai-conform.json")); let _ = c.kill(); assert_eq!(code, 0, "{v:#}"); // A copy of the objects with one byte of one object changed. let copy = d.join("objects"); std::fs::create_dir_all(©).unwrap(); let mut changed = None; for e in std::fs::read_dir(&objects).unwrap().flatten() { if e.path().is_file() { let mut b = std::fs::read(e.path()).unwrap(); let name = e.file_name().to_string_lossy().into_owned(); if changed.is_none() && name.ends_with(".wai") && b.len() > 2000 { let n = b.len(); b[n - 1] ^= 1; changed = Some(name); } std::fs::write(copy.join(e.file_name()), b).unwrap(); } } let (mut c, url) = origin(©); let (code, v) = conform(&d, "origin", "http", &url, &d.join("bad.wai-conform.json")); let _ = c.kill(); assert_eq!(code, 1, "{v:#}"); assert_eq!(v["data"]["failed"], 1, "{v:#}"); assert!(v["data"]["failures"][0]["got"].as_str().unwrap().contains("the body is not the object"), "{v:#}"); } /// A verifier behind HTTP: the reference's verdicts, served one case a request. #[test] fn a_verifier_behind_http_conforms() { let d = tmp("http-verifier"); let corpus = wai_dir().join("signer-conformance").join("vectors.json"); let o = Command::new(env!("CARGO_BIN_EXE_wai_signer_vectors")).arg("verdicts").arg(&corpus).output().unwrap(); let verdicts: std::collections::HashMap = String::from_utf8_lossy(&o.stdout).lines().filter_map(|l| l.split_once('\t')).map(|(a, b)| (a.to_string(), b.to_string())).collect(); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}/verify", listener.local_addr().unwrap()); std::thread::spawn(move || { for s in listener.incoming().flatten() { let mut s = s; let mut buf = Vec::new(); let mut chunk = [0u8; 65536]; let body = loop { let n = s.read(&mut chunk).unwrap_or(0); if n == 0 { break None; } buf.extend_from_slice(&chunk[..n]); if let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") { let head = String::from_utf8_lossy(&buf[..end]).to_ascii_lowercase(); let len: usize = head.lines().find_map(|l| l.strip_prefix("content-length:")).and_then(|v| v.trim().parse().ok()).unwrap_or(0); if buf.len() >= end + 4 + len { break Some(buf[end + 4..end + 4 + len].to_vec()); } } }; let v: Value = body.and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or(Value::Null); let verdict = verdicts.get(v["name"].as_str().unwrap_or("")).cloned().unwrap_or_default(); let reply = serde_json::json!({"verdict": verdict}).to_string(); let _ = s.write_all(format!("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{reply}", reply.len()).as_bytes()); } }); let (code, v) = conform(&d, "verifier", "signer", &url, &d.join("signer.wai-conform.json")); assert_eq!(code, 0, "{v:#}"); let rec: Value = serde_json::from_slice(&std::fs::read(d.join("signer.wai-conform.json")).unwrap()).unwrap(); assert_eq!(rec["implementation"]["adapter"], "http"); } #[test] fn a_dry_run_writes_nothing_and_each_failure_has_its_exit_status() { let d = tmp("statuses"); let root = wai_dir(); let r = root.to_str().unwrap(); let w = q(env!("CARGO_BIN_EXE_wai")); let imp = format!("{w} score render {{in}} --pcm {{out}}"); let o = wai(&d, &["conform", "decoder", "--suite", "score", "--impl", &imp, "--root", r, "--dry-run", "--json"]); assert!(o.status.success(), "{}", text(&o)); let v: Value = serde_json::from_slice(&o.stdout).unwrap(); assert_eq!(v["dry_run"], true); assert!(std::fs::read_dir(&d).unwrap().next().is_none(), "a dry run writes nothing"); // The record exists: not replaced without --yes (4), the proposal and the confirming command // in the envelope. std::fs::write(d.join("score.wai-conform.json"), "{}").unwrap(); let o = wai(&d, &["conform", "decoder", "--suite", "score", "--impl", &imp, "--root", r, "--json"]); assert_eq!(o.status.code(), Some(4), "{}", text(&o)); let v: Value = serde_json::from_slice(&o.stdout).unwrap(); assert_eq!((v["error"]["code"].as_str(), v["proposed"][0]["replaces"].as_bool()), (Some("confirmation-required"), Some(true)), "{v:#}"); assert!(v["confirm"].as_str().unwrap().ends_with(" --yes"), "{v:#}"); assert_eq!(std::fs::read(d.join("score.wai-conform.json")).unwrap(), b"{}"); let code = |args: &[&str]| wai(&d, args).status.code(); assert_eq!(code(&["conform", "decoder", "--suite", "score", "--root", r]), Some(2), "no --impl"); assert_eq!(code(&["conform", "relay", "--impl", "x", "--root", r]), Some(5), "no such kind"); assert_eq!(code(&["conform", "decoder", "--suite", "nope", "--impl", &imp, "--root", r]), Some(5), "no such suite"); assert_eq!(code(&["conform", "decoder", "--suite", "score", "--impl", "no-such-program-here {in}", "--root", r]), Some(5), "no such program"); let free = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); assert_eq!(code(&["conform", "origin", "--impl", &format!("http://127.0.0.1:{free}"), "--root", r, "--out", d.join("o.wai-conform.json").to_str().unwrap()]), Some(6), "nothing listening"); let o = wai(&d, &["conform", "--list", "--root", r, "--json"]); let v: Value = serde_json::from_slice(&o.stdout).unwrap(); assert!(v["data"]["suites"].as_array().unwrap().iter().all(|s| s["cases"].as_u64().unwrap() > 0), "{v:#}"); }