//! `wai breaking` on a small repository laid out as this one is: each kind of break the gate //! exists for is reported, fails the run, and passes only once a line the change adds to //! `wai/CHANGELOG.md` names it in an entry that says "breaking": //! //! - a registered code removed (a capability, a wire identifier's magic); //! - a record's signing preimage changed (its domain-separation prefix; and its signed //! vectors, which the corpus then carries); //! - a public Rust function removed (documented by rustdoc, in a scratch directory); //! - a C function's signature changed; //! - an OpenAPI route removed. //! //! Additions are reported as compatible and pass; an acknowledgement already in the base //! acknowledges nothing; the categories are cumulative; a file the measurement list names is //! not compared, and a vector beside it is. use serde_json::{Value, json}; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; fn tmp(name: &str) -> PathBuf { let d = std::env::temp_dir().join(format!("wai-breaking-{name}-{}", std::process::id())); let _ = std::fs::remove_dir_all(&d); std::fs::create_dir_all(&d).unwrap(); d } fn git(dir: &Path, args: &[&str]) { let o = Command::new("git").arg("-C").arg(dir).args(args).output().expect("git runs"); assert!(o.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&o.stderr)); } fn write(dir: &Path, rel: &str, text: &str) { let p = dir.join(rel); std::fs::create_dir_all(p.parent().unwrap()).unwrap(); std::fs::write(p, text).unwrap(); } fn edit(dir: &Path, rel: &str, from: &str, to: &str) { let p = dir.join(rel); let t = std::fs::read_to_string(&p).unwrap(); assert_eq!(t.matches(from).count(), 1, "{rel}: {from:?} once"); std::fs::write(p, t.replace(from, to)).unwrap(); } /// `wai breaking --against --json ` in `dir`: the exit status and the envelope. fn breaking(dir: &Path, base: &str, extra: &[&str]) -> (i32, Value) { let o: Output = Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(dir).args(["breaking", "--against", base, "--json"]).args(extra).output().expect("wai runs"); let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("JSON: {}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))); (o.status.code().unwrap_or(-1), v) } fn finding<'a>(v: &'a Value, id: &str) -> Option<&'a Value> { v["data"]["findings"].as_array().unwrap().iter().find(|f| f["id"] == id) } const REGISTRY: &str = r#"{"schema": "x", "capabilities": [ {"capability": "wai.test.kept", "kind": "payload", "status": "registered", "media": "image", "role": "standalone", "tier": "lossless", "licence": "royalty-free", "replicate": "standard-defined", "sets": [], "spec_section": "§5", "edition": null, "registered": "2026-01-01"}, {"capability": "wai.test.removed", "kind": "payload", "status": "registered", "media": "image", "role": "standalone", "tier": "lossless", "licence": "royalty-free", "replicate": "standard-defined", "sets": [], "spec_section": "§5", "edition": null, "registered": "2026-01-01"}], "identifiers": [], "unregistered_labels": [], "wire_identifiers": [{"name": "WTS1", "kind": "magic", "document": "extensions/t.md", "section": "§2", "status": "defined"}, {"name": "WTS1", "kind": "fourcc", "document": "extensions/t.md", "section": "§3", "status": "provisional"}]} "#; const HEADER: &str = "#ifndef WAI_H\n#define WAI_H\n#include \n#define WAI_OK 0\n\ typedef struct wai_buffer_t {\n uint8_t *data;\n uintptr_t len;\n} wai_buffer_t;\n\ #ifdef __cplusplus\nextern \"C\" {\n#endif\n\ int wai_pack(const char *manifest_json,\n const uint8_t *payload,\n uintptr_t payload_len,\n struct wai_buffer_t *out);\n\ #ifdef __cplusplus\n}\n#endif\n#endif\n"; fn openapi() -> String { json!({"openapi": "3.1.0", "info": {"title": "t", "version": "1"}, "paths": { "/room": {"get": {"operationId": "room", "responses": {"200": {"description": "ok"}}}}, "/grant": {"post": {"operationId": "grant", "requestBody": {"required": true, "content": {"application/json": {"schema": {"type": "object"}}}}, "responses": {"200": {"description": "ok"}}}} }}) .to_string() } /// A repository with one of everything the check reads, committed; its commit. fn repo(name: &str) -> (PathBuf, String) { let d = tmp(name); git(&d, &["init", "-q"]); git(&d, &["config", "user.email", "t@example.invalid"]); git(&d, &["config", "user.name", "t"]); write(&d, "wai/CHANGELOG.md", "# changelog\n\n## v0.4.0 — current\n\n- **Breaking:** `api:rust:wai::old` was acknowledged long ago.\n"); write(&d, "wai/registry/capabilities.json", REGISTRY); write(&d, "wai/claim-conformance/vectors.json", r#"{"about": "a", "vectors": [{"name": "v1", "receipt": {"sig": "aa11"}, "expected": {"outcome": "accept"}}]}"#); write(&d, "wai/claim-conformance/claim_verify.py", "print('a verifier, not a vector')\n"); write(&d, "wai/format-probe-conformance/provenance.json", r#"{"decodes": [{"host": "arm64", "identical": true}]}"#); write(&d, "wai/format-probe-conformance/vectors.json", r#"{"vectors": [{"name": "h266", "expect": "probe"}]}"#); write(&d, "wai/wai-rs/include/wai.h", HEADER); write(&d, "joule-code/joule-code-rs/crates/joule-moq-room/openapi.json", &openapi()); write(&d, "wai/wai-rs/Cargo.toml", "[package]\nname = \"wai\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[lib]\npath = \"src/lib.rs\"\n\n[workspace]\n"); write(&d, "wai/wai-rs/Cargo.lock", "version = 4\n\n[[package]]\nname = \"wai\"\nversion = \"0.1.0\"\n"); write(&d, "wai/wai-rs/src/lib.rs", "pub mod rec;\n\n/// Kept.\npub fn keep(x: u8) -> u8 { x }\n\n/// Removed by a test.\npub fn gone() {}\n\npub struct Open { pub a: u8 }\n"); write(&d, "wai/wai-rs/src/rec.rs", "/// The record's signing domain.\npub const SIGN_DOMAIN: &[u8] = b\"wai:test-record\\x01\";\npub const MAGIC: [u8; 4] = *b\"WTS1\";\n"); write(&d, ".gitignore", "target/\n"); git(&d, &["add", "-A"]); git(&d, &["commit", "-q", "-m", "base"]); let o = Command::new("git").arg("-C").arg(&d).args(["rev-parse", "HEAD"]).output().unwrap(); (d, String::from_utf8_lossy(&o.stdout).trim().to_string()) } fn reset(d: &Path) { git(d, &["checkout", "-q", "--", "."]); git(d, &["clean", "-q", "-fdx", "-e", "target"]); } /// The break `id` is reported and fails the run; with a changelog entry naming it, the /// run passes and reports it acknowledged. fn holds_until_acknowledged(d: &Path, base: &str, only: &str, id: &str) { let (code, v) = breaking(d, base, &["--only", only]); assert_eq!(code, 1, "{id}: {v:#}"); let f = finding(&v, id).unwrap_or_else(|| panic!("{id} reported: {v:#}")); assert_eq!(f["breaking"], true); assert_eq!(f["acknowledged"], false); assert!(!f["why"].as_str().unwrap().is_empty() && !f["what"].as_str().unwrap().is_empty(), "{f}"); assert_eq!(v["ok"], false); // Named, but in an entry that does not say it breaks: not acknowledged. let log = std::fs::read_to_string(d.join("wai/CHANGELOG.md")).unwrap(); std::fs::write(d.join("wai/CHANGELOG.md"), log.replace("## v0.4.0 — current\n", &format!("## v0.4.0 — current\n\n- Renamed `{id}`.\n"))).unwrap(); assert_eq!(breaking(d, base, &["--only", only]).0, 1, "{id}: an entry without the word acknowledges nothing"); std::fs::write(d.join("wai/CHANGELOG.md"), log.replace("## v0.4.0 — current\n", &format!("## v0.4.0 — current\n\n- **Breaking:** `{id}`: intended, for the test.\n"))).unwrap(); let (code, v) = breaking(d, base, &["--only", only]); assert_eq!(code, 0, "{id} acknowledged: {v:#}"); assert_eq!(finding(&v, id).unwrap()["acknowledged"], true); assert_eq!(v["ok"], true); std::fs::write(d.join("wai/CHANGELOG.md"), log).unwrap(); } #[test] fn nothing_changed_nothing_breaks() { let (d, base) = repo("none"); let (code, v) = breaking(&d, &base, &["--skip", "rust"]); assert_eq!(code, 0, "{v:#}"); assert_eq!(v["op"], "compat.check"); assert_eq!(v["data"]["breaking"], 0); assert!(v["data"]["findings"].as_array().unwrap().is_empty(), "{v:#}"); } #[test] fn a_removed_registered_code_breaks_until_acknowledged() { let (d, base) = repo("code"); let reg = REGISTRY.replace( r#"{"capability": "wai.test.removed", "kind": "payload", "status": "registered", "media": "image", "role": "standalone", "tier": "lossless", "licence": "royalty-free", "replicate": "standard-defined", "sets": [], "spec_section": "§5", "edition": null, "registered": "2026-01-01"}"#, r#"{"capability": "wai.test.added", "kind": "payload", "status": "registered", "media": "image", "role": "standalone", "tier": "lossless", "licence": "royalty-free", "replicate": "standard-defined", "sets": [], "spec_section": "§5", "edition": null, "registered": "2026-01-01"}"#, ); assert_ne!(reg, REGISTRY); write(&d, "wai/registry/capabilities.json", ®); holds_until_acknowledged(&d, &base, "registry", "wire:capability:wai.test.removed"); let (_, v) = breaking(&d, &base, &["--only", "registry"]); let added = finding(&v, "wire:capability:wai.test.added").expect("the addition is listed"); assert_eq!(added["breaking"], false); // A magic removed: the wire identifier of that kind goes; the 4CC of the same name stays. write(&d, "wai/registry/capabilities.json", ®ISTRY.replace(r#"{"name": "WTS1", "kind": "magic", "document": "extensions/t.md", "section": "§2", "status": "defined"},"#, "")); holds_until_acknowledged(&d, &base, "registry", "wire:wire-identifier:magic/WTS1"); let (_, v) = breaking(&d, &base, &["--only", "registry"]); assert!(finding(&v, "wire:wire-identifier:fourcc/WTS1").is_none(), "{v:#}"); reset(&d); } #[test] fn a_changed_signing_preimage_breaks_until_acknowledged() { let (d, base) = repo("preimage"); edit(&d, "wai/wai-rs/src/rec.rs", "b\"wai:test-record\\x01\"", "b\"wai:test-record\\x02\""); holds_until_acknowledged(&d, &base, "domains", "record:domain:wai/wai-rs/src/rec.rs::SIGN_DOMAIN"); reset(&d); // The preimage's shape changed, and the corpus was regenerated: its signed vector differs. edit(&d, "wai/claim-conformance/vectors.json", "\"sig\": \"aa11\"", "\"sig\": \"bb22\""); holds_until_acknowledged(&d, &base, "corpora", "record:corpus:wai/claim-conformance/vectors.json"); reset(&d); // A new vector and a changed verifier script are not breaks. edit(&d, "wai/claim-conformance/vectors.json", "\"expected\": {\"outcome\": \"accept\"}}]", "\"expected\": {\"outcome\": \"accept\"}}, {\"name\": \"v2\", \"expected\": {\"outcome\": \"accept\"}}]"); write(&d, "wai/claim-conformance/claim_verify.py", "print('changed')\n"); let (code, v) = breaking(&d, &base, &["--only", "corpora"]); assert_eq!(code, 0, "{v:#}"); assert_eq!(finding(&v, "record:corpus:wai/claim-conformance/vectors.json").unwrap()["breaking"], false); reset(&d); // A domain moved to another file, same name and value: not a break. write(&d, "wai/wai-rs/src/rec.rs", "pub const MAGIC: [u8; 4] = *b\"WTS1\";\n"); write(&d, "wai/wai-rs/src/rec2.rs", "pub const SIGN_DOMAIN: &[u8] = b\"wai:test-record\\x01\";\n"); assert_eq!(breaking(&d, &base, &["--only", "domains"]).0, 0); reset(&d); } #[test] fn a_removed_public_rust_function_breaks_until_acknowledged() { let (d, base) = repo("rust"); edit(&d, "wai/wai-rs/src/lib.rs", "/// Removed by a test.\npub fn gone() {}\n", "/// New.\npub fn added() {}\n"); holds_until_acknowledged(&d, &base, "rust", "api:rust:wai::gone"); let (_, v) = breaking(&d, &base, &["--only", "rust"]); assert_eq!(finding(&v, "api:rust:wai::added").unwrap()["breaking"], false); reset(&d); // A changed signature, and a field added to a struct a literal builds. edit(&d, "wai/wai-rs/src/lib.rs", "pub fn keep(x: u8) -> u8 { x }", "pub fn keep(x: u16) -> u16 { x }"); edit(&d, "wai/wai-rs/src/lib.rs", "pub struct Open { pub a: u8 }", "pub struct Open { pub a: u8, pub b: u8 }"); let (code, v) = breaking(&d, &base, &["--only", "rust"]); assert_eq!(code, 1); let keep = finding(&v, "api:rust:wai::keep").expect("the signature change"); assert!(keep["what"].as_str().unwrap().contains("u16"), "{keep}"); assert_eq!(finding(&v, "api:rust:wai::Open::b").unwrap()["breaking"], true); reset(&d); } /// The base is documented with every standard its crate reaches by path, whole: a path /// dependency in another workspace inherits its version and edition from that workspace's /// root manifest, so the dependency's own directory alone does not build. #[test] fn the_base_holds_a_path_dependency_in_another_workspace() { let (d, _) = repo("rust-dep"); write(&d, "dep/dep-rs/Cargo.toml", "[workspace]\nmembers = [\"crates/core\"]\nresolver = \"2\"\n\n[workspace.package]\nversion = \"0.1.0\"\nedition = \"2021\"\n"); write(&d, "dep/dep-rs/crates/core/Cargo.toml", "[package]\nname = \"dep-core\"\nversion.workspace = true\nedition.workspace = true\n\n[lib]\npath = \"src/lib.rs\"\n"); write(&d, "dep/dep-rs/crates/core/src/lib.rs", "pub fn helper() {}\n"); edit(&d, "wai/wai-rs/Cargo.toml", "\n[workspace]\n", "\n[dependencies]\ndep-core = { path = \"../../dep/dep-rs/crates/core\" }\n\n[workspace]\n"); write(&d, "wai/wai-rs/Cargo.lock", "version = 4\n\n[[package]]\nname = \"dep-core\"\nversion = \"0.1.0\"\n\n[[package]]\nname = \"wai\"\nversion = \"0.1.0\"\ndependencies = [\n \"dep-core\",\n]\n"); git(&d, &["add", "-A"]); git(&d, &["commit", "-q", "-m", "a path dependency in another workspace"]); let o = Command::new("git").arg("-C").arg(&d).args(["rev-parse", "HEAD"]).output().unwrap(); let base = String::from_utf8_lossy(&o.stdout).trim().to_string(); edit(&d, "wai/wai-rs/src/lib.rs", "/// Removed by a test.\npub fn gone() {}\n", ""); let (code, v) = breaking(&d, &base, &["--only", "rust"]); assert_eq!(code, 1, "the base documents, and the removal breaks: {v:#}"); assert_eq!(finding(&v, "api:rust:wai::gone").unwrap()["breaking"], true); reset(&d); } #[test] fn a_changed_c_signature_breaks_until_acknowledged() { let (d, base) = repo("c"); // A parameter renamed is the same signature. edit(&d, "wai/wai-rs/include/wai.h", "uintptr_t payload_len,", "uintptr_t len,"); assert_eq!(breaking(&d, &base, &["--only", "c"]).0, 0); edit(&d, "wai/wai-rs/include/wai.h", "uintptr_t len,", "uint32_t len,"); holds_until_acknowledged(&d, &base, "c", "api:c:wai_pack"); reset(&d); edit(&d, "wai/wai-rs/include/wai.h", " uintptr_t len;\n", " uint32_t len;\n"); holds_until_acknowledged(&d, &base, "c", "api:c:wai_buffer_t"); reset(&d); } #[test] fn a_removed_openapi_route_breaks_until_acknowledged() { let (d, base) = repo("openapi"); let mut doc: Value = serde_json::from_str(&openapi()).unwrap(); doc["paths"].as_object_mut().unwrap().remove("/room"); doc["paths"]["/new"] = json!({"get": {"operationId": "new", "responses": {"200": {"description": "ok"}}}}); write(&d, "joule-code/joule-code-rs/crates/joule-moq-room/openapi.json", &doc.to_string()); holds_until_acknowledged(&d, &base, "openapi", "api:openapi:joule-moq-room:GET /room"); let (_, v) = breaking(&d, &base, &["--only", "openapi"]); assert_eq!(finding(&v, "api:openapi:joule-moq-room:GET /new").unwrap()["breaking"], false); reset(&d); } /// A file the measurement list names changes without a finding (a note says so); a vector /// file beside it, which the list does not name, still breaks. #[test] fn a_listed_measurement_is_not_compared_and_a_vector_beside_it_is() { let (d, base) = repo("measurements"); write(&d, "wai/format-probe-conformance/provenance.json", r#"{"decodes": [{"host": "x86-64", "identical": false}]}"#); let (code, v) = breaking(&d, &base, &["--only", "corpora"]); assert_eq!(code, 0, "{v:#}"); assert!(v["data"]["findings"].as_array().unwrap().is_empty(), "{v:#}"); assert!(v["data"]["notes"].as_array().unwrap().iter().any(|n| n.as_str().unwrap().contains("provenance.json changed, a measurement")), "{v:#}"); edit(&d, "wai/format-probe-conformance/vectors.json", "\"expect\": \"probe\"", "\"expect\": \"refuse\""); let (code, v) = breaking(&d, &base, &["--only", "corpora"]); assert_eq!(code, 1, "{v:#}"); assert_eq!(finding(&v, "wire:corpus:wai/format-probe-conformance/vectors.json").unwrap()["breaking"], true); assert!(finding(&v, "wire:corpus:wai/format-probe-conformance/provenance.json").is_none()); reset(&d); } #[test] fn categories_are_cumulative_and_an_old_acknowledgement_counts_for_nothing() { let (d, base) = repo("cats"); edit(&d, "wai/wai-rs/include/wai.h", "uintptr_t payload_len,", "uint32_t payload_len,"); edit(&d, "wai/wai-rs/src/rec.rs", "b\"wai:test-record\\x01\"", "b\"wai:test-record\\x02\""); let (code, v) = breaking(&d, &base, &["--category", "wire", "--skip", "rust"]); assert_eq!(code, 0, "{v:#}"); let (code, v) = breaking(&d, &base, &["--category", "record", "--skip", "rust"]); assert_eq!(code, 1); assert!(finding(&v, "api:c:wai_pack").is_none() && finding(&v, "record:domain:wai/wai-rs/src/rec.rs::SIGN_DOMAIN").is_some(), "{v:#}"); let (code, v) = breaking(&d, &base, &["--skip", "rust"]); assert_eq!(code, 1); assert!(finding(&v, "api:c:wai_pack").is_some()); reset(&d); // The base's changelog already names a break: that entry acknowledges nothing now. edit(&d, "wai/wai-rs/src/lib.rs", "pub mod rec;\n", "pub mod rec;\n/// Old.\npub fn old() {}\n"); git(&d, &["commit", "-q", "-am", "old"]); let o = Command::new("git").arg("-C").arg(&d).args(["rev-parse", "HEAD"]).output().unwrap(); let base2 = String::from_utf8_lossy(&o.stdout).trim().to_string(); edit(&d, "wai/wai-rs/src/lib.rs", "/// Old.\npub fn old() {}\n", ""); let (code, v) = breaking(&d, &base2, &["--only", "rust"]); assert_eq!(code, 1, "{v:#}"); assert_eq!(finding(&v, "api:rust:wai::old").unwrap()["acknowledged"], false); reset(&d); } #[test] fn usage_and_missing_refs_have_their_exit_codes() { let (d, _) = repo("usage"); let o = Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(&d).args(["breaking", "--json"]).output().unwrap(); assert_eq!(o.status.code(), Some(2)); let v: Value = serde_json::from_slice(&o.stdout).unwrap(); assert_eq!(v["error"]["code"], "usage"); let (code, v) = breaking(&d, "no-such-ref", &[]); assert_eq!(code, 5); assert_eq!(v["error"]["code"], "ref-not-found"); let o = Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(&d).args(["breaking", "--against", "HEAD", "--only", "nonsense"]).output().unwrap(); assert_eq!(o.status.code(), Some(2)); }