//! `wai breaking` on a small repository laid out as this one is: each kind of break the gate //! exists for is reported, fails the run, and passes only once a line the change adds to //! `wai/CHANGELOG.md`, or to a fragment of it under `wai/changes/`, names it in an entry that //! says "breaking": //! //! - a registered code removed (a capability, a wire identifier's magic); //! - a record's signing preimage changed (its domain-separation prefix; and its signed //! vectors, which the corpus then carries); //! - a public Rust function removed (documented by rustdoc, in a scratch directory); //! - a C function's signature changed; //! - an OpenAPI route removed; //! - a JavaScript binding, a Swift initializer's label and a Kotlin parameter changed. //! //! Additions are reported as compatible and pass; an acknowledgement already in the base //! acknowledges nothing; the categories are cumulative; a file the measurement list names is //! not compared, and a vector beside it is. //! //! The catalogue: a renamed stable operation fails, and no line acknowledges it until the old id //! is kept, deprecated, for its window; a preview operation's change fails until an added entry //! names it, with or without the word; an MCP tool's input schema is held as the operation's. //! //! A project: an accepted break with no reason is refused with the project file; a break in the //! API it names under `breaking.api` fails until `breaking.allow` accepts it, and the reason is //! reported; its package's sink classes are compared with a package as published, and a run that //! has no package to compare with stops rather than passing. use serde_json::{Value, json}; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; fn tmp(name: &str) -> PathBuf { let d = std::env::temp_dir().join(format!("wai-breaking-{name}-{}", std::process::id())); let _ = std::fs::remove_dir_all(&d); std::fs::create_dir_all(&d).unwrap(); d } fn git(dir: &Path, args: &[&str]) { let o = Command::new("git").arg("-C").arg(dir).args(args).output().expect("git runs"); assert!(o.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&o.stderr)); } fn write(dir: &Path, rel: &str, text: &str) { let p = dir.join(rel); std::fs::create_dir_all(p.parent().unwrap()).unwrap(); std::fs::write(p, text).unwrap(); } fn edit(dir: &Path, rel: &str, from: &str, to: &str) { let p = dir.join(rel); let t = std::fs::read_to_string(&p).unwrap(); assert_eq!(t.matches(from).count(), 1, "{rel}: {from:?} once"); std::fs::write(p, t.replace(from, to)).unwrap(); } /// `wai breaking --against --json ` in `dir`: the exit status and the envelope. fn breaking(dir: &Path, base: &str, extra: &[&str]) -> (i32, Value) { let o: Output = Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(dir).args(["breaking", "--against", base, "--json"]).args(extra).output().expect("wai runs"); let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("JSON: {}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))); (o.status.code().unwrap_or(-1), v) } fn finding<'a>(v: &'a Value, id: &str) -> Option<&'a Value> { v["data"]["findings"].as_array().unwrap().iter().find(|f| f["id"] == id) } const REGISTRY: &str = r#"{"schema": "x", "capabilities": [ {"capability": "wai.test.kept", "kind": "payload", "status": "registered", "media": "image", "role": "standalone", "tier": "lossless", "licence": "royalty-free", "replicate": "standard-defined", "sets": [], "spec_section": "§5", "edition": null, "registered": "2026-01-01"}, {"capability": "wai.test.removed", "kind": "payload", "status": "registered", "media": "image", "role": "standalone", "tier": "lossless", "licence": "royalty-free", "replicate": "standard-defined", "sets": [], "spec_section": "§5", "edition": null, "registered": "2026-01-01"}], "identifiers": [], "unregistered_labels": [], "wire_identifiers": [{"name": "WTS1", "kind": "magic", "document": "extensions/t.md", "section": "§2", "status": "defined"}, {"name": "WTS1", "kind": "fourcc", "document": "extensions/t.md", "section": "§3", "status": "provisional"}]} "#; const HEADER: &str = "#ifndef WAI_H\n#define WAI_H\n#include \n#define WAI_OK 0\n\ typedef struct wai_buffer_t {\n uint8_t *data;\n uintptr_t len;\n} wai_buffer_t;\n\ #ifdef __cplusplus\nextern \"C\" {\n#endif\n\ int wai_pack(const char *manifest_json,\n const uint8_t *payload,\n uintptr_t payload_len,\n struct wai_buffer_t *out);\n\ #ifdef __cplusplus\n}\n#endif\n#endif\n"; fn openapi() -> String { json!({"openapi": "3.1.0", "info": {"title": "t", "version": "1"}, "paths": { "/room": {"get": {"operationId": "room", "responses": {"200": {"description": "ok"}}}}, "/grant": {"post": {"operationId": "grant", "requestBody": {"required": true, "content": {"application/json": {"schema": {"type": "object"}}}}, "responses": {"200": {"description": "ok"}}}} }}) .to_string() } /// A repository with one of everything the check reads, committed; its commit. fn repo(name: &str) -> (PathBuf, String) { let d = tmp(name); git(&d, &["init", "-q"]); git(&d, &["config", "user.email", "t@example.invalid"]); git(&d, &["config", "user.name", "t"]); write(&d, "wai/CHANGELOG.md", "# changelog\n\n## v0.4.0 — current\n\n- **Breaking:** `api:rust:wai::old` was acknowledged long ago.\n"); write(&d, "wai/registry/capabilities.json", REGISTRY); write(&d, "wai/claim-conformance/vectors.json", r#"{"about": "a", "vectors": [{"name": "v1", "receipt": {"sig": "aa11"}, "expected": {"outcome": "accept"}}]}"#); write(&d, "wai/claim-conformance/claim_verify.py", "print('a verifier, not a vector')\n"); write(&d, "wai/format-probe-conformance/provenance.json", r#"{"decodes": [{"host": "arm64", "identical": true}]}"#); write(&d, "wai/format-probe-conformance/vectors.json", r#"{"vectors": [{"name": "h266", "expect": "probe"}]}"#); write(&d, "wai/wai-rs/include/wai.h", HEADER); write(&d, "joule-code/joule-code-rs/crates/joule-moq-room/openapi.json", &openapi()); write(&d, "wai/wai-rs/Cargo.toml", "[package]\nname = \"wai\"\nversion = \"0.1.0\"\nedition = \"2021\"\n\n[lib]\npath = \"src/lib.rs\"\n\n[workspace]\n"); write(&d, "wai/wai-rs/Cargo.lock", "version = 4\n\n[[package]]\nname = \"wai\"\nversion = \"0.1.0\"\n"); write(&d, "wai/wai-rs/src/lib.rs", "pub mod rec;\n\n/// Kept.\npub fn keep(x: u8) -> u8 { x }\n\n/// Removed by a test.\npub fn gone() {}\n\npub struct Open { pub a: u8 }\n"); write(&d, "wai/wai-rs/src/rec.rs", "/// The record's signing domain.\npub const SIGN_DOMAIN: &[u8] = b\"wai:test-record\\x01\";\npub const MAGIC: [u8; 4] = *b\"WTS1\";\n"); write(&d, ".gitignore", "target/\n"); git(&d, &["add", "-A"]); git(&d, &["commit", "-q", "-m", "base"]); let o = Command::new("git").arg("-C").arg(&d).args(["rev-parse", "HEAD"]).output().unwrap(); (d, String::from_utf8_lossy(&o.stdout).trim().to_string()) } fn reset(d: &Path) { git(d, &["checkout", "-q", "--", "."]); git(d, &["clean", "-q", "-fdx", "-e", "target"]); } /// The break `id` is reported and fails the run; with a changelog entry naming it, the /// run passes and reports it acknowledged. fn holds_until_acknowledged(d: &Path, base: &str, only: &str, id: &str) { let (code, v) = breaking(d, base, &["--only", only]); assert_eq!(code, 1, "{id}: {v:#}"); let f = finding(&v, id).unwrap_or_else(|| panic!("{id} reported: {v:#}")); assert_eq!(f["breaking"], true); assert_eq!(f["acknowledged"], false); assert!(!f["why"].as_str().unwrap().is_empty() && !f["what"].as_str().unwrap().is_empty(), "{f}"); assert_eq!(v["ok"], false); // Named, but in an entry that does not say it breaks: not acknowledged. let log = std::fs::read_to_string(d.join("wai/CHANGELOG.md")).unwrap(); std::fs::write(d.join("wai/CHANGELOG.md"), log.replace("## v0.4.0 — current\n", &format!("## v0.4.0 — current\n\n- Renamed `{id}`.\n"))).unwrap(); assert_eq!(breaking(d, base, &["--only", only]).0, 1, "{id}: an entry without the word acknowledges nothing"); std::fs::write(d.join("wai/CHANGELOG.md"), log.replace("## v0.4.0 — current\n", &format!("## v0.4.0 — current\n\n- **Breaking:** `{id}`: intended, for the test.\n"))).unwrap(); let (code, v) = breaking(d, base, &["--only", only]); assert_eq!(code, 0, "{id} acknowledged: {v:#}"); assert_eq!(finding(&v, id).unwrap()["acknowledged"], true); assert_eq!(v["ok"], true); std::fs::write(d.join("wai/CHANGELOG.md"), log).unwrap(); } #[test] fn nothing_changed_nothing_breaks() { let (d, base) = repo("none"); let (code, v) = breaking(&d, &base, &["--skip", "rust"]); assert_eq!(code, 0, "{v:#}"); assert_eq!(v["op"], "compat.check"); assert_eq!(v["data"]["breaking"], 0); assert!(v["data"]["findings"].as_array().unwrap().is_empty(), "{v:#}"); } #[test] fn a_removed_registered_code_breaks_until_acknowledged() { let (d, base) = repo("code"); let reg = REGISTRY.replace( r#"{"capability": "wai.test.removed", "kind": "payload", "status": "registered", "media": "image", "role": "standalone", "tier": "lossless", "licence": "royalty-free", "replicate": "standard-defined", "sets": [], "spec_section": "§5", "edition": null, "registered": "2026-01-01"}"#, r#"{"capability": "wai.test.added", "kind": "payload", "status": "registered", "media": "image", "role": "standalone", "tier": "lossless", "licence": "royalty-free", "replicate": "standard-defined", "sets": [], "spec_section": "§5", "edition": null, "registered": "2026-01-01"}"#, ); assert_ne!(reg, REGISTRY); write(&d, "wai/registry/capabilities.json", ®); holds_until_acknowledged(&d, &base, "registry", "wire:capability:wai.test.removed"); let (_, v) = breaking(&d, &base, &["--only", "registry"]); let added = finding(&v, "wire:capability:wai.test.added").expect("the addition is listed"); assert_eq!(added["breaking"], false); // A magic removed: the wire identifier of that kind goes; the 4CC of the same name stays. write(&d, "wai/registry/capabilities.json", ®ISTRY.replace(r#"{"name": "WTS1", "kind": "magic", "document": "extensions/t.md", "section": "§2", "status": "defined"},"#, "")); holds_until_acknowledged(&d, &base, "registry", "wire:wire-identifier:magic/WTS1"); let (_, v) = breaking(&d, &base, &["--only", "registry"]); assert!(finding(&v, "wire:wire-identifier:fourcc/WTS1").is_none(), "{v:#}"); reset(&d); } /// A class of covering statement (jwp-receipts §4.1) that is no longer admitted, or whose form /// changed, breaks until acknowledged: a relay that fans out on it no longer conforms. One /// newly admitted is compatible. #[test] fn a_covering_statement_withdrawn_breaks_until_acknowledged() { let (d, _) = repo("covering"); let class = |name: &str, form: &str| format!(r#"{{"statement": "{name}", "form": "{form}", "document": "extensions/t.md", "section": "§2"}}"#); let with = |classes: &[String]| format!("{}, \"covering_statements\": [{}]}}\n", REGISTRY.trim_end().strip_suffix('}').unwrap(), classes.join(", ")); write(&d, "wai/registry/capabilities.json", &with(&[class("test-run", "WTS1")])); git(&d, &["add", "-A"]); git(&d, &["commit", "-q", "-m", "a covering statement"]); let o = Command::new("git").arg("-C").arg(&d).args(["rev-parse", "HEAD"]).output().unwrap(); let base = String::from_utf8_lossy(&o.stdout).trim().to_string(); assert_eq!(breaking(&d, &base, &["--only", "registry"]).0, 0, "nothing changed"); write(&d, "wai/registry/capabilities.json", &with(&[class("test-other", "WTS1")])); holds_until_acknowledged(&d, &base, "registry", "wire:covering-statement:test-run"); let (_, v) = breaking(&d, &base, &["--only", "registry"]); assert_eq!(finding(&v, "wire:covering-statement:test-other").expect("the admission is listed")["breaking"], false); write(&d, "wai/registry/capabilities.json", &with(&[class("test-run", "WTS2")])); holds_until_acknowledged(&d, &base, "registry", "wire:covering-statement:test-run"); reset(&d); } /// A fragment of the changelog (a file of `wai/changes/`, one entry) acknowledges a break as a /// line added to the changelog does; a fragment the base already holds acknowledges nothing, /// and neither does the fragments' README. #[test] fn a_changelog_fragment_acknowledges_a_break() { let (d, base) = repo("fragment"); let id = "wire:capability:wai.test.removed"; write(&d, "wai/registry/capabilities.json", ®ISTRY.replace( r#", {"capability": "wai.test.removed", "kind": "payload", "status": "registered", "media": "image", "role": "standalone", "tier": "lossless", "licence": "royalty-free", "replicate": "standard-defined", "sets": [], "spec_section": "§5", "edition": null, "registered": "2026-01-01"}"#, "", )); assert_eq!(breaking(&d, &base, &["--only", "registry"]).0, 1, "the removal breaks"); write(&d, "wai/changes/README.md", &format!("An example entry:\n\n- **Breaking:** `{id}`: an example.\n")); assert_eq!(breaking(&d, &base, &["--only", "registry"]).0, 1, "the README is no entry"); write(&d, "wai/changes/2026-01-02-removed.md", &format!("### A capability goes\n\n- Renamed `{id}`.\n")); assert_eq!(breaking(&d, &base, &["--only", "registry"]).0, 1, "an entry without the word acknowledges nothing"); write(&d, "wai/changes/2026-01-02-removed.md", &format!("### A capability goes\n\n- **Breaking:** `{id}`: intended, for the test.\n")); let (code, v) = breaking(&d, &base, &["--only", "registry"]); assert_eq!(code, 0, "acknowledged by the fragment: {v:#}"); assert_eq!(finding(&v, id).unwrap()["acknowledged"], true); // Committed into the base, the same fragment acknowledges nothing a later change breaks. git(&d, &["add", "-A"]); git(&d, &["commit", "-q", "-m", "the removal and its entry"]); let o = Command::new("git").arg("-C").arg(&d).args(["rev-parse", "HEAD"]).output().unwrap(); let base2 = String::from_utf8_lossy(&o.stdout).trim().to_string(); write(&d, "wai/registry/capabilities.json", ®ISTRY.replace(r#"{"name": "WTS1", "kind": "magic", "document": "extensions/t.md", "section": "§2", "status": "defined"},"#, "")); write(&d, "wai/changes/2026-01-02-removed.md", &format!("### A capability goes\n\n- **Breaking:** `{id}`: intended, for the test.\n- **Breaking:** `wire:wire-identifier:magic/WTS1`, said in the base.\n")); let (code, v) = breaking(&d, &base2, &["--only", "registry"]); assert_eq!(code, 0, "a line added to a fragment the base holds acknowledges: {v:#}"); write(&d, "wai/changes/2026-01-02-removed.md", &format!("### A capability goes\n\n- **Breaking:** `{id}`: intended, for the test.\n")); assert_eq!(breaking(&d, &base2, &["--only", "registry"]).0, 1, "the base's fragment alone acknowledges nothing new"); } #[test] fn a_changed_signing_preimage_breaks_until_acknowledged() { let (d, base) = repo("preimage"); edit(&d, "wai/wai-rs/src/rec.rs", "b\"wai:test-record\\x01\"", "b\"wai:test-record\\x02\""); holds_until_acknowledged(&d, &base, "domains", "record:domain:wai/wai-rs/src/rec.rs::SIGN_DOMAIN"); reset(&d); // The preimage's shape changed, and the corpus was regenerated: its signed vector differs. edit(&d, "wai/claim-conformance/vectors.json", "\"sig\": \"aa11\"", "\"sig\": \"bb22\""); holds_until_acknowledged(&d, &base, "corpora", "record:corpus:wai/claim-conformance/vectors.json"); reset(&d); // A new vector and a changed verifier script are not breaks. edit(&d, "wai/claim-conformance/vectors.json", "\"expected\": {\"outcome\": \"accept\"}}]", "\"expected\": {\"outcome\": \"accept\"}}, {\"name\": \"v2\", \"expected\": {\"outcome\": \"accept\"}}]"); write(&d, "wai/claim-conformance/claim_verify.py", "print('changed')\n"); let (code, v) = breaking(&d, &base, &["--only", "corpora"]); assert_eq!(code, 0, "{v:#}"); assert_eq!(finding(&v, "record:corpus:wai/claim-conformance/vectors.json").unwrap()["breaking"], false); reset(&d); // A domain moved to another file, same name and value: not a break. write(&d, "wai/wai-rs/src/rec.rs", "pub const MAGIC: [u8; 4] = *b\"WTS1\";\n"); write(&d, "wai/wai-rs/src/rec2.rs", "pub const SIGN_DOMAIN: &[u8] = b\"wai:test-record\\x01\";\n"); assert_eq!(breaking(&d, &base, &["--only", "domains"]).0, 0); reset(&d); } #[test] fn a_removed_public_rust_function_breaks_until_acknowledged() { let (d, base) = repo("rust"); edit(&d, "wai/wai-rs/src/lib.rs", "/// Removed by a test.\npub fn gone() {}\n", "/// New.\npub fn added() {}\n"); holds_until_acknowledged(&d, &base, "rust", "api:rust:wai::gone"); let (_, v) = breaking(&d, &base, &["--only", "rust"]); assert_eq!(finding(&v, "api:rust:wai::added").unwrap()["breaking"], false); reset(&d); // A changed signature, and a field added to a struct a literal builds. edit(&d, "wai/wai-rs/src/lib.rs", "pub fn keep(x: u8) -> u8 { x }", "pub fn keep(x: u16) -> u16 { x }"); edit(&d, "wai/wai-rs/src/lib.rs", "pub struct Open { pub a: u8 }", "pub struct Open { pub a: u8, pub b: u8 }"); let (code, v) = breaking(&d, &base, &["--only", "rust"]); assert_eq!(code, 1); let keep = finding(&v, "api:rust:wai::keep").expect("the signature change"); assert!(keep["what"].as_str().unwrap().contains("u16"), "{keep}"); assert_eq!(finding(&v, "api:rust:wai::Open::b").unwrap()["breaking"], true); reset(&d); } /// The base is documented with every standard its crate reaches by path, whole: a path /// dependency in another workspace inherits its version and edition from that workspace's /// root manifest, so the dependency's own directory alone does not build. #[test] fn the_base_holds_a_path_dependency_in_another_workspace() { let (d, _) = repo("rust-dep"); write(&d, "dep/dep-rs/Cargo.toml", "[workspace]\nmembers = [\"crates/core\"]\nresolver = \"2\"\n\n[workspace.package]\nversion = \"0.1.0\"\nedition = \"2021\"\n"); write(&d, "dep/dep-rs/crates/core/Cargo.toml", "[package]\nname = \"dep-core\"\nversion.workspace = true\nedition.workspace = true\n\n[lib]\npath = \"src/lib.rs\"\n"); write(&d, "dep/dep-rs/crates/core/src/lib.rs", "pub fn helper() {}\n"); edit(&d, "wai/wai-rs/Cargo.toml", "\n[workspace]\n", "\n[dependencies]\ndep-core = { path = \"../../dep/dep-rs/crates/core\" }\n\n[workspace]\n"); write(&d, "wai/wai-rs/Cargo.lock", "version = 4\n\n[[package]]\nname = \"dep-core\"\nversion = \"0.1.0\"\n\n[[package]]\nname = \"wai\"\nversion = \"0.1.0\"\ndependencies = [\n \"dep-core\",\n]\n"); git(&d, &["add", "-A"]); git(&d, &["commit", "-q", "-m", "a path dependency in another workspace"]); let o = Command::new("git").arg("-C").arg(&d).args(["rev-parse", "HEAD"]).output().unwrap(); let base = String::from_utf8_lossy(&o.stdout).trim().to_string(); edit(&d, "wai/wai-rs/src/lib.rs", "/// Removed by a test.\npub fn gone() {}\n", ""); let (code, v) = breaking(&d, &base, &["--only", "rust"]); assert_eq!(code, 1, "the base documents, and the removal breaks: {v:#}"); assert_eq!(finding(&v, "api:rust:wai::gone").unwrap()["breaking"], true); reset(&d); } #[test] fn a_changed_c_signature_breaks_until_acknowledged() { let (d, base) = repo("c"); // A parameter renamed is the same signature. edit(&d, "wai/wai-rs/include/wai.h", "uintptr_t payload_len,", "uintptr_t len,"); assert_eq!(breaking(&d, &base, &["--only", "c"]).0, 0); edit(&d, "wai/wai-rs/include/wai.h", "uintptr_t len,", "uint32_t len,"); holds_until_acknowledged(&d, &base, "c", "api:c:wai_pack"); reset(&d); edit(&d, "wai/wai-rs/include/wai.h", " uintptr_t len;\n", " uint32_t len;\n"); holds_until_acknowledged(&d, &base, "c", "api:c:wai_buffer_t"); reset(&d); } #[test] fn a_removed_openapi_route_breaks_until_acknowledged() { let (d, base) = repo("openapi"); let mut doc: Value = serde_json::from_str(&openapi()).unwrap(); doc["paths"].as_object_mut().unwrap().remove("/room"); doc["paths"]["/new"] = json!({"get": {"operationId": "new", "responses": {"200": {"description": "ok"}}}}); write(&d, "joule-code/joule-code-rs/crates/joule-moq-room/openapi.json", &doc.to_string()); holds_until_acknowledged(&d, &base, "openapi", "api:openapi:joule-moq-room:GET /room"); let (_, v) = breaking(&d, &base, &["--only", "openapi"]); assert_eq!(finding(&v, "api:openapi:joule-moq-room:GET /new").unwrap()["breaking"], false); reset(&d); } /// A file the measurement list names changes without a finding (a note says so); a vector /// file beside it, which the list does not name, still breaks. #[test] fn a_listed_measurement_is_not_compared_and_a_vector_beside_it_is() { let (d, base) = repo("measurements"); write(&d, "wai/format-probe-conformance/provenance.json", r#"{"decodes": [{"host": "x86-64", "identical": false}]}"#); let (code, v) = breaking(&d, &base, &["--only", "corpora"]); assert_eq!(code, 0, "{v:#}"); assert!(v["data"]["findings"].as_array().unwrap().is_empty(), "{v:#}"); assert!(v["data"]["notes"].as_array().unwrap().iter().any(|n| n.as_str().unwrap().contains("provenance.json changed, a measurement")), "{v:#}"); edit(&d, "wai/format-probe-conformance/vectors.json", "\"expect\": \"probe\"", "\"expect\": \"refuse\""); let (code, v) = breaking(&d, &base, &["--only", "corpora"]); assert_eq!(code, 1, "{v:#}"); assert_eq!(finding(&v, "wire:corpus:wai/format-probe-conformance/vectors.json").unwrap()["breaking"], true); assert!(finding(&v, "wire:corpus:wai/format-probe-conformance/provenance.json").is_none()); reset(&d); } #[test] fn categories_are_cumulative_and_an_old_acknowledgement_counts_for_nothing() { let (d, base) = repo("cats"); edit(&d, "wai/wai-rs/include/wai.h", "uintptr_t payload_len,", "uint32_t payload_len,"); edit(&d, "wai/wai-rs/src/rec.rs", "b\"wai:test-record\\x01\"", "b\"wai:test-record\\x02\""); let (code, v) = breaking(&d, &base, &["--category", "wire", "--skip", "rust"]); assert_eq!(code, 0, "{v:#}"); let (code, v) = breaking(&d, &base, &["--category", "record", "--skip", "rust"]); assert_eq!(code, 1); assert!(finding(&v, "api:c:wai_pack").is_none() && finding(&v, "record:domain:wai/wai-rs/src/rec.rs::SIGN_DOMAIN").is_some(), "{v:#}"); let (code, v) = breaking(&d, &base, &["--skip", "rust"]); assert_eq!(code, 1); assert!(finding(&v, "api:c:wai_pack").is_some()); reset(&d); // The base's changelog already names a break: that entry acknowledges nothing now. edit(&d, "wai/wai-rs/src/lib.rs", "pub mod rec;\n", "pub mod rec;\n/// Old.\npub fn old() {}\n"); git(&d, &["commit", "-q", "-am", "old"]); let o = Command::new("git").arg("-C").arg(&d).args(["rev-parse", "HEAD"]).output().unwrap(); let base2 = String::from_utf8_lossy(&o.stdout).trim().to_string(); edit(&d, "wai/wai-rs/src/lib.rs", "/// Old.\npub fn old() {}\n", ""); let (code, v) = breaking(&d, &base2, &["--only", "rust"]); assert_eq!(code, 1, "{v:#}"); assert_eq!(finding(&v, "api:rust:wai::old").unwrap()["acknowledged"], false); reset(&d); } #[test] fn usage_and_missing_refs_have_their_exit_codes() { let (d, _) = repo("usage"); let o = Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(&d).args(["breaking", "--json"]).output().unwrap(); assert_eq!(o.status.code(), Some(2)); let v: Value = serde_json::from_slice(&o.stdout).unwrap(); assert_eq!(v["error"]["code"], "usage"); let (code, v) = breaking(&d, "no-such-ref", &[]); assert_eq!(code, 5); assert_eq!(v["error"]["code"], "ref-not-found"); let o = Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(&d).args(["breaking", "--against", "HEAD", "--only", "nonsense"]).output().unwrap(); assert_eq!(o.status.code(), Some(2)); } fn commit(d: &Path, msg: &str) -> String { git(d, &["add", "-A"]); git(d, &["commit", "-q", "-m", msg]); let o = Command::new("git").arg("-C").arg(d).args(["rev-parse", "HEAD"]).output().unwrap(); String::from_utf8_lossy(&o.stdout).trim().to_string() } /// A catalogue export of two operations, one stable and one preview, as `wai ops export` /// writes them. fn ops(stable_id: &str, extra: &[Value]) -> String { let op = |id: &str, stability: &str| { json!({"id": id, "summary": "s", "cli": format!("wai {}", id.replace('.', " ")), "aliases": [], "usage": [], "placement": "local", "effect": "reads", "dry_run": false, "confirm": "never", "route": null, "doors": {"cli": true, "mcp": true, "http": false}, "toolset": "core", "stability": stability, "since": "0.3.0", "deprecated": null, "receipt": false, "needs": null, "input": {"type": "object", "properties": {"path": {"type": "string", "x-cli": {"positional": true}}, "out": {"type": "string", "x-cli": {"flag": "--out"}}}, "required": ["path"], "additionalProperties": false}, "output": {"type": "object", "required": ["text"], "properties": {"text": {"type": "string"}}}, "stream": null, "names": {"cli": "x", "mcp": id.replace('.', "_"), "c": "x", "rust": "x", "sdk": "x"}}) }; let mut list = vec![op(stable_id, "stable"), op("project.lock", "preview")]; list.extend(extra.iter().cloned()); serde_json::to_string_pretty(&json!({"format": "wai.ops/1", "toolchain": "0.5.0", "envelope": "envelope/1", "exit_codes": [{"exit": 0, "meaning": "done"}, {"exit": 1, "meaning": "a check failed"}], "agent_flags": [{"flag": "--yes", "summary": "s", "aliases": [{"flag": "--confirm", "deprecated": {"since": "0.3.0", "use_instead": "--yes"}}]}], "operations": list})) .unwrap() } fn mcp(required: &[&str]) -> String { json!({"format": "wai.mcp-tools/1", "tools": [{"name": "media_inspect", "op": "media.inspect", "toolset": "core", "inputSchema": {"type": "object", "properties": {"path": {"type": "string", "description": "A file."}, "limit": {"type": "integer"}}, "required": required, "additionalProperties": false}}]}) .to_string() } /// The operation catalogue: a stable operation renamed fails, and no changelog line acknowledges /// it; kept under its old id, deprecated, it passes; its deprecated alias removed before its /// window closes fails, named or not. (Planted fault: the rename.) #[test] fn a_renamed_stable_operation_fails_until_its_old_id_is_kept() { let (d, _) = repo("catalogue-rename"); write(&d, "wai/catalogue/ops.json", &ops("media.inspect", &[])); write(&d, "wai/catalogue/mcp.json", &mcp(&["path"])); let base = commit(&d, "a catalogue"); write(&d, "wai/catalogue/ops.json", &ops("media.identify", &[])); let (code, v) = breaking(&d, &base, &["--only", "catalogue"]); assert_eq!(code, 1, "{v:#}"); let f = finding(&v, "api:op:media.inspect").unwrap_or_else(|| panic!("{v:#}")); assert_eq!((f["breaking"].as_bool(), f["acknowledged"].as_bool()), (Some(true), Some(false))); assert!(f["window"].as_str().unwrap().contains("never deprecated"), "{f}"); assert_eq!(finding(&v, "api:op:media.identify").unwrap()["breaking"], false); // A Breaking line does not acknowledge it: the old id has to stay. write(&d, "wai/changes/2026-01-02-renamed.md", "### Renamed\n\n- **Breaking:** `api:op:media.inspect` is media.identify now.\n"); let (code, v) = breaking(&d, &base, &["--only", "catalogue"]); assert_eq!(code, 1, "no line acknowledges a stable rename: {v:#}"); assert_eq!(finding(&v, "api:op:media.inspect").unwrap()["acknowledged"], false); assert!(v["data"]["notes"].as_array().unwrap().iter().any(|n| n.as_str().unwrap().contains("no line acknowledges it")), "{v:#}"); // Kept under its old id, deprecated in this release: no break. let mut kept: Value = serde_json::from_str(&ops("media.identify", &[])).unwrap(); let mut old = kept["operations"][0].clone(); old["id"] = json!("media.inspect"); old["cli"] = json!("wai media inspect"); old["deprecated"] = json!({"since": "0.5.0", "use_instead": "media.identify"}); kept["operations"].as_array_mut().unwrap().push(old); write(&d, "wai/catalogue/ops.json", &serde_json::to_string_pretty(&kept).unwrap()); let (code, v) = breaking(&d, &base, &["--only", "catalogue"]); assert_eq!(code, 0, "the old id kept, deprecated: {v:#}"); let base2 = commit(&d, "renamed, the old id kept"); // Removing it in the same release: its window is open, and no line acknowledges that. write(&d, "wai/catalogue/ops.json", &ops("media.identify", &[])); write(&d, "wai/changes/2026-01-03-dropped.md", "### Dropped\n\n- **Breaking:** `api:op:media.inspect` is gone.\n"); let (code, v) = breaking(&d, &base2, &["--only", "catalogue"]); assert_eq!(code, 1, "{v:#}"); assert!(finding(&v, "api:op:media.inspect").unwrap()["window"].as_str().unwrap().contains("window is open"), "{v:#}"); reset(&d); } /// A preview operation's change is no break, and fails until a line the change adds names it, /// with or without the word "breaking". (Planted fault: a preview flag made required.) #[test] fn a_preview_change_passes_once_a_line_names_it() { let (d, _) = repo("catalogue-preview"); write(&d, "wai/catalogue/ops.json", &ops("media.inspect", &[])); let base = commit(&d, "a catalogue"); let mut v: Value = serde_json::from_str(&ops("media.inspect", &[])).unwrap(); v["operations"][1]["input"]["required"] = json!(["path", "out"]); write(&d, "wai/catalogue/ops.json", &serde_json::to_string_pretty(&v).unwrap()); let id = "api:flag:project.lock --out"; let (code, r) = breaking(&d, &base, &["--only", "catalogue"]); assert_eq!(code, 1, "{r:#}"); let f = finding(&r, id).unwrap_or_else(|| panic!("{r:#}")); assert_eq!((f["breaking"].as_bool(), f["preview"].as_bool(), f["acknowledged"].as_bool()), (Some(false), Some(true), Some(false))); assert_eq!(r["data"]["preview"], 1); write(&d, "wai/changes/2026-01-02-lock.md", &format!("### wai lock takes --out\n\n- `{id}`: it is required now.\n")); let (code, r) = breaking(&d, &base, &["--only", "catalogue"]); assert_eq!(code, 0, "{r:#}"); assert_eq!(finding(&r, id).unwrap()["acknowledged"], true); // The same change to the stable operation is a break, which a line without the word does not // acknowledge. v["operations"][0]["input"]["required"] = json!(["path", "out"]); write(&d, "wai/catalogue/ops.json", &serde_json::to_string_pretty(&v).unwrap()); write(&d, "wai/changes/2026-01-03-inspect.md", "### inspect\n\n- `api:flag:media.inspect --out`: required.\n"); let (code, r) = breaking(&d, &base, &["--only", "catalogue"]); assert_eq!(code, 1, "{r:#}"); assert_eq!(finding(&r, "api:flag:media.inspect --out").unwrap()["breaking"], true); reset(&d); } /// An MCP tool's input schema is held as an operation's inputs: an argument made required breaks; /// a reworded description does not. #[test] fn an_mcp_tools_input_schema_breaks_until_acknowledged() { let (d, _) = repo("catalogue-mcp"); write(&d, "wai/catalogue/ops.json", &ops("media.inspect", &[])); write(&d, "wai/catalogue/mcp.json", &mcp(&["path"])); let base = commit(&d, "a catalogue"); write(&d, "wai/catalogue/mcp.json", &mcp(&["path"]).replace("A file.", "Any file.")); assert_eq!(breaking(&d, &base, &["--only", "catalogue"]).0, 0); write(&d, "wai/catalogue/mcp.json", &mcp(&["path", "limit"])); holds_until_acknowledged(&d, &base, "catalogue", "api:mcp:media_inspect"); reset(&d); } const BINDINGS: &str = "use wasm_bindgen::prelude::*;\n#[wasm_bindgen]\npub struct Sink { n: u8 }\n#[wasm_bindgen]\nimpl Sink {\n #[wasm_bindgen(constructor)]\n pub fn new(bytes: &[u8]) -> Sink { Sink { n: 0 } }\n pub fn present(&mut self, at_ms: u32) -> bool { true }\n}\n#[wasm_bindgen(js_name = verifyPin)]\npub fn verify_pin(pin: &str) -> bool { true }\n"; const SWIFT: &str = "import Foundation\npublic final class WaiSink {\n public init(path: String) {}\n public func present(_ digest: String, atMs: UInt64 = 0) -> Bool { true }\n func internalOnly() {}\n}\n"; const KOTLIN: &str = "package wai.sink\n\nclass WaiSink(val path: String) {\n fun present(objectDigest: String, atMs: Long = 0): Boolean = true\n private fun secret() {}\n}\n"; /// The JavaScript, Swift and Kotlin APIs: a binding's parameter type changed, a Swift /// initializer's label changed and a Kotlin function's new required parameter each break until /// acknowledged; an optional parameter added is compatible. #[test] fn the_javascript_swift_and_kotlin_apis_break_until_acknowledged() { let (d, _) = repo("languages"); write(&d, "wai/wai-web/src/lib.rs", BINDINGS); write(&d, "wai/wai-sink-swift/Sources/WaiSink/Sink.swift", SWIFT); write(&d, "wai/wai-sink-android/core/src/main/kotlin/wai/sink/WaiSink.kt", KOTLIN); let base = commit(&d, "the native and browser sinks"); assert_eq!(breaking(&d, &base, &["--only", "js,swift,kotlin"]).0, 0); edit(&d, "wai/wai-web/src/lib.rs", "pub fn present(&mut self, at_ms: u32)", "pub fn present(&mut self, at_ms: u64)"); holds_until_acknowledged(&d, &base, "js", "api:js:wai-web:Sink.present"); reset(&d); edit(&d, "wai/wai-web/src/lib.rs", "#[wasm_bindgen(js_name = verifyPin)]", "#[wasm_bindgen(js_name = checkPin)]"); holds_until_acknowledged(&d, &base, "js", "api:js:wai-web:verifyPin"); reset(&d); edit(&d, "wai/wai-sink-swift/Sources/WaiSink/Sink.swift", "public init(path: String)", "public init(file: String)"); holds_until_acknowledged(&d, &base, "swift", "api:swift:WaiSink.WaiSink.init"); reset(&d); edit(&d, "wai/wai-sink-swift/Sources/WaiSink/Sink.swift", "atMs: UInt64 = 0)", "atMs: UInt64 = 0, label: String? = nil)"); let (code, v) = breaking(&d, &base, &["--only", "swift"]); assert_eq!(code, 0, "a parameter with a default is compatible: {v:#}"); reset(&d); edit(&d, "wai/wai-sink-android/core/src/main/kotlin/wai/sink/WaiSink.kt", "atMs: Long = 0)", "atMs: Long = 0, label: String)"); holds_until_acknowledged(&d, &base, "kotlin", "api:kotlin:wai.sink.WaiSink.present"); reset(&d); } /// `wai breaking` in a project, in `dir`: the exit status and the envelope. fn in_project(dir: &Path, args: &[&str]) -> (i32, Value) { let o: Output = Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(dir).args(["breaking", "--json"]).args(args).output().expect("wai runs"); let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|_| panic!("JSON: {}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))); (o.status.code().unwrap_or(-1), v) } const SDK: &str = "export function play(video, url, onState = () => {}) {}\nexport function stop(video) {}\nexport const VERSION = 2;\n"; /// A project's accepted breaks each say why: one with no reason is refused with the project file /// (exit 2), before anything is compared. (Planted fault: an allow with no reason.) #[test] fn an_accepted_break_with_no_reason_is_refused() { let d = tmp("project-reason"); write(&d, "wai.json", r#"{"wai":2,"name":"p","breaking":{"api":["sdk/index.js"],"allow":[{"id":"api:js:sdk/index.js:stop"}]}}"#); write(&d, "sdk/index.js", SDK); std::fs::create_dir_all(d.join("old")).unwrap(); let (code, v) = in_project(&d, &["--baseline", "old"]); assert_eq!(code, 2, "{v:#}"); assert_eq!(v["error"]["code"], "project-invalid"); assert!(v["error"]["message"].as_str().unwrap().contains("breaking.allow[0].reason"), "{v:#}"); write(&d, "wai.json", r#"{"wai":2,"name":"p","breaking":{"api":["sdk/index.js"],"allow":[{"id":"api:js:sdk/index.js:stop","reason":" "}]}}"#); assert_eq!(in_project(&d, &["--baseline", "old"]).0, 2, "an empty reason is none"); // `breaking` is a member of version 2: a file of version 1 that names it is told so. write(&d, "wai.json", r#"{"name":"p","breaking":{"api":["sdk/index.js"]}}"#); let (code, v) = in_project(&d, &["--baseline", "old"]); assert_eq!(code, 2, "{v:#}"); assert!(v["error"]["message"].as_str().unwrap().contains("a member of version 2"), "{v:#}"); } /// A project's API, against the project at a git ref: a removed export fails until /// breaking.allow accepts it with a reason, which the report carries; an allow that matches /// nothing is a note. #[test] fn a_projects_api_breaks_until_its_project_accepts_it() { let (d, _) = repo("project-api"); let p = d.join("apps/player"); write(&p, "wai.json", r#"{"wai":2,"name":"player","breaking":{"api":["sdk"]}}"#); write(&p, "sdk/index.js", SDK); write(&p, "sdk/README.md", "not an API\n"); let base = commit(&d, "a project"); edit(&p, "sdk/index.js", "export function stop(video) {}\n", ""); let (code, v) = in_project(&p, &["--against", &base]); assert_eq!(code, 1, "{v:#}"); assert_eq!(v["data"]["project"], "."); let f = finding(&v, "api:js:sdk/index.js:stop").unwrap_or_else(|| panic!("{v:#}")); assert_eq!(f["breaking"], true); write(&p, "wai.json", r#"{"wai":2,"name":"player","breaking":{"api":["sdk"],"allow":[{"id":"api:js:sdk/index.js:stop","reason":"pausing replaces stopping; call pause"},{"id":"api:js:sdk/index.js:seek","reason":"never there"}]}}"#); let (code, v) = in_project(&p, &["--against", &base]); assert_eq!(code, 0, "{v:#}"); let f = finding(&v, "api:js:sdk/index.js:stop").unwrap(); assert_eq!((f["acknowledged"].as_bool(), f["reason"].as_str()), (Some(true), Some("pausing replaces stopping; call pause"))); assert!(v["data"]["notes"].as_array().unwrap().iter().any(|n| n.as_str().unwrap().contains("api:js:sdk/index.js:seek")), "{v:#}"); // A required parameter added: another break, not accepted. edit(&p, "sdk/index.js", "export function play(video, url, onState = () => {})", "export function play(video, url, onState = () => {}, room)"); assert_eq!(in_project(&p, &["--against", &base]).0, 1); // --project names it from elsewhere, as the same run. let (code, v) = in_project(&d, &["--project", "apps/player", "--against", &base]); assert_eq!(code, 1, "{v:#}"); assert!(finding(&v, "api:js:sdk/index.js:play").is_some(), "{v:#}"); reset(&d); } fn copy_dir(from: &Path, to: &Path) { std::fs::create_dir_all(to).unwrap(); for e in std::fs::read_dir(from).unwrap().flatten() { let p = e.path(); if p.is_dir() { copy_dir(&p, &to.join(e.file_name())); } else { std::fs::copy(&p, to.join(e.file_name())).unwrap(); } } } /// A project's package, against the package as published: the sink-class check. A class gone /// breaks; without the package built, or with no package in the baseline, the run stops (exit 5) /// instead of passing. #[test] fn a_projects_package_is_held_to_its_sink_classes() { let pack = Path::new(env!("CARGO_MANIFEST_DIR")).join("../pack-conformance/packs/video-base-01/package"); let d = tmp("project-package"); write(&d, "wai.json", r#"{"name":"title","package":{"policy":"policy.toml"}}"#); write(&d, "policy.toml", "schema = \"wai.pack.policy/1\"\n"); copy_dir(&pack, &d.join("published")); // Not built yet: no package to compare. let (code, v) = in_project(&d, &["--baseline", "published"]); assert_eq!(code, 5, "{v:#}"); assert_eq!(v["error"]["code"], "package-not-built"); copy_dir(&pack, &d.join("dist/package")); let (code, v) = in_project(&d, &["--baseline", "published"]); assert_eq!(code, 0, "the same package: {v:#}"); assert_eq!(v["data"]["surfaces"], json!(["package", "schemas", "openapi", "js", "swift", "kotlin"])); let policy = d.join("dist/package/policy.json"); let mut p: Value = serde_json::from_slice(&std::fs::read(&policy).unwrap()).unwrap(); p["class"].as_array_mut().unwrap().retain(|c| c["id"] != "tv"); std::fs::write(&policy, p.to_string()).unwrap(); std::fs::remove_dir_all(d.join("dist/package/hls/tv")).unwrap(); let (code, v) = in_project(&d, &["--baseline", "published"]); assert_eq!(code, 1, "{v:#}"); let f = finding(&v, "wire:sink-class:tv").unwrap_or_else(|| panic!("{v:#}")); assert_eq!(f["breaking"], true); assert_eq!(in_project(&d, &["--baseline", "published", "--skip", "package"]).0, 0); // A baseline that is neither a project nor a package; a project baseline with no package. std::fs::create_dir_all(d.join("empty")).unwrap(); assert_eq!(in_project(&d, &["--baseline", "empty"]).1["error"]["code"], "baseline-not-found"); write(&d, "old-project/wai.json", r#"{"name":"title","package":{"policy":"policy.toml"}}"#); let (code, v) = in_project(&d, &["--baseline", "old-project"]); assert_eq!((code, v["error"]["code"].as_str()), (5, Some("baseline-package-not-found")), "{v:#}"); } /// Outside a project, --baseline is refused, and so is --root with --project. #[test] fn a_baseline_is_a_projects() { let (d, base) = repo("baseline-usage"); let o = Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(&d).args(["breaking", "--json", "--baseline", "."]).output().unwrap(); assert_eq!(o.status.code(), Some(2), "{}", String::from_utf8_lossy(&o.stdout)); let o = Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(&d).args(["breaking", "--json", "--against", &base, "--baseline", "."]).output().unwrap(); assert_eq!(o.status.code(), Some(2)); let o = Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(&d).args(["breaking", "--json", "--against", &base, "--only", "package"]).output().unwrap(); assert_eq!(o.status.code(), Some(2), "package is a project's surface"); }