open-standards/wai/wai-rs/tests/cli_mcp.rs

//! `wai mcp` as a coding agent meets it: the binary started as an MCP server, and a client in
//! this test speaking the protocol to it over standard input and output (newline-delimited
//! JSON-RPC 2.0, MCP 2025-06-18): the handshake, the tool list, and every tool through
//! `tools/call`.
//!
//! - Every tool is an operation of the toolchain's catalogue, named by it: the tool list is
//!   exactly the operations below, each tool's name its operation's with `_` for the dot, each
//!   operation of the catalogue (`wai::catalogue`). `core` is offered by default,
//!   `dev` when it is asked for, and a toolset that does not exist is refused.
//! - `receipt_verify` agrees with the committed corpora vector by vector: every group and object
//!   vector of jwp-conformance (kind delivery), every claim vector whose context is one the tool
//!   takes (none, the previous claim, the object, an OER receipt, a gap's object; kind claim),
//!   every room of live-conformance/rooms (kind room_session) and every session of
//!   live-conformance/audit (kind audit_bundle); a receipt changed by one character is refused.
//! - `envelope_wrap` writes an envelope inside the project directory only, never over a file
//!   without `overwrite`; `envelope_inspect` reads it back, and the directory as a package.
//! - `conformance_run` runs a corpus's independent verifier and reports it passed.
//! - The `dev` tools refuse a service that is not on a loopback address, before any
//!   connection, and any target but `local`; with no room they say how to start one. (The room
//!   itself, joined, published to and verified through these tools, is joule-moq-room's
//!   tests/mcp_dev_room.rs.)
//! - `--read-only` offers no tool that writes, serves or asks for a grant.
#![cfg(feature = "mcp")]

use serde_json::{json, Value};
use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};

/// The `wai/` directory: the specification and the corpora.
fn wai_dir() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR")).join("..").canonicalize().unwrap()
}

/// An MCP client of a `wai mcp` process.
struct Client {
    child: Child,
    stdin: ChildStdin,
    out: BufReader<ChildStdout>,
    next: i64,
}

impl Client {
    fn start(project: &Path, extra: &[&str]) -> Client {
        let mut child = Command::new(env!("CARGO_BIN_EXE_wai"))
            .arg("mcp")
            .arg("--spec-root")
            .arg(wai_dir())
            .arg("--project")
            .arg(project)
            .args(extra)
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(Stdio::inherit())
            .spawn()
            .expect("wai mcp starts");
        let stdin = child.stdin.take().unwrap();
        let out = BufReader::new(child.stdout.take().unwrap());
        let mut c = Client { child, stdin, out, next: 1 };
        let init = c.request("initialize", json!({"protocolVersion": "2025-06-18", "capabilities": {}, "clientInfo": {"name": "cli_mcp test", "version": "1"}}));
        assert_eq!(init["result"]["protocolVersion"], "2025-06-18", "{init}");
        c.send(&json!({"jsonrpc": "2.0", "method": "notifications/initialized"}));
        c
    }

    fn send(&mut self, v: &Value) {
        writeln!(self.stdin, "{v}").unwrap();
        self.stdin.flush().unwrap();
    }

    fn line(&mut self) -> Value {
        let mut l = String::new();
        assert!(self.out.read_line(&mut l).unwrap() > 0, "the server closed its output");
        serde_json::from_str(&l).unwrap_or_else(|e| panic!("a line that is not JSON on the protocol channel: {e}: {l:?}"))
    }

    fn request(&mut self, method: &str, params: Value) -> Value {
        let id = self.next;
        self.next += 1;
        self.send(&json!({"jsonrpc": "2.0", "id": id, "method": method, "params": params}));
        let v = self.line();
        assert_eq!(v["id"], json!(id), "answers come in order: {v}");
        v
    }

    /// A tool's structured result, or its error text.
    fn call(&mut self, name: &str, args: Value) -> Result<Value, String> {
        let v = self.request("tools/call", json!({"name": name, "arguments": args}));
        let r = &v["result"];
        assert!(r.is_object(), "{name}: {v}");
        let text = r["content"][0]["text"].as_str().unwrap_or("").to_string();
        if r["isError"] == json!(true) {
            return Err(text);
        }
        let s = r["structuredContent"].clone();
        assert!(s.is_object(), "{name}: a structured result: {v}");
        assert!(serde_json::from_str::<Value>(&text).is_ok_and(|t| t.is_object()), "{name}: the text block is the result's JSON");
        Ok(s)
    }

    fn ok(&mut self, name: &str, args: Value) -> Value {
        self.call(name, args.clone()).unwrap_or_else(|e| panic!("{name} {args}: {e}"))
    }
}

impl Drop for Client {
    fn drop(&mut self) {
        let _ = self.child.kill();
        let _ = self.child.wait();
    }
}

fn scratch(tag: &str) -> PathBuf {
    let d = std::env::temp_dir().join(format!("wai-cli-mcp-{tag}-{}", std::process::id()));
    let _ = std::fs::remove_dir_all(&d);
    std::fs::create_dir_all(&d).unwrap();
    d.canonicalize().unwrap()
}

fn corpus(rel: &str) -> Value {
    serde_json::from_str(&std::fs::read_to_string(wai_dir().join(rel)).unwrap()).unwrap()
}

/// The operations of each toolset, in the order the server lists them. A tool is its
/// operation's name with `_` for the dot.
const CORE: &[&str] = &["envelope.inspect", "registry.list", "registry.show", "receipt.verify", "envelope.wrap", "conformance.list", "conformance.run", "docs.search"];
const DEV: &[&str] = &["dev.start", "dev.stop", "room.grant", "room.join", "room.leave", "session.show"];

fn tool_names(c: &mut Client) -> Vec<String> {
    let list = c.request("tools/list", json!({}));
    list["result"]["tools"].as_array().unwrap().iter().map(|t| t["name"].as_str().unwrap().to_string()).collect()
}

fn names_of(ops: &[&[&str]]) -> Vec<String> {
    ops.iter().flat_map(|s| s.iter()).map(|op| op.replace('.', "_")).collect()
}

#[test]
fn the_handshake_and_the_tool_list() {
    let d = scratch("handshake");
    // core by default.
    let mut c = Client::start(&d, &[]);
    assert_eq!(tool_names(&mut c), names_of(&[CORE]));
    // core and dev when asked: every tool an operation named by the rule.
    let mut c = Client::start(&d, &["--toolset", "core,dev"]);
    let list = c.request("tools/list", json!({}));
    let tools = list["result"]["tools"].as_array().unwrap();
    let names: Vec<String> = tools.iter().map(|t| t["name"].as_str().unwrap().to_string()).collect();
    assert_eq!(names, names_of(&[CORE, DEV]));
    for t in tools {
        let op = t["_meta"]["wai/operation"].as_str().unwrap();
        assert!(wai::catalogue::op(op).is_some_and(|o| o.mcp), "{t}: an operation of the catalogue that wai mcp serves");
        assert_eq!(t["name"].as_str().unwrap(), op.replace('.', "_"), "{t}: the tool is its operation, named by it");
        let set = t["_meta"]["wai/toolset"].as_str().unwrap();
        assert_eq!(set, if DEV.contains(&op) { "dev" } else { "core" }, "{t}");
        let d = t["description"].as_str().unwrap();
        assert!(!d.is_empty() && !d.contains('\n'), "{t}: a one-line description");
        assert_eq!(t["inputSchema"]["type"], "object", "{t}");
        assert!(t["annotations"]["readOnlyHint"].is_boolean(), "{t}");
        assert_eq!(d.starts_with("DEVELOPMENT ONLY."), set == "dev", "{t}: a development tool is marked so, and only one");
    }
    let mut only_dev = Client::start(&d, &["--toolset", "dev"]);
    assert_eq!(tool_names(&mut only_dev), names_of(&[DEV]));
    // A toolset this server does not have is refused before it serves.
    let out = Command::new(env!("CARGO_BIN_EXE_wai")).args(["mcp", "--toolset", "core,nonesuch"]).stdin(Stdio::null()).output().unwrap();
    assert_eq!(out.status.code(), Some(2));
    assert!(String::from_utf8_lossy(&out.stderr).contains("nonesuch"), "{out:?}");
    // The protocol's own answers.
    assert_eq!(c.request("ping", json!({}))["result"], json!({}));
    assert_eq!(c.request("nonsense/method", json!({}))["error"]["code"], -32601);
    assert_eq!(c.request("tools/call", json!({"name": "no_such_tool", "arguments": {}}))["error"]["code"], -32602);
    c.send(&json!([{"jsonrpc": "2.0", "id": 99, "method": "ping"}]));
    assert_eq!(c.line()["error"]["code"], -32600, "a batch is refused");
    writeln!(c.stdin, "{{not json").unwrap();
    c.stdin.flush().unwrap();
    assert_eq!(c.line()["error"]["code"], -32700);
    // An older revision a client asks for is answered in it.
    let mut old = Client::start(&d, &[]);
    assert_eq!(old.request("initialize", json!({"protocolVersion": "2024-11-05"}))["result"]["protocolVersion"], "2024-11-05");
}

#[test]
fn read_only_offers_no_tool_that_writes_serves_or_mints() {
    let d = scratch("readonly");
    let mut c = Client::start(&d, &["--toolset", "core,dev", "--read-only"]);
    let names = tool_names(&mut c);
    for gone in ["envelope_wrap", "dev_start", "dev_stop", "room_grant", "room_join", "room_leave", "session_show"] {
        assert!(!names.iter().any(|n| n == gone), "{gone} is offered read-only");
    }
    assert!(names.iter().any(|n| n == "receipt_verify"));
    let r = c.request("tools/call", json!({"name": "envelope_wrap", "arguments": {"input": "x", "capability": "wai.text.zstd"}}));
    assert_eq!(r["error"]["code"], -32602, "{r}");
}

#[test]
fn the_registry_and_the_specification() {
    let d = scratch("registry");
    let mut c = Client::start(&d, &[]);
    let all = c.ok("registry_list", json!({}));
    let n = all["count"].as_u64().unwrap();
    assert!(n > 50 && all["capabilities"].as_array().unwrap().len() as u64 == n);
    let rf = c.ok("registry_list", json!({"licence": "royalty-free", "media": "image"}));
    assert!(rf["count"].as_u64().unwrap() > 0);
    for e in rf["capabilities"].as_array().unwrap() {
        assert_eq!((e["licence"].as_str(), e["media"].as_str()), (Some("royalty-free"), Some("image")), "{e}");
    }
    let png = c.ok("registry_show", json!({"capability": "wai.image.png"}));
    assert_eq!(png["registry"]["capability"], "wai.image.png");
    assert_eq!(png["this_build"]["licence"], png["registry"]["licence"]);
    assert!(!png["spec_lines"].as_array().unwrap().is_empty(), "{png}");
    let e = c.call("registry_show", json!({"capability": "wai.image.nothing"})).unwrap_err();
    assert!(e.contains("not in the registry") && e.contains("registry_list"), "{e}");
    let s = c.ok("docs_search", json!({"query": "capability fallback", "limit": 5}));
    let hits = s["hits"].as_array().unwrap();
    assert_eq!(hits.len(), 5);
    for h in hits {
        let t = h["text"].as_str().unwrap().to_ascii_lowercase();
        assert!(t.contains("capability") && t.contains("fallback"), "{h}");
        assert!(h["line"].as_u64().unwrap() > 0 && h["file"].as_str().unwrap().ends_with(".md"));
    }
    assert_eq!(hits[0]["file"], "SPEC.md", "SPEC.md is searched first");
    assert!(c.call("docs_search", json!({"query": "  "})).is_err());
}

#[test]
fn packaging_and_inspecting_stay_inside_the_project() {
    let d = scratch("package");
    let mut c = Client::start(&d, &[]);
    let input = d.join("hello.txt");
    std::fs::write(&input, b"hello, world").unwrap();
    let p = c.ok("envelope_wrap", json!({"input": input, "capability": "wai.text.zstd", "intent": "improve"}));
    assert_eq!((p["format"].as_str(), p["components"][0]["capability"].as_str()), (Some("WAI1"), Some("wai.text.zstd")), "{p}");
    let out = d.join("hello.txt.wai");
    assert!(out.is_file());
    assert!(c.call("envelope_wrap", json!({"input": input, "capability": "wai.text.zstd", "intent": "improve"})).unwrap_err().contains("exists"));
    c.ok("envelope_wrap", json!({"input": input, "capability": "wai.text.zstd", "intent": "improve", "overwrite": true}));
    let outside = std::env::temp_dir().join(format!("wai-cli-mcp-outside-{}.wai", std::process::id()));
    let e = c.call("envelope_wrap", json!({"input": input, "capability": "wai.text.zstd", "output": outside})).unwrap_err();
    assert!(e.contains("outside the project directory") && !outside.exists(), "{e}");
    assert!(c.call("envelope_wrap", json!({"input": input, "capability": "wai.text.zstd", "output": "../escape.wai"})).is_err());
    assert!(c.call("envelope_wrap", json!({"input": input, "capability": "not.a.capability"})).is_err());
    #[cfg(unix)]
    {
        std::os::unix::fs::symlink(&outside, d.join("link.wai")).unwrap();
        assert!(c.call("envelope_wrap", json!({"input": input, "capability": "wai.text.zstd", "output": "link.wai", "overwrite": true})).unwrap_err().contains("a link"));
        assert!(!outside.exists(), "nothing written through a link");
        std::fs::remove_file(d.join("link.wai")).unwrap();
    }
    let i = c.ok("envelope_inspect", json!({"path": out}));
    assert_eq!((i["kind"].as_str(), i["format"].as_str(), i["intent"].as_str()), (Some("envelope"), Some("WAI1"), Some("improve")), "{i}");
    let comp = &i["components"][0];
    assert_eq!(comp["capability"], "wai.text.zstd");
    assert_eq!(comp["bytes"], 12);
    assert!(comp["licence"].is_string() && comp["tier"].is_string(), "{comp}");
    assert_eq!(i["checks"]["component_roles"], "ok");
    let pkg = c.ok("envelope_inspect", json!({"path": d}));
    assert_eq!(pkg["kind"], "package");
    assert_eq!(pkg["envelopes"].as_array().unwrap().len(), 1);
    assert_eq!(pkg["listing_sha256"].as_str().unwrap().len(), 64);
    assert!(pkg["capabilities"]["wai.text.zstd"].is_object(), "{pkg}");
    assert!(c.call("envelope_inspect", json!({"path": input})).unwrap_err().contains("not a WAI envelope"));
    let _ = std::fs::remove_dir_all(&d);
}

/// `reject step=N reason=X` or `accept` as the corpus states it.
fn expected_receipt(e: &Value) -> (String, Option<u64>, Option<String>) {
    (e["outcome"].as_str().unwrap().replace("reject", "refuse"), e["step"].as_u64(), e["reason"].as_str().map(String::from))
}

#[test]
fn receipts_agree_with_the_jwp_corpus_and_a_changed_one_is_refused() {
    let d = scratch("receipts");
    let mut c = Client::start(&d, &[]);
    let doc = corpus("jwp-conformance/vectors.json");
    let mut n = 0;
    for v in doc["vectors"].as_array().unwrap() {
        let i = &v["input"];
        let args = match v["procedure"].as_str() {
            Some("group") => {
                let mut a = json!({"receipt": i["group"]});
                if let Some(p) = i.get("previous") {
                    a["previous"] = p.clone();
                }
                a
            }
            Some("object") => json!({"receipt": i["object"], "group": i["group"], "envelope_hex": i["envelope"]}),
            _ => continue,
        };
        let got = c.ok("receipt_verify", args);
        assert_eq!(got["kind"], "delivery", "{}: the kind read from the arguments", v["name"]);
        let (want, step, reason) = expected_receipt(&v["expected"]);
        assert_eq!(got["verdict"].as_str(), Some(want.as_str()), "{}: {got}", v["name"]);
        if want == "refuse" {
            assert_eq!((got["step"].as_u64(), got["reason"].as_str().map(String::from)), (step, reason), "{}: {got}", v["name"]);
        } else if let Some(h) = v["expected"]["link_hash"].as_str() {
            assert_eq!(got["link_hash"], h, "{}", v["name"]);
        }
        n += 1;
    }
    assert_eq!(n, 54, "every group and object vector");
    // A group receipt that verifies, its figure changed: refused.
    let good = doc["vectors"].as_array().unwrap().iter().find(|v| v["procedure"] == "group" && v["expected"]["outcome"] == "accept" && v["input"].get("previous").is_none()).unwrap();
    let mut bad = good["input"]["group"].clone();
    bad["joules_total"] = json!(bad["joules_total"].as_u64().unwrap() + 1);
    let r = c.ok("receipt_verify", json!({"receipt": bad}));
    assert_eq!((r["verdict"].as_str(), r["reason"].as_str()), (Some("refuse"), Some("signature_verify_failed")), "{r}");
    let r = c.ok("receipt_verify", json!({"kind": "delivery", "receipt": good["input"]["group"].to_string()}));
    assert_eq!((r["kind"].as_str(), r["receipt"].as_str(), r["verdict"].as_str()), (Some("delivery"), Some("group"), Some("accept")), "a receipt given as its text: {r}");
    assert!(c.call("receipt_verify", json!({"receipt": {"kind": "elsewhere"}})).is_err());
    // A kind the tool does not take, arguments naming none, and arguments naming two.
    assert!(c.call("receipt_verify", json!({"kind": "letter", "receipt": bad})).unwrap_err().contains("kind"));
    assert!(c.call("receipt_verify", json!({})).is_err());
    assert!(c.call("receipt_verify", json!({"receipt": bad, "claim": "x"})).unwrap_err().contains("give kind"));
}

#[test]
fn claims_agree_with_the_claim_corpus() {
    let d = scratch("claims");
    let mut c = Client::start(&d, &[]);
    let doc = corpus("claim-conformance/vectors.json");
    let mut n = 0;
    let taken = ["previous", "object", "oer", "gap_object"];
    for v in doc["vectors"].as_array().unwrap() {
        let ctx = v.get("context").cloned().unwrap_or(json!({}));
        if v["procedure"] != "claim" || ctx.as_object().unwrap().keys().any(|k| !taken.contains(&k.as_str())) {
            continue;
        }
        let mut args = ctx.clone();
        args["claim"] = v["receipt"].clone();
        let got = c.ok("receipt_verify", args);
        assert_eq!(got["kind"], "claim", "{}", v["name"]);
        let e = &v["expected"];
        if e["outcome"] == "accept" {
            assert_eq!(got["verdict"], "accept", "{}: {got}", v["name"]);
            if let Some(h) = e["link_hash"].as_str() {
                assert_eq!(got["link_hash"], h, "{}", v["name"]);
            }
        } else {
            assert_eq!((got["verdict"].as_str(), got["step"].as_u64(), got["reason"].as_str()), (Some("refuse"), e["step"].as_u64(), e["reason"].as_str()), "{}: {got}", v["name"]);
        }
        n += 1;
    }
    assert!(n > 340, "{n} claim vectors, alone or with the context the tool takes");
}

#[test]
fn rooms_agree_with_the_live_rooms_corpus() {
    let d = scratch("rooms");
    let mut c = Client::start(&d, &[]);
    let doc = corpus("live-conformance/rooms/vectors.json");
    for v in doc["vectors"].as_array().unwrap() {
        let got = c.ok("receipt_verify", json!({"kind": "room_session", "descriptor": v["descriptor"], "grants": v["grants"], "key_log_head": v["key_log_head"], "chains": v["chains"]}));
        assert_eq!(got["kind"], "room_session");
        assert_eq!(got["outcome"], v["outcome"], "{}: {got}", v["name"]);
        match v["outcome"].as_str().unwrap() {
            "refuse" => assert_eq!((&got["reason"], &got["at"]), (&v["reason"], &v["at"]), "{}", v["name"]),
            "gap" => assert_eq!((&got["gaps"], &got["room"]), (&v["gaps"], &v["room"]), "{}", v["name"]),
            _ => assert_eq!(got["room"], v["room"], "{}", v["name"]),
        }
    }
}

#[test]
fn audit_bundles_agree_with_the_stream_audit_corpus() {
    let d = scratch("audit");
    let mut c = Client::start(&d, &[]);
    let doc = corpus("live-conformance/audit/vectors.json");
    for s in doc["sessions"].as_array().unwrap() {
        let got = c.ok("receipt_verify", json!({"bundle": s["bundle"], "owner": doc["owner"], "channels": doc["channels"]}));
        assert_eq!(got["kind"], "audit_bundle");
        assert_eq!(got["outcome"], s["outcome"]["outcome"], "{}: {got}", s["name"]);
        if s["outcome"]["outcome"] == "refuse" {
            assert_eq!(got["reason"], s["outcome"]["reason"], "{}", s["name"]);
        }
    }
}

#[test]
fn a_corpus_runs_through_its_independent_verifier() {
    let d = scratch("conformance");
    let mut c = Client::start(&d, &[]);
    let l = c.ok("conformance_list", json!({}));
    let pairs: Vec<(String, String)> = l["corpora"].as_array().unwrap().iter().map(|p| (p["corpus"].as_str().unwrap().into(), p["verifier"].as_str().unwrap().into())).collect();
    assert!(pairs.contains(&("live-conformance/rooms".into(), "live-conformance/live_rooms_verify.py".into())), "{pairs:?}");
    assert!(pairs.contains(&("claim-conformance".into(), "claim-conformance/claim_verify.py".into())), "{pairs:?}");
    let r = c.ok("conformance_run", json!({"corpus": "live-conformance/rooms", "timeout_s": 600}));
    assert_eq!(r["passed"], true, "{r}");
    assert_eq!(r["exit_code"], 0);
    assert!(c.call("conformance_run", json!({"corpus": "no-such-conformance"})).unwrap_err().contains("no verifier"));
    let e = c.call("conformance_run", json!({"corpus": "live-conformance/rooms", "verifier": "../scripts/check-spec-refs.py"})).unwrap_err();
    assert!(e.contains("outside") || e.contains("_verify.py"), "{e}");
}

#[test]
fn development_tools_refuse_any_service_off_this_machine() {
    let d = scratch("dev");
    let mut c = Client::start(&d, &["--toolset", "core,dev"]);
    for remote in ["http://10.0.0.5:9682", "https://127.0.0.1:9682", "http://room.example/room-api", "http://127.0.0.1.example:80"] {
        let e = c.call("room_grant", json!({"principal": "alice", "role": "publisher", "service": remote})).unwrap_err();
        assert!(e.starts_with("DEVELOPMENT ONLY"), "{remote}: {e}");
    }
    // A target other than local is refused, whatever the service.
    for t in ["room_grant", "room_join", "room_leave", "session_show"] {
        let e = c.call(t, json!({"target": "cloud", "principal": "alice", "role": "publisher"})).unwrap_err();
        assert!(e.contains("target local alone"), "{t}: {e}");
    }
    // A loopback service is asked: with nothing listening, the connection is what fails.
    let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
    let e = c.call("room_grant", json!({"target": "local", "principal": "alice", "role": "publisher", "service": format!("http://127.0.0.1:{port}")})).unwrap_err();
    assert!(!e.starts_with("DEVELOPMENT ONLY") && e.contains(&port.to_string()), "{e}");
    // With no room running, each tool says how to start one.
    for t in ["room_join", "room_leave"] {
        assert!(c.call(t, json!({"principal": "alice"})).is_err());
    }
    assert!(c.call("session_show", json!({})).unwrap_err().contains("dev_start"));
    assert!(c.call("dev_stop", json!({})).is_err());
    assert!(c.call("room_grant", json!({"principal": "alice", "role": "publisher"})).unwrap_err().contains("dev_start"));
    let e = c.call("dev_start", json!({"room_bin": d.join("no-such-program")})).unwrap_err();
    assert!(e.contains("no-such-program"), "{e}");
}