open-standards/wai/wai-rs/tests/cli_toolchain.rs

//! The toolchain's own surface through the `wai` binary (`--features quickstart --test
//! cli_toolchain`):
//!
//! - `wai --version` and `--json` name one version, the digests of the catalogue and the registry
//!   as they are committed, the release of the standard and the build's features; the crate, the
//!   export and the changelog's newest release say the same version;
//! - `wai docs search` and `wai registry show` give what `wai mcp`'s tools give, with a refusal's
//!   exit code and stable code;
//! - a deprecated flag still works and says so in one line, and `wai/DEPRECATIONS.md` is held to
//!   the catalogue, both ways.
#![cfg(feature = "quickstart")]

use serde_json::{Value, json};
use sha2::{Digest, Sha256};
use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Output, Stdio};
use wai::catalogue::{self, OPS};

fn wai_dir() -> PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR")).join("..").canonicalize().unwrap()
}

fn scratch(name: &str) -> PathBuf {
    let d = Path::new(env!("CARGO_TARGET_TMPDIR")).join("cli_toolchain").join(name);
    let _ = std::fs::remove_dir_all(&d);
    std::fs::create_dir_all(&d).unwrap();
    d
}

/// `wai`, with no specification root from its environment.
fn wai(d: &Path, args: &[&str]) -> Output {
    Command::new(env!("CARGO_BIN_EXE_wai")).current_dir(d).args(args).env_remove("WAI_SPEC_ROOT").stdin(Stdio::null()).output().expect("wai runs")
}

fn text(o: &Output) -> String {
    format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr))
}

/// The envelope of a `--json` run.
fn envelope(d: &Path, args: &[&str]) -> (Option<i32>, Value) {
    let mut a = args.to_vec();
    a.push("--json");
    let o = wai(d, &a);
    let v: Value = serde_json::from_slice(&o.stdout).unwrap_or_else(|e| panic!("{args:?}: not one envelope ({e}): {}", text(&o)));
    assert_eq!(v["wai"], catalogue::ENVELOPE, "{v}");
    (o.status.code(), v)
}

fn sha256(b: &[u8]) -> String {
    Sha256::digest(b).iter().map(|x| format!("{x:02x}")).collect()
}

#[test]
fn version_names_the_toolchain_its_catalogue_its_registry_and_the_standard() {
    let d = scratch("version");
    let (code, v) = envelope(&d, &["--version"]);
    assert_eq!(code, Some(0), "{v}");
    assert_eq!((v["ok"].as_bool(), v["op"].as_str()), (Some(true), Some("toolchain.show")), "{v}");
    let data = &v["data"];
    let version = env!("CARGO_PKG_VERSION");
    assert_eq!(data["toolchain"], version);
    assert_eq!(data["specification"]["version"], version);
    assert_eq!(data["specification"]["formats"], json!(["WAI1", "WAI2"]));
    assert_eq!(data["envelope"], catalogue::ENVELOPE);
    assert_eq!(data["catalogue"]["format"], catalogue::FORMAT);
    assert_eq!(data["catalogue"]["operations"], OPS.len());
    // The digests are of the committed files, byte for byte.
    let (ops, reg) = (std::fs::read(wai_dir().join("catalogue/ops.json")).unwrap(), std::fs::read(wai_dir().join("registry/capabilities.json")).unwrap());
    assert_eq!(data["catalogue"]["sha256"], sha256(&ops));
    assert_eq!(data["registry"]["sha256"], sha256(&reg));
    let caps: Value = serde_json::from_slice(&reg).unwrap();
    assert_eq!(data["registry"]["capabilities"], caps["capabilities"].as_array().unwrap().len());
    // The features named are this build's: each answers on, and quickstart is among them here.
    let features: Vec<&str> = data["features"].as_array().unwrap().iter().map(|f| f.as_str().unwrap()).collect();
    assert!(features.contains(&"quickstart"), "{features:?}");
    assert!(features.iter().all(|f| catalogue::feature_on(f) == Some(true)), "{features:?}");

    // Text, for a person; the other spellings run the same operation.
    let o = wai(&d, &["--version"]);
    assert!(o.status.success(), "{}", text(&o));
    let t = String::from_utf8_lossy(&o.stdout).into_owned();
    assert_eq!(t.lines().next(), Some(format!("wai {version}").as_str()), "{t}");
    assert!(t.contains(&sha256(&ops)) && t.contains(&sha256(&reg)), "{t}");
    for spelling in [&["version"][..], &["toolchain", "show"]] {
        assert_eq!(wai(&d, spelling).stdout, o.stdout, "{spelling:?}");
    }
    assert_eq!(wai(&d, &["--version", "extra"]).status.code(), Some(2));
    let h = wai(&d, &["--version", "--help"]);
    assert!(h.status.success() && String::from_utf8_lossy(&h.stdout).contains("toolchain.show"), "{}", text(&h));
}

/// One version source: what the binary says is the crate's, the export's and the newest release of
/// the changelog (WAI toolchain v1, U1).
#[test]
fn the_crate_the_export_and_the_changelog_name_one_version() {
    let version = env!("CARGO_PKG_VERSION");
    let ops: Value = serde_json::from_slice(&std::fs::read(wai_dir().join("catalogue/ops.json")).unwrap()).unwrap();
    assert_eq!(ops["toolchain"], version, "wai/catalogue/ops.json");
    let log = std::fs::read_to_string(wai_dir().join("CHANGELOG.md")).unwrap();
    let newest = log.lines().find_map(|l| l.strip_prefix("## v")).expect("a release heading");
    let newest = newest.split_whitespace().next().unwrap();
    assert_eq!(newest, version, "wai/CHANGELOG.md's newest release is not this crate's version");
}

/// A `wai mcp` server's answer to one tool, as `structuredContent`.
fn tool(args: Value, name: &str, project: &Path) -> Value {
    let mut c = Command::new(env!("CARGO_BIN_EXE_wai"))
        .args(["mcp", "--spec-root", wai_dir().to_str().unwrap(), "--project", project.to_str().unwrap()])
        .stdin(Stdio::piped())
        .stdout(Stdio::piped())
        .stderr(Stdio::inherit())
        .spawn()
        .expect("wai mcp starts");
    let mut stdin = c.stdin.take().unwrap();
    let mut out = BufReader::new(c.stdout.take().unwrap());
    let init = json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2025-06-18", "capabilities": {}}});
    let call = json!({"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": name, "arguments": args}});
    writeln!(stdin, "{init}\n{}\n{call}", json!({"jsonrpc": "2.0", "method": "notifications/initialized"})).unwrap();
    let mut line = String::new();
    out.read_line(&mut line).unwrap();
    line.clear();
    out.read_line(&mut line).unwrap();
    drop(stdin);
    let _ = c.wait();
    let r: Value = serde_json::from_str(&line).unwrap();
    r["result"]["structuredContent"].clone()
}

#[test]
fn docs_search_is_the_tool_wai_mcp_serves() {
    let d = scratch("docs");
    let root = wai_dir();
    let root = root.to_str().unwrap();
    let args = ["docs", "search", "signer_id", "origin-bound", "--spec-root", root];
    let (code, v) = envelope(&d, &args);
    assert_eq!(code, Some(0), "{v}");
    assert_eq!(v["op"], "docs.search");
    let hits = v["data"]["hits"].as_array().unwrap();
    assert!(!hits.is_empty(), "{v}");
    for h in hits {
        let t = h["text"].as_str().unwrap().to_ascii_lowercase();
        assert!(t.contains("signer_id") && t.contains("origin-bound"), "{h}");
        assert!(h["file"].is_string() && h["line"].is_u64() && h["section"].is_string(), "{h}");
    }
    // The door gives what the tool gives.
    assert_eq!(v["data"], tool(json!({"query": "signer_id origin-bound"}), "docs_search", &d));

    // Text names each hit's file and line.
    let o = wai(&d, &args);
    let t = String::from_utf8_lossy(&o.stdout).into_owned();
    assert!(t.contains(&format!("{}:{}", hits[0]["file"].as_str().unwrap(), hits[0]["line"])), "{t}");

    let (_, l) = envelope(&d, &["docs", "search", "capability", "--limit", "2", "--spec-root", root]);
    assert_eq!((l["data"]["count"].as_u64(), l["data"]["truncated"].as_bool()), (Some(2), Some(true)), "{l}");
    let (code, none) = envelope(&d, &["docs", "search", "zzzqqqzzz", "--spec-root", root]);
    assert_eq!((code, none["data"]["count"].as_u64()), (Some(0), Some(0)), "no line holds it, and that is an answer");

    // The refusals.
    assert_eq!(wai(&d, &["docs", "search"]).status.code(), Some(2));
    assert_eq!(wai(&d, &["docs", "search", "x", "--nope"]).status.code(), Some(2));
    assert_eq!(wai(&d, &["docs", "search", "x", "--limit", "0"]).status.code(), Some(2));
    let (code, e) = envelope(&d, &["docs", "search", "x", "--spec-root", d.to_str().unwrap()]);
    assert_eq!((code, e["error"]["code"].as_str()), (Some(5), Some("spec-not-found")), "{e}");
    assert!(e["error"]["hint"].as_str().unwrap().contains("--spec-root"), "{e}");
}

#[test]
fn registry_show_is_the_tool_wai_mcp_serves() {
    let d = scratch("registry");
    let root = wai_dir();
    let root = root.to_str().unwrap();
    let args = ["registry", "show", "wai.image.png", "--spec-root", root];
    let (code, v) = envelope(&d, &args);
    assert_eq!(code, Some(0), "{v}");
    assert_eq!(v["op"], "registry.show");
    assert_eq!(v["data"]["registry"]["capability"], "wai.image.png", "{v}");
    assert_eq!(v["data"]["decoded_natively_here"], true, "the floor, in this build: {v}");
    assert!(!v["data"]["spec_lines"].as_array().unwrap().is_empty(), "{v}");
    assert_eq!(v["data"], tool(json!({"capability": "wai.image.png"}), "registry_show", &d));

    let t = String::from_utf8_lossy(&wai(&d, &args).stdout).into_owned();
    assert!(t.starts_with("wai.image.png\n") && t.contains("decoded natively by this build: yes") && t.contains("in the specification:"), "{t}");

    // Without the specification's text it still answers from the registry, and says so.
    let (_, no_text) = envelope(&d, &["registry", "show", "wai.image.png", "--spec-root", d.to_str().unwrap()]);
    assert!(no_text["data"]["spec_lines"].is_null(), "{no_text}");
    assert_eq!(no_text["data"]["registry"]["capability"], "wai.image.png");

    let (code, e) = envelope(&d, &["registry", "show", "wai.image.none", "--spec-root", root]);
    assert_eq!((code, e["error"]["code"].as_str()), (Some(5), Some("capability-not-found")), "{e}");
    assert_eq!(wai(&d, &["registry", "show"]).status.code(), Some(2));
    assert_eq!(wai(&d, &["registry", "show", "a", "b"]).status.code(), Some(2));
    // The operations beside it still answer as they did.
    assert!(wai(&d, &["registry", "list"]).status.success());
}

#[test]
fn a_deprecated_flag_works_and_says_so_in_one_line() {
    let d = scratch("deprecated");
    std::fs::write(d.join("p.bin"), b"hello").unwrap();
    let notice = "wai: deprecated: --confirm, use --yes instead";
    let o = wai(&d, &["wrap", "wai.text.zstd", "p.bin", "--confirm"]);
    assert!(o.status.success(), "{}", text(&o));
    assert_eq!(String::from_utf8_lossy(&o.stderr).matches(notice).count(), 1, "{}", text(&o));
    assert!(!String::from_utf8_lossy(&o.stdout).contains("deprecated"), "standard error alone");
    // With --json the envelope stays one document, and the notice is said once.
    let o = wai(&d, &["wrap", "wai.text.zstd", "p.bin", "--confirm", "--json"]);
    assert!(o.status.success(), "{}", text(&o));
    assert_eq!(String::from_utf8_lossy(&o.stderr).matches(notice).count(), 1, "{}", text(&o));
    let v: Value = serde_json::from_slice(&o.stdout).unwrap();
    assert_eq!(v["ok"], true, "{v}");
    let o = wai(&d, &["wrap", "wai.text.zstd", "p.bin", "--yes"]);
    assert!(o.status.success() && !text(&o).contains("deprecated"), "{}", text(&o));
}

/// `wai/DEPRECATIONS.md` lists what the catalogue marks deprecated, and nothing it does not, unless
/// the row names its removal; `wai/VERSIONING.md` states the window and links the list.
#[test]
fn the_deprecations_page_is_the_catalogues_list() {
    let page = std::fs::read_to_string(wai_dir().join("DEPRECATIONS.md")).unwrap();
    // (kind, item, deprecated in, replacement, removed in)
    let rows: Vec<Vec<String>> = page
        .lines()
        .filter(|l| l.starts_with('|'))
        .skip(2)
        .map(|l| l.trim().trim_matches('|').split('|').map(|c| c.trim().trim_matches('`').to_string()).collect())
        .collect();
    assert!(rows.iter().all(|r| r.len() == 5), "{rows:?}");
    let mut marked: Vec<(String, String, String)> = Vec::new();
    for f in catalogue::AGENT_FLAGS {
        if let Some((alias, since)) = f.deprecated_alias {
            marked.push((alias.to_string(), since.to_string(), f.flag.to_string()));
        }
    }
    for o in OPS {
        if let Some(d) = o.deprecated {
            marked.push((o.id.to_string(), d.since.to_string(), d.use_instead.to_string()));
        }
        // A flag of one operation renamed: listed as that operation's command and the flag.
        for a in o.input {
            if let (Some((was, since)), catalogue::Cli::Flag(f)) = (a.was, a.cli) {
                let cmd = o.cli.expect("a renamed flag is a command line's");
                marked.push((format!("{cmd} {was}"), since.to_string(), format!("{cmd} {f}")));
            }
        }
    }
    for (item, since, instead) in &marked {
        let r = rows.iter().find(|r| r[1] == *item).unwrap_or_else(|| panic!("the catalogue marks {item} deprecated, and wai/DEPRECATIONS.md does not list it"));
        assert_eq!((&r[2], &r[3]), (since, instead), "{item}");
    }
    for r in &rows {
        assert!(marked.iter().any(|(i, _, _)| *i == r[1]) || !r[4].is_empty(), "wai/DEPRECATIONS.md lists {}, which the catalogue does not mark deprecated and no release removed", r[1]);
    }

    let v = std::fs::read_to_string(wai_dir().join("VERSIONING.md")).unwrap().split_whitespace().collect::<Vec<_>>().join(" ");
    for needle in ["(DEPRECATIONS.md)", "at least two minor releases and six months", "wai --version --json", "wai: deprecated: --confirm, use --yes instead"] {
        assert!(v.contains(needle), "wai/VERSIONING.md does not state {needle:?}");
    }
    assert!(page.contains("(VERSIONING.md)"), "wai/DEPRECATIONS.md links the policy");
}