open-standards/wai/wai-rs/tests/cli_setup_agent.rs
Transaction Science db358c7fd5 The skill lists every core tool, and setup-agent keeps to its own block
The skill's core toolset names toolchain_doctor, and says where the specification's
text comes from for an installed binary. A test holds each toolset's line to that
toolset's tools both ways, and each command's flags to the catalogue. The 9 templates
carry the new text.

setup-agent takes no marker from a closed code fence, adds frontmatter only with the
block, sees frontmatter after a byte order mark, and leaves alone a path with a file
where it needs a directory, setting up the rest of the project.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-11-07 03:08:05 -05:00

414 lines
20 KiB
Rust

//! `wai setup-agent` in a scratch project, run as an agent runs it (no terminal and `--json`, so
//! its output is the toolchain's JSON envelope, `envelope/1`, and it asks nothing):
//!
//! - a fresh project gets AGENTS.md, the skill and `.mcp.json`, and the configuration files of the
//! agents whose directories it holds, and no CLAUDE.md; a second run changes nothing;
//! - a CLAUDE.md the project has gains an import of AGENTS.md;
//! - a rerun replaces only its own block: what a person wrote outside it survives, byte for byte,
//! and a marker named in prose or shown in a code fence is no block;
//! - every template's skill is the one this binary writes, with its own section after the block;
//! - a file that exists is changed only with `--yes` (or `--confirm`): without it nothing that
//! exists is written, the command exits 4 (confirmation required) with each proposed change and
//! the command that confirms; with it the block is added and everything else in the file is
//! kept; `--dry-run` writes nothing;
//! - `--toolset` registers the server with those toolsets;
//! - a JSON file it cannot read as JSON is left alone, with the entry to add by hand;
//! - nothing outside the project directory is touched, through a link or otherwise;
//! - a file where a path needs a directory leaves that path alone, and the rest is set up;
//! - its errors carry the table's exit status and a stable code: 2 usage, 5 not found.
use serde_json::{json, Value};
use std::path::{Path, PathBuf};
use std::process::Command;
fn scratch(tag: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("wai-cli-setup-agent-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d.canonicalize().unwrap()
}
/// Run it in `dir`: its exit status and its JSON.
fn run(dir: &Path, extra: &[&str]) -> (i32, Value) {
let out = Command::new(env!("CARGO_BIN_EXE_wai")).arg("setup-agent").arg("--dir").arg(dir).args(extra).arg("--json").output().expect("run wai");
let text = String::from_utf8_lossy(&out.stdout);
let v: Value = serde_json::from_str(text.trim()).unwrap_or_else(|e| panic!("one JSON envelope with --json: {e}: {text}"));
(out.status.code().unwrap(), v)
}
fn files(v: &Value) -> &Vec<Value> {
assert_eq!((v["wai"].as_str(), v["op"].as_str()), (Some("envelope/1"), Some("agent.setup")), "the toolchain's envelope: {v}");
v["data"]["files"].as_array().unwrap()
}
fn action(v: &Value, path: &str) -> String {
files(v).iter().find(|f| f["path"] == path).map(|f| f["action"].as_str().unwrap().to_string()).unwrap_or_else(|| "absent".into())
}
fn read(p: PathBuf) -> String {
std::fs::read_to_string(p).unwrap()
}
/// The skill's file, in the directory every project gets.
const SKILL: &str = ".agents/skills/wai/SKILL.md";
/// The same skill, in the directory of an agent that keeps its own.
const OWN_SKILL: &str = ".claude/skills/wai/SKILL.md";
const BEGIN: &str = "<!-- wai agent instructions: begin -->";
const END: &str = "<!-- wai agent instructions: end -->";
/// The block of `text`, markers included.
fn block(text: &str) -> &str {
let (b, e) = (text.find(BEGIN).unwrap(), text.find(END).unwrap());
&text[b..e + END.len()]
}
#[test]
fn a_fresh_project_is_set_up_and_a_second_run_changes_nothing() {
let d = scratch("fresh");
std::fs::create_dir(d.join(".vscode")).unwrap();
let (code, v) = run(&d, &[]);
assert_eq!(code, 0, "{v}");
assert_eq!(v["ok"], true);
for p in ["AGENTS.md", ".mcp.json", ".vscode/mcp.json"] {
assert_eq!(action(&v, p), "created", "{p}: {v}");
}
assert_eq!(action(&v, ".cursor/mcp.json"), "absent", "an agent whose directory is not there is left out");
assert_eq!(action(&v, "CLAUDE.md"), "absent", "no CLAUDE.md is made where the project has none");
assert!(!d.join("CLAUDE.md").exists());
let agents = read(d.join("AGENTS.md"));
assert!(agents.contains("## WAI") && agents.contains("wai mcp") && agents.contains("receipt_verify"), "{agents}");
let mcp: Value = serde_json::from_str(&read(d.join(".mcp.json"))).unwrap();
assert_eq!(mcp["mcpServers"]["wai"], json!({"command": "wai", "args": ["mcp"]}));
let vs: Value = serde_json::from_str(&read(d.join(".vscode/mcp.json"))).unwrap();
assert_eq!(vs["servers"]["wai"], json!({"type": "stdio", "command": "wai", "args": ["mcp"]}));
let (code, v) = run(&d, &[]);
assert_eq!(code, 0);
assert!(files(&v).iter().all(|f| f["action"] == "unchanged"), "{v}");
// --all writes every agent's file, a directory it needs made inside the project.
let (code, v) = run(&d, &["--all", "--confirm"]);
assert_eq!(code, 0, "{v}");
for p in [".cursor/mcp.json", ".gemini/settings.json", "opencode.json"] {
assert_eq!(action(&v, p), "created", "{p}");
}
let g: Value = serde_json::from_str(&read(d.join(".gemini/settings.json"))).unwrap();
assert_eq!(g["context"]["fileName"][0], "AGENTS.md", "{g}");
let oc: Value = serde_json::from_str(&read(d.join("opencode.json"))).unwrap();
assert_eq!(oc["mcp"]["wai"], json!({"type": "local", "command": ["wai", "mcp"], "enabled": true}));
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn an_existing_file_changes_only_when_confirmed() {
let d = scratch("existing");
let mine = "# My project\n\nRun `make test` before you commit.\n";
let claude = "# Notes for one agent\n";
std::fs::write(d.join("AGENTS.md"), mine).unwrap();
std::fs::write(d.join("CLAUDE.md"), claude).unwrap();
std::fs::write(d.join(".mcp.json"), r#"{"mcpServers": {"other": {"command": "other-server"}}}"#).unwrap();
// A dry run writes nothing, existing or not.
let (code, v) = run(&d, &["--dry-run"]);
assert_eq!(code, 0, "{v}");
assert_eq!((action(&v, "AGENTS.md"), action(&v, "CLAUDE.md")), ("would_update".into(), "would_update".into()));
assert_eq!(read(d.join("AGENTS.md")), mine);
// Without --confirm: what exists waits, and the command says how to confirm (exit 4).
let (code, v) = run(&d, &[]);
assert_eq!(code, 4, "confirmation required: {v}");
assert_eq!((v["ok"].as_bool(), v["error"]["code"].as_str(), v["error"]["retryable"].as_bool()), (Some(false), Some("confirmation-required"), Some(false)), "{v}");
for p in ["AGENTS.md", "CLAUDE.md", ".mcp.json"] {
assert_eq!(action(&v, p), "needs_confirmation", "{p}");
}
assert_eq!(read(d.join("AGENTS.md")), mine, "nothing that exists was changed");
assert_eq!(read(d.join("CLAUDE.md")), claude, "nothing that exists was changed");
let proposed = v["proposed"].as_array().unwrap();
assert_eq!(proposed.len(), 3, "{v}");
let diff = proposed.iter().find(|f| f["path"] == "AGENTS.md").unwrap()["diff"].as_str().unwrap().to_string();
assert!(diff.starts_with("--- a/AGENTS.md\n+++ b/AGENTS.md\n") && diff.contains("+## WAI"), "{diff}");
let confirm = v["confirm"].as_str().unwrap();
assert!(confirm.ends_with("--yes") && confirm.contains(d.to_str().unwrap()), "{confirm}");
// With it, the block is added and the rest kept.
let (code, v) = run(&d, &["--yes"]);
assert_eq!(code, 0, "{v}");
assert_eq!(action(&v, "AGENTS.md"), "updated");
let c = read(d.join("CLAUDE.md"));
assert!(c.starts_with(claude) && c.lines().any(|l| l == "@AGENTS.md"), "a CLAUDE.md the project has imports AGENTS.md: {c}");
let agents = read(d.join("AGENTS.md"));
assert!(agents.starts_with(mine) && agents.contains("## WAI"), "{agents}");
let mcp: Value = serde_json::from_str(&read(d.join(".mcp.json"))).unwrap();
assert_eq!(mcp["mcpServers"]["other"]["command"], "other-server", "the other server is kept");
assert_eq!(mcp["mcpServers"]["wai"]["args"][0], "mcp");
// Once more: the block is replaced in place, not added twice.
let (code, _) = run(&d, &["--confirm"]);
assert_eq!(code, 0);
assert_eq!(read(d.join("AGENTS.md")).matches("## WAI").count(), 1);
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn the_skill_is_written_where_agents_load_it() {
let d = scratch("skill");
let (code, v) = run(&d, &[]);
assert_eq!(code, 0, "{v}");
assert_eq!(action(&v, SKILL), "created");
assert_eq!(action(&v, OWN_SKILL), "absent", "an agent's own directory is written only where the project has one");
let s = read(d.join(SKILL));
assert!(s.starts_with("---\nname: wai\ndescription: "), "a skill opens with its frontmatter: {s}");
assert!(s.contains("`wai verify <path>...`") && s.contains("`receipt_verify`") && !s.contains("{{"), "{s}");
// An agent with its own directory gets the same file, and --only and --all reach it.
std::fs::create_dir(d.join(".claude")).unwrap();
let (code, v) = run(&d, &[]);
assert_eq!(code, 0, "{v}");
assert_eq!(action(&v, OWN_SKILL), "created");
assert_eq!(read(d.join(OWN_SKILL)), s);
let d2 = scratch("skill-only");
let (code, v) = run(&d2, &["--only", OWN_SKILL]);
assert_eq!(code, 0, "{v}");
assert_eq!((action(&v, OWN_SKILL), action(&v, SKILL), action(&v, "AGENTS.md")), ("created".into(), "absent".into(), "absent".into()));
let d3 = scratch("skill-all");
let (code, v) = run(&d3, &["--all"]);
assert_eq!(code, 0, "{v}");
assert_eq!((action(&v, OWN_SKILL), action(&v, SKILL)), ("created".into(), "created".into()));
// The program registered is the program the skill names.
let d4 = scratch("skill-command");
let (code, _) = run(&d4, &["--command", "tools/wai", "--only", SKILL]);
assert_eq!(code, 0);
assert!(read(d4.join(SKILL)).contains("`tools/wai verify <path>...`"));
for x in [d, d2, d3, d4] {
let _ = std::fs::remove_dir_all(x);
}
}
/// A rerun replaces the block and nothing else: a person's frontmatter, what they wrote above and
/// below the block, and a change they made inside it, which is the block's and goes.
#[test]
fn a_rerun_replaces_only_its_own_block() {
let d = scratch("rerun");
let (code, _) = run(&d, &[]);
assert_eq!(code, 0);
for file in [SKILL, "AGENTS.md"] {
let p = d.join(file);
let made = read(p.clone());
let ours = block(&made).to_string();
let edited = made.replace("name: wai\n", "name: wai\nlicense: mine\n").replace(BEGIN, &format!("My note above, kept.\n\n{BEGIN}"));
let edited = edited.replace(END, &format!("{END}\n\nMy note below, kept."));
let changed = edited.replace("open media standard", "thing I edited");
assert_ne!(changed, edited, "{file}");
std::fs::write(&p, &changed).unwrap();
// Unconfirmed, the file is as the person left it.
let (code, v) = run(&d, &[]);
assert_eq!(code, 4, "{file}: {v}");
assert_eq!(read(p.clone()), changed, "{file}: nothing that exists changes unconfirmed");
let proposed = v["proposed"].as_array().unwrap();
assert_eq!(proposed.len(), 1, "{file}: {v}");
let (code, v) = run(&d, &["--yes"]);
assert_eq!(code, 0, "{file}: {v}");
assert_eq!(action(&v, file), "updated");
assert_eq!(read(p.clone()), edited, "{file}: the block is the binary's again, and everything outside it the person's");
assert_eq!(block(&read(p.clone())), ours);
// And it stays: a further run has nothing to do.
let (code, v) = run(&d, &[]);
assert_eq!(code, 0, "{file}: {v}");
assert_eq!(action(&v, file), "unchanged");
}
let _ = std::fs::remove_dir_all(&d);
}
/// A marker named in prose is no block: the old search took the first mention for the block's
/// start and replaced everything from there to the real end.
#[test]
fn a_marker_named_in_prose_is_not_a_block() {
let d = scratch("prose");
run(&d, &[]);
let p = d.join("AGENTS.md");
let made = read(p.clone());
let mine = format!("# Mine\n\nThe tool's text starts at `{BEGIN}`.\n\nEverything from here down is mine.\n\n");
std::fs::write(&p, format!("{mine}{made}")).unwrap();
let (code, v) = run(&d, &[]);
assert_eq!(code, 0, "{v}");
assert_eq!(action(&v, "AGENTS.md"), "unchanged", "the block is the binary's already");
std::fs::write(&p, format!("{mine}{}", made.replace("## WAI", "## An edit"))).unwrap();
let (code, v) = run(&d, &["--yes"]);
assert_eq!(code, 0, "{v}");
assert_eq!(read(p), format!("{mine}{made}"));
let _ = std::fs::remove_dir_all(&d);
}
/// A person's fenced example of both markers is theirs: the block goes after it, and stays there.
#[test]
fn a_marker_shown_in_a_code_fence_is_not_a_block() {
let d = scratch("fence");
let mine = format!("# Mine\n\nThe block looks like this:\n\n```text\n{BEGIN}\nmy example\n{END}\n```\n");
std::fs::write(d.join("AGENTS.md"), &mine).unwrap();
let (code, v) = run(&d, &["--yes"]);
assert_eq!(code, 0, "{v}");
let once = read(d.join("AGENTS.md"));
assert!(once.starts_with(&mine) && once.contains("## WAI"), "{once}");
let (code, v) = run(&d, &[]);
assert_eq!(code, 0, "{v}");
assert_eq!(action(&v, "AGENTS.md"), "unchanged", "the block is the one after the fence");
let _ = std::fs::remove_dir_all(&d);
}
/// The block's frontmatter comes once: a file that holds the block and no frontmatter has only its
/// block replaced, as the block's own text says.
#[test]
fn a_skill_that_holds_its_block_gains_nothing_outside_it() {
let d = scratch("no-frontmatter");
run(&d, &[]);
let p = d.join(SKILL);
let made = read(p.clone());
let bare = format!("{}\n", block(&made));
std::fs::write(&p, bare.replace("open media standard", "thing I edited")).unwrap();
let (code, v) = run(&d, &["--yes"]);
assert_eq!(code, 0, "{v}");
assert_eq!(read(p), bare);
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn markers_that_make_no_one_block_are_left_alone() {
let d = scratch("two-blocks");
run(&d, &[]);
let p = d.join(SKILL);
let made = read(p.clone());
let twice = format!("{made}\nmine\n\n{}\n", block(&made));
std::fs::write(&p, &twice).unwrap();
let (code, v) = run(&d, &["--yes"]);
assert_eq!(code, 0, "{v}");
assert_eq!(action(&v, SKILL), "skipped");
let f = files(&v).iter().find(|f| f["path"] == SKILL).unwrap();
assert!(f["reason"].as_str().unwrap().contains("mend it by hand"), "{f}");
assert_eq!(read(p), twice);
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn a_file_with_windows_line_endings_keeps_them() {
let d = scratch("crlf");
std::fs::write(d.join("AGENTS.md"), "# Mine\r\n\r\nkeep\r\n").unwrap();
let (code, v) = run(&d, &["--yes"]);
assert_eq!(code, 0, "{v}");
let a = read(d.join("AGENTS.md"));
assert!(a.starts_with("# Mine\r\n\r\nkeep\r\n\r\n<!-- wai") && !a.replace("\r\n", "").contains('\n'), "{a:?}");
let (code, v) = run(&d, &[]);
assert_eq!(code, 0, "{v}");
assert_eq!(action(&v, "AGENTS.md"), "unchanged");
let _ = std::fs::remove_dir_all(&d);
}
/// What a template carries is what this binary writes: the file of a fresh project, then a section
/// of the template's own. So a rerun in a project made from it has nothing to do, and any change
/// to the skill's text fails here until the examples are written again.
#[test]
fn every_template_carries_the_skill_this_binary_writes() {
let fresh = scratch("template-fresh");
let (code, _) = run(&fresh, &["--only", SKILL]);
assert_eq!(code, 0);
let written = read(fresh.join(SKILL));
let examples = Path::new(env!("CARGO_MANIFEST_DIR")).join("../examples");
let mut seen = 0;
for e in std::fs::read_dir(&examples).unwrap().flatten().filter(|e| e.path().is_dir()) {
let name = e.file_name().to_string_lossy().into_owned();
let have = read(e.path().join(SKILL));
assert!(have.starts_with(&written), "{name}: its skill is not the one this binary writes; write the examples again");
let own = &have[written.len()..];
assert!(own.starts_with("\n## This project\n\n") && own.contains(&format!("the `{name}` template")) && own.contains("`AGENTS.md`"), "{name}: {own}");
let d = scratch(&format!("template-{name}"));
std::fs::create_dir_all(d.join(".agents/skills/wai")).unwrap();
std::fs::write(d.join(SKILL), &have).unwrap();
let (code, v) = run(&d, &["--only", SKILL]);
assert_eq!((code, action(&v, SKILL)), (0, "unchanged".to_string()), "{name}: {v}");
let _ = std::fs::remove_dir_all(&d);
seen += 1;
}
assert!(seen > 0);
let _ = std::fs::remove_dir_all(&fresh);
}
#[test]
fn a_file_that_is_not_plain_json_is_left_alone() {
let d = scratch("jsonc");
std::fs::create_dir(d.join(".vscode")).unwrap();
let jsonc = "{\n // the user's comment\n \"servers\": {}\n}\n";
std::fs::write(d.join(".vscode/mcp.json"), jsonc).unwrap();
let (code, v) = run(&d, &["--confirm"]);
assert_eq!(code, 0, "{v}");
assert_eq!(action(&v, ".vscode/mcp.json"), "skipped");
let f = files(&v).iter().find(|f| f["path"] == ".vscode/mcp.json").unwrap();
assert!(f["reason"].as_str().unwrap().contains("add by hand"), "{f}");
assert_eq!(read(d.join(".vscode/mcp.json")), jsonc);
let _ = std::fs::remove_dir_all(&d);
}
#[cfg(unix)]
#[test]
fn nothing_outside_the_project_is_touched() {
let d = scratch("links");
let outside = scratch("links-outside");
std::fs::write(outside.join("AGENTS.md"), "outside\n").unwrap();
std::os::unix::fs::symlink(outside.join("AGENTS.md"), d.join("AGENTS.md")).unwrap();
std::os::unix::fs::symlink(&outside, d.join(".cursor")).unwrap();
std::os::unix::fs::symlink(&outside, d.join(".agents")).unwrap();
let (code, v) = run(&d, &["--confirm", "--all"]);
assert_eq!(code, 0, "{v}");
assert_eq!(action(&v, "AGENTS.md"), "skipped");
assert_eq!(action(&v, ".cursor/mcp.json"), "skipped");
assert_eq!(action(&v, SKILL), "skipped");
assert!(!outside.join("skills").exists(), "nothing written through the linked directory");
assert_eq!(read(outside.join("AGENTS.md")), "outside\n");
assert!(!outside.join("mcp.json").exists(), "nothing written through the linked directory");
let names: Vec<_> = std::fs::read_dir(&outside).unwrap().map(|e| e.unwrap().file_name()).collect();
assert_eq!(names.len(), 1, "{names:?}");
let _ = std::fs::remove_dir_all(&d);
let _ = std::fs::remove_dir_all(&outside);
}
/// A file named as a directory it needs is left alone, with the reason, and the project is set up.
#[test]
fn a_file_where_a_directory_goes_is_left_alone() {
let d = scratch("not-a-directory");
for name in [".agents", ".claude", ".cursor"] {
std::fs::write(d.join(name), "mine\n").unwrap();
}
let (code, v) = run(&d, &[]);
assert_eq!(code, 0, "{v}");
assert_eq!((action(&v, "AGENTS.md"), action(&v, ".mcp.json")), ("created".to_string(), "created".to_string()), "{v}");
for path in [SKILL, OWN_SKILL, ".cursor/mcp.json"] {
assert_eq!(action(&v, path), "skipped", "{path}: {v}");
let f = files(&v).iter().find(|f| f["path"] == path).unwrap();
assert!(f["reason"].as_str().unwrap().contains("needs a directory"), "{f}");
}
for name in [".agents", ".claude", ".cursor"] {
assert_eq!(read(d.join(name)), "mine\n", "{name}");
}
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn the_toolsets_named_are_registered() {
let d = scratch("toolsets");
std::fs::write(d.join("opencode.json"), "{}\n").unwrap();
let (code, v) = run(&d, &["--toolset", "core,dev", "--confirm"]);
assert_eq!(code, 0, "{v}");
let mcp: Value = serde_json::from_str(&read(d.join(".mcp.json"))).unwrap();
assert_eq!(mcp["mcpServers"]["wai"], json!({"command": "wai", "args": ["mcp", "--toolset", "core,dev"]}));
let oc: Value = serde_json::from_str(&read(d.join("opencode.json"))).unwrap();
assert_eq!(oc["mcp"]["wai"]["command"], json!(["wai", "mcp", "--toolset", "core,dev"]));
assert!(read(d.join("AGENTS.md")).contains("`wai mcp --toolset core,dev`"));
let (code, v) = run(&d, &["--toolset", "quantum"]);
assert_eq!((code, v["error"]["code"].as_str()), (2, Some("usage")), "{v}");
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn a_usage_error_is_structured() {
let d = scratch("usage");
let (code, v) = run(&d, &["--frobnicate"]);
assert_eq!(code, 2);
assert_eq!((v["wai"].as_str(), v["op"].as_str(), v["ok"].as_bool(), v["error"]["code"].as_str()), (Some("envelope/1"), Some("agent.setup"), Some(false), Some("usage")), "{v}");
assert!(v["error"]["message"].as_str().unwrap().contains("--frobnicate") && v["error"]["hint"].is_string(), "{v}");
let (code, v) = run(&d, &["--only", "README.md"]);
assert_eq!((code, v["error"]["code"].as_str()), (2, Some("usage")), "{v}");
let (code, v) = run(&d.join("not-there"), &[]);
assert_eq!((code, v["error"]["code"].as_str()), (5, Some("not-found")), "{v}");
let _ = std::fs::remove_dir_all(&d);
}