open-standards/wai/wai-quantum-cli/tests/providers.rs

//! `wai-quantum providers …`, `run --provider` and the MCP tools, as a person
//! and an agent use them, against the reference provider that this binary
//! serves itself (`wai-quantum providers reference`), over a local process
//! and over HTTP. Each run has its own configuration directory, and its
//! credentials go to files there (`WAI_QUANTUM_CREDENTIALS=file`), so no
//! keychain is touched. Nothing leaves this machine and nothing spends money:
//! the priced provider is the reference with a price in its configuration.
#![cfg(feature = "providers")]

use serde_json::{Value, json};
use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Output, Stdio};

const SECRET: &str = "wai-cli-test-credential-93ad1b";
const BELL: &str = "OPENQASM 3.0;\nqubit[2] q;\nbit[2] c;\nh q[0];\ncx q[0], q[1];\nc = measure q;\n";

fn bin() -> &'static str {
    env!("CARGO_BIN_EXE_wai-quantum")
}

/// A configuration directory of its own, and everything printed in it.
struct Home {
    dir: PathBuf,
    seen: std::cell::RefCell<String>,
}

impl Home {
    fn new(tag: &str) -> Home {
        let dir = std::env::temp_dir().join(format!("wai-quantum-cli-byoq-{tag}-{}", std::process::id()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        Home { dir, seen: Default::default() }
    }

    fn cmd(&self) -> Command {
        let mut c = Command::new(bin());
        c.env("XDG_CONFIG_HOME", self.dir.join("config")).env("HOME", &self.dir).env("WAI_QUANTUM_CREDENTIALS", "file").env_remove("WAI_QUANTUM_PROVIDERS").env_remove("WAI_QUANTUM_KEY");
        c
    }

    /// Runs `args` with `stdin`, keeping what it printed.
    fn run(&self, args: &[&str], stdin: &str) -> Output {
        let mut child = self.cmd().args(args).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap();
        child.stdin.take().unwrap().write_all(stdin.as_bytes()).unwrap();
        let out = child.wait_with_output().unwrap();
        let mut seen = self.seen.borrow_mut();
        seen.push_str(&String::from_utf8_lossy(&out.stdout));
        seen.push_str(&String::from_utf8_lossy(&out.stderr));
        out
    }

    fn ok(&self, args: &[&str], stdin: &str) -> String {
        let out = self.run(args, stdin);
        assert!(out.status.success(), "{args:?}: {}\n{}", String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
        String::from_utf8(out.stdout).unwrap()
    }

    fn program(&self) -> String {
        let p = self.dir.join("bell.qasm");
        std::fs::write(&p, BELL).unwrap();
        p.display().to_string()
    }

    /// Every file under the directory, as text.
    fn files(&self) -> String {
        fn walk(p: &Path, out: &mut String) {
            for e in std::fs::read_dir(p).into_iter().flatten().flatten() {
                let p = e.path();
                if p.is_dir() {
                    walk(&p, out);
                } else if let Ok(t) = std::fs::read_to_string(&p) {
                    out.push_str(&format!("== {}\n{t}\n", p.display()));
                }
            }
        }
        let mut s = String::new();
        walk(&self.dir, &mut s);
        s
    }
}

impl Drop for Home {
    fn drop(&mut self) {
        let _ = std::fs::remove_dir_all(&self.dir);
    }
}

fn reference_command(extra: &[&str]) -> Vec<String> {
    let mut c = vec![bin().to_string(), "providers".into(), "reference".into(), "--stdio".into()];
    c.extend(extra.iter().map(|s| s.to_string()));
    c
}

fn add_process(h: &Home, name: &str, extra: &[&str], credential: bool) -> String {
    let mut args = vec!["providers", "add", name, "--kind", "process"];
    if credential {
        args.push("--credential-stdin");
    }
    args.push("--");
    let command = reference_command(extra);
    let refs: Vec<&str> = command.iter().map(String::as_str).collect();
    args.extend(refs);
    h.ok(&args, if credential { SECRET } else { "" })
}

#[test]
fn a_person_adds_tests_plans_and_runs_on_a_provider_and_the_receipt_verifies() {
    let h = Home::new("person");
    let prog = h.program();
    assert!(h.ok(&["providers", "list"], "").contains("no providers yet"));
    add_process(&h, "lab", &[], false);
    let list = h.ok(&["providers", "list", "--json"], "");
    let v: Value = serde_json::from_str(&list).unwrap();
    assert_eq!((v["providers"][0]["name"].as_str(), v["providers"][0]["protocol"].as_str()), (Some("lab"), Some("wai.quantum.provider/1")));
    let d = h.ok(&["providers", "describe", "lab"], "");
    assert!(d.contains("reference:line-5, 5 qubits") && d.contains("billing     free"), "{d}");
    let t = h.ok(&["providers", "test", "lab"], "");
    assert!(t.contains("31 passed, 0 failed, 0 skipped"), "{t}");
    let free = h.ok(&["providers", "free"], "");
    assert!(free.contains("exact") && free.contains("quantum-jobs") && free.contains("lab") && free.contains("free, as it states"), "{free}");
    // The dry run: the compile, its check and the estimate, nothing sent.
    let dry = h.ok(&["run", &prog, "--shots", "300", "--provider", "lab", "--dry-run"], "");
    assert!(dry.contains("compile     2 qubits, 300 shots") && dry.contains("estimate    free") && dry.contains("nothing has been sent"), "{dry}");
    // The run, sealed with this user's key, and checked by verify.
    let body = h.ok(&["run", &prog, "--shots", "300", "--seed", "4", "--provider", "lab", "--json"], "");
    let job: Value = serde_json::from_str(&body).unwrap();
    assert_eq!((job["method"].as_str(), job["result"]["accuracy"]["class"].as_str()), (Some("provider"), Some("measured")));
    assert_eq!(job["request"]["provider"], "lab");
    let saved = h.dir.join("job.json");
    std::fs::write(&saved, &body).unwrap();
    let check = h.ok(&["verify", saved.to_str().unwrap()], "");
    assert!(check.contains("signature    VALID"), "{check}");
    // As a person reads it.
    let text = h.ok(&["run", &prog, "--provider", "lab"], "");
    assert!(text.contains("accuracy    measured") && text.contains("ideal       total variation distance") && text.contains("cost        free") && text.contains("receipt     verifies"), "{text}");
    // A job the engine's methods would take and a device cannot is refused.
    let out = h.run(&["run", &prog, "--provider", "lab", "-o", "Z0"], "");
    assert!(!out.status.success() && String::from_utf8_lossy(&out.stderr).contains("counts"));
    assert!(h.ok(&["providers", "remove", "lab"], "").contains("removed lab"));
    assert!(!h.run(&["providers", "describe", "lab"], "").status.success());
}

#[test]
fn a_run_that_is_not_free_is_refused_until_the_person_confirms_it() {
    let h = Home::new("priced");
    let prog = h.program();
    add_process(&h, "paid", &["--price", "0.0003", "--currency", "USD"], false);
    let out = h.run(&["run", &prog, "--shots", "1000", "--provider", "paid"], "");
    assert_eq!(out.status.code(), Some(3), "{}", String::from_utf8_lossy(&out.stderr));
    let (o, e) = (String::from_utf8_lossy(&out.stdout), String::from_utf8_lossy(&out.stderr));
    assert!(o.contains("estimate    0.3 USD (billed by the shot)") && e.contains("--yes"), "{o}\n{e}");
    let body: Value = serde_json::from_str(&h.ok(&["run", &prog, "--shots", "1000", "--provider", "paid", "--yes", "--json"], "")).unwrap();
    assert_eq!((body["result"]["cost"]["confirmed"].as_bool(), body["result"]["cost"]["charged"]["amount"].as_str()), (Some(true), Some("0.3")));
    // Its conformance check runs no job without leave.
    let t = h.ok(&["providers", "test", "paid"], "");
    assert!(t.contains("skip  the programs run: not every run is free"), "{t}");
}

#[test]
fn a_credential_is_kept_in_a_file_only_its_owner_reads_and_appears_in_nothing_printed() {
    let h = Home::new("secret");
    let prog = h.program();
    let hash = wai_quantum_qpu::reference::credential_hash(SECRET);
    let added = add_process(&h, "lab", &["--credential-blake3", &hash], true);
    assert!(added.contains("kept in a file only you can read"), "{added}");
    let list = h.ok(&["providers", "list"], "");
    assert!(list.contains("credential: file (readable by you only; there is no keychain here)"), "{list}");
    let run = h.ok(&["run", &prog, "--provider", "lab", "--json"], "");
    assert!(run.contains("\"receipt\""));
    h.ok(&["providers", "describe", "lab"], "");
    h.ok(&["providers", "test", "lab", "--json"], "");
    h.ok(&["run", &prog, "--provider", "lab", "--dry-run", "--json"], "");
    // The credential is in its own file, readable by its owner only, and in
    // nothing else: not the list, not a receipt, not anything printed.
    let file = h.dir.join("config/wai/quantum-provider-credentials/lab");
    assert_eq!(std::fs::read_to_string(&file).unwrap(), SECRET);
    #[cfg(unix)]
    {
        use std::os::unix::fs::PermissionsExt;
        assert_eq!(std::fs::metadata(&file).unwrap().permissions().mode() & 0o777, 0o600);
    }
    std::fs::write(h.dir.join("run.json"), &run).unwrap();
    let everything = h.files().replace(&format!("== {}\n{SECRET}\n", file.display()), "");
    assert!(!everything.contains(SECRET), "the credential is in a file other than its own");
    assert!(!h.seen.borrow().contains(SECRET), "the credential was printed");
    // A credential named by an environment variable is read from it, and kept nowhere.
    h.ok(&["providers", "add", "env", "--kind", "process", "--credential-env", "MY_PROVIDER_TOKEN", "--", bin(), "providers", "reference", "--stdio", "--credential-blake3", &hash], "");
    let mut c = h.cmd();
    let out = c.env("MY_PROVIDER_TOKEN", SECRET).args(["run", &prog, "--provider", "env", "--json"]).output().unwrap();
    assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr));
    assert!(!String::from_utf8_lossy(&out.stdout).contains(SECRET) && !String::from_utf8_lossy(&out.stderr).contains(SECRET));
    // Without it, the provider refuses.
    let out = h.run(&["run", &prog, "--provider", "env"], "");
    assert!(!out.status.success() && String::from_utf8_lossy(&out.stderr).contains("unauthorized"), "{}", String::from_utf8_lossy(&out.stderr));
    // A credential is never a setting.
    let out = h.run(&["providers", "add", "bad", "--kind", "http", "--endpoint", "https://example.invalid", "--set", "api_token=x"], "");
    assert!(!out.status.success() && String::from_utf8_lossy(&out.stderr).contains("looks like a credential"));
}

/// The reference served over HTTP by this binary, and its URL.
fn serve_http(h: &Home, extra: &[&str]) -> (Child, String) {
    let mut child = h.cmd().args(["providers", "reference", "--http", "127.0.0.1:0"]).args(extra).stdout(Stdio::piped()).spawn().unwrap();
    let mut line = String::new();
    BufReader::new(child.stdout.take().unwrap()).read_line(&mut line).unwrap();
    let url = line.trim().strip_prefix("listening ").unwrap_or_else(|| panic!("{line}")).to_string();
    (child, url)
}

#[test]
fn a_provider_over_http_runs_with_its_credential_in_the_header() {
    if Command::new("curl").arg("--version").output().is_err() {
        println!("no curl on this host: the HTTP transport runs where curl is");
        return;
    }
    let h = Home::new("http");
    let prog = h.program();
    let hash = wai_quantum_qpu::reference::credential_hash(SECRET);
    let (mut child, url) = serve_http(&h, &["--credential-blake3", &hash, "--basis", "prxcz", "--all-to-all"]);
    h.ok(&["providers", "add", "web", "--kind", "http", "--endpoint", &url, "--credential-stdin"], SECRET);
    let t = h.ok(&["providers", "test", "web"], "");
    let body = h.ok(&["run", &prog, "--provider", "web", "--json"], "");
    let _ = child.kill();
    let _ = child.wait();
    assert!(t.contains("31 passed, 0 failed"), "{t}");
    let job: Value = serde_json::from_str(&body).unwrap();
    assert_eq!(job["result"]["compiled"]["basis"], "prxcz");
    assert!(!h.seen.borrow().contains(SECRET));
    // A credential over plain http to another machine is refused.
    h.ok(&["providers", "add", "far", "--kind", "http", "--endpoint", "http://192.0.2.1/q", "--credential-stdin"], SECRET);
    let out = h.run(&["run", &prog, "--provider", "far", "--dry-run"], "");
    assert!(!out.status.success() && String::from_utf8_lossy(&out.stderr).contains("only over https"), "{}", String::from_utf8_lossy(&out.stderr));
}

/// One MCP message and its answer.
fn rpc(stdin: &mut impl Write, stdout: &mut impl BufRead, id: u64, method: &str, params: Value) -> Value {
    writeln!(stdin, "{}", json!({ "jsonrpc": "2.0", "id": id, "method": method, "params": params })).unwrap();
    stdin.flush().unwrap();
    let mut line = String::new();
    stdout.read_line(&mut line).unwrap();
    serde_json::from_str(&line).unwrap_or_else(|e| panic!("{e}: {line}"))
}

#[test]
fn an_agent_lists_estimates_is_refused_and_runs_only_with_confirm_cost() {
    let h = Home::new("agent");
    add_process(&h, "free-lab", &[], false);
    add_process(&h, "paid", &["--price", "0.25", "--currency", "EUR", "--per", "job"], false);
    let mut child = h.cmd().args(["mcp", "--demo-key"]).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()).spawn().unwrap();
    let mut stdin = child.stdin.take().unwrap();
    let mut stdout = BufReader::new(child.stdout.take().unwrap());
    rpc(&mut stdin, &mut stdout, 1, "initialize", json!({ "protocolVersion": "2025-06-18" }));
    let tools = rpc(&mut stdin, &mut stdout, 2, "tools/list", json!({}));
    let names: Vec<String> = tools["result"]["tools"].as_array().unwrap().iter().map(|t| t["name"].as_str().unwrap().to_string()).collect();
    for n in ["providers_list", "provider_estimate", "run_on_provider"] {
        assert!(names.iter().any(|x| x == n), "{n} not in {names:?}");
    }
    let call = |stdin: &mut std::process::ChildStdin, stdout: &mut BufReader<std::process::ChildStdout>, id: u64, name: &str, args: Value| rpc(stdin, stdout, id, "tools/call", json!({ "name": name, "arguments": args }))["result"].clone();
    let list = call(&mut stdin, &mut stdout, 3, "providers_list", json!({ "free": true }));
    assert_eq!(list["isError"], false, "{list}");
    assert_eq!(list["structuredContent"]["providers"].as_array().unwrap().len(), 2);
    assert_eq!(list["structuredContent"]["free"]["providers"][0]["name"], "free-lab", "{list}");
    let est = call(&mut stdin, &mut stdout, 4, "provider_estimate", json!({ "provider": "paid", "qasm": BELL, "shots": 100 }));
    assert_eq!((est["structuredContent"]["free"].as_bool(), est["structuredContent"]["estimate"]["amount"].as_str()), (Some(false), Some("0.25")), "{est}");
    let refused = call(&mut stdin, &mut stdout, 5, "run_on_provider", json!({ "provider": "paid", "qasm": BELL, "shots": 100 }));
    assert_eq!(refused["isError"], true, "{refused}");
    assert!(refused["content"][0]["text"].as_str().unwrap().contains("confirm_cost: true"), "{refused}");
    let ran = call(&mut stdin, &mut stdout, 6, "run_on_provider", json!({ "provider": "paid", "qasm": BELL, "shots": 100, "confirm_cost": true }));
    assert_eq!(ran["isError"], false, "{ran}");
    let job = ran["structuredContent"].clone();
    let free = call(&mut stdin, &mut stdout, 7, "run_on_provider", json!({ "provider": "free-lab", "qasm": BELL }));
    assert_eq!(free["isError"], false, "a free run needs no confirmation: {free}");
    let checked = call(&mut stdin, &mut stdout, 8, "verify_quantum_result", json!({ "job": job }));
    assert_eq!(checked["structuredContent"]["valid"], true, "{checked}");
    // The engine's own run tool does not stand in for a provider.
    let stand_in = call(&mut stdin, &mut stdout, 9, "run_quantum_circuit", json!({ "qasm": BELL, "provider": "paid" }));
    assert_eq!(stand_in["isError"], true, "{stand_in}");
    drop(stdin);
    let _ = child.wait();
}