open-standards/wai/wai-quantum/tests/intent_conformance.rs

//! The intent conformance set (`wai/quantum-conformance/intents/`): each
//! program run as a job with its intents, and its verdict held to what the
//! manifest says: the level, whether the intents hold, each intent's
//! outcome, strength, check and distance, the repairs made and the refusal's
//! code.
//!
//! The cases come from `bundle.json`, which holds every program and
//! reference inline, so the same test runs where no file can be read (under
//! wasm32-wasip2, by wasmtime); natively, a second test holds the bundle
//! equal to the files. Run with `WAI_INTENT_BUNDLE_WRITE=1` to write the
//! bundle from the files.

#![cfg(all(feature = "quantum_engine", feature = "quantum_receipt"))]

use serde_json::{Value, json};
use wai_quantum::engine::{job, refusal, verdict};

const BUNDLE: &str = include_str!("../../quantum-conformance/intents/bundle.json");

/// The job a case runs: its program, its run, and its intents.
fn request(case: &Value) -> Value {
    let mut req = json!({ "qasm": case["qasm"] });
    for (k, v) in case["run"].as_object().into_iter().flatten() {
        req[k] = v.clone();
    }
    if let Some(i) = case.get("intent") {
        req["intent"] = i.clone();
    }
    req
}

fn check(case: &Value) -> Result<(), String> {
    let e = &case["expect"];
    let req = request(case);
    let outcome = job::plan_and_run(&req, 1 << 30);
    let v = match (&outcome, e["code"].as_str()) {
        (Err(err), Some(code)) => {
            let got = refusal::codes_in(err).last().copied();
            if got != Some(code) {
                return Err(format!("refused with {got:?}, not {code}: {err}"));
            }
            verdict::of_refusal(&req, err)
        }
        (Err(err), None) => return Err(format!("refused: {err}")),
        (Ok(_), Some(code)) => return Err(format!("ran, where it must be refused with {code}")),
        (Ok((_, result)), None) => {
            if let Some(m) = e["method"].as_str()
                && result["method"] != m
            {
                return Err(format!("ran on {}, not {m}", result["method"]));
            }
            if let Some(want) = case["repaired_qasm"].as_str()
                && result["repaired"]["qasm"] != want
            {
                return Err(format!("repaired to {}, not {want:?}", result["repaired"]["qasm"]));
            }
            result["verdict"].clone()
        }
    };
    if v["schema"] != verdict::SCHEMA {
        return Err(format!("no verdict: {v}"));
    }
    if v["level"] != e["level"] {
        return Err(format!("level {}, not {}: {}", v["level"], e["level"], v["statement"]));
    }
    if e.get("holds").is_some() && v["holds"] != e["holds"] {
        return Err(format!("holds {}, not {}: {}", v["holds"], e["holds"], v["statement"]));
    }
    if let Some(want) = e["intents"].as_array() {
        let got = v["intent"].as_array().ok_or("no intents")?;
        if got.len() != want.len() {
            return Err(format!("{} intents checked, not {}", got.len(), want.len()));
        }
        for (i, (g, w)) in got.iter().zip(want).enumerate() {
            for k in ["outcome", "strength", "how"] {
                if w.get(k).is_some() && g[k] != w[k] {
                    return Err(format!("intent {}: {k} {}, not {}: {}", i + 1, g[k], w[k], g["why"]));
                }
            }
            if let Some(d) = w["distance"].as_f64() {
                let got = g["distance"].as_f64().ok_or_else(|| format!("intent {}: no distance", i + 1))?;
                if (got - d).abs() > 1e-12 {
                    return Err(format!("intent {}: distance {got}, not {d}", i + 1));
                }
            }
        }
    }
    if let Some(want) = e["repairs"].as_array() {
        let got: Vec<&Value> = v["repairs"].as_array().into_iter().flatten().map(|r| &r["code"]).collect();
        if got != want.iter().collect::<Vec<_>>() {
            return Err(format!("repairs {got:?}, not {want:?}"));
        }
    }
    let s = v["statement"].as_str().unwrap_or("");
    if s.is_empty() || !s.ends_with('.') {
        return Err(format!("no plain statement: {v}"));
    }
    Ok(())
}

#[test]
fn every_case_gets_the_verdict_its_manifest_gives() {
    let bundle: Value = serde_json::from_str(BUNDLE).unwrap();
    let cases = bundle["cases"].as_array().unwrap();
    assert!(cases.len() >= 20, "{} cases", cases.len());
    let mut failed = Vec::new();
    let mut levels = std::collections::BTreeSet::new();
    for c in cases {
        if let Err(e) = check(c) {
            failed.push(format!("{}: {e}", c["name"].as_str().unwrap_or("?")));
        }
        levels.insert(c["expect"]["level"].as_str().unwrap_or("").to_string());
    }
    assert!(failed.is_empty(), "{} of {} cases failed:\n{}", failed.len(), cases.len(), failed.join("\n"));
    // Every level of the ladder is held to a case.
    assert_eq!(levels.into_iter().collect::<Vec<_>>(), ["evidence", "executed", "parsed", "proof", "refused"]);
}

/// The bundle is the manifest with each file read in: the program as
/// `qasm`, each reference's text in its intent, and the repaired program.
#[cfg(not(target_os = "wasi"))]
#[test]
fn the_bundle_is_the_corpus() {
    let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../quantum-conformance/intents");
    let read = |f: &str| std::fs::read_to_string(dir.join(f)).unwrap_or_else(|e| panic!("{f}: {e}"));
    let manifest: Value = serde_json::from_str(&read("manifest.json")).unwrap();
    let mut cases = Vec::new();
    for c in manifest["cases"].as_array().unwrap() {
        let mut b = c.clone();
        b["qasm"] = json!(read(c["program"].as_str().unwrap()));
        if let Some(list) = c.get("intent").and_then(Value::as_array) {
            let mut resolved = Vec::new();
            for i in list {
                let mut i = i.clone();
                for k in ["equivalent", "prepares"] {
                    if let Some(f) = i.get(k).and_then(|r| r.get("file")).and_then(Value::as_str).map(str::to_string) {
                        i[k] = json!(read(&f));
                        i["name"] = json!(f);
                    }
                }
                resolved.push(i);
            }
            b["intent"] = Value::Array(resolved);
        }
        if let Some(r) = c["expect"]["repaired"].as_str() {
            b["repaired_qasm"] = json!(read(&format!("repaired/{r}.qasm")));
        }
        cases.push(b);
    }
    let want = json!({
        "format": "wai.quantum-intent-conformance-bundle/1",
        "about": "manifest.json's cases with every program and reference inline, for builds that read no files (the browser, the edge); a test holds it equal to the files",
        "cases": cases,
    });
    if std::env::var_os("WAI_INTENT_BUNDLE_WRITE").is_some() {
        std::fs::write(dir.join("bundle.json"), serde_json::to_string_pretty(&want).unwrap() + "\n").unwrap();
        return;
    }
    let have: Value = serde_json::from_str(BUNDLE).unwrap();
    assert!(have == want, "bundle.json is not the corpus: run with WAI_INTENT_BUNDLE_WRITE=1 to write it");
}