open-standards/wai/wai-quantum/tests/answer_conformance.rs

//! The answer view layer's conformance set (`wai/quantum-conformance/answers/`):
//! each answer read as the layer must read it (the codes of its errors and
//! warnings in the order of their lines, its root, the order its views are
//! drawn in), its views run on the engine to what they must show, and a
//! stored answer's receipt checked by running the answer again.
//!
//! The cases come from `bundle.json`, which holds every answer inline, so
//! the same test runs where no file can be read; natively, a second test
//! holds the bundle equal to the files, and the stored cases equal to what
//! `bell` and its seal give. Run with `WAI_ANSWER_BUNDLE_WRITE=1` to write
//! them.

#![cfg(all(feature = "quantum_engine", feature = "quantum_receipt"))]

use serde_json::{Value, json};
use wai_quantum::ed25519_dalek::SigningKey;
use wai_quantum::engine::answer::{
    self,
    eval::{self, Host},
};

const BUNDLE: &str = include_str!("../../quantum-conformance/answers/bundle.json");

fn key() -> SigningKey {
    SigningKey::from_bytes(&[3; 32])
}

fn check(case: &Value, all: &[Value]) -> Result<(), String> {
    let e = &case["expect"];
    let text = case["answer"].as_str().ok_or("no answer")?;
    let a = answer::parse(text);
    let errors: Vec<&str> = a.diags.iter().filter(|d| d.is_error()).map(|d| d.code).collect();
    let warnings: Vec<&str> = a.diags.iter().filter(|d| !d.is_error()).map(|d| d.code).collect();
    fn codes(v: &Value) -> Vec<&str> {
        v.as_array().map(|a| a.iter().filter_map(Value::as_str).collect()).unwrap_or_default()
    }
    if errors != codes(&e["errors"]) {
        return Err(format!("errors {errors:?}, not {:?}: {:?}", codes(&e["errors"]), a.errors()));
    }
    if warnings != codes(&e["warnings"]) {
        return Err(format!("warnings {warnings:?}, not {:?}: {:?}", codes(&e["warnings"]), a.diags));
    }
    if json!(a.root) != e["root"] {
        return Err(format!("root {:?}, not {}", a.root, e["root"]));
    }
    if json!(a.order) != e["order"] {
        return Err(format!("order {:?}, not {}", a.order, e["order"]));
    }
    if let Some(other) = e["same_as"].as_str() {
        let o = all.iter().find(|c| c["name"] == other).ok_or("no such case")?;
        if answer::parse(o["answer"].as_str().unwrap_or("")).to_json() != a.to_json() {
            return Err(format!("not the same answer as {other}"));
        }
    }
    if let Some(run) = e["run"].as_object() {
        let k = key();
        let r = eval::evaluate(&a, &Host { job_bytes: 1 << 30, key: Some(&k) });
        for (name, want) in run {
            let got = &r["results"][name];
            if got["ok"] != true {
                return Err(format!("{name} did not run: {got}"));
            }
            let v = &got["value"];
            if let Some(t) = want["total"].as_u64() {
                let counts = v["result"]["counts"].as_object().ok_or_else(|| format!("{name}: no counts"))?;
                let total: u64 = counts.values().filter_map(Value::as_u64).sum();
                let outcomes: Vec<&String> = counts.keys().collect();
                if total != t || outcomes.iter().any(|o| !want["outcomes"].as_array().is_some_and(|w| w.iter().any(|x| x == o.as_str()))) {
                    return Err(format!("{name}: counts {counts:?}, not {t} over {}", want["outcomes"]));
                }
            }
            for k in ["level", "holds", "verdict"] {
                if want.get(k).is_some() && v[k] != want[k] {
                    return Err(format!("{name}: {k} {}, not {}", v[k], want[k]));
                }
            }
            if let Some(xs) = want["expectations"].as_array() {
                let got: Vec<f64> = v["result"]["expectations"].as_array().into_iter().flatten().filter_map(|x| x["value"].as_f64()).collect();
                if got.len() != xs.len() || got.iter().zip(xs).any(|(g, w)| (g - w.as_f64().unwrap_or(f64::NAN)).abs() > 1e-9) {
                    return Err(format!("{name}: expectations {got:?}, not {xs:?}"));
                }
            }
            if let Some(x) = want["bloch_x"].as_f64()
                && (v["bloch"]["x"].as_f64().unwrap_or(f64::NAN) - x).abs() > 1e-9
            {
                return Err(format!("{name}: Bloch x {}, not {x}", v["bloch"]["x"]));
            }
            if want.get("sealed").is_some() {
                if v["request"]["sealed"] != want["sealed"] {
                    return Err(format!("{name}: sealed {}, not {}", v["request"]["sealed"], want["sealed"]));
                }
                if wai_quantum::engine::job::verify(v).map(|x| x.0) != Ok(true) {
                    return Err(format!("{name}: the seal does not verify"));
                }
            }
        }
    }
    if let Some(valid) = e["stored"]["valid"].as_bool() {
        let v = eval::verify(text, 1 << 30)?;
        if v["valid"].as_bool() != Some(valid) {
            return Err(format!("the stored receipt: {v}, not valid = {valid}"));
        }
    }
    Ok(())
}

#[test]
fn every_answer_reads_and_runs_as_its_manifest_says() {
    let bundle: Value = serde_json::from_str(BUNDLE).unwrap();
    let cases = bundle["cases"].as_array().unwrap();
    assert!(cases.len() >= 18, "{} cases", cases.len());
    let mut failed = Vec::new();
    let mut seen = std::collections::BTreeSet::new();
    for c in cases {
        if let Err(e) = check(c, cases) {
            failed.push(format!("{}: {e}", c["name"].as_str().unwrap_or("?")));
        }
        for k in ["errors", "warnings"] {
            for code in c["expect"][k].as_array().into_iter().flatten() {
                seen.insert(code.as_str().unwrap_or("").to_string());
            }
        }
    }
    assert!(failed.is_empty(), "{} of {} cases failed:\n{}", failed.len(), cases.len(), failed.join("\n"));
    // Every code of the layer has a case.
    for code in wai_quantum::engine::refusal::codes().iter().filter(|c| c.code >= "J100" && c.code < "J200") {
        assert!(seen.contains(code.code), "no case gives {}", code.code);
    }
}

/// The bundle is the manifest with each answer read in, and the stored
/// cases are `bell` with the marker its seal gives.
#[cfg(not(target_os = "wasi"))]
#[test]
fn the_bundle_is_the_corpus() {
    let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../quantum-conformance/answers");
    let write = std::env::var_os("WAI_ANSWER_BUNDLE_WRITE").is_some();
    let read = |f: &str| std::fs::read_to_string(dir.join(f)).unwrap_or_else(|e| panic!("{f}: {e}"));
    // The stored answer: bell, run with the key, and its seal's marker after it.
    let bell = read("cases/bell.wqa");
    let k = key();
    let sealed = eval::value(&answer::parse(&bell), "seal", &Host { job_bytes: 1 << 30, key: Some(&k) }).unwrap();
    let stored = format!("{bell}{}\n", eval::marker("seal", &sealed));
    let changed = stored.replacen("@Sample(bell, 1000)", "@Sample(bell, 999)", 1);
    for (f, want) in [("cases/bell-stored.wqa", &stored), ("cases/bell-stored-changed.wqa", &changed)] {
        if write {
            std::fs::write(dir.join(f), want).unwrap();
        } else {
            assert!(read(f) == *want, "{f} is not bell with its seal's marker: run with WAI_ANSWER_BUNDLE_WRITE=1 to write it");
        }
    }
    let manifest: Value = serde_json::from_str(&read("manifest.json")).unwrap();
    let cases: Vec<Value> = manifest["cases"]
        .as_array()
        .unwrap()
        .iter()
        .map(|c| {
            let mut b = c.clone();
            b["answer"] = json!(read(c["file"].as_str().unwrap()));
            b
        })
        .collect();
    let want = json!({
        "format": "wai.quantum-answer-conformance-bundle/1",
        "about": "manifest.json's cases with every answer inline, for builds that read no files (the browser, the edge); a test holds it equal to the files",
        "cases": cases,
    });
    if write {
        std::fs::write(dir.join("bundle.json"), serde_json::to_string_pretty(&want).unwrap() + "\n").unwrap();
        return;
    }
    let have: Value = serde_json::from_str(BUNDLE).unwrap();
    assert!(have == want, "bundle.json is not the corpus: run with WAI_ANSWER_BUNDLE_WRITE=1 to write it");
}