//! The answer view layer's conformance set (`wai/quantum-conformance/answers/`): //! each answer read as the layer must read it (the codes of its errors and //! warnings in the order of their lines, its root, the order its views are //! drawn in), its views run on the engine to what they must show, and a //! stored answer's receipt checked by running the answer again. //! //! The cases come from `bundle.json`, which holds every answer inline, so //! the same test runs where no file can be read; natively, a second test //! holds the bundle equal to the files, and the stored cases equal to what //! `bell` and its seal give. Run with `WAI_ANSWER_BUNDLE_WRITE=1` to write //! them. #![cfg(all(feature = "quantum_engine", feature = "quantum_receipt"))] use serde_json::{Value, json}; use wai_quantum::ed25519_dalek::SigningKey; use wai_quantum::engine::answer::{ self, eval::{self, Host}, }; const BUNDLE: &str = include_str!("../../quantum-conformance/answers/bundle.json"); fn key() -> SigningKey { SigningKey::from_bytes(&[3; 32]) } fn check(case: &Value, all: &[Value]) -> Result<(), String> { let e = &case["expect"]; let text = case["answer"].as_str().ok_or("no answer")?; let a = answer::parse(text); let errors: Vec<&str> = a.diags.iter().filter(|d| d.is_error()).map(|d| d.code).collect(); let warnings: Vec<&str> = a.diags.iter().filter(|d| !d.is_error()).map(|d| d.code).collect(); fn codes(v: &Value) -> Vec<&str> { v.as_array().map(|a| a.iter().filter_map(Value::as_str).collect()).unwrap_or_default() } if errors != codes(&e["errors"]) { return Err(format!("errors {errors:?}, not {:?}: {:?}", codes(&e["errors"]), a.errors())); } if warnings != codes(&e["warnings"]) { return Err(format!("warnings {warnings:?}, not {:?}: {:?}", codes(&e["warnings"]), a.diags)); } if json!(a.root) != e["root"] { return Err(format!("root {:?}, not {}", a.root, e["root"])); } if json!(a.order) != e["order"] { return Err(format!("order {:?}, not {}", a.order, e["order"])); } if let Some(other) = e["same_as"].as_str() { let o = all.iter().find(|c| c["name"] == other).ok_or("no such case")?; if answer::parse(o["answer"].as_str().unwrap_or("")).to_json() != a.to_json() { return Err(format!("not the same answer as {other}")); } } if let Some(run) = e["run"].as_object() { let k = key(); let r = eval::evaluate(&a, &Host { job_bytes: 1 << 30, key: Some(&k) }); for (name, want) in run { let got = &r["results"][name]; if got["ok"] != true { return Err(format!("{name} did not run: {got}")); } let v = &got["value"]; if let Some(t) = want["total"].as_u64() { let counts = v["result"]["counts"].as_object().ok_or_else(|| format!("{name}: no counts"))?; let total: u64 = counts.values().filter_map(Value::as_u64).sum(); let outcomes: Vec<&String> = counts.keys().collect(); if total != t || outcomes.iter().any(|o| !want["outcomes"].as_array().is_some_and(|w| w.iter().any(|x| x == o.as_str()))) { return Err(format!("{name}: counts {counts:?}, not {t} over {}", want["outcomes"])); } } for k in ["level", "holds", "verdict"] { if want.get(k).is_some() && v[k] != want[k] { return Err(format!("{name}: {k} {}, not {}", v[k], want[k])); } } if let Some(xs) = want["expectations"].as_array() { let got: Vec = v["result"]["expectations"].as_array().into_iter().flatten().filter_map(|x| x["value"].as_f64()).collect(); if got.len() != xs.len() || got.iter().zip(xs).any(|(g, w)| (g - w.as_f64().unwrap_or(f64::NAN)).abs() > 1e-9) { return Err(format!("{name}: expectations {got:?}, not {xs:?}")); } } if let Some(x) = want["bloch_x"].as_f64() && (v["bloch"]["x"].as_f64().unwrap_or(f64::NAN) - x).abs() > 1e-9 { return Err(format!("{name}: Bloch x {}, not {x}", v["bloch"]["x"])); } if want.get("sealed").is_some() { if v["request"]["sealed"] != want["sealed"] { return Err(format!("{name}: sealed {}, not {}", v["request"]["sealed"], want["sealed"])); } if wai_quantum::engine::job::verify(v).map(|x| x.0) != Ok(true) { return Err(format!("{name}: the seal does not verify")); } } } } if let Some(valid) = e["stored"]["valid"].as_bool() { let v = eval::verify(text, 1 << 30)?; if v["valid"].as_bool() != Some(valid) { return Err(format!("the stored receipt: {v}, not valid = {valid}")); } } Ok(()) } #[test] fn every_answer_reads_and_runs_as_its_manifest_says() { let bundle: Value = serde_json::from_str(BUNDLE).unwrap(); let cases = bundle["cases"].as_array().unwrap(); assert!(cases.len() >= 18, "{} cases", cases.len()); let mut failed = Vec::new(); let mut seen = std::collections::BTreeSet::new(); for c in cases { if let Err(e) = check(c, cases) { failed.push(format!("{}: {e}", c["name"].as_str().unwrap_or("?"))); } for k in ["errors", "warnings"] { for code in c["expect"][k].as_array().into_iter().flatten() { seen.insert(code.as_str().unwrap_or("").to_string()); } } } assert!(failed.is_empty(), "{} of {} cases failed:\n{}", failed.len(), cases.len(), failed.join("\n")); // Every code of the layer has a case. for code in wai_quantum::engine::refusal::codes().iter().filter(|c| c.code >= "J100" && c.code < "J200") { assert!(seen.contains(code.code), "no case gives {}", code.code); } } /// The bundle is the manifest with each answer read in, and the stored /// cases are `bell` with the marker its seal gives. #[cfg(not(target_os = "wasi"))] #[test] fn the_bundle_is_the_corpus() { let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../quantum-conformance/answers"); let write = std::env::var_os("WAI_ANSWER_BUNDLE_WRITE").is_some(); let read = |f: &str| std::fs::read_to_string(dir.join(f)).unwrap_or_else(|e| panic!("{f}: {e}")); // The stored answer: bell, run with the key, and its seal's marker after it. let bell = read("cases/bell.wqa"); let k = key(); let sealed = eval::value(&answer::parse(&bell), "seal", &Host { job_bytes: 1 << 30, key: Some(&k) }).unwrap(); let stored = format!("{bell}{}\n", eval::marker("seal", &sealed)); let changed = stored.replacen("@Sample(bell, 1000)", "@Sample(bell, 999)", 1); for (f, want) in [("cases/bell-stored.wqa", &stored), ("cases/bell-stored-changed.wqa", &changed)] { if write { std::fs::write(dir.join(f), want).unwrap(); } else { assert!(read(f) == *want, "{f} is not bell with its seal's marker: run with WAI_ANSWER_BUNDLE_WRITE=1 to write it"); } } let manifest: Value = serde_json::from_str(&read("manifest.json")).unwrap(); let cases: Vec = manifest["cases"] .as_array() .unwrap() .iter() .map(|c| { let mut b = c.clone(); b["answer"] = json!(read(c["file"].as_str().unwrap())); b }) .collect(); let want = json!({ "format": "wai.quantum-answer-conformance-bundle/1", "about": "manifest.json's cases with every answer inline, for builds that read no files (the browser, the edge); a test holds it equal to the files", "cases": cases, }); if write { std::fs::write(dir.join("bundle.json"), serde_json::to_string_pretty(&want).unwrap() + "\n").unwrap(); return; } let have: Value = serde_json::from_str(BUNDLE).unwrap(); assert!(have == want, "bundle.json is not the corpus: run with WAI_ANSWER_BUNDLE_WRITE=1 to write it"); }