//! The intent conformance set (`wai/quantum-conformance/intents/`): each //! program run as a job with its intents, and its verdict held to what the //! manifest says: the level, whether the intents hold, each intent's //! outcome, strength, check and distance, the repairs made and the refusal's //! code. //! //! The cases come from `bundle.json`, which holds every program and //! reference inline, so the same test runs where no file can be read (under //! wasm32-wasip2, by wasmtime); natively, a second test holds the bundle //! equal to the files. Run with `WAI_INTENT_BUNDLE_WRITE=1` to write the //! bundle from the files. #![cfg(all(feature = "quantum_engine", feature = "quantum_receipt"))] use serde_json::{Value, json}; use wai_quantum::engine::{job, refusal, verdict}; const BUNDLE: &str = include_str!("../../quantum-conformance/intents/bundle.json"); /// The job a case runs: its program, its run, and its intents. fn request(case: &Value) -> Value { let mut req = json!({ "qasm": case["qasm"] }); for (k, v) in case["run"].as_object().into_iter().flatten() { req[k] = v.clone(); } if let Some(i) = case.get("intent") { req["intent"] = i.clone(); } req } fn check(case: &Value) -> Result<(), String> { let e = &case["expect"]; let req = request(case); let outcome = job::plan_and_run(&req, 1 << 30); let v = match (&outcome, e["code"].as_str()) { (Err(err), Some(code)) => { let got = refusal::codes_in(err).last().copied(); if got != Some(code) { return Err(format!("refused with {got:?}, not {code}: {err}")); } verdict::of_refusal(&req, err) } (Err(err), None) => return Err(format!("refused: {err}")), (Ok(_), Some(code)) => return Err(format!("ran, where it must be refused with {code}")), (Ok((_, result)), None) => { if let Some(m) = e["method"].as_str() && result["method"] != m { return Err(format!("ran on {}, not {m}", result["method"])); } if let Some(want) = case["repaired_qasm"].as_str() && result["repaired"]["qasm"] != want { return Err(format!("repaired to {}, not {want:?}", result["repaired"]["qasm"])); } result["verdict"].clone() } }; if v["schema"] != verdict::SCHEMA { return Err(format!("no verdict: {v}")); } if v["level"] != e["level"] { return Err(format!("level {}, not {}: {}", v["level"], e["level"], v["statement"])); } if e.get("holds").is_some() && v["holds"] != e["holds"] { return Err(format!("holds {}, not {}: {}", v["holds"], e["holds"], v["statement"])); } if let Some(want) = e["intents"].as_array() { let got = v["intent"].as_array().ok_or("no intents")?; if got.len() != want.len() { return Err(format!("{} intents checked, not {}", got.len(), want.len())); } for (i, (g, w)) in got.iter().zip(want).enumerate() { for k in ["outcome", "strength", "how"] { if w.get(k).is_some() && g[k] != w[k] { return Err(format!("intent {}: {k} {}, not {}: {}", i + 1, g[k], w[k], g["why"])); } } if let Some(d) = w["distance"].as_f64() { let got = g["distance"].as_f64().ok_or_else(|| format!("intent {}: no distance", i + 1))?; if (got - d).abs() > 1e-12 { return Err(format!("intent {}: distance {got}, not {d}", i + 1)); } } } } if let Some(want) = e["repairs"].as_array() { let got: Vec<&Value> = v["repairs"].as_array().into_iter().flatten().map(|r| &r["code"]).collect(); if got != want.iter().collect::>() { return Err(format!("repairs {got:?}, not {want:?}")); } } let s = v["statement"].as_str().unwrap_or(""); if s.is_empty() || !s.ends_with('.') { return Err(format!("no plain statement: {v}")); } Ok(()) } #[test] fn every_case_gets_the_verdict_its_manifest_gives() { let bundle: Value = serde_json::from_str(BUNDLE).unwrap(); let cases = bundle["cases"].as_array().unwrap(); assert!(cases.len() >= 20, "{} cases", cases.len()); let mut failed = Vec::new(); let mut levels = std::collections::BTreeSet::new(); for c in cases { if let Err(e) = check(c) { failed.push(format!("{}: {e}", c["name"].as_str().unwrap_or("?"))); } levels.insert(c["expect"]["level"].as_str().unwrap_or("").to_string()); } assert!(failed.is_empty(), "{} of {} cases failed:\n{}", failed.len(), cases.len(), failed.join("\n")); // Every level of the ladder is held to a case. assert_eq!(levels.into_iter().collect::>(), ["evidence", "executed", "parsed", "proof", "refused"]); } /// The bundle is the manifest with each file read in: the program as /// `qasm`, each reference's text in its intent, and the repaired program. #[cfg(not(target_os = "wasi"))] #[test] fn the_bundle_is_the_corpus() { let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../quantum-conformance/intents"); let read = |f: &str| std::fs::read_to_string(dir.join(f)).unwrap_or_else(|e| panic!("{f}: {e}")); let manifest: Value = serde_json::from_str(&read("manifest.json")).unwrap(); let mut cases = Vec::new(); for c in manifest["cases"].as_array().unwrap() { let mut b = c.clone(); b["qasm"] = json!(read(c["program"].as_str().unwrap())); if let Some(list) = c.get("intent").and_then(Value::as_array) { let mut resolved = Vec::new(); for i in list { let mut i = i.clone(); for k in ["equivalent", "prepares"] { if let Some(f) = i.get(k).and_then(|r| r.get("file")).and_then(Value::as_str).map(str::to_string) { i[k] = json!(read(&f)); i["name"] = json!(f); } } resolved.push(i); } b["intent"] = Value::Array(resolved); } if let Some(r) = c["expect"]["repaired"].as_str() { b["repaired_qasm"] = json!(read(&format!("repaired/{r}.qasm"))); } cases.push(b); } let want = json!({ "format": "wai.quantum-intent-conformance-bundle/1", "about": "manifest.json's cases with every program and reference inline, for builds that read no files (the browser, the edge); a test holds it equal to the files", "cases": cases, }); if std::env::var_os("WAI_INTENT_BUNDLE_WRITE").is_some() { std::fs::write(dir.join("bundle.json"), serde_json::to_string_pretty(&want).unwrap() + "\n").unwrap(); return; } let have: Value = serde_json::from_str(BUNDLE).unwrap(); assert!(have == want, "bundle.json is not the corpus: run with WAI_INTENT_BUNDLE_WRITE=1 to write it"); }