open-standards/wai/wai-quantum/tests/formats_conformance.rs

//! The conformance set for other formats (`wai/quantum-conformance/formats/`):
//! every QIR and jeff module read, and every OpenQASM source written, held
//! to what its manifest says: the program it reads to (the same circuit by
//! the equivalence ladder), the outcomes it gives, the notes, a round trip
//! back to its own format, the formats it writes to (the bytes of QIR
//! written pinned by their BLAKE3), and each refusal with its code and its
//! place.
//!
//! The cases come from `bundle.json`, which holds each file inline (text, or
//! base64 for jeff), so the same test runs where no file can be read (under
//! wasm32-wasip2, by wasmtime); natively, a second test holds the bundle
//! equal to the files. Run with `WAI_FORMATS_BUNDLE_WRITE=1` to write the
//! bundle from the files.

#![cfg(feature = "quantum_engine")]

use serde_json::{Value, json};
use std::collections::BTreeSet;
use wai_quantum::engine::formats::{self, Format, Profile};
use wai_quantum::engine::{equiv, job};
use wai_quantum::qasm;

const BUNDLE: &str = include_str!("../../quantum-conformance/formats/bundle.json");

fn input(case: &Value) -> Result<Vec<u8>, String> {
    match (case["text"].as_str(), case["base64"].as_str()) {
        (Some(t), None) => Ok(t.as_bytes().to_vec()),
        (None, Some(b)) => formats::decode_base64(b),
        _ => Err("a case holds its file as text or base64".into()),
    }
}

fn codes(notes: &[wai_quantum::qasm::Diagnostic]) -> Vec<String> {
    let mut v: Vec<String> = notes.iter().map(|d| d.code.to_string()).collect();
    v.sort();
    v.dedup();
    v
}

fn strings(v: &Value) -> Vec<String> {
    v.as_array().map(|a| a.iter().filter_map(Value::as_str).map(String::from).collect()).unwrap_or_default()
}

fn check(case: &Value) -> Result<(), String> {
    let format = Format::parse(case["format"].as_str().ok_or("no format")?).ok_or("an unknown format")?;
    let bytes = input(case)?;
    let e = &case["expect"];
    let reading = formats::read(format, &bytes);
    if let Some(code) = e["refused"].as_str() {
        let d = match reading {
            Err(d) => d,
            Ok(_) => return Err(format!("read, where it must be refused with {code}")),
        };
        if d.code != code {
            return Err(format!("refused with {}, not {code}: {d}", d.code));
        }
        if let Some(l) = e["line"].as_u64()
            && d.line() != Some(l as usize)
        {
            return Err(format!("refused on line {:?}, not {l}: {d}", d.line()));
        }
        if let Some(at) = e["at"].as_str()
            && !d.message.contains(at)
        {
            return Err(format!("the refusal does not say where ({at}): {d}"));
        }
        return Ok(());
    }
    let r = reading.map_err(|d| format!("does not read: {d}"))?;
    let (got, want) = (codes(&r.notes), strings(&e["notes"]));
    if got != want {
        return Err(format!("notes {got:?}, not {want:?}: {:?}", r.notes.iter().map(ToString::to_string).collect::<Vec<_>>()));
    }
    if let Some(p) = e["profile"].as_str()
        && r.summary["profile"] != json!(p)
    {
        return Err(format!("read under the {} profile, not {p}", r.summary["profile"]));
    }
    let p = qasm::read(&r.qasm).map_err(|d| format!("the program read does not read: {d}"))?.program;
    if let Some(d) = e["dynamic"].as_bool()
        && p.dynamic.is_some() != d
    {
        return Err(format!("dynamic is {}, not {d}", p.dynamic.is_some()));
    }
    if let Some(n) = e["qubits"].as_u64()
        && u64::from(p.n_qubits) != n
    {
        return Err(format!("{} qubits, not {n}", p.n_qubits));
    }
    if let Some(reference) = case["reference"].as_str() {
        let v = equiv::equivalent(&r.qasm, reference);
        if v["verdict"] != "equivalent" {
            return Err(format!("not the reference: {v}\n{}", r.qasm));
        }
        if let Some(how) = e["how"].as_str()
            && v["how"] != json!(how)
        {
            return Err(format!("the reference decided by {}, not {how}", v["how"]));
        }
    }
    if e["exact"] == json!(true) {
        qasm::exact_circuit(&r.qasm).map_err(|d| format!("the bit-exact method does not take it: {d}"))?;
    }
    if e.get("outcomes").is_some() || e.get("equal_bits").is_some() {
        let (_, res) = job::plan_and_run(&json!({ "qasm": r.qasm, "shots": 512, "seed": 7 }), 1 << 30)?;
        let seen: BTreeSet<String> = res["counts"].as_object().ok_or("no counts")?.keys().cloned().collect();
        if let Some(o) = e.get("outcomes") {
            let want: BTreeSet<String> = strings(o).into_iter().collect();
            if seen != want {
                return Err(format!("outcomes {seen:?}, not {want:?}"));
            }
        }
        for pair in e["equal_bits"].as_array().into_iter().flatten() {
            let (a, b) = (pair[0].as_u64().ok_or("a bit")? as usize, pair[1].as_u64().ok_or("a bit")? as usize);
            for k in &seen {
                let bit = |i: usize| k.as_bytes()[k.len() - 1 - i];
                if bit(a) != bit(b) {
                    return Err(format!("bits {a} and {b} differ in {k}"));
                }
            }
        }
    }
    if e["round_trip"] == json!(true) {
        let c = formats::convert(format, Format::Qasm, &bytes, None, true).map_err(|d| d.to_string())?;
        if c.check["verdict"]["verdict"] != "equivalent" {
            return Err(format!("written back to {} and read again, not the same: {}", format.name(), c.check));
        }
    }
    for w in e["writes"].as_array().into_iter().flatten() {
        let to = Format::parse(w["to"].as_str().ok_or("writes: to")?).ok_or("writes: an unknown format")?;
        let profile = w["profile_asked"].as_str().map(|s| Profile::parse(s).ok_or("an unknown profile")).transpose()?;
        let c = formats::convert(format, to, &bytes, profile, true);
        if let Some(code) = w["refused"].as_str() {
            match c {
                Err(d) if d.code == code => continue,
                Err(d) => return Err(format!("writing {}: refused with {}, not {code}: {d}", to.name(), d.code)),
                Ok(_) => return Err(format!("writing {}: written, where it must be refused with {code}", to.name())),
            }
        }
        let c = c.map_err(|d| format!("writing {}: {d}", to.name()))?;
        if c.check["verdict"]["verdict"] != "equivalent" {
            return Err(format!("written to {} and read back, not the same: {}", to.name(), c.check));
        }
        if let Some(how) = w["how"].as_str()
            && c.check["verdict"]["how"] != json!(how)
        {
            return Err(format!("written to {}: checked by {}, not {how}", to.name(), c.check["verdict"]["how"]));
        }
        if let Some(p) = w["profile"].as_str()
            && c.written["profile"] != json!(p)
        {
            return Err(format!("written under the {} profile, not {p}", c.written["profile"]));
        }
        let got = codes(&c.notes);
        let want = strings(&w["notes"]);
        if got != want {
            return Err(format!("writing {}: notes {got:?}, not {want:?}", to.name()));
        }
        if let Some(h) = w["blake3"].as_str() {
            let got = blake3::hash(&c.output).to_hex().to_string();
            if got != h {
                return Err(format!("written to {}: BLAKE3 {got}, not {h}", to.name()));
            }
        }
    }
    Ok(())
}

#[test]
fn every_module_reads_and_writes_as_its_manifest_says() {
    let bundle: Value = serde_json::from_str(BUNDLE).expect("bundle.json reads");
    let cases = bundle["cases"].as_array().expect("cases");
    assert!(cases.len() >= 40, "{} cases", cases.len());
    let failures: Vec<String> = cases.iter().filter_map(|c| check(c).err().map(|e| format!("{}: {e}", c["file"].as_str().unwrap_or("?")))).collect();
    assert!(failures.is_empty(), "{} of {} cases:\n{}", failures.len(), cases.len(), failures.join("\n"));
}

/// The bundle is the files: each case's file (text, or base64 for jeff) and
/// reference as the manifest names them, and the manifest's expectations.
#[cfg(not(target_os = "wasi"))]
#[test]
fn the_bundle_is_the_corpus() {
    let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../quantum-conformance/formats");
    let manifest: Value = serde_json::from_str(&std::fs::read_to_string(dir.join("manifest.json")).unwrap()).unwrap();
    let cases: Vec<Value> = manifest["cases"]
        .as_array()
        .unwrap()
        .iter()
        .map(|c| {
            let f = c["file"].as_str().unwrap();
            let bytes = std::fs::read(dir.join(f)).unwrap_or_else(|e| panic!("{f}: {e}"));
            let mut d = json!({ "file": f, "format": c["format"], "expect": c["expect"] });
            if c["format"] == "jeff" {
                d["base64"] = json!(formats::encode_base64(&bytes));
            } else {
                d["text"] = json!(String::from_utf8(bytes).unwrap());
            }
            if let Some(r) = c["reference"].as_str() {
                d["reference"] = json!(std::fs::read_to_string(dir.join(r)).unwrap());
            }
            d
        })
        .collect();
    if std::env::var_os("WAI_FORMATS_BUNDLE_WRITE").is_some() {
        let bundle = json!({ "format": manifest["format"], "about": "The manifest's cases with each file inline (text, or base64 for jeff), for builds that read no files.", "cases": cases });
        std::fs::write(dir.join("bundle.json"), serde_json::to_string_pretty(&bundle).unwrap() + "\n").unwrap();
        return;
    }
    let bundle: Value = serde_json::from_str(BUNDLE).unwrap();
    assert_eq!(bundle["cases"], Value::Array(cases), "bundle.json differs from the files; write it with WAI_FORMATS_BUNDLE_WRITE=1");
}