open-standards/wai/wai-ops-gpu/tests/corpus.rs

//! Every graph vector of `ops-conformance/` that `wai-ops/1-i32` covers, on every adapter
//! this process finds: each decode's output integers, or its refusal code, must be the
//! committed ones, which are the CPU interpreter's (checked here too, decode by decode).
//! Each graph runs on the 64-bit kernels, and on the 32-bit kernels too where its
//! certificate meets `acc32`.
//!
//! **No adapter is not a pass.** With no adapter the test prints `ops-gpu corpus:
//! SKIPPED` and the reason, and fails if `WAI_OPS_GPU_REQUIRE` is set. With
//! `WAI_OPS_GPU_EXPECT` (a comma-separated list of adapter keys, `Adapter::key`), the
//! keys found must be exactly those. Each adapter's result is one line, `ops-gpu
//! corpus: key=…`, also appended to `WAI_OPS_GPU_REPORT` when it is set; a CI step
//! requires that line with its counts.

#[path = "support/corpus.rs"]
mod corpus;

use std::io::Write;

use wai_ops_gpu::blocking;

fn report(line: &str) {
    println!("{line}");
    if let Ok(p) = std::env::var("WAI_OPS_GPU_REPORT") {
        let mut f = std::fs::OpenOptions::new().create(true).append(true).open(&p).unwrap_or_else(|e| panic!("{p}: {e}"));
        writeln!(f, "{line}").unwrap();
    }
}

#[test]
fn every_vector_the_subset_covers_is_the_interpreters_on_every_adapter() {
    let d = blocking::discover();
    for (a, why) in &d.refused {
        report(&format!("ops-gpu adapter refused: key={} {} reason={why:?}", a.key(), a.describe()));
    }
    let mut keys: Vec<String> = d.found.iter().map(|g| g.adapter().key()).collect();
    keys.sort();
    if let Ok(want) = std::env::var("WAI_OPS_GPU_EXPECT") {
        let mut want: Vec<String> = want.split(',').filter(|s| !s.is_empty()).map(str::to_string).collect();
        want.sort();
        assert_eq!(keys, want, "the adapters found are not the ones WAI_OPS_GPU_EXPECT declares");
        report(&format!("ops-gpu adapters: as declared ({})", keys.join(",")));
    }
    if d.found.is_empty() {
        let why = if d.refused.is_empty() { "the platform offers no adapter".to_string() } else { format!("{} adapter(s) refused, above", d.refused.len()) };
        report(&format!("ops-gpu corpus: SKIPPED: no adapter ran the corpus: {why}"));
        assert!(std::env::var_os("WAI_OPS_GPU_REQUIRE").is_none(), "WAI_OPS_GPU_REQUIRE is set and no adapter ran the corpus");
        return;
    }
    let vs = corpus::vectors();
    for gpu in &d.found {
        let a = gpu.adapter();
        let t0 = std::time::Instant::now();
        let t = corpus::run_all(gpu, &vs, 20);
        for m in &t.mismatches {
            println!("MISMATCH {m}");
        }
        for e in &t.errors {
            println!("DEVICE ERROR {e}");
        }
        report(&format!(
            "ops-gpu corpus: key={} {} acc32 vectors={} decodes={}; acc64 vectors={} decodes={}; refusals={}; cpu={} load_refused={} mismatches={} errors={} secs={:.1}",
            a.key(),
            a.describe(),
            t.vectors32,
            t.decodes32,
            t.vectors64,
            t.decodes64,
            t.refusals,
            t.cpu,
            t.load_refused,
            t.mismatches.len(),
            t.errors.len(),
            t0.elapsed().as_secs_f64()
        ));
        assert!(t.errors.is_empty(), "{}: device errors: {:?}", a.key(), t.errors);
        assert!(t.mismatches.is_empty(), "{}: {} mismatches, the first: {:?}", a.key(), t.mismatches.len(), t.mismatches.first());
        assert!(t.vectors64 > 1000 && t.vectors32 > 500, "{}: too few vectors ran ({} at acc64, {} at acc32)", a.key(), t.vectors64, t.vectors32);
    }
}

/// Why the width is chosen by the certificate: the 32-bit kernels, run on the graphs whose
/// certificate does not meet `acc32` (which [`wai_ops_gpu::Gpu::run_at`] refuses, and
/// which only this hidden entry point allows), give results that differ from the
/// interpreter's on some of them. On every one of those the 64-bit kernels agree
/// (the test above).
#[test]
fn the_32_bit_kernels_are_not_exact_where_the_certificate_does_not_admit_them() {
    use wai_ops::Profile;
    use wai_ops_gpu::{GpuSequence, Width, support};
    let d = blocking::discover();
    if d.found.is_empty() {
        println!("ops-gpu width: SKIPPED: no adapter");
        assert!(std::env::var_os("WAI_OPS_GPU_REQUIRE").is_none(), "WAI_OPS_GPU_REQUIRE is set and no adapter was found");
        return;
    }
    let vs = corpus::vectors();
    for gpu in &d.found {
        let (mut ran, mut differ, mut unrunnable) = (0u32, 0u32, 0u32);
        for v in &vs {
            let Ok(l) = wai_ops::graph::load(&v.graph, Profile::Acc64) else { continue };
            if l.meets(Profile::Acc32) || support(&l).is_err() {
                continue;
            }
            assert_eq!(
                blocking::run_at(gpu, &l, Width::Acc32, &[], &[], &mut GpuSequence::new()).err(),
                Some(wai_ops_gpu::GpuError::Fallback(wai_ops_gpu::Fallback::Profile)),
                "{}: run_at refuses the 32-bit kernels for a graph that does not meet acc32",
                v.id
            );
            ran += 1;
            let mut seq = GpuSequence::new();
            for dd in &v.decodes {
                match wai_ops_gpu::blocking::resolve(gpu.run_unchecked(&l, Width::Acc32, &dd.inputs, &dd.scalars, &mut seq)) {
                    Ok(Ok(r)) => {
                        let got = match r {
                            Ok(t) => corpus::Expect::Outputs(t),
                            Err(r) => corpus::Expect::Refuse(r.code.as_str().to_string()),
                        };
                        if got != dd.expect {
                            differ += 1;
                            break;
                        }
                    }
                    _ => {
                        unrunnable += 1;
                        break;
                    }
                }
            }
        }
        println!("ops-gpu width: key={} acc64-only graphs={ran}: on the 32-bit kernels {differ} differ from the interpreter, {unrunnable} have parameters past those kernels' widths", gpu.adapter().key());
        assert!(differ > 0, "{}: the 32-bit kernels agreed on every graph the certificate keeps from them", gpu.adapter().key());
    }
}