d7425f89a2
- ops-npu-bench: time and energy of the corpus and of an int_synth graph on the device and on the CPU, every device result compared with the interpreter's first; the package's energy counter (a vendor counter) and the device runtime's own estimate (a model), each named as what it is. - wai-ops-npu.yml: on the linux-amd runner, the probes, the corpus, the twelve kernel edits and the width test on the device, with the device declared and required; on linux-arm64, the crate builds and an absent device is SKIPPED without WAI_OPS_NPU_REQUIRE and a failure with it. - The corpus test requires the vectors it runs to be exactly those npu.jsonl lists in the subset. Co-Authored-By: Claude Opus 5.5 <[email protected]>
98 lines
4.4 KiB
Rust
98 lines
4.4 KiB
Rust
//! Every graph vector of `ops-conformance/` whose graph is in `wai-ops/1-npu` (§9.2),
|
|
//! with its arithmetic operators on the device: each decode's output integers, or its
|
|
//! refusal code, must be the committed ones, which are the CPU interpreter's (checked
|
|
//! here too, decode by decode).
|
|
//!
|
|
//! **No device is not a pass.** With no device the test prints `ops-npu corpus:
|
|
//! SKIPPED` and the reason, and fails if `WAI_OPS_NPU_REQUIRE` is set. With
|
|
//! `WAI_OPS_NPU_EXPECT` (the device's key, or `none`), the device found must be that
|
|
//! one. The result is one line, `ops-npu corpus: key=…`, also appended to
|
|
//! `WAI_OPS_NPU_REPORT` when it is set; a CI step requires that line with its counts.
|
|
|
|
#[path = "support/corpus.rs"]
|
|
mod corpus;
|
|
|
|
use std::collections::BTreeMap;
|
|
|
|
use corpus::report;
|
|
|
|
#[test]
|
|
fn every_vector_in_the_subset_is_the_interpreters_on_the_device() {
|
|
let vs = corpus::vectors();
|
|
let (rs, load_refused) = corpus::ready(&vs);
|
|
// Where the corpus stands against the subset, whether or not a device is here.
|
|
let mut outside: BTreeMap<String, u32> = BTreeMap::new();
|
|
for r in &rs {
|
|
for why in &r.plan.reasons {
|
|
*outside.entry(why.clone()).or_default() += 1;
|
|
}
|
|
}
|
|
let inside: Vec<&corpus::Ready<'_>> = rs.iter().filter(|r| r.plan.in_subset()).collect();
|
|
let mut flags: BTreeMap<String, u32> = BTreeMap::new();
|
|
for r in &inside {
|
|
for f in &r.plan.flags {
|
|
*flags.entry(f.clone()).or_default() += 1;
|
|
}
|
|
}
|
|
report(&format!(
|
|
"ops-npu subset: vectors={} loaded={} load_refused={load_refused} in_subset={} outside={}; reasons {:?}; flags in the subset {:?}",
|
|
vs.len(),
|
|
rs.len(),
|
|
inside.len(),
|
|
rs.len() - inside.len(),
|
|
outside,
|
|
flags
|
|
));
|
|
assert!(inside.len() > 500, "too few vectors in the subset: {}", inside.len());
|
|
// The vectors run here are exactly those npu.jsonl lists in the subset (§14.5).
|
|
let listed: std::collections::BTreeSet<(String, String)> = std::fs::read_to_string(corpus::conformance().join("npu.jsonl"))
|
|
.expect("ops-conformance/npu.jsonl")
|
|
.split_terminator('\n')
|
|
.filter_map(|l| {
|
|
let v = wai_ops::json::parse(l.as_bytes()).expect("a canonical line");
|
|
let on = v.get("npu").and_then(wai_ops::json::Value::as_bool).expect("npu");
|
|
on.then(|| (v.get("file").and_then(wai_ops::json::Value::as_str).unwrap().to_string(), v.get("id").and_then(wai_ops::json::Value::as_str).unwrap().to_string()))
|
|
})
|
|
.collect();
|
|
let ran: std::collections::BTreeSet<(String, String)> = inside.iter().map(|r| (r.v.file.clone(), r.v.id.clone())).collect();
|
|
assert_eq!(ran, listed, "the vectors in the subset are not the ones npu.jsonl lists");
|
|
let Some(npu) = corpus::device("corpus") else { return };
|
|
let t0 = std::time::Instant::now();
|
|
let t = corpus::run_all(&npu, &inside, 20);
|
|
for m in &t.mismatches {
|
|
println!("MISMATCH {m}");
|
|
}
|
|
for e in &t.stats.errors {
|
|
println!("DEVICE ERROR {e}");
|
|
}
|
|
for d in t.stats.declined.iter().take(20) {
|
|
println!("DECLINED {d}");
|
|
}
|
|
report(&format!(
|
|
"ops-npu corpus: key={} {}; vectors={} decodes={} refusals={} movement_only={} npu_ops={} moved={} lines mac8={} mac16={} mac32={} rq={} ew={} lut={} dispatches={} declined={} mismatches={} errors={} secs={:.1}",
|
|
npu.key(),
|
|
npu.identity().describe(),
|
|
t.vectors,
|
|
t.decodes,
|
|
t.refusals,
|
|
t.movement_only,
|
|
t.stats.npu_ops,
|
|
t.stats.moved,
|
|
t.stats.mac[0],
|
|
t.stats.mac[1],
|
|
t.stats.mac[2],
|
|
t.stats.rq,
|
|
t.stats.ew,
|
|
t.stats.lut,
|
|
npu.dispatches(),
|
|
t.stats.declined.len(),
|
|
t.mismatched,
|
|
t.stats.errors.len(),
|
|
t0.elapsed().as_secs_f64()
|
|
));
|
|
assert!(t.stats.errors.is_empty(), "{}: device errors: {:?}", npu.key(), &t.stats.errors[..t.stats.errors.len().min(5)]);
|
|
assert!(t.stats.declined.is_empty(), "{}: operators of the subset declined: {:?}", npu.key(), &t.stats.declined[..t.stats.declined.len().min(5)]);
|
|
assert_eq!(t.mismatched, 0, "{}: {} vectors differ, the first: {:?}", npu.key(), t.mismatched, t.mismatches.first());
|
|
// Every path ran: each MAC width, and each other job.
|
|
assert!(t.stats.mac.iter().all(|&n| n > 0) && t.stats.rq > 0 && t.stats.ew > 0 && t.stats.lut > 0, "{}: a path never ran: {:?}", npu.key(), t.stats);
|
|
}
|