67ac8c544b
Each decoder-level equivalence test now runs int_kernels::equivalence_policies: the reference, the fast kernels on every tiling, and each kernel set the processor runs. tests/int_ops_conformance.rs decodes every graph vector of ops-conformance/ (vectors/ and graphs/) through the interpreter with the fast kernels behind it, under each of those policies, and holds every decode to the integers or refusal the vector states. Co-Authored-By: Claude Opus 5.5 <[email protected]>
84 lines
4 KiB
Rust
84 lines
4 KiB
Rust
//! Every graph vector of `ops-conformance/` (`vectors/` and `graphs/`: each operator on
|
|
//! its own, and the multi-operator graphs, the lowerings and the state vectors among them)
|
|
//! decoded through the integer operator set's interpreter with this crate's fast kernels
|
|
//! behind it (`int_ops::Host`), under every fast policy and every kernel set this
|
|
//! processor runs (`int_kernels::equivalence_policies`): each decode must give the output
|
|
//! integers the vector states, or its refusal code. The vectors state the reference
|
|
//! interpreter's results (`wai-ops`' `vectors verify` recomputes them), so this holds
|
|
//! each fast kernel, operator by operator, to the reference on the whole corpus.
|
|
#![cfg(feature = "neural_int")]
|
|
|
|
use std::path::PathBuf;
|
|
|
|
use serde_json::Value;
|
|
use wai::int_kernels::{with_policy, KernelPath};
|
|
use wai_ops::{Profile, Sequence, Tensor};
|
|
|
|
fn corpus() -> PathBuf {
|
|
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../ops-conformance")
|
|
}
|
|
|
|
fn int(v: &Value) -> i64 {
|
|
v.as_str().and_then(|s| s.parse().ok()).unwrap_or_else(|| panic!("not a decimal integer: {v}"))
|
|
}
|
|
|
|
fn tensor(v: &Value) -> Tensor {
|
|
let s: Vec<i64> = v["shape"].as_array().unwrap().iter().map(int).collect();
|
|
let data = v["data"].as_array().unwrap().iter().map(int).collect();
|
|
Tensor::new(s[0] as u32, s[1] as u32, s[2] as u32, data)
|
|
}
|
|
|
|
fn unhex(s: &str) -> Vec<u8> {
|
|
(0..s.len()).step_by(2).map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()).collect()
|
|
}
|
|
|
|
/// What a decode gave: its outputs, or its refusal's code.
|
|
type Got = Result<Vec<Tensor>, String>;
|
|
|
|
fn stated(d: &Value) -> Got {
|
|
let e = &d["expect"];
|
|
match e.get("refuse") {
|
|
Some(c) => Err(c.as_str().unwrap().to_string()),
|
|
None => Ok(e["outputs"].as_array().unwrap().iter().map(tensor).collect()),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn every_graph_vector_decodes_as_stated_under_every_fast_kernel_set() {
|
|
let policies: Vec<_> = wai::int_kernels::equivalence_policies().into_iter().filter(|p| p.path == KernelPath::Fast).collect();
|
|
let (mut files, mut lines, mut decodes) = (0, 0, 0);
|
|
for dir in ["vectors", "graphs"] {
|
|
let mut names: Vec<_> = std::fs::read_dir(corpus().join(dir)).unwrap().map(|e| e.unwrap().path()).collect();
|
|
names.sort();
|
|
for path in names.into_iter().filter(|p| p.extension().is_some_and(|e| e == "jsonl")) {
|
|
files += 1;
|
|
for line in std::fs::read_to_string(&path).unwrap().lines().filter(|l| !l.trim().is_empty()) {
|
|
let v: Value = serde_json::from_str(line).unwrap();
|
|
let id = v["id"].as_str().unwrap().to_string();
|
|
let loaded = wai_ops::graph::load(&unhex(v["graph"].as_str().unwrap()), Profile::Acc64);
|
|
let load = match &loaded {
|
|
Ok(_) => "ok".to_string(),
|
|
Err(r) => r.code.as_str().to_string(),
|
|
};
|
|
assert_eq!(load, v["load"].as_str().unwrap(), "{}: {id}: load", path.display());
|
|
lines += 1;
|
|
let Ok(g) = loaded else { continue };
|
|
let ds = v["decodes"].as_array().unwrap();
|
|
for &pol in &policies {
|
|
// a sequence carries state across the line's decodes (§10.3)
|
|
let mut seq = Sequence::new();
|
|
for (k, d) in ds.iter().enumerate() {
|
|
let inputs: Vec<Tensor> = d["inputs"].as_array().unwrap().iter().map(tensor).collect();
|
|
let scalars: Vec<i64> = d["scalars"].as_array().unwrap().iter().map(int).collect();
|
|
let got: Got = with_policy(pol, || wai_ops::run_with(&g, inputs, &scalars, &mut seq, &wai::int_ops::Host))
|
|
.map_err(|r| r.code.as_str().to_string());
|
|
assert!(got == stated(d), "{}: {id}: decode {k} under {pol:?}", path.display());
|
|
decodes += 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
assert!(files >= 30 && lines >= 1000, "the corpus is all there: {files} files, {lines} vectors");
|
|
assert!(decodes > 0);
|
|
}
|