open-standards/wai/wai-rs/tests/det_cost_bin.rs

//! `wai_det_cost`, the decoder behind the determinism-cost record
//! (`research/determinism-cost.md`), decodes through the sink's own integer path: the
//! committed `wai.neural.int_hyper` vector of `int-payload-conformance/` gives its pinned
//! RGB8 digest, every `WIH1` refusal fixture there is refused with its code, and the
//! timed child protocol reports the decodes it ran.
//!
//! One fixture is not refused by the tool: `wih1_json_form`, the exporter's JSON bitstream of
//! `tiny`, which the sink's decode refuses (`bad_magic`). Since 3f9c3ef64 the tool reads a
//! payload path ending `.json` as that JSON form and converts it to `WIH1` before anything is
//! timed (its module docs), so the trained sets' records can time the exporters' output; that
//! fixture's line is the decode of the vector it is the JSON form of (`converted_from`).
#![cfg(feature = "neural_int")]

use std::io::{BufRead, BufReader, Write};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};

use serde_json::Value;

fn wai() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..")
}

fn index() -> Value {
    serde_json::from_slice(&std::fs::read(wai().join("int-payload-conformance/vectors.json")).unwrap()).unwrap()
}

/// A set directory holding an index entry's members under the names the decode reads.
fn set_dir(entry: &Value, dir: &Path) -> PathBuf {
    std::fs::create_dir_all(dir).unwrap();
    for (name, rel) in entry["set"].as_object().unwrap() {
        std::fs::copy(wai().join(rel.as_str().unwrap()), dir.join(name)).unwrap();
    }
    dir.to_path_buf()
}

fn scratch(name: &str) -> PathBuf {
    let d = std::env::temp_dir().join(format!("wai-det-cost-{}-{name}", std::process::id()));
    let _ = std::fs::remove_dir_all(&d);
    std::fs::create_dir_all(&d).unwrap();
    d
}

fn bin() -> Command {
    Command::new(env!("CARGO_BIN_EXE_wai_det_cost"))
}

#[test]
fn the_committed_vector_and_every_refusal_fixture_decode_as_the_corpus_states() {
    let idx = index();
    let tmp = scratch("decode");
    let mut jobs = String::new();
    let mut want = Vec::new();
    for (i, v) in idx["vectors"].as_array().unwrap().iter().filter(|v| v["capability"] == "wai.neural.int_hyper").enumerate() {
        let side: Value = serde_json::from_slice(&std::fs::read(wai().join(v["sidecar"].as_str().unwrap())).unwrap()).unwrap();
        let set = set_dir(v, &tmp.join(format!("v{i}")));
        let payload = wai().join(v["payload"].as_str().unwrap());
        jobs.push_str(&format!("{}\t{}\t-\n", set.display(), payload.display()));
        let w = side[v["name"].as_str().unwrap()]["sha256"].as_str().unwrap().to_string();
        // A decoded line also gives the picture's width and height; the digest is what is held.
        want.push((payload.display().to_string(), vec![w]));
    }
    assert!(!want.is_empty(), "the corpus has an int_hyper vector");
    for (i, r) in idx["refusals"].as_array().unwrap().iter().filter(|r| r["capability"] == "wai.neural.int_hyper").enumerate() {
        let set = set_dir(r, &tmp.join(format!("r{i}")));
        let rel = r["payload"].as_str().unwrap();
        let payload = wai().join(rel);
        jobs.push_str(&format!("{}\t{}\t-\n", set.display(), payload.display()));
        if rel.ends_with(".json") {
            // The tool converts the JSON form before it decodes (module docs): the digest of the
            // vector it is the JSON form of.
            let v = idx["vectors"].as_array().unwrap().iter().find(|v| v["converted_from"]["json"] == rel).unwrap_or_else(|| panic!("{rel}: no vector is its JSON form's"));
            let side: Value = serde_json::from_slice(&std::fs::read(wai().join(v["sidecar"].as_str().unwrap())).unwrap()).unwrap();
            want.push((payload.display().to_string(), vec![side[v["name"].as_str().unwrap()]["sha256"].as_str().unwrap().to_string()]));
            continue;
        }
        want.push((payload.display().to_string(), vec!["REFUSED".to_string(), r["code"].as_str().unwrap().to_string()]));
    }
    assert!(want.iter().any(|(p, w)| p.ends_with(".json") && w[0] != "REFUSED"), "the JSON-form fixture is among the jobs");
    std::fs::write(tmp.join("jobs.tsv"), &jobs).unwrap();
    let out = bin().arg("decode").arg(tmp.join("jobs.tsv")).output().unwrap();
    assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr));
    let text = String::from_utf8(out.stdout).unwrap();
    let lines: Vec<&str> = text.lines().collect();
    assert_eq!(lines.len(), want.len());
    for (line, (payload, w)) in lines.iter().zip(&want) {
        let f: Vec<&str> = line.split('\t').collect();
        assert_eq!(f[0], payload);
        if w[0] == "REFUSED" {
            assert_eq!(&f[1..], &w[..], "{payload}: a refusal is its code, exactly");
        } else {
            assert_eq!(f.len(), 4, "{payload}: {line:?}");
            assert_eq!(f[1], w[0], "{payload}: the RGB8 digest");
        }
    }
}

#[test]
fn the_timed_child_reports_every_decode_it_ran() {
    let idx = index();
    let v = idx["vectors"].as_array().unwrap().iter().find(|v| v["capability"] == "wai.neural.int_hyper").unwrap();
    let tmp = scratch("serve");
    let set = set_dir(v, &tmp.join("set"));
    std::fs::write(tmp.join("jobs.tsv"), format!("{}\t{}\t-\n", set.display(), wai().join(v["payload"].as_str().unwrap()).display())).unwrap();
    let mut child = bin().arg("serve").arg(tmp.join("jobs.tsv")).stdin(Stdio::piped()).stdout(Stdio::piped()).spawn().unwrap();
    let mut out = BufReader::new(child.stdout.take().unwrap());
    let mut stdin = child.stdin.take().unwrap();
    let mut line = String::new();
    out.read_line(&mut line).unwrap();
    assert_eq!(line.trim(), "ready 1");
    writeln!(stdin, "go 3").unwrap();
    line.clear();
    out.read_line(&mut line).unwrap();
    let w: Vec<&str> = line.split_whitespace().collect();
    assert_eq!(&w[..2], ["done", "3"], "{line:?}");
    assert!(w[2].parse::<u128>().unwrap() > 0);
    writeln!(stdin, "exit").unwrap();
    assert!(child.wait().unwrap().success());
}