open-standards/wai/wai-pack/src/conformance.rs

//! The packaging conformance corpus (`wai/pack-conformance/`): what this crate
//! computes from its inputs, written by [`regen`] and checked by [`check`]. An
//! independent verifier (`pack-conformance/pack_verify.py`) recomputes the same
//! values from the extension text.
//!
//! - `policy/`: policies a packager accepts, and `expected.json` (each one's canonical
//!   JSON and digest, with any overrides; and the pairs `eq-<name>-a.toml`,
//!   `eq-<name>-b.toml`, two spellings of one policy, which must have one digest).
//! - `refusals/`: policies and sources a packager refuses, and `expected.json` (each
//!   one's reason). The binary sources are written by [`regen`] from
//!   [`source_refusals`].
//! - `conversions/` and `conversions.json`: the unit conversions (milliseconds to a
//!   timebase, alignment to the lead) and the Y4M-to-RGB conversion, by the SHA-256 of
//!   each RGB8 frame.
//! - `scenecut/`: a policy and `expected.json`: per fixture, its SHA-256, every frame's
//!   score, and the pieces with their flags, statistics and kinds, and the plan digest.
//! - `encode/`: `WIV1` payloads of the integer video encoder (`wai::int_video_enc`),
//!   written by [`regen`], and `expected.json`: for each, the fixture frames it codes,
//!   the encoder's parameters, the payload's BLAKE3, the SSE of its reconstruction
//!   against the source, and the SHA-256 of every decoded frame.
//! - `qchoice/expected.json`: the step the base encode chooses for every piece of every
//!   fixture ([`crate::encode::choose_q`]): each candidate's SSE and its SSE per sample
//!   in thousandths, the step chosen, and the chosen payload's BLAKE3.
//! - `packs/`, `mutations/` and `receipts/`: the golden packages, the stage index,
//!   object and receipt cases with their reasons, and version-1 receipts as JSON
//!   ([`crate::golden`]).

use crate::classify::{changed_permille, classify, mad_milli};
use crate::policy::{Overrides, Policy, Timebase};
use crate::scenecut::{plan_digest, segment, SegParams};
use crate::source::{wav::WavReader, y4m};
use crate::{fixture, hex, sha256_hex};
use crate::encode::{choose_q, sse_px_milli, MotionParams};
use wai::int_video_enc::{self, EncParams};
use serde_json::{json, Value};
use std::path::Path;

/// One difference between the corpus and what this crate computes.
pub type Mismatch = String;

fn read(p: &Path) -> std::io::Result<Vec<u8>> {
    std::fs::read(p).map_err(|e| std::io::Error::new(e.kind(), format!("{}: {e}", p.display())))
}

fn read_json(p: &Path) -> std::io::Result<Value> {
    serde_json::from_slice(&read(p)?).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{}: {e}", p.display())))
}

fn write_json(p: &Path, v: &Value) -> std::io::Result<()> {
    let mut s = serde_json::to_string_pretty(v).expect("JSON");
    s.push('\n');
    std::fs::write(p, s)
}

/// The `.toml` files of `dir`, sorted bytewise.
fn tomls(dir: &Path) -> std::io::Result<Vec<String>> {
    let mut v: Vec<String> = std::fs::read_dir(dir)?
        .filter_map(|e| e.ok())
        .map(|e| e.file_name().to_string_lossy().into_owned())
        .filter(|n| n.ends_with(".toml"))
        .collect();
    v.sort();
    Ok(v)
}

// ─────────────────────────────── policy ───────────────────────────────

/// The overrides a policy case applies: `eq-defaults-a.toml`, which leaves
/// `royalty_free_only` off, is also digested with it forced on (§3.2: an override only
/// forces it on).
fn policy_cases(names: &[String]) -> Vec<(String, Overrides)> {
    let mut v: Vec<(String, Overrides)> = names.iter().map(|n| (n.clone(), Overrides::default())).collect();
    if names.iter().any(|n| n == "eq-defaults-a.toml") {
        v.push(("eq-defaults-a.toml".into(), Overrides { royalty_free_only: true }));
    }
    v
}

fn policy_expected(dir: &Path) -> std::io::Result<Value> {
    let names = tomls(dir)?;
    let mut cases = Vec::new();
    for (name, o) in policy_cases(&names) {
        let p = Policy::parse_bytes(&read(&dir.join(&name))?)
            .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{name}: {e}")))?
            .effective(&o);
        let mut c = json!({"file": name, "canonical": p.canonical_json(), "digest": hex(&p.digest())});
        if o.royalty_free_only {
            c["overrides"] = json!({"royalty_free_only": true});
        }
        cases.push(c);
    }
    let equal: Vec<Value> = names
        .iter()
        .filter_map(|n| n.strip_suffix("-a.toml").filter(|s| s.starts_with("eq-")))
        .map(|s| json!([format!("{s}-a.toml"), format!("{s}-b.toml")]))
        .collect();
    Ok(json!({"cases": cases, "equal": equal}))
}

// ─────────────────────────────── refusals ───────────────────────────────

/// The source refusal cases, written by [`regen`]: a file name and its bytes. Each
/// is refused with the reason its name carries.
pub fn source_refusals() -> Vec<(&'static str, Vec<u8>)> {
    let frame = [16u8; 6];
    let y4m = |head: &[u8], body: &[u8]| [head, body].concat();
    // A WAV file with its block alignment and data length stated, so a case can make
    // either disagree with the rest.
    let wav_with = |tag: u16, channels: u16, bits: u16, align: u16, data_len: u32, data: &[u8]| {
        let mut b = b"RIFF\0\0\0\0WAVEfmt \x10\0\0\0".to_vec();
        b.extend_from_slice(&tag.to_le_bytes());
        b.extend_from_slice(&channels.to_le_bytes());
        b.extend_from_slice(&8000u32.to_le_bytes());
        b.extend_from_slice(&(8000u32.wrapping_mul(align as u32)).to_le_bytes());
        b.extend_from_slice(&align.to_le_bytes());
        b.extend_from_slice(&bits.to_le_bytes());
        b.extend_from_slice(b"data");
        b.extend_from_slice(&data_len.to_le_bytes());
        b.extend_from_slice(data);
        b
    };
    let wav = |tag: u16, channels: u16, bits: u16, data: &[u8]| {
        wav_with(tag, channels, bits, (channels as u32 * bits as u32 / 8) as u16, data.len() as u32, data)
    };
    vec![
        ("y4m-header.y4m", y4m(b"YUV4MPEG W2 H2 F25:1\n", &[])),
        ("y4m-interlaced.y4m", y4m(b"YUV4MPEG2 W2 H2 F25:1 It C420jpeg\nFRAME\n", &frame)),
        ("y4m-interlaced-frame.y4m", y4m(b"YUV4MPEG2 W2 H2 F25:1 C420jpeg\nFRAME Ib\n", &frame)),
        ("y4m-colorspace.y4m", y4m(b"YUV4MPEG2 W2 H2 F25:1 C422\nFRAME\n", &[16; 8])),
        ("y4m-size-change.y4m", y4m(b"YUV4MPEG2 W2 H2 F25:1 C420jpeg\nFRAME\n", &[&frame[..], b"FRAME W4\n", &frame].concat())),
        ("y4m-truncated.y4m", y4m(b"YUV4MPEG2 W2 H2 F25:1 C420jpeg\nFRAME\n", &frame[..5])),
        ("wav-format-float.wav", wav(3, 1, 32, &[0; 8])),
        ("wav-format-8bit.wav", wav(1, 1, 8, &[0; 4])),
        ("wav-truncated.wav", {
            let mut w = wav(1, 2, 16, &[0; 8]);
            w.truncate(w.len() - 2);
            w
        }),
        // 2 · channels past 2^16: a block alignment equal to the wrapped product.
        ("wav-format-channels-32768.wav", wav_with(1, 32768, 16, 0, 0, &[])),
        ("wav-format-channels-32769.wav", wav_with(1, 32769, 16, 2, 0, &[])),
        // A data chunk that declares 4 GiB − 2 bytes and holds 4.
        ("wav-truncated-huge-data.wav", wav_with(1, 1, 16, 2, 0xffff_fffe, &[0; 4])),
    ]
}

/// The reason a source file is refused with, or `ok`.
pub fn source_verdict(name: &str, bytes: &[u8]) -> &'static str {
    if name.ends_with(".y4m") {
        y4m::read_all(bytes).map(|_| "ok").unwrap_or_else(|e| e.code())
    } else {
        let read = || -> Result<(), crate::source::SourceError> {
            let mut r = WavReader::new(bytes)?;
            while r.remaining() > 0 {
                r.next_block(1 << 16)?;
            }
            Ok(())
        };
        read().map(|_| "ok").unwrap_or_else(|e| e.code())
    }
}

/// A packager's verdict on a policy's bytes: `ok`, or the reason it refuses it (the
/// policy's, or the first rung's).
pub fn policy_verdict(bytes: &[u8]) -> String {
    crate::capcheck::admit(bytes, &Overrides::default()).map(|_| "ok".to_string()).unwrap_or_else(|e| e.code())
}

fn refusals_expected(dir: &Path) -> std::io::Result<Value> {
    let mut cases = Vec::new();
    for name in tomls(&dir.join("policy"))? {
        let bytes = read(&dir.join("policy").join(&name))?;
        cases.push(json!({"file": format!("policy/{name}"), "reason": policy_verdict(&bytes)}));
    }
    for (name, bytes) in source_refusals() {
        cases.push(json!({"file": format!("source/{name}"), "reason": source_verdict(name, &bytes)}));
    }
    Ok(json!({"cases": cases}))
}

// ─────────────────────────────── conversions ───────────────────────────────

const MS_CASES: &[(u64, u32, u32)] = &[
    (1000, 25, 1),
    (40, 25, 1),
    (39, 25, 1),
    (0, 25, 1),
    (1000, 30000, 1001),
    (2000, 30000, 1001),
    (1, 48000, 1),
    (4000, 90000, 1),
    (9_007_199_254_740_991, 4_294_967_295, 1),
];

/// A timebase as `(num, den)`.
type Rate = (u32, u32);

const ALIGN_CASES: &[(u64, Rate, Rate)] = &[
    (50, (25, 1), (48000, 1)),
    (3, (25, 1), (30000, 1001)),
    (29, (30000, 1001), (25, 1)),
    (48000, (48000, 1), (25, 1)),
    (47999, (48000, 1), (25, 1)),
    (u64::MAX, (1, 4_294_967_295), (4_294_967_295, 1)),
];

/// The Y4M conversion cases, written by [`regen`]: 4:2:0 with odd dimensions, and 4:4:4,
/// whose samples sweep the range so that every clip of the conversion is reached.
pub fn conversion_sources() -> Vec<(&'static str, Vec<u8>)> {
    let planes = |w: usize, h: usize, cw: usize, ch: usize, frames: usize| {
        let mut b = Vec::new();
        for t in 0..frames {
            b.extend_from_slice(b"FRAME\n");
            for i in 0..w * h {
                b.push(((i * 37 + t * 101) % 256) as u8);
            }
            for i in 0..cw * ch {
                b.push(((i * 59 + t * 7 + 13) % 256) as u8);
            }
            for i in 0..cw * ch {
                b.push(((i * 83 + t * 29 + 101) % 256) as u8);
            }
        }
        b
    };
    vec![
        ("bt709-420.y4m", [&b"YUV4MPEG2 W7 H5 F25:1 Ip C420jpeg\n"[..], &planes(7, 5, 4, 3, 3)].concat()),
        ("bt709-444.y4m", [&b"YUV4MPEG2 W6 H4 F25:1 C444\n"[..], &planes(6, 4, 6, 4, 2)].concat()),
    ]
}

fn conversions_expected() -> Value {
    let ms: Vec<Value> = MS_CASES
        .iter()
        .map(|&(ms, num, den)| json!({"ms": ms, "num": num, "den": den, "units": Timebase { num, den }.units_from_ms(ms)}))
        .collect();
    let align: Vec<Value> = ALIGN_CASES
        .iter()
        .map(|&(start, (ln, ld), (cn, cd))| {
            let units = Timebase { num: cn, den: cd }.align_from(start, Timebase { num: ln, den: ld });
            json!({"start": start, "lead": [ln, ld], "component": [cn, cd], "units": units})
        })
        .collect();
    let rgb: Vec<Value> = conversion_sources()
        .into_iter()
        .map(|(name, b)| {
            let (_, frames) = y4m::read_all(&b).expect("a conversion source reads");
            let shas: Vec<String> = frames.iter().map(|f| sha256_hex(&f.rgb)).collect();
            json!({"file": format!("conversions/{name}"), "rgb_sha256": shas})
        })
        .collect();
    json!({"ms_to_units": ms, "align": align, "y4m_rgb": rgb})
}

// ─────────────────────────────── scene cut ───────────────────────────────

/// The scene-cut vectors of every fixture under the policy at `policy`, component `v`.
pub fn scenecut_expected(policy: &[u8]) -> Result<Value, String> {
    let p = Policy::parse_bytes(policy).map_err(|e| e.to_string())?;
    let c = p.component("v").ok_or("the scene-cut policy has no component v")?;
    let sp = SegParams::from_policy(&p.segmentation, c.timebase);
    let mut out = Vec::new();
    for name in fixture::SCENECUT_NAMES {
        let bytes = fixture::generate(name).expect("a listed fixture");
        let (hd, frames) = y4m::read_all(&bytes).map_err(|e| e.to_string())?;
        let (pieces, scores) = segment(sp, hd.w, hd.h, frames.iter().map(|f| &f.rgb));
        let starts: Vec<u64> = pieces.iter().map(|x| x.start).collect();
        let pieces: Vec<Value> = pieces
            .iter()
            .map(|x| {
                json!({
                    "start": x.start, "len": x.len, "flags": x.flags,
                    "kind": classify(&x.stats, &p.classify).label(),
                    "mad_milli": mad_milli(&x.stats), "changed_permille": changed_permille(&x.stats),
                    "occupancy": x.stats.occupancy,
                })
            })
            .collect();
        out.push(json!({
            "name": name, "sha256": sha256_hex(&bytes), "frames": frames.len(),
            "scores": scores, "pieces": pieces, "plan_digest": hex(&plan_digest(&starts)),
        }));
    }
    Ok(json!({"policy": "policy.toml", "component": "v", "fixtures": out}))
}

// ─────────────────────────────── encode ───────────────────────────────

/// The encoder parameter sets the encode vectors run: whole-pel and half-pel, two
/// block sizes, two steps.
pub const ENCODE_SETS: &[(&str, EncParams)] = &[
    ("a", EncParams { block: 8, search: 3, mv_precision: 1, q: 6 }),
    ("b", EncParams { block: 16, search: 2, mv_precision: 2, q: 12 }),
];

/// The frames of each piece an encode vector codes: the first of the piece's frames.
pub const ENCODE_FRAMES: usize = 12;

/// One encode vector: its file name, the payload, and its record.
pub struct EncodeVector {
    pub file: String,
    pub wiv1: Vec<u8>,
    pub record: Value,
    /// The source frames it codes.
    pub clip: Vec<Vec<u8>>,
}

/// Every encode vector: each piece of each fixture (its first [`ENCODE_FRAMES`]
/// frames, cut by the scene-cut policy), under each of [`ENCODE_SETS`].
pub fn encode_vectors(scenecut_policy: &[u8]) -> Result<Vec<EncodeVector>, String> {
    let p = Policy::parse_bytes(scenecut_policy).map_err(|e| e.to_string())?;
    let c = p.component("v").ok_or("the scene-cut policy has no component v")?;
    let sp = SegParams::from_policy(&p.segmentation, c.timebase);
    let mut out = Vec::new();
    for name in fixture::SCENECUT_NAMES {
        let bytes = fixture::generate(name).expect("a listed fixture");
        let (hd, frames) = y4m::read_all(&bytes).map_err(|e| e.to_string())?;
        let (pieces, _) = segment(sp, hd.w, hd.h, frames.iter().map(|f| &f.rgb));
        for (i, piece) in pieces.iter().enumerate() {
            let first = piece.start as usize;
            let n = (piece.len as usize).min(ENCODE_FRAMES);
            let clip: Vec<Vec<u8>> = frames[first..first + n].iter().map(|f| f.rgb.clone()).collect();
            for (set, ep) in ENCODE_SETS {
                let e = int_video_enc::encode(&clip, hd.h, hd.w, ep).map_err(|e| e.to_string())?;
                let file = format!("{name}-p{i}-{set}.wiv1");
                let shas: Vec<String> = e.recon.iter().map(|f| sha256_hex(f)).collect();
                let record = json!({
                    "file": file, "fixture": name, "first": first, "frames": n,
                    "params": {"block": ep.block, "search": ep.search, "mv_precision": ep.mv_precision, "q": ep.q},
                    "blake3": hex(blake3::hash(&e.wiv1).as_bytes()), "sse": e.sse, "frame_sha256": shas,
                });
                out.push(EncodeVector { file, wiv1: e.wiv1, record, clip: clip.clone() });
            }
        }
    }
    Ok(out)
}

/// Decode a payload through `wai.video.int_motion` and state what test 4 asserts of
/// it: the decode equals the encoder's reconstruction (each frame's SHA-256 in the
/// record), and the record's SSE is the one recomputed from the source.
fn check_encode_vector(v: &EncodeVector, out: &mut Vec<Mismatch>) {
    use wai::int_payload::{decode_int, IntDecoded, IntParams};
    let frames = match decode_int(wai::codecs::CAP_VIDEO_INT_MOTION, &v.wiv1, &IntParams::none()) {
        Ok(IntDecoded::Video { frames, .. }) => frames,
        Ok(_) | Err(_) => {
            out.push(format!("encode/{}: does not decode as wai.video.int_motion", v.file));
            return;
        }
    };
    let shas: Vec<Value> = frames.iter().map(|f| Value::from(sha256_hex(f))).collect();
    if Value::from(shas) != v.record["frame_sha256"] {
        out.push(format!("encode/{}: the decode is not the encoder's reconstruction", v.file));
    }
    let sse: u64 = v.clip.iter().flatten().zip(frames.iter().flatten()).map(|(&a, &b)| (a as i64 - b as i64).pow(2) as u64).sum();
    if v.record["sse"] != sse {
        out.push(format!("encode/{}: the stated SSE is not the decode's against the source", v.file));
    }
}

// ─────────────────────────────── the choice of step ───────────────────────────────

/// The rung parameters the step goldens run: one set whose bound some pieces meet at 8
/// and others only at 4, and one whose bound most pieces meet at no step.
pub fn qchoice_params() -> Vec<MotionParams> {
    [5000, 1300]
        .into_iter()
        .map(|max_sse_px_milli| MotionParams { block: 8, search: 1, mv_precision: 1, q_candidates: vec![16, 12, 8, 4], max_sse_px_milli })
        .collect()
}

/// Every piece of every fixture (its first [`ENCODE_FRAMES`] frames): each candidate's
/// SSE, the step chosen and the chosen payload's BLAKE3.
pub fn qchoice_expected(scenecut_policy: &[u8]) -> Result<Value, String> {
    let p = Policy::parse_bytes(scenecut_policy).map_err(|e| e.to_string())?;
    let c = p.component("v").ok_or("the scene-cut policy has no component v")?;
    let sp = SegParams::from_policy(&p.segmentation, c.timebase);
    let mut sets = Vec::new();
    for mp in qchoice_params() {
        let mut out = Vec::new();
        for name in fixture::SCENECUT_NAMES {
            let bytes = fixture::generate(name).expect("a listed fixture");
            let (hd, frames) = y4m::read_all(&bytes).map_err(|e| e.to_string())?;
            let (pieces, _) = segment(sp, hd.w, hd.h, frames.iter().map(|f| &f.rgb));
            for (i, piece) in pieces.iter().enumerate() {
                let first = piece.start as usize;
                let n = (piece.len as usize).min(ENCODE_FRAMES);
                let clip: Vec<Vec<u8>> = frames[first..first + n].iter().map(|f| f.rgb.clone()).collect();
                let mut candidates = Vec::new();
                for &q in &mp.q_candidates {
                    let ep = EncParams { block: mp.block, search: mp.search, mv_precision: mp.mv_precision, q };
                    let e = int_video_enc::encode(&clip, hd.h, hd.w, &ep).map_err(|e| e.to_string())?;
                    candidates.push(json!({"q": q, "sse": e.sse, "sse_px_milli": sse_px_milli(e.sse, hd.h, hd.w, n)}));
                }
                let chosen = choose_q(&clip, hd.h, hd.w, &mp).map_err(|e| e.to_string())?;
                out.push(json!({
                    "fixture": name, "piece": i, "first": first, "frames": n, "candidates": candidates,
                    "q": chosen.q, "blake3": hex(blake3::hash(&chosen.encoded.wiv1).as_bytes()),
                }));
            }
        }
        sets.push(json!({
            "params": {"block": mp.block, "search": mp.search, "mv_precision": mp.mv_precision, "q_candidates": mp.q_candidates, "max_sse_px_milli": mp.max_sse_px_milli},
            "pieces": out,
        }));
    }
    Ok(json!({"policy": "scenecut/policy.toml", "sets": sets}))
}

// ─────────────────────────────── regen and check ───────────────────────────────

/// Write every expected file, and the generated inputs, under `dir`.
pub fn regen(dir: &Path) -> std::io::Result<()> {
    write_json(&dir.join("policy/expected.json"), &policy_expected(&dir.join("policy"))?)?;
    std::fs::create_dir_all(dir.join("refusals/source"))?;
    for (name, bytes) in source_refusals() {
        std::fs::write(dir.join("refusals/source").join(name), bytes)?;
    }
    write_json(&dir.join("refusals/expected.json"), &refusals_expected(&dir.join("refusals"))?)?;
    std::fs::create_dir_all(dir.join("conversions"))?;
    for (name, bytes) in conversion_sources() {
        std::fs::write(dir.join("conversions").join(name), bytes)?;
    }
    write_json(&dir.join("conversions.json"), &conversions_expected())?;
    let text = read(&dir.join("scenecut/policy.toml"))?;
    let sc = scenecut_expected(&text).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
    write_json(&dir.join("scenecut/expected.json"), &sc)?;
    let vectors = encode_vectors(&text).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
    std::fs::create_dir_all(dir.join("encode"))?;
    for v in &vectors {
        std::fs::write(dir.join("encode").join(&v.file), &v.wiv1)?;
    }
    let records: Vec<Value> = vectors.into_iter().map(|v| v.record).collect();
    write_json(&dir.join("encode/expected.json"), &json!({"policy": "scenecut/policy.toml", "vectors": records}))?;
    std::fs::create_dir_all(dir.join("qchoice"))?;
    let qc = qchoice_expected(&text).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
    write_json(&dir.join("qchoice/expected.json"), &qc)?;
    crate::golden::regen(dir)
}

fn same(what: &str, want: &Value, got: &Value, out: &mut Vec<Mismatch>) {
    if want != got {
        out.push(format!("{what}: the corpus and this implementation differ"));
    }
}

/// Recompute the corpus under `dir` and list every difference; empty means it holds.
/// Generated inputs are compared byte for byte with what [`regen`] would write.
pub fn check(dir: &Path) -> std::io::Result<Vec<Mismatch>> {
    let mut out = Vec::new();
    // The equal pairs are checked on the digests this implementation computes, not on
    // the committed ones, which an implementation that diverged would still match.
    let pol = policy_expected(&dir.join("policy"))?;
    same("policy/expected.json", &read_json(&dir.join("policy/expected.json"))?, &pol, &mut out);
    let digest_of = |f: &str| {
        pol["cases"].as_array().and_then(|c| c.iter().find(|x| x["file"] == f && x.get("overrides").is_none())).map(|x| x["digest"].clone())
    };
    // Every policy of the corpus is one a packager accepts, its rungs included (§3.6):
    // the example is a policy someone can pack under.
    for name in tomls(&dir.join("policy"))? {
        let v = policy_verdict(&read(&dir.join("policy").join(&name))?);
        if v != "ok" {
            out.push(format!("policy/{name}: a packager refuses it ({v})"));
        }
    }
    for pair in pol["equal"].as_array().into_iter().flatten() {
        let (a, b) = (pair[0].as_str().unwrap_or(""), pair[1].as_str().unwrap_or(""));
        if digest_of(a).is_none() || digest_of(a) != digest_of(b) {
            out.push(format!("policy: {a} and {b} spell one policy but their digests differ"));
        }
    }
    for (name, bytes) in source_refusals() {
        if read(&dir.join("refusals/source").join(name))? != bytes {
            out.push(format!("refusals/source/{name}: not the bytes the corpus generator writes"));
        }
    }
    let refusals = read_json(&dir.join("refusals/expected.json"))?;
    same("refusals/expected.json", &refusals, &refusals_expected(&dir.join("refusals"))?, &mut out);
    for c in refusals["cases"].as_array().into_iter().flatten() {
        if c["reason"] == "ok" {
            out.push(format!("refusals: {} is not refused", c["file"]));
        }
    }
    for (name, bytes) in conversion_sources() {
        if read(&dir.join("conversions").join(name))? != bytes {
            out.push(format!("conversions/{name}: not the bytes the corpus generator writes"));
        }
    }
    same("conversions.json", &read_json(&dir.join("conversions.json"))?, &conversions_expected(), &mut out);
    let text = read(&dir.join("scenecut/policy.toml"))?;
    match scenecut_expected(&text) {
        Ok(sc) => same("scenecut/expected.json", &read_json(&dir.join("scenecut/expected.json"))?, &sc, &mut out),
        Err(e) => out.push(format!("scenecut/policy.toml: {e}")),
    }
    match encode_vectors(&text) {
        Ok(vectors) => {
            for v in &vectors {
                if read(&dir.join("encode").join(&v.file))? != v.wiv1 {
                    out.push(format!("encode/{}: not the bytes the encoder writes", v.file));
                }
                check_encode_vector(v, &mut out);
            }
            let records: Vec<Value> = vectors.into_iter().map(|v| v.record).collect();
            let want = json!({"policy": "scenecut/policy.toml", "vectors": records});
            same("encode/expected.json", &read_json(&dir.join("encode/expected.json"))?, &want, &mut out);
        }
        Err(e) => out.push(format!("encode: {e}")),
    }
    match qchoice_expected(&text) {
        Ok(qc) => same("qchoice/expected.json", &read_json(&dir.join("qchoice/expected.json"))?, &qc, &mut out),
        Err(e) => out.push(format!("qchoice: {e}")),
    }
    out.extend(crate::golden::check(dir)?);
    Ok(out)
}

/// One line of `wai-pack stage-json` (the staged differential, `stage_diff.py`): the
/// result of the line's `op`, or `{"error": …}`.
///
/// - `refine` `{h, w, q, below, src}`: the WIR1 layer that brings each frame of `below`
///   toward `src` (§5.4): `{"payload": hex, "recon": [SHA-256 of each frame]}`.
/// - `still` `{w, h, rgb}`: the still (§5.7), `{"png": hex}`, and its decode's SHA-256.
/// - `read-png` `{hex}`: what a still's reader (§5.7) makes of the bytes:
///   `{"verdict": "ok", "w", "h", "rgb_sha256"}`, or `{"verdict": "refused"}`.
/// - `plan` `{policy, pieces: [{start, duration, kind}], layer_bytes: [[…]]}`: the policy's
///   verdict, and per piece its rules and moments, its ladder's rung ids, its deadline and
///   each class's plan (§5.7, §8.1).
/// - `deliver` `{policy, class_plan, published: [{piece, start, duration, at_frame,
///   layer, deadline_ms, bytes}], plan: {rtt_us, bytes_per_ms, chunk_bytes, enc_us,
///   startup_us, layer_cap, fifo}}`: the simulated delivery (§8.9).
pub fn stage_json(v: &Value) -> Value {
    match stage_op(v) {
        Ok(r) => r,
        Err(e) => json!({"error": e}),
    }
}

fn u64_of(v: &Value, k: &str) -> Result<u64, String> {
    v[k].as_u64().ok_or_else(|| format!("{k}: not an unsigned integer"))
}

fn bytes_of(v: &Value) -> Result<Vec<u8>, String> {
    v.as_array().ok_or("not an array")?.iter().map(|x| x.as_u64().filter(|b| *b < 256).map(|b| b as u8).ok_or_else(|| "not a byte".to_string())).collect()
}

fn frames_of(v: &Value) -> Result<Vec<Vec<u8>>, String> {
    v.as_array().ok_or("frames: not an array")?.iter().map(bytes_of).collect()
}

fn stage_op(v: &Value) -> Result<Value, String> {
    match v["op"].as_str() {
        Some("refine") => {
            let (h, w, q) = (u64_of(v, "h")? as u32, u64_of(v, "w")? as u32, u64_of(v, "q")? as u32);
            let (payload, recon) = crate::encode::encode_refinement(&frames_of(&v["below"])?, &frames_of(&v["src"])?, h, w, q)?;
            Ok(json!({"payload": hex(&payload), "recon": recon.iter().map(|f| sha256_hex(f)).collect::<Vec<_>>()}))
        }
        Some("still") => {
            let (w, h) = (u64_of(v, "w")? as u32, u64_of(v, "h")? as u32);
            let rgb = bytes_of(&v["rgb"])?;
            if rgb.len() as u64 != 3 * w as u64 * h as u64 {
                return Err("rgb: not 3·w·h bytes".into());
            }
            let png = crate::png::encode(&rgb, w, h);
            let (_, _, back) = crate::png::decode(&png)?;
            Ok(json!({"png": hex(&png), "decoded_sha256": sha256_hex(&back)}))
        }
        Some("read-png") => {
            let b = crate::unhex(v["hex"].as_str().ok_or("hex: not a string")?).ok_or("hex: not hexadecimal")?;
            Ok(match crate::png::decode(&b) {
                Ok((w, h, rgb)) => json!({"verdict": "ok", "w": w, "h": h, "rgb_sha256": sha256_hex(&rgb)}),
                Err(_) => json!({"verdict": "refused"}),
            })
        }
        Some("plan") => {
            let text = v["policy"].as_str().ok_or("policy: not a string")?;
            let p = match Policy::parse(text) {
                Ok(p) => p,
                Err(e) => return Ok(json!({"verdict": e.code()})),
            };
            let comp = &p.components[0];
            let mut out = Vec::new();
            for (pc, lb) in v["pieces"].as_array().ok_or("pieces")?.iter().zip(v["layer_bytes"].as_array().ok_or("layer_bytes")?) {
                let kind = crate::classify::PieceKind::from_code(u64_of(pc, "kind")? as u8).ok_or("kind")?;
                let (start, duration) = (u64_of(pc, "start")?, u64_of(pc, "duration")?);
                let pp = crate::plan::plan_piece(&p, comp.timebase, start, duration, kind);
                let ladder = crate::plan::piece_ladder(&p, &comp.id, &pp).ok_or("no base")?;
                let mut ids = vec![ladder.base.id.clone()];
                ids.extend(ladder.refinements.iter().map(|r| r.id.clone()));
                ids.extend(ladder.floors.iter().map(|r| r.id.clone()));
                let lb: Vec<u64> = lb.as_array().ok_or("layer_bytes")?.iter().map(|x| x.as_u64().ok_or("a byte count")).collect::<Result<_, _>>()?;
                let plan: Vec<u8> = p.classes.iter().map(|c| crate::plan::class_plan(c, &lb, pp.max_layer, duration, comp.timebase)).collect();
                out.push(json!({"rules": pp.rules, "moments": pp.moments, "ladder": ids, "deadline_ms": pp.deadline_ms, "class_plan": plan}));
            }
            Ok(json!({"verdict": "ok", "pieces": out}))
        }
        Some("deliver") => {
            let text = v["policy"].as_str().ok_or("policy: not a string")?;
            let p = Policy::parse(text).map_err(|e| e.code().to_string())?;
            let row = bytes_of(&v["class_plan"])?;
            let mut published = Vec::new();
            for x in v["published"].as_array().ok_or("published")? {
                let env = vec![0u8; u64_of(x, "bytes")? as usize];
                published.push(crate::live::Published {
                    piece: u64_of(x, "piece")?,
                    start: u64_of(x, "start")?,
                    duration: u64_of(x, "duration")?,
                    at_frame: u64_of(x, "at_frame")?,
                    layer: u64_of(x, "layer")? as u8,
                    role: if u64_of(x, "layer")? == 0 { wai::stage_index::Role::Base } else { wai::stage_index::Role::Refinement },
                    capability: String::new(),
                    deadline_ms: u64_of(x, "deadline_ms")? as u32,
                    hash: crate::objects::object_hash(&env),
                    envelope: env,
                });
            }
            let q = &v["plan"];
            let plan = crate::delivery::DeliveryPlan {
                rtt_us: u64_of(q, "rtt_us")?,
                bytes_per_ms: u64_of(q, "bytes_per_ms")?,
                chunk_bytes: u64_of(q, "chunk_bytes")?,
                enc_us: u64_of(q, "enc_us")?,
                startup_us: u64_of(q, "startup_us")?,
                layer_cap: q["layer_cap"].as_u64().map(|c| c as u8),
                schedule: if q["fifo"].as_bool() == Some(true) { wai::stage_measure::sim::Schedule::Fifo } else { wai::stage_measure::sim::Schedule::StrictPriority },
            };
            let key = ed25519_dalek::SigningKey::from_bytes(&[0x5a; 32]);
            let d = crate::delivery::simulate(&published, &p, &row, &plan, &key)?;
            let mut j = d.to_json();
            j.as_object_mut().expect("an object").remove("plan");
            Ok(j)
        }
        _ => Err("op: refine, still, plan or deliver".into()),
    }
}