//! What a test does when the out-of-tree inputs behind a headline decode claim are //! absent. //! //! Some decode claims are tested against inputs too large, or not redistributable, //! to commit: the `int_hyper` vectors (the pretrained mbt2018_mean, exported to //! integers) and the JPEG-AI whole-decoder bundle. A test that `return`s when its //! inputs are missing is reported as *passed*, so a run with nothing to check looks //! exactly like a run that checked everything. These helpers remove that ambiguity: //! //! - `WAI_REQUIRE_SOTA_VECTORS` unset (or `0`/`false`/`no`): the test still skips, //! but writes a banner to the process's real stderr — past libtest's output //! capture, so it shows in an ordinary `cargo test` run, not only under //! `--nocapture`. //! - `WAI_REQUIRE_SOTA_VECTORS=1`: a missing input is a test failure naming the //! tool that produces it. A run that claims to verify these decodes sets it. //! //! Either way, with or without the switch, a variable that names a test's inputs //! makes them required: pointing a test at inputs and having it pass because they //! are not there is the failure this module exists to prevent. So the test fails //! when a variable read through [`env_paths`] names a path that does not exist, //! and when a directory variable read through [`env_dir`] (`WAI_SOTA_INT_DIR`, //! `WAI_JPEGAI_ASSETS`) names something that is not a directory, or a directory //! missing a file the test reads ([`InputDir::absent`]). The JPEG-AI tests that //! take a directory through [`env_paths`] fail the same way on a missing file, //! because their loaders do not skip. So without the switch a test skips only when //! no variable names what is missing: when a variable read through [`env_paths`] //! is unset (the ones that are set must still name existing paths), or when the //! variable of [`env_dir`] is unset or empty and the default location it falls //! back to is absent or incomplete. [`env_paths`] has no default, so an empty value //! there names no existing path and fails. `tests/require_vectors_rule.rs` checks //! this rule by running these helpers in child processes, with the switch unset and //! set. //! //! Where each covered input comes from, and who verifies it: //! //! | test | inputs | produced by | verified in CI | //! |---|---|---|---| //! | `int_hyper_synth` (pretrained-model cases) | `WAI_SOTA_INT_DIR` | `tools/wai_sota_vectors.sh` | yes — `wai-sota-vectors.yml` regenerates and requires them | //! | `int_video_sota_keyframe` | `WAI_SOTA_INT_DIR` | `tools/wai_sota_vectors.sh` | yes — same job | //! | `jpegai_e2e_full` | `WAI_JPEGAI_{CODESTREAM,WEIGHTS_DIR,NORM}` | `tools/wai_jpegai_fetch_e2e_inputs.sh` (the models root's `jpegai/v1/`, pinned by `tools/jpegai-e2e-v1.sha256`) | yes — `wai-jpegai-e2e.yml` fetches, checks and requires them | //! | `jpegai_reconstruction_binding` | same, plus optional `WAI_JPEGAI_EXPECT_SHA256` | same | yes — same job, with the decoded-pixel hash pinned | //! | `jpegai_live` | `WAI_JPEGAI_ASSETS` (default `../jpegai-vm-assets`) | the local JPEG-AI asset directory (not in the repository) | no — local only; also needs the `neural` + `codecs` features | //! //! Tests that still skip silently on absent inputs and are NOT covered by this //! switch — an open gap, listed here so it stays visible: //! `jpegai_*_calibration.rs`, `jpegai_e2e.rs`, `jpegai_e2e_{chain,chroma,dequant, //! frontend,hyperlatent,synthesis}.rs`, `jpegai_mans_golden.rs`, //! `jpegai_postfilter_chain.rs` (per-stage and staged JPEG-AI fixtures traced from //! the reference software), `neural_native_decode.rs` (float ONNX decoders, models //! and `/tmp` fixtures built per machine), and the `WAI_JPEGAI_*` unit tests in //! `src/jpegai_synthesis.rs` and `src/neural/mod.rs`. use std::io::Write; use std::path::{Path, PathBuf}; pub const SWITCH: &str = "WAI_REQUIRE_SOTA_VECTORS"; /// Whether this run has declared that the out-of-tree inputs must be present. pub fn required() -> bool { match std::env::var(SWITCH) { Ok(v) => !matches!(v.trim().to_ascii_lowercase().as_str(), "" | "0" | "false" | "no"), Err(_) => false, } } /// Call at the point a test would otherwise return early for lack of inputs. /// Panics when [`required`]; otherwise announces the skip where it cannot be /// missed. The caller returns (or skips the sub-check) after this. pub fn absent(test: &str, missing: &str, produce: &str) { let msg = format!("{test}: NOT VERIFIED — {missing}\n produce with: {produce}"); if required() { panic!("{msg}\n {SWITCH} is set, so a missing input is a failure, not a skip"); } // Deliberately not eprintln!: libtest captures the print macros of a passing // test, which is how skips became invisible in the first place. // wai-sota-vectors.yml and wai-jpegai-e2e.yml fail on "SKIPPED" or // "NOT VERIFIED" in their test log, so keep both words in this banner. let _ = writeln!( std::io::stderr(), "\n!!! SKIPPED — {msg}\n set {SWITCH}=1 to make a missing input fail\n" ); } /// Read a JSON input. `None` only when the file does not exist; a file that exists /// but cannot be read or parsed is a broken input and fails the test either way. #[allow(dead_code)] pub fn load_json(p: &Path) -> Option { if !p.exists() { return None; } let bytes = std::fs::read(p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())); Some(serde_json::from_slice(&bytes).unwrap_or_else(|e| panic!("parse {}: {e}", p.display()))) } /// Every named JSON file from `dir`, or `None` after [`InputDir::absent`] names /// the ones missing (which fails instead when a variable named `dir`). #[allow(dead_code)] pub fn load_all( test: &str, dir: &InputDir, names: [&str; N], produce: &str, ) -> Option<[serde_json::Value; N]> { let loaded = names.map(|n| load_json(&dir.join(n))); let missing: Vec<&str> = names.iter().zip(&loaded).filter(|(_, v)| v.is_none()).map(|(n, _)| *n).collect(); if !missing.is_empty() { dir.absent(test, &format!("{} not in {}", missing.join(", "), dir.display()), produce); return None; } Some(loaded.map(Option::unwrap)) } /// The environment variables a test reads its input paths from. `None` (after /// [`absent`]) when any is unset. A variable that IS set but names a path that does /// not exist is a misconfigured run — pointing a test at a file and having it pass /// because the file is missing is the failure this module exists to prevent — so /// that fails regardless of the switch, and whether or not the other variables are /// set. #[allow(dead_code)] pub fn env_paths(test: &str, vars: [&str; N], produce: &str) -> Option<[String; N]> { let got = vars.map(|v| std::env::var(v).ok()); // The set ones first: a variable naming a missing path fails even when another // variable the test reads is unset, which would otherwise end the test as a skip. for (v, p) in vars.iter().zip(&got) { if let Some(p) = p { assert!(Path::new(p).exists(), "{test}: {v}={p} does not exist (produce with: {produce})"); } } let unset: Vec<&str> = vars.iter().zip(&got).filter(|(_, g)| g.is_none()).map(|(v, _)| *v).collect(); if !unset.is_empty() { absent(test, &format!("{} not set", unset.join(" + ")), produce); return None; } Some(got.map(Option::unwrap)) } /// The directory a test reads its inputs from, and whether a variable named it. /// Made by [`env_dir`]. #[allow(dead_code)] pub struct InputDir { path: PathBuf, /// The variable that named `path`; `None` when `path` is the default. named_by: Option, } #[allow(dead_code)] impl InputDir { pub fn path(&self) -> &Path { &self.path } pub fn join(&self, name: impl AsRef) -> PathBuf { self.path.join(name) } pub fn display(&self) -> std::path::Display<'_> { self.path.display() } /// [`absent`] for an input that belongs in this directory. When a variable /// named the directory, the run pointed the test at its inputs, so a missing /// one fails whether or not the switch is set. Only the default directory may /// be absent or incomplete and skip. pub fn absent(&self, test: &str, missing: &str, produce: &str) { if let Some(var) = &self.named_by { panic!( "{test}: NOT VERIFIED — {missing}\n produce with: {produce}\n \ {var} names this directory, so a missing input is a failure with or \ without {SWITCH}" ); } absent(test, missing, produce); } } impl AsRef for InputDir { fn as_ref(&self) -> &Path { &self.path } } /// The directory a test reads its inputs from: the variable `var` when it is set /// and non-empty, else `default`. Empty counts as unset, as it does for /// `tools/wai_sota_vectors.sh`'s own `${1:-/tmp/sota_int}`, so the script and the /// tests fall back to the same place. A variable that IS set names the test's /// inputs: it fails the test, regardless of the switch, when it does not name a /// directory (as in [`env_paths`]), and the returned [`InputDir`] fails it when a /// file is missing from that directory. The default may be absent or incomplete: /// the test then goes through [`absent`]. #[allow(dead_code)] pub fn env_dir(test: &str, var: &str, default: &str, produce: &str) -> InputDir { match std::env::var(var) { Ok(p) if !p.is_empty() => { assert!( Path::new(&p).is_dir(), "{test}: {var}={p} is not a directory (produce with: {produce})" ); InputDir { path: PathBuf::from(p), named_by: Some(var.to_owned()) } } _ => InputDir { path: PathBuf::from(default), named_by: None }, } }