diff --git a/.github/workflows/wai-neural-e2e.yml b/.github/workflows/wai-neural-e2e.yml index fe28ce1fe..cebedadc4 100644 --- a/.github/workflows/wai-neural-e2e.yml +++ b/.github/workflows/wai-neural-e2e.yml @@ -30,6 +30,20 @@ jobs: neural-e2e: name: synthetic fixtures → Rust decode → reference (±4) runs-on: linux + # The Rust decoders link `ort`, whose `download-binaries` feature pulls a + # prebuilt static onnxruntime (1.28.0 for ort 2.0.0-rc.13). That library was + # built against glibc >= 2.38 and GCC 13's libstdc++. The runner's default + # job image, catthehacker/ubuntu:act-22.04, has glibc 2.35 and GCC 11, so the + # test binary fails to LINK with five undefined symbols: + # __isoc23_strtol __isoc23_strtoll __isoc23_strtoull (glibc 2.38+) + # basic_string::_M_replace_cold basic_string::_M_replace_cold + # (libstdc++, GCC 13+) + # That link error is why this job failed at ~1 minute on every run: the + # Python steps pass on a warm cache and the job dies at its last step. Checked + # in both images: act-22.04 lacks all five, act-24.04 (glibc 2.39, g++ 13) + # provides all five. + container: + image: catthehacker/ubuntu:act-24.04 steps: - uses: actions/checkout@v5 - uses: actions/setup-python@v5 @@ -60,4 +74,32 @@ jobs: working-directory: wai/wai-rs # --no-default-features: the neural decoders need only `ort`, NOT the C # codec chain (AV1/JXL/opus), so the job needs no system codec deps. - run: cargo test --no-default-features --features neural --test neural_native_decode -- _matches_reference --nocapture + # + # The grep is the assertion, not decoration. Every `*_matches_reference` + # test opens with `if !wai.exists() || !ref.exists() || !model.exists() { + # eprintln!("skipping ..."); return; }` — so a missing fixture makes the + # test SKIP, and a skip is reported as a PASS. Verified by running this + # exact command on a machine with no /tmp fixtures: it prints + # "6 passed; 0 failed" while comparing nothing. + # + # That means a green `cargo test` here does NOT mean the decoders were + # checked against their references. If the Python fixture step ever + # succeeds while writing nothing, this job goes green having verified + # nothing at all — the failure mode is silent and looks like success. + # Failing on "skipping" is what makes the job's name true. + # + # One test is excluded by name, not tolerated as a skip: + # `snac_real_model_matches_reference` needs real pretrained SNAC weights, + # which this synthetic-fixture job never builds, so it would skip on every + # run and the guard would fail a job whose five in-scope decoders all + # match. Excluding it keeps the guard strict for everything in scope. + run: | + set -o pipefail + cargo test --no-default-features --features neural --test neural_native_decode \ + -- _matches_reference --skip snac_real_model_matches_reference --nocapture \ + 2>&1 | tee /tmp/neural_e2e.log + if grep -q "skipping" /tmp/neural_e2e.log; then + echo "::error::a reference comparison was SKIPPED — its fixtures are missing, so this job verified nothing" + grep "skipping" /tmp/neural_e2e.log + exit 1 + fi diff --git a/wai/tools/wai_validate_synthetic.py b/wai/tools/wai_validate_synthetic.py index 8f5dc70ab..f71c39572 100644 --- a/wai/tools/wai_validate_synthetic.py +++ b/wai/tools/wai_validate_synthetic.py @@ -120,6 +120,31 @@ def write_wai(path, cap, kind, header, z, y, H, W): path.write_bytes(b"WAI1" + struct.pack(" int: # Each harness is independent; a failure in one must not block the others # (e.g. a synthetic model that only traces at certain sizes). Report a - # per-decoder PASS/FAIL summary and exit non-zero only if ALL failed. + # per-decoder PASS/FAIL summary, and exit non-zero unless EVERY harness succeeded. harnesses = [ ("elic", lambda: roundtrip_elic(build(MiniElic, 0), x, Path("/tmp/wai_elic"), args.size)), ("mlicpp", lambda: roundtrip_mlicpp(build(MiniMlic, 1), x, Path("/tmp/wai_mlic"), args.size)), @@ -397,7 +422,19 @@ def main() -> int: print(f" ✗ {name}: {type(e).__name__}: {e}") print(f"generated {ok}/{len(harnesses)} synthetic neural fixtures into /tmp") print("now: cargo test --features neural --test neural_native_decode -- _matches_reference") - return 0 if ok else 1 + # Every harness must succeed, not merely one of them. + # + # `0 if ok else 1` reported success whenever a single harness worked, and the + # loop above deliberately catches per-harness exceptions to keep going. So a + # 1-of-5 run exited 0, the CI step went green, and the four decoders whose + # fixtures were never written then SKIPPED in the Rust suite — where a skip is + # reported as a pass. The job came out fully green having checked one decoder + # out of five, and nothing in the output said so. + # + # Partial generation is still useful interactively: the per-harness ✓/✗ lines + # are printed either way, and a caller who wants one fixture can ignore the + # exit code. What is not acceptable is CI reading a partial run as a pass. + return 0 if ok == len(harnesses) else 1 if __name__ == "__main__":