diff --git a/.github/workflows/tests-in-ci.yml b/.github/workflows/tests-in-ci.yml new file mode 100644 index 000000000..8fc926dde --- /dev/null +++ b/.github/workflows/tests-in-ci.yml @@ -0,0 +1,41 @@ +name: tests in CI + +# Every integration test of the reference crate is run by a job, and none can run +# empty. +# +# The conformance workflows name the tests they run, one `--test NAME` at a time. A test +# file nobody adds to a step never runs, and a feature-gated one compiled in a step that +# lacks its feature reports `ok` with no test in it. Both have happened here, silently. +# scripts/check-tests-in-ci.py requires that every test file is named by a step or +# listed, with its reason, in the crate's tests/NOT-IN-CI.txt, and that every +# feature-gated one declares its features in Cargo.toml, so that cargo refuses to build +# it without them. +# +# No `paths:` filter, on purpose. On this forge a filter matches the files of the push, +# not of the pull request, so a later push that touches no test would leave the head with +# no run of this check. The job takes seconds: it reads files and builds nothing. + +# One run per branch at a time. A push supersedes the run before it. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: [main] + pull_request: + +jobs: + covered: + name: every test file is run by a job or listed with its reason + runs-on: linux + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + # The check must be able to fail, or it is decoration. + - name: The check fails closed (self-test) + run: ./scripts/check-tests-in-ci-selftest.sh + + - name: Every test file is run or listed, and none can run empty + run: python3 -I scripts/check-tests-in-ci.py diff --git a/.github/workflows/wai-conformance.yml b/.github/workflows/wai-conformance.yml index 0f8049376..060996c2e 100644 --- a/.github/workflows/wai-conformance.yml +++ b/.github/workflows/wai-conformance.yml @@ -254,12 +254,17 @@ jobs: # the readers to the derived reader's result on every committed envelope. Both run in # the release step only: the first writes manifests of 2^27 bytes, which takes a debug # build most of a minute. + # tests/int_video.rs, int_video_halfpel.rs and int_video_keyframe.rs hold the integer + # video codec to its goldens (whole clips, half-pel motion, the keyframe through the + # integer hyperprior); int_world.rs and int_world2.rs the integer world renderers; + # jpegai_bitstream_calibration.rs the JPEG AI bitstream reader against the committed + # codestream. No step named any of them until scripts/check-tests-in-ci.py asked. - name: Fast integer kernels ≡ reference kernels (overflow checks on) working-directory: wai/wai-rs - run: cargo test --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds + run: cargo test --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds --test int_video --test int_video_halfpel --test int_video_keyframe --test int_world --test int_world2 --test jpegai_bitstream_calibration - name: Fast integer kernels ≡ reference kernels (release, overflow checks off) working-directory: wai/wai-rs - run: cargo test --release --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds --test manifest_parse_memory --test manifest_reader_equivalence + run: cargo test --release --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds --test manifest_parse_memory --test manifest_reader_equivalence --test int_video --test int_video_halfpel --test int_video_keyframe --test int_world --test int_world2 --test jpegai_bitstream_calibration # The integer decode path builds under each of its features alone, so a sink can take # one medium without the others, and for wasm32 (a browser sink); `int_payload` arms a # medium only under its feature, and none needs `neural`. So does the staged-measurement @@ -323,6 +328,13 @@ jobs: # by inclusion and named by the relay's delivery-gap claims, and each base's stage claim, # which seals and verifies; the subscriber of two tracks holds exactly their chains. # Cargo.toml declares it with required-features = ["staged", "video_int", "receipt_claims"]. + # The interactive-world tests, which no step named: world_replay.rs (the replay + # class, under `world`), world_model.rs (the signed action trace and per-frame + # provenance, under `world_model`) and int_world_shared.rs (two sinks converging on + # one integer world, under both). `world_model` is enabled by no other step. + - name: World replay, the world model and the shared integer world + working-directory: wai/wai-rs + run: cargo test --no-default-features --features world_model,world_int --test world_replay --test world_model --test int_world_shared - name: Staged tracks over MoQ, with signed receipts working-directory: wai/wai-rs run: cargo test --no-default-features --features staged,video_int,receipt_claims --test staged_moq @@ -897,7 +909,7 @@ jobs: # is how the wrappers treat hostile bytes and where their limits fall, which is the # same code on every host. A missing library fails the build step by name. codec-wrappers: - name: codec wrappers · fuzz smoke, text decode limits, C entry points + name: codec wrappers · fuzz smoke, text decode limits, C entry points, provenance, the semantic path runs-on: macos steps: - uses: actions/checkout@v5 @@ -927,6 +939,43 @@ jobs: exit 1 fi + # wai_prov needs `codecs`, so no other job builds it. Its --carry path writes the + # provenance asset and receipt that wai-web's reach vectors commit: carried onto + # its own receipt, a run must write both again byte for byte. A receipt changed + # by one byte must be refused, which shows the comparison is not vacuous. + - name: wai_prov carries the committed provenance receipt, byte for byte + working-directory: wai/wai-rs + shell: bash + run: | + set -euo pipefail + cargo build --locked --features provenance,codecs,receipt_claims --bin wai_prov + bin="${CARGO_TARGET_DIR:-target}/debug/wai_prov" + v=../wai-web/reach/test/vectors/provenance + "$bin" "$RUNNER_TEMP/prov" --carry "$v/captured.receipt.json" + cmp "$RUNNER_TEMP/prov/captured.jpeg.wai" "$v/captured.jpeg.wai" + cmp "$RUNNER_TEMP/prov/captured.receipt.json" "$v/captured.receipt.json" + sed 's/"encode.jpeg"/"encode.jpeh"/' "$v/captured.receipt.json" > "$RUNNER_TEMP/changed.receipt.json" + if cmp -s "$RUNNER_TEMP/changed.receipt.json" "$v/captured.receipt.json"; then + echo "::error::the changed receipt is not changed"; exit 1 + fi + if "$bin" "$RUNNER_TEMP/prov-changed" --carry "$RUNNER_TEMP/changed.receipt.json"; then + echo "::error::a changed receipt was carried"; exit 1 + fi + + # tests/semantic_path.rs needs `semantic`, which builds on `codecs` and binds the + # ONNX runtime, so this is the one job that can run it. + - name: The semantic path + working-directory: wai/wai-rs + shell: bash + run: | + set -o pipefail + cargo test --locked --features codecs,semantic --test semantic_path 2>&1 | tee "$RUNNER_TEMP/semantic-path.log" + ran=$(grep -cE '^test result: ok\. [1-9][0-9]* passed' "$RUNNER_TEMP/semantic-path.log" || true) + if [ "$ran" -ne 1 ]; then + echo "::error::the semantic path's test binary ran no test" + exit 1 + fi + # The C2PA emitter is gated behind an off-by-default feature, so the four-platform # job above — which builds a fixed feature list — never compiles it. A feature no # workflow enables has no CI coverage at all: the suite passes, and the tests in it diff --git a/.github/workflows/wai-energy-receipts.yml b/.github/workflows/wai-energy-receipts.yml index 9dec45179..f0dc3538e 100644 --- a/.github/workflows/wai-energy-receipts.yml +++ b/.github/workflows/wai-energy-receipts.yml @@ -46,7 +46,8 @@ name: WAI energy receipts # - run a meter against a power reader. The report path is tested in the # library (energy_figure::meter_report), and the protocol against # scripted readers (energy_burst, energy_meter). -# - build wai_prov (needs `codecs`, which links system libjxl) or the +# - build wai_prov (needs `codecs`, which links system libjxl; the +# codec-wrappers job of wai-conformance.yml builds and runs it) or the # wai-quantum CLI bin (needs quantum_toolchain, quantum_qir, # quantum_channel, quantum_comm and quantum_qudit). Cargo skips a bin whose # required features are off without saying so. @@ -151,12 +152,12 @@ jobs: # reference meters (wai_video_meter, wai_meter, wai_world_meter, # wai_quantum_meter), whose report path needs receipt_claims — so one of # those that stops building fails here. - # The fifth, wai_prov, is NOT built by this or any workflow: it requires - # the `codecs` feature, which links system libjxl (>= 0.11), and no - # workflow enables `codecs` or installs libjxl. Cargo skips a bin whose - # required features are off without saying so. Its metering is - # energy_burst, whose tests run in the lib suite; the bin itself is - # checked only by hand. + # The fifth, wai_prov, is not built here: it requires the `codecs` + # feature, which links system libjxl (>= 0.11). Cargo skips a bin whose + # required features are off without saying so. The codec-wrappers job + # of wai-conformance.yml, on the macOS runner, builds it and runs its + # --carry path against the committed provenance receipt. Its metering is + # energy_burst, whose tests run in the lib suite. - name: wai-rs — energy classes, labels and legacy pins working-directory: wai/wai-rs shell: bash diff --git a/scripts/check-tests-in-ci-selftest.sh b/scripts/check-tests-in-ci-selftest.sh new file mode 100755 index 000000000..a9f6dbb9f --- /dev/null +++ b/scripts/check-tests-in-ci-selftest.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# +# check-tests-in-ci-selftest.sh — check-tests-in-ci.py can fail. +# +# A check that cannot fail is decoration. This builds a scratch tree shaped like the +# repository (one workflow, one crate with a plain test, a feature-gated test and a +# test listed as not run) and asserts that the check passes it, and fails each way the +# property can break, saying which. +set -uo pipefail + +here=$(cd "$(dirname "$0")" && pwd) +check="$here/check-tests-in-ci.py" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +C="$tmp/wai/wai-rs" + +mk() { + rm -rf "$tmp/.github" "$tmp/wai" + mkdir -p "$tmp/.github/workflows" "$C/tests" + printf 'jobs:\n t:\n steps:\n - run: cargo test --features f --test alpha --test gated\n' > "$tmp/.github/workflows/w.yml" + printf '[package]\nname = "x"\n\n[features]\nf = []\ng = []\n\n[[test]]\nname = "gated"\npath = "tests/gated.rs"\nrequired-features = ["f"]\n' > "$C/Cargo.toml" + printf '#[test]\nfn a() {}\n' > "$C/tests/alpha.rs" + printf '#![cfg(feature = "f")]\n#[test]\nfn g() {}\n' > "$C/tests/gated.rs" + printf '#[test]\nfn l() {}\n' > "$C/tests/local.rs" + printf '# name, then why no job runs it\nlocal needs a device no runner has\n' > "$C/tests/NOT-IN-CI.txt" +} + +fails=0 +expect() { # expect [text the output must hold] + local want=$1 what=$2 needle=${3:-} out rc + out=$(python3 -I "$check" --root "$tmp" 2>&1); rc=$? + if [ "$want" = pass ] && [ $rc -ne 0 ]; then echo "FAIL $what: the check failed"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi + if [ "$want" = fail ] && [ $rc -eq 0 ]; then echo "FAIL $what: the check passed"; fails=$((fails+1)); return; fi + if [ -n "$needle" ] && ! printf '%s' "$out" | grep -q -- "$needle"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi + echo "ok $what" +} + +mk; expect pass "a tree where every test is run or listed passes" + +mk; printf '#[test]\nfn b() {}\n' > "$C/tests/beta.rs" +expect fail "a test file no step names and nothing lists" "beta.rs: no workflow step names it" + +mk; printf '[package]\nname = "x"\n\n[features]\nf = []\n' > "$C/Cargo.toml" +expect fail "a feature-gated test with no required-features" "runs it empty" + +mk; sed -i.bak 's/required-features = \["f"\]/required-features = ["g"]/' "$C/Cargo.toml" +expect fail "required-features that are not the file's features" "is gated on" + +mk; printf '#![cfg(any(feature = "f", feature = "g"))]\n#[test]\nfn g() {}\n' > "$C/tests/gated.rs" +expect fail "a gate the check cannot read" "cannot tell what it needs" + +mk; printf 'alpha no reason at all, a step runs it\n' >> "$C/tests/NOT-IN-CI.txt" +expect fail "a test listed as not run that a step names" "remove the line" + +mk; printf 'ghost it was deleted\n' >> "$C/tests/NOT-IN-CI.txt" +expect fail "a listed name that is no test file" "ghost is not a test file" + +mk; printf 'local\n' > "$C/tests/NOT-IN-CI.txt" +expect fail "a listed test with no reason" "not .name reason." + +mk; printf '\n[[test]]\nname = "gone"\npath = "tests/gone.rs"\n' >> "$C/Cargo.toml" +expect fail "a [[test]] entry for a file that does not exist" "which does not exist" + +if [ $fails -ne 0 ]; then echo "check-tests-in-ci-selftest: $fails failure(s)"; exit 1; fi +echo "check-tests-in-ci-selftest: the check passes a covered tree and fails each uncovered one." diff --git a/scripts/check-tests-in-ci.py b/scripts/check-tests-in-ci.py new file mode 100755 index 000000000..b604c5385 --- /dev/null +++ b/scripts/check-tests-in-ci.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""check-tests-in-ci.py: every integration test of a crate is run by a job, and none +can run empty. + +The conformance workflows name the tests they run, one `--test NAME` at a time: the +crate is too large to run whole on every leg. So a test file nobody adds to a step never +runs, and nothing says so. And a file that is `#![cfg(feature = "...")]` compiles, in a +step that lacks the feature, to a binary with no tests in it that reports `ok`. + +For each crate listed in CRATES this requires that + + 1. every `tests/*.rs` is named by `--test NAME` in a workflow, or is listed in the + crate's `tests/NOT-IN-CI.txt` with the reason no job runs it; + 2. every `tests/*.rs` whose crate-level `#![cfg(...)]` names features has a `[[test]]` + entry in Cargo.toml whose `required-features` are exactly those features, so that + naming it in a step without them stops cargo with an error; + 3. `NOT-IN-CI.txt` names only test files that exist and that no workflow names, each + with a reason; + 4. every `[[test]]` entry names a file that exists. + +A crate whose workflow runs `cargo test` over all its targets needs no entry in CRATES: +cargo finds its tests itself. + + check-tests-in-ci.py [--root DIR] + +Exit 0 when all four hold, 1 with one line a finding otherwise. Standard library only. +""" +import os +import re +import sys + +# Crates whose workflows name their tests one by one. +CRATES = ["wai/wai-rs"] + +NOT_IN_CI = "NOT-IN-CI.txt" + + +def workflow_test_names(root): + """Every NAME of a `--test NAME` in a workflow file.""" + names = set() + wf = os.path.join(root, ".github", "workflows") + for f in sorted(os.listdir(wf)): + if f.endswith((".yml", ".yaml")): + with open(os.path.join(wf, f), encoding="utf-8") as fh: + names.update(re.findall(r"--test[ =]([A-Za-z0-9_-]+)", fh.read())) + return names + + +def crate_cfg_features(path): + """The features a test file's crate-level `#![cfg(...)]` requires, as a sorted list; + [] when it has none. None when the attribute is a form this check does not read + (`any`, `not`, or something that is not a feature), which is a finding.""" + with open(path, encoding="utf-8") as fh: + src = fh.read() + feats = set() + for m in re.finditer(r"^#!\[cfg\((.*)\)\]\s*$", src, re.M): + inner = m.group(1).strip() + body = inner + am = re.fullmatch(r"all\((.*)\)", inner, re.S) + if am: + body = am.group(1) + parts = [p.strip() for p in body.split(",") if p.strip()] + for p in parts: + fm = re.fullmatch(r'feature\s*=\s*"([A-Za-z0-9_-]+)"', p) + if not fm: + return None + feats.add(fm.group(1)) + return sorted(feats) + + +def cargo_tests(path): + """The `[[test]]` entries of a Cargo.toml: {name: (path, required-features or None)}.""" + with open(path, encoding="utf-8") as fh: + text = fh.read() + out = {} + for block in re.split(r"^\[\[", text, flags=re.M)[1:]: + if not block.startswith("test]]"): + continue + body = re.split(r"^\[", block[len("test]]"):], flags=re.M)[0] + name = re.search(r'^name\s*=\s*"([^"]+)"', body, re.M) + if not name: + continue + p = re.search(r'^path\s*=\s*"([^"]+)"', body, re.M) + rf = re.search(r"^required-features\s*=\s*\[(.*?)\]", body, re.M | re.S) + feats = sorted(re.findall(r'"([^"]+)"', rf.group(1))) if rf else None + out[name.group(1)] = (p.group(1) if p else f"tests/{name.group(1)}.rs", feats) + return out + + +def not_in_ci(path): + """`NOT-IN-CI.txt`: {name: reason}. A line is a name, blank space, and the reason; + `#` starts a comment line.""" + out, bad = {}, [] + if not os.path.exists(path): + return out, bad + with open(path, encoding="utf-8") as fh: + for n, line in enumerate(fh, 1): + line = line.rstrip("\n") + if not line.strip() or line.lstrip().startswith("#"): + continue + m = re.fullmatch(r"([A-Za-z0-9_-]+)\s+(\S.*)", line) + if not m: + bad.append(f"{path}:{n}: not `name reason`") + continue + if m.group(1) in out: + bad.append(f"{path}:{n}: {m.group(1)} is listed twice") + out[m.group(1)] = m.group(2).strip() + return out, bad + + +def check(root): + findings = [] + named = workflow_test_names(root) + for crate in CRATES: + tdir = os.path.join(root, crate, "tests") + files = sorted(f[:-3] for f in os.listdir(tdir) if f.endswith(".rs") and os.path.isfile(os.path.join(tdir, f))) + declared = cargo_tests(os.path.join(root, crate, "Cargo.toml")) + listed, bad = not_in_ci(os.path.join(tdir, NOT_IN_CI)) + findings += bad + for t in files: + if t not in named and t not in listed: + findings.append(f"{crate}/tests/{t}.rs: no workflow step names it (`--test {t}`), and {NOT_IN_CI} does not list it") + feats = crate_cfg_features(os.path.join(tdir, t + ".rs")) + if feats is None: + findings.append(f"{crate}/tests/{t}.rs: its #![cfg(...)] is not a feature or an all(...) of features; this check cannot tell what it needs") + elif feats: + entry = declared.get(t) + if entry is None or entry[1] is None: + findings.append(f'{crate}/tests/{t}.rs: gated on {", ".join(feats)} with no [[test]] required-features in Cargo.toml, so a step without the feature runs it empty') + elif entry[1] != feats: + findings.append(f'{crate}/Cargo.toml: [[test]] {t} requires {entry[1]}, and the file is gated on {feats}') + for t, why in sorted(listed.items()): + if t not in files: + findings.append(f"{crate}/tests/{NOT_IN_CI}: {t} is not a test file") + elif t in named: + findings.append(f"{crate}/tests/{NOT_IN_CI}: {t} is listed as not run, and a workflow names it; remove the line") + for t, (path, _) in sorted(declared.items()): + if not os.path.isfile(os.path.join(root, crate, path)): + findings.append(f"{crate}/Cargo.toml: [[test]] {t} names {path}, which does not exist") + return findings + + +def main(argv): + root = "." + if len(argv) >= 2 and argv[0] == "--root": + root = argv[1] + findings = check(root) + for f in findings: + print(f) + if findings: + print(f"check-tests-in-ci: {len(findings)} finding(s)") + return 1 + print("check-tests-in-ci: every test file is run by a job or listed with its reason, and none can run empty.") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/wai/wai-rs/Cargo.toml b/wai/wai-rs/Cargo.toml index 6d7a75d85..510f7e6ec 100644 --- a/wai/wai-rs/Cargo.toml +++ b/wai/wai-rs/Cargo.toml @@ -879,3 +879,252 @@ required-features = ["stage_measure"] name = "wai_telemetry" path = "src/bin/wai_telemetry.rs" required-features = ["telemetry"] + +# Every other feature-gated integration test, declared with its features for the same +# reason as those above: named in a step that lacks them, cargo stops with an error +# instead of compiling the file to an empty binary that reports ok. A test file gated +# on a feature without an entry here fails scripts/check-tests-in-ci.py. +[[test]] +name = "dash_read_bounds" +path = "tests/dash_read_bounds.rs" +required-features = ["http"] + +[[test]] +name = "ffi_robustness" +path = "tests/ffi_robustness.rs" +required-features = ["codecs"] + +[[test]] +name = "fuzz_smoke" +path = "tests/fuzz_smoke.rs" +required-features = ["codecs"] + +[[test]] +name = "int_hyper_synth" +path = "tests/int_hyper_synth.rs" +required-features = ["neural_int"] + +[[test]] +name = "int_video" +path = "tests/int_video.rs" +required-features = ["video_int"] + +[[test]] +name = "int_video_halfpel" +path = "tests/int_video_halfpel.rs" +required-features = ["video_int"] + +[[test]] +name = "int_video_keyframe" +path = "tests/int_video_keyframe.rs" +required-features = ["neural_int", "video_int"] + +[[test]] +name = "int_video_sota_keyframe" +path = "tests/int_video_sota_keyframe.rs" +required-features = ["neural_int", "video_int"] + +[[test]] +name = "int_world" +path = "tests/int_world.rs" +required-features = ["world_int"] + +[[test]] +name = "int_world2" +path = "tests/int_world2.rs" +required-features = ["world_int"] + +[[test]] +name = "int_world_shared" +path = "tests/int_world_shared.rs" +required-features = ["world_int", "world_model"] + +[[test]] +name = "jpegai_bitstream_calibration" +path = "tests/jpegai_bitstream_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_color_calibration" +path = "tests/jpegai_color_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_cross_platform" +path = "tests/jpegai_cross_platform.rs" +required-features = ["neural_int", "provenance"] + +[[test]] +name = "jpegai_dequant_calibration" +path = "tests/jpegai_dequant_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_derive_calibration" +path = "tests/jpegai_derive_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_e2e" +path = "tests/jpegai_e2e.rs" +required-features = ["codecs", "neural"] + +[[test]] +name = "jpegai_e2e_chain" +path = "tests/jpegai_e2e_chain.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_e2e_chroma" +path = "tests/jpegai_e2e_chroma.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_e2e_dequant" +path = "tests/jpegai_e2e_dequant.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_e2e_frontend" +path = "tests/jpegai_e2e_frontend.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_e2e_hyperlatent" +path = "tests/jpegai_e2e_hyperlatent.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_e2e_synthesis" +path = "tests/jpegai_e2e_synthesis.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_efl_calibration" +path = "tests/jpegai_efl_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_efn_calibration" +path = "tests/jpegai_efn_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_hyper_calibration" +path = "tests/jpegai_hyper_calibration.rs" +required-features = ["neural"] + +[[test]] +name = "jpegai_hyperscale_calibration" +path = "tests/jpegai_hyperscale_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_icci_calibration" +path = "tests/jpegai_icci_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_icci_net_calibration" +path = "tests/jpegai_icci_net_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_lef_calibration" +path = "tests/jpegai_lef_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_live" +path = "tests/jpegai_live.rs" +required-features = ["codecs", "neural"] + +[[test]] +name = "jpegai_lsbs_calibration" +path = "tests/jpegai_lsbs_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_mans_calibration" +path = "tests/jpegai_mans_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_mans_golden" +path = "tests/jpegai_mans_golden.rs" +required-features = ["neural"] + +[[test]] +name = "jpegai_mcm_calibration" +path = "tests/jpegai_mcm_calibration.rs" +required-features = ["neural"] + +[[test]] +name = "jpegai_postfilter_chain" +path = "tests/jpegai_postfilter_chain.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_reconstruct_calibration" +path = "tests/jpegai_reconstruct_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_reconstruction_binding" +path = "tests/jpegai_reconstruction_binding.rs" +required-features = ["neural_int", "provenance"] + +[[test]] +name = "jpegai_scope_ort" +path = "tests/jpegai_scope_ort.rs" +required-features = ["neural"] + +[[test]] +name = "jpegai_sigma_index_calibration" +path = "tests/jpegai_sigma_index_calibration.rs" +required-features = ["neural_int"] + +[[test]] +name = "jpegai_synthesis_calibration" +path = "tests/jpegai_synthesis_calibration.rs" +required-features = ["neural"] + +[[test]] +name = "jpegai_uv_calibration" +path = "tests/jpegai_uv_calibration.rs" +required-features = ["neural"] + +[[test]] +name = "neural_cost_limit" +path = "tests/neural_cost_limit.rs" +required-features = ["ffi", "neural", "video_int"] + +[[test]] +name = "semantic_path" +path = "tests/semantic_path.rs" +required-features = ["semantic"] + +[[test]] +name = "text_decode_bounds" +path = "tests/text_decode_bounds.rs" +required-features = ["codecs"] + +[[test]] +name = "text_decode_memory" +path = "tests/text_decode_memory.rs" +required-features = ["codecs"] + +[[test]] +name = "video_receipt" +path = "tests/video_receipt.rs" +required-features = ["video_receipt"] + +[[test]] +name = "world_model" +path = "tests/world_model.rs" +required-features = ["world_model"] + +[[test]] +name = "world_replay" +path = "tests/world_replay.rs" +required-features = ["world"] diff --git a/wai/wai-rs/tests/NOT-IN-CI.txt b/wai/wai-rs/tests/NOT-IN-CI.txt new file mode 100644 index 000000000..171a826fc --- /dev/null +++ b/wai/wai-rs/tests/NOT-IN-CI.txt @@ -0,0 +1,38 @@ +# Integration tests of this crate that no CI job runs, and why. +# +# scripts/check-tests-in-ci.py requires every tests/*.rs to be named by a workflow step +# (`--test NAME`) or listed here. A line is the test's name, blank space, and the reason. +# A test leaves this list by being added to a step, which the check then requires. +# +# Every test below compares this crate's JPEG AI decoder, stage by stage, with values +# taken from the reference decoder. Those values are not in the tree, so on a runner each +# of these would report ok without checking anything, which is worse than not running. + +jpegai_color_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_dequant_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_derive_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_e2e reads the JPEG AI assets root (wai/jpegai-vm-assets, not committed); without it the test returns without checking anything +jpegai_e2e_chain reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_e2e_chroma reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_e2e_dequant reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_e2e_frontend reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_e2e_hyperlatent reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_e2e_synthesis reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_efl_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_efn_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_hyper_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_hyperscale_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_icci_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_icci_net_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_lef_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_live decodes a codestream against the reference decoder's picture, both in the JPEG AI assets root (not committed); without them it skips, and fails only under WAI_REQUIRE_SOTA_VECTORS=1 +jpegai_lsbs_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_mans_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_mans_golden reads the JPEG AI assets root (wai/jpegai-vm-assets, not committed); without it the test returns without checking anything +jpegai_mcm_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_postfilter_chain reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_reconstruct_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_scope_ort reads the JPEG AI assets root (wai/jpegai-vm-assets, not committed); without it the test returns without checking anything +jpegai_sigma_index_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_synthesis_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything +jpegai_uv_calibration reads a fixture of the reference decoder's intermediate values, named by an environment variable and held outside the tree; without it the test returns without checking anything