cdbe206cc7
The conformance workflows name the tests they run. Thirty-eight test files of wai-rs were named by no step, so no job ran them, and forty-nine feature-gated ones had no required-features, so a step without the feature would have run them empty and reported ok. - scripts/check-tests-in-ci.py, with a self-test, in a new unfiltered workflow: every tests/*.rs is named by a step or listed with its reason in tests/NOT-IN-CI.txt, and every feature-gated one declares its features. - Ten real tests join steps: the integer video and world tests and the JPEG AI bitstream reader (fast kernels, both builds), world replay, the world model and the shared integer world (a new step), the semantic path (codec-wrappers). - Twenty-eight are listed: each compares the JPEG AI decoder with reference values that are not in the tree, and returns without checking when they are absent. - Cargo.toml declares required-features for every feature-gated test. Co-Authored-By: Claude Opus 5.5 <[email protected]>
65 lines
3.3 KiB
Shell
Executable file
65 lines
3.3 KiB
Shell
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# check-tests-in-ci-selftest.sh — check-tests-in-ci.py can fail.
|
|
#
|
|
# A check that cannot fail is decoration. This builds a scratch tree shaped like the
|
|
# repository (one workflow, one crate with a plain test, a feature-gated test and a
|
|
# test listed as not run) and asserts that the check passes it, and fails each way the
|
|
# property can break, saying which.
|
|
set -uo pipefail
|
|
|
|
here=$(cd "$(dirname "$0")" && pwd)
|
|
check="$here/check-tests-in-ci.py"
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
C="$tmp/wai/wai-rs"
|
|
|
|
mk() {
|
|
rm -rf "$tmp/.github" "$tmp/wai"
|
|
mkdir -p "$tmp/.github/workflows" "$C/tests"
|
|
printf 'jobs:\n t:\n steps:\n - run: cargo test --features f --test alpha --test gated\n' > "$tmp/.github/workflows/w.yml"
|
|
printf '[package]\nname = "x"\n\n[features]\nf = []\ng = []\n\n[[test]]\nname = "gated"\npath = "tests/gated.rs"\nrequired-features = ["f"]\n' > "$C/Cargo.toml"
|
|
printf '#[test]\nfn a() {}\n' > "$C/tests/alpha.rs"
|
|
printf '#![cfg(feature = "f")]\n#[test]\nfn g() {}\n' > "$C/tests/gated.rs"
|
|
printf '#[test]\nfn l() {}\n' > "$C/tests/local.rs"
|
|
printf '# name, then why no job runs it\nlocal needs a device no runner has\n' > "$C/tests/NOT-IN-CI.txt"
|
|
}
|
|
|
|
fails=0
|
|
expect() { # expect <pass|fail> <what> [text the output must hold]
|
|
local want=$1 what=$2 needle=${3:-} out rc
|
|
out=$(python3 -I "$check" --root "$tmp" 2>&1); rc=$?
|
|
if [ "$want" = pass ] && [ $rc -ne 0 ]; then echo "FAIL $what: the check failed"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi
|
|
if [ "$want" = fail ] && [ $rc -eq 0 ]; then echo "FAIL $what: the check passed"; fails=$((fails+1)); return; fi
|
|
if [ -n "$needle" ] && ! printf '%s' "$out" | grep -q -- "$needle"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi
|
|
echo "ok $what"
|
|
}
|
|
|
|
mk; expect pass "a tree where every test is run or listed passes"
|
|
|
|
mk; printf '#[test]\nfn b() {}\n' > "$C/tests/beta.rs"
|
|
expect fail "a test file no step names and nothing lists" "beta.rs: no workflow step names it"
|
|
|
|
mk; printf '[package]\nname = "x"\n\n[features]\nf = []\n' > "$C/Cargo.toml"
|
|
expect fail "a feature-gated test with no required-features" "runs it empty"
|
|
|
|
mk; sed -i.bak 's/required-features = \["f"\]/required-features = ["g"]/' "$C/Cargo.toml"
|
|
expect fail "required-features that are not the file's features" "is gated on"
|
|
|
|
mk; printf '#![cfg(any(feature = "f", feature = "g"))]\n#[test]\nfn g() {}\n' > "$C/tests/gated.rs"
|
|
expect fail "a gate the check cannot read" "cannot tell what it needs"
|
|
|
|
mk; printf 'alpha no reason at all, a step runs it\n' >> "$C/tests/NOT-IN-CI.txt"
|
|
expect fail "a test listed as not run that a step names" "remove the line"
|
|
|
|
mk; printf 'ghost it was deleted\n' >> "$C/tests/NOT-IN-CI.txt"
|
|
expect fail "a listed name that is no test file" "ghost is not a test file"
|
|
|
|
mk; printf 'local\n' > "$C/tests/NOT-IN-CI.txt"
|
|
expect fail "a listed test with no reason" "not .name reason."
|
|
|
|
mk; printf '\n[[test]]\nname = "gone"\npath = "tests/gone.rs"\n' >> "$C/Cargo.toml"
|
|
expect fail "a [[test]] entry for a file that does not exist" "which does not exist"
|
|
|
|
if [ $fails -ne 0 ]; then echo "check-tests-in-ci-selftest: $fails failure(s)"; exit 1; fi
|
|
echo "check-tests-in-ci-selftest: the check passes a covered tree and fails each uncovered one."
|