b582725df3
The change of wai/conform-output (69f88e62f..ce85900ca, 11 commits), on the fix branch: its
changelog entries as files of wai/changes/, and every other file as the branch has it, its
conflicts with main resolved (the sink kind and the native sink suite, the {case}
placeholder, the entry files and the wasmtime pin check landed since its base).
Its table of independent verifiers is no hand-kept list: wai-rs/build.rs reads it from the
`# independent:` lines of the step files (.github/wai-conformance/verifiers/), which
take the verifiers' command lines out of the steps' bodies; a step with only verifiers to
run keeps its file as a declaration. scripts/wai-conformance-lists.py reads and checks the
same lines, and the table's tests read the step files instead of the workflow.
- 0b653a415 wai conform: JUnit and HTML reports, the independent verifiers beside an implementation or alone, and the corpus union as one command; wai doctor
- c37330ac8 wai-release: the quickstart from every archive, as a new user runs it, and its PowerShell twin for Windows
- 0e1f3eb8a wai conform: the union's rule reads the workflow's commands, not its comments
- 741bd2bd1 wai: the README names wai doctor, toolchain_doctor and wai conform's independent verifiers
- ba302c439 wai doctor: a shim is the program its .shim file names, and 0.3.0 is WAI's
- bb7451190 wai conform: two passing answers that differ are a disagreement, each record digests what its run reads, and the rules are held
- a60f59c8f wai mcp: conformance_run on a corpus's directory runs its own verifier, as before the claims runs joined the table
- 710490dfa check-wai-ops-doors: the verbs and their near-synonyms are the catalogue's, read from it
- fb0ae3151 check-wai-release-quickstart: the release's quickstart is held in the repository checks, gated and ordered too
- 7fa7b506d wai/quickstart/test/clean-machine: the help is the header, its first line to its last
- ce85900ca wai: the README says a run beside the independent verifier passes only when the two agree
Co-Authored-By: Claude Opus 5.5 <[email protected]>
251 lines
17 KiB
Shell
Executable file
251 lines
17 KiB
Shell
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# wai-conformance-lists-selftest.sh — the lists wai-conformance.yml derives can fail.
|
|
#
|
|
# The workflow's feature lists, test lists, steps, corpora and gate paths are read from files
|
|
# (scripts/wai-conformance-lists.py). A derivation that silently loses an entry drops coverage
|
|
# with every job still green, so this copies the files it reads into a scratch tree and
|
|
# requires: the tree as committed holds every entry of the hand lists it replaced; taking a
|
|
# step, a feature, a tag or a corpus away fails that; a new step's file brings its feature,
|
|
# test, binary and path into every list; each malformed file fails the check, saying why; and
|
|
# a step that fails stops the run with its status. Each check holds whatever else the tree
|
|
# holds, since this runs on every pull request, one that adds a step too: no check looks for an
|
|
# entry at a place in a sorted list, a step at a place in the run, or a line made by a file the
|
|
# check does not change itself.
|
|
set -uo pipefail
|
|
|
|
here=$(cd "$(dirname "$0")" && pwd)
|
|
repo=$(cd "$here/.." && pwd)
|
|
L="$here/wai-conformance-lists.py"
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
T="$tmp/t"
|
|
|
|
mk() {
|
|
rm -rf "$T"
|
|
mkdir -p "$T/scripts" "$T/wai/wai-rs/tests" "$T/.github"
|
|
cp -R "$repo/.github/workflows" "$repo/.github/wai-conformance" "$T/.github/"
|
|
cp "$repo/scripts/pr-touches-paths.py" "$T/scripts/"
|
|
cp "$repo/wai/wai-rs/Cargo.toml" "$T/wai/wai-rs/"
|
|
cp "$repo"/wai/wai-rs/tests/*.rs "$T/wai/wai-rs/tests/"
|
|
}
|
|
V="$T/.github/wai-conformance/verifiers"
|
|
ed() { perl -pi -e "$2" "$1"; } # ed <file> <perl substitution>, the same on GNU and BSD systems
|
|
|
|
fails=0
|
|
expect() { # expect <0|1|n> <what> <text the output must hold or ""> -- <lists.py args>
|
|
local want=$1 what=$2 needle=$3 out rc
|
|
shift 4
|
|
out=$(python3 -I "$L" --root "$T" "$@" 2>&1); rc=$?
|
|
if [ "$want" = 0 ] && [ $rc -ne 0 ]; then echo "FAIL $what: exit $rc"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /'; fails=$((fails+1)); return; fi
|
|
if [ "$want" != 0 ] && [ $rc -eq 0 ]; then echo "FAIL $what: it passed"; fails=$((fails+1)); return; fi
|
|
if [ -n "$needle" ] && ! grep -qF -- "$needle" <<<"$out"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /'; fails=$((fails+1)); return; fi
|
|
echo "ok $what"
|
|
}
|
|
# The lists are printed sorted, so what else the tree holds (a step a pull request adds,
|
|
# wherever its name sorts) moves an entry along a line: these find the entry among the line's
|
|
# words instead of at a fixed place in it.
|
|
named() { # named <output> <lead> <entry>: a line `FAIL <lead> ...` names <entry> among its words
|
|
local lead="FAIL $2 " l w words
|
|
while IFS= read -r l; do
|
|
[[ $l == "$lead"* ]] || continue
|
|
read -ra words <<<"${l#"$lead"}"
|
|
for w in "${words[@]}"; do [ "$w" = "$3" ] && return 0; done
|
|
done <<<"$1"
|
|
return 1
|
|
}
|
|
expect_named() { # expect_named <what> <lead> <entry> [<lead> <entry>]... -- <lists.py args>: it fails, naming each
|
|
local what=$1 out rc miss="" pairs=() i
|
|
shift
|
|
while [ "$1" != -- ]; do pairs+=("$1" "$2"); shift 2; done
|
|
shift
|
|
out=$(python3 -I "$L" --root "$T" "$@" 2>&1); rc=$?
|
|
if [ $rc -eq 0 ]; then echo "FAIL $what: it passed"; fails=$((fails+1)); return; fi
|
|
for ((i = 0; i < ${#pairs[@]}; i += 2)); do
|
|
named "$out" "${pairs[i]}" "${pairs[i+1]}" || miss="$miss; no line \`${pairs[i]} ...\` names ${pairs[i+1]}"
|
|
done
|
|
if [ -n "$miss" ]; then echo "FAIL $what${miss/;/:}"; grep '^FAIL [A-Za-z_]*: ' <<<"$out" | head -5 | sed 's/^/ /'; fails=$((fails+1)); return; fi
|
|
echo "ok $what"
|
|
}
|
|
|
|
# The tree as committed.
|
|
mk
|
|
expect 0 "the committed files hold every entry of the hand lists" "steps of the hand list run" -- selftest
|
|
expect 0 "and the check finds nothing" "well formed" -- check
|
|
a=$(python3 -I "$L" --root "$T" dry-run); b=$(python3 -I "$L" --root "$T" dry-run)
|
|
if [ "$a" = "$b" ] && [ -n "$a" ]; then echo "ok dry-run prints the same twice"; else echo "FAIL dry-run is not stable"; fails=$((fails+1)); fi
|
|
python3 -I "$L" --root "$T" prebuild > "$tmp/prebuild.sh" && bash -n "$tmp/prebuild.sh" \
|
|
&& echo "ok the build step's script is shell" || { echo "FAIL the build step's script does not parse"; fails=$((fails+1)); }
|
|
|
|
# Each way coverage can leave a list fails.
|
|
mk; rm "$V/live-events.sh"
|
|
expect_named "a verifier step's file taken away" "WAI_FEATURES: lost" live_events "union_tests: lost" live_events -- selftest
|
|
expect 1 "and its step named" "step \`live events · wai-rs and live_events_verify.py\` no longer runs" -- selftest
|
|
mk; ed "$V/live-rooms.sh" 's/^# features: live_room$/# features: live_compose/'
|
|
expect_named "a feature taken out of a step's header" "WAI_FEATURES: lost" live_room -- selftest
|
|
mk; perl -ni -e 'print unless $. == 1' "$T/wai/wai-rs/tests/int_kernels_vectors.rs"
|
|
expect_named "a tag taken off a test file" "int_kernels_debug_tests: lost" int_kernels_vectors -- selftest
|
|
mk; rm "$T/.github/wai-conformance/corpora/worlds.txt"
|
|
expect 1 "a corpus file taken away" "corpora: lost" -- selftest
|
|
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/--paths-from .github\/wai-conformance\/verifiers //'
|
|
expect 1 "the gate given its wide glob without the files' paths" "it would run on every change" -- selftest
|
|
|
|
# A new step brings its own entries into every list, and edits no other file.
|
|
mk
|
|
# What `selftest --exact` says of the steps and of the order, for the tree as committed.
|
|
exact_steps() { python3 -I "$L" --root "$T" selftest --exact 2>&1 | grep -E '^FAIL (step |the steps |the corpora )' | LC_ALL=C sort; }
|
|
s0=$(exact_steps)
|
|
python3 - "$T/wai/wai-rs/Cargo.toml" <<'P'
|
|
import sys
|
|
p = sys.argv[1]
|
|
t = open(p).read().replace("[features]\n", "[features]\nfresh_step = []\n", 1)
|
|
open(p, "w").write(t)
|
|
P
|
|
printf '#[test]\nfn f() {}\n' > "$T/wai/wai-rs/tests/fresh_step.rs"
|
|
# Its order is below every step's in the tree, so it runs first whatever orders the tree holds.
|
|
lo=$(awk '/^# order: -?[0-9]+$/ { print $3 }' "$V"/*.sh | sort -n | head -1)
|
|
cat > "$V/fresh-step.sh" <<S
|
|
# step: fresh step · wai-rs and fresh_verify.py
|
|
# order: $((lo - 1))
|
|
# features: fresh_step
|
|
# bins: wai_claim_vectors
|
|
# paths: wai/extensions/fresh-step.md
|
|
#
|
|
# A step the self-test adds.
|
|
set -euo pipefail
|
|
echo "the step ran with \$WAI_FEATURES"
|
|
(cd wai-rs && cargo test --no-default-features --features "\$WAI_FEATURES" --test fresh_step)
|
|
S
|
|
expect 0 "a new step's file: the lists still hold the hand lists" "steps of the hand list run" -- selftest
|
|
expect_named "and are more than them" "WAI_FEATURES: more than the hand list:" fresh_step \
|
|
"union_tests: more than the hand list:" fresh_step "alone: more than the hand list:" fresh_step \
|
|
"paths: more than the hand list:" wai/extensions/fresh-step.md -- selftest --exact
|
|
expect 0 "its feature joins WAI_FEATURES, first by its order" "WAI_FEATURES=fresh_step," -- env
|
|
expect 0 "its test joins the build" "--test fresh_step" -- prebuild
|
|
expect 0 "its feature list is checked alone" "fresh_step" -- alone
|
|
expect 0 "its path joins the gate" "wai/extensions/fresh-step.md" -- paths
|
|
expect 0 "its step runs first, by its order" " - step: fresh step" -- dry-run
|
|
first=$(python3 -I "$L" --root "$T" dry-run | grep -m1 '^ - step:')
|
|
[ "$first" = " - step: fresh step · wai-rs and fresh_verify.py (working-directory: wai)" ] \
|
|
&& echo "ok and before every other" || { echo "FAIL the new step is not first: $first"; fails=$((fails+1)); }
|
|
# --exact compares each step by its name, not its place: the step planted first is one
|
|
# addition, and every step of the hand list keeps its body and its order.
|
|
s1=$(exact_steps)
|
|
more=$(LC_ALL=C comm -13 <(printf '%s\n' "$s0") <(printf '%s\n' "$s1"))
|
|
less=$(LC_ALL=C comm -23 <(printf '%s\n' "$s0") <(printf '%s\n' "$s1"))
|
|
[ "$more" = 'FAIL step `fresh step · wai-rs and fresh_verify.py`: more than the hand list' ] && [ -z "$less" ] \
|
|
&& echo "ok --exact names the step planted first as one addition, and no step of the hand list as changed or moved" \
|
|
|| { echo "FAIL --exact, a step planted first:"; printf '%s\n' "$more" | head -5 | sed 's/^/ + /'; printf '%s\n' "$less" | head -5 | sed 's/^/ - /'; fails=$((fails+1)); }
|
|
# And a corpus planted first is one addition, not a change of the corpora's order.
|
|
C="$T/.github/wai-conformance/corpora"
|
|
lo=$(awk '/^# order: -?[0-9]+$/ { print $3 }' "$C"/*.txt | sort -n | head -1)
|
|
printf '# corpus: fresh\n# order: %s\n# bin: wai_world_vectors\n# dir: ../fresh-conformance\n' "$((lo - 1))" > "$C/fresh.txt"
|
|
out=$(python3 -I "$L" --root "$T" selftest --exact 2>&1)
|
|
if grep -qE '^FAIL corpora: more than the hand list: (.* )?fresh wai_world_vectors \.\./fresh-conformance( |$)' <<<"$out" && [ "$(exact_steps)" = "$s1" ]; then
|
|
echo "ok --exact names a corpus planted first as an addition, and not as a change of order"
|
|
else
|
|
echo "FAIL --exact, a corpus planted first:"; grep '^FAIL' <<<"$out" | head -5 | sed 's/^/ /'; fails=$((fails+1))
|
|
fi
|
|
|
|
# The dry-run prints the command lines of the derived steps outside the verifiers' job, so a
|
|
# change to one of them shows in a diff of two dry-runs, and it refuses a step it cannot read.
|
|
mk
|
|
d0=$(python3 -I "$L" --root "$T" dry-run)
|
|
expect 0 "the dry-run prints the release kernel step's command" "$ cargo test --lib --no-default-features --release --features " -- dry-run
|
|
expect 0 "and int-kernels-amd's, both profiles" "$ nice -n 10 cargo test --lib --locked --no-default-features --release --features " -- dry-run
|
|
# Each of whatever corpora and feature lists the tree holds, by its own line.
|
|
tails=$(grep -o -- '--bin [^ ]* -- verify [^ ]*$' <<<"$d0")
|
|
cs=$(python3 -I "$L" --root "$T" corpora)
|
|
miss=$(while read -r c b dir; do [ -n "$c" ] || continue; grep -qxF -- "--bin $b -- verify $dir" <<<"$tails" || printf ' %s' "$c"; done <<<"$cs")
|
|
[ -n "$cs" ] && [ -z "$miss" ] && echo "ok and each corpus's" \
|
|
|| { echo "FAIL the dry-run prints no command for the corpora:${miss:- (none listed)}"; fails=$((fails+1)); }
|
|
checks=$(grep -o -- '\$ cargo check --no-default-features --tests --features [^ ]*$' <<<"$d0")
|
|
as=$(python3 -I "$L" --root "$T" alone)
|
|
miss=$(while read -r a; do
|
|
[ -n "$a" ] || continue
|
|
n=$(tr ',' '\n' <<<"$a" | LC_ALL=C sort | paste -sd, -)
|
|
grep -qxF -- "\$ cargo check --no-default-features --tests --features $n" <<<"$checks" || printf ' %s' "$a"
|
|
done <<<"$as")
|
|
[ -n "$as" ] && [ -z "$miss" ] && echo "ok and each list features-alone checks" \
|
|
|| { echo "FAIL the dry-run prints no features-alone check for the lists:${miss:- (none listed)}"; fails=$((fails+1)); }
|
|
ed "$T/.github/workflows/wai-conformance.yml" 's/(int-kernels int-kernels-release --extra-features i32_forms -- )--release /$1/'
|
|
d1=$(python3 -I "$L" --root "$T" dry-run)
|
|
dd=$(diff <(printf '%s\n' "$d0") <(printf '%s\n' "$d1"))
|
|
n=$(grep -c '^[<>]' <<<"$dd")
|
|
[ "$n" = 2 ] && grep -q '^< .*--release --features' <<<"$dd" \
|
|
&& echo "ok a flag taken off a kernel step is one changed line of the dry-run" \
|
|
|| { echo "FAIL taking --release off the release step changed $n line(s) of the dry-run"; fails=$((fails+1)); }
|
|
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/^(\s+for f in \$lists; do)$/$1\n cargo check \$NOT_SET/'
|
|
expect 1 "a command reading a value the dry-run cannot know fails it" "\`\$NOT_SET\` has no value the dry-run knows" -- dry-run
|
|
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/(run: python3 \.\.\/\.\.\/scripts\/wai-conformance-lists\.py corpora --verify.*)$/$1\n if: matrix.os != '"'"'windows'"'"'/'
|
|
expect 0 "a step one shell family skips prints each family's commands" "every cargo-class corpus (the Windows leg; working-directory: -)" -- dry-run
|
|
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/^(\s+- name: )wai-rs, each feature list alone$/$1wai-rs, every feature list alone/'
|
|
expect 1 "a derived step the dry-run cannot find fails it" "has no step \`wai-rs, each feature list alone...\`" -- dry-run
|
|
|
|
# Each malformed file fails the check, saying why.
|
|
mk; ed "$V/live-events.sh" 's/^# order: 100$/# order: 100\n# feature: live_events/'
|
|
expect 1 "a header key no step file has" "\`feature\` is not a key" -- check
|
|
mk; perl -ni -e 'print unless /^# step: /' "$V/live-events.sh"
|
|
expect 1 "a step file with no step name" "no \`step:\` line" -- check
|
|
mk; ed "$V/live-events.sh" 's/^# order: 100$/# order: soon/'
|
|
expect 1 "an order that is no integer" "is not an integer" -- check
|
|
mk; ed "$V/live-events.sh" 's#^\# paths: wai/extensions/live-events.md#\# paths: docs/live-events.md#'
|
|
expect 1 "a path the forge's filter does not cover" "so the forge would never start the workflow for it" -- check
|
|
mk; ed "$V/live-events.sh" 's/^# features: live_events$/# features: live_eventz/'
|
|
expect 1 "a feature Cargo.toml does not have" "\`live_eventz\` is not in" -- check
|
|
mk; ed "$V/live-events.sh" 's/--test live_events\)/--test live_eventz)/'
|
|
expect 1 "a test the crate does not have" "--test live_eventz: wai/wai-rs has no such test" -- check
|
|
mk; ed "$V/http-differential.sh" 's#wai-bins/union/wai_http_vectors#wai-bins/union/wai_http_vectorz#'
|
|
expect 1 "a binary the crate does not have" "\`wai_http_vectorz\` is not a [[bin]]" -- check
|
|
mk; ed "$T/wai/wai-rs/tests/int_kernels_vectors.rs" 's{^// wai-conformance: int-kernels }{// wai-conformance: int-kernel }'
|
|
expect 1 "a tag no step takes" "is no tag a step takes" -- check
|
|
mk; cp "$V/live-events.sh" "$V/live-events-again.sh"
|
|
expect 1 "two files with one step name" "is also" -- check
|
|
mk; printf '# step: empty\n# order: 1\n#\n# nothing to run\n' > "$V/empty.sh"
|
|
expect 1 "a step file with no body" "no body" -- check
|
|
mk; printf '# corpus: x\n# order: 1\n# bin: wai_world_vectors\n' > "$T/.github/wai-conformance/corpora/x.txt"
|
|
expect 1 "a corpus file with no directory" "no \`dir:\` line" -- check
|
|
|
|
# The independent verifiers: one `independent:` line an entry, run by the union step alone.
|
|
mk
|
|
expect 0 "a declaration, entries and no body, is no step" "a declaration; its verifiers run in the union step" -- run --only loc
|
|
expect 0 "the entries print in order, each with its command line" "loc live python3 -I live-conformance/loc_verify.py live-conformance/loc" -- independent
|
|
mk; ed "$V/loc.sh" 's/^# independent: loc live /# independent: loc nowhere /'
|
|
expect 1 "an entry of no area" "is none of trust, media, delivery, live" -- check
|
|
mk; ed "$V/loc.sh" 's/^# independent: loc live python3 /# independent: loc live /'
|
|
expect 1 "an entry with no command line" "before \`python3\`" -- check
|
|
mk; ed "$V/loc.sh" 's/^# independent: loc live /# independent: e2ee live /'
|
|
expect 1 "an entry's name another file gives" "is also" -- check
|
|
mk; printf 'python3 -I live-conformance/loc_verify.py live-conformance/loc\n' >> "$V/loc.sh"
|
|
expect 1 "a body that runs an entry's command line itself" "itself; the union step runs it" -- check
|
|
mk; perl -ni -e 'print unless /^# independent: /' "$V/loc.sh"
|
|
expect 1 "a declaration whose entries are taken away" "no body" -- check
|
|
|
|
# The commands refuse what they cannot answer, rather than answering with nothing.
|
|
mk
|
|
expect 1 "a tag no step takes, asked for" "a tag is one of" -- tests int-kernel
|
|
for f in "$T"/wai/wai-rs/tests/*.rs; do perl -ni -e 'print unless m{^// wai-conformance:}' "$f"; done
|
|
expect 1 "tests for a tag no file carries" "the step would run no test" -- tests int-kernels
|
|
expect 1 "and test-tagged with it" "the step would run no test" -- test-tagged int-kernels -- --lib
|
|
|
|
# A step that fails stops the run with its status, and names its file.
|
|
mk
|
|
printf '# step: passes\n# order: 1\nset -euo pipefail\necho "features: $WAI_FEATURES"\n' > "$V/aa-passes.sh"
|
|
printf '# step: fails\n# order: 2\nset -euo pipefail\nexit 3\n' > "$V/ab-fails.sh"
|
|
# The features it is given are the whole derived list, as `env` prints it for the tree.
|
|
wf=$(python3 -I "$L" --root "$T" env | sed -n 's/^WAI_FEATURES=//p')
|
|
out=$(python3 -I "$L" --root "$T" run --only aa-passes 2>&1); rc=$?
|
|
if [ $rc -eq 0 ] && [ -n "$wf" ] && grep -qxF -- "features: $wf" <<<"$out"; then echo "ok a step that passes runs, with the derived features"
|
|
else echo "FAIL a passing step: exit $rc, or not given WAI_FEATURES=$wf"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /'; fails=$((fails+1)); fi
|
|
out=$(python3 -I "$L" --root "$T" run --only aa-passes --only ab-fails 2>&1); rc=$?
|
|
if [ $rc -eq 3 ] && grep -qF "::error::fails failed with exit status 3 (.github/wai-conformance/verifiers/ab-fails.sh)" <<<"$out"; then
|
|
echo "ok a step that fails stops the run with its status, naming its file"
|
|
else
|
|
echo "FAIL a failing step: exit $rc"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /'; fails=$((fails+1))
|
|
fi
|
|
printf '# step: pipe\n# order: 3\nfalse | true\n' > "$V/ac-pipe.sh"
|
|
expect 1 "a step whose pipe fails fails, as under a workflow's bash" "" -- run --only ac-pipe
|
|
expect 1 "a step --only names that no file holds" "names a step no file holds" -- run --only no-such-step
|
|
|
|
if [ $fails -ne 0 ]; then echo "wai-conformance-lists-selftest: $fails failure(s)"; exit 1; fi
|
|
echo "wai-conformance-lists-selftest: the derived lists hold the hand lists, and fail each way they can lose an entry."
|