95a5e4a43a
Two more plants passed `check`: the union's line inside a function the step
never calls, and inside a quoted string (`: '` ... `'`). Each is still "a
line's command" to a reader of lines, but the step does not run it as written,
so the union never runs and the step passes.
Now the union's line and the `wai verify` line after it must be at the step's
top level: the lines before each leave no quote, here-document, `{` or `(` open.
`${...}` and `${{ ... }}` are no block, and a comment is skipped. The
docstring and the workflow's comment name the rule.
The selftest plants four cases: the union in a function never called, in a
quoted string, in a here-document, and the records' check in a function never
called. Each passed the previous check and fails this one (97 cases, every
one held).
Changelog: none
Co-Authored-By: Claude Opus 5.5 <[email protected]>
410 lines
26 KiB
Shell
Executable file
410 lines
26 KiB
Shell
Executable file
#!/usr/bin/env bash
|
|
#
|
|
# wai-conformance-lists-selftest.sh — the lists wai-conformance.yml derives can fail.
|
|
#
|
|
# The workflow's feature lists, test lists, steps, corpora and gate paths are read from files
|
|
# (scripts/wai-conformance-lists.py). A derivation that silently loses an entry drops coverage
|
|
# with every job still green, so this copies the files it reads into a scratch tree and
|
|
# requires: the tree as committed holds every entry of the hand lists it replaced; taking a
|
|
# step, a feature, a tag or a corpus away fails that; a new step's file brings its feature,
|
|
# test, binary and path into every list; each malformed file fails the check, saying why; a
|
|
# step that fails stops the run with its status; each way the corpus union's step has been
|
|
# planted to pass a failed run (set -e off in any spelling, a flag that narrows or stops the
|
|
# union or changes what runs it, a `wai` it did not build, the union or its check in a function
|
|
# never called, a string or a here-document) fails the check; a trigger of wai-breaking.yml
|
|
# that does not start on a change to what the table's tests read fails the check; and an entry
|
|
# whose needs= a run without packages does not bear out fails `needs`. Each check holds
|
|
# whatever else the tree holds, since this runs on every pull request, one that adds a step
|
|
# too: no check looks for an entry at a place in a sorted list, a step at a place in the run,
|
|
# or a line made by a file the check does not change itself.
|
|
set -uo pipefail
|
|
|
|
here=$(cd "$(dirname "$0")" && pwd)
|
|
repo=$(cd "$here/.." && pwd)
|
|
L="$here/wai-conformance-lists.py"
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
T="$tmp/t"
|
|
|
|
mk() {
|
|
rm -rf "$T"
|
|
mkdir -p "$T/scripts" "$T/wai/wai-rs/tests" "$T/.github"
|
|
cp -R "$repo/.github/workflows" "$repo/.github/wai-conformance" "$T/.github/"
|
|
cp "$repo/scripts/pr-touches-paths.py" "$T/scripts/"
|
|
cp "$repo/wai/wai-rs/Cargo.toml" "$T/wai/wai-rs/"
|
|
cp "$repo"/wai/wai-rs/tests/*.rs "$T/wai/wai-rs/tests/"
|
|
}
|
|
V="$T/.github/wai-conformance/verifiers"
|
|
ed() { perl -pi -e "$2" "$1"; } # ed <file> <perl substitution>, the same on GNU and BSD systems
|
|
|
|
fails=0
|
|
expect() { # expect <0|1|n> <what> <text the output must hold or ""> -- <lists.py args>
|
|
local want=$1 what=$2 needle=$3 out rc
|
|
shift 4
|
|
out=$(python3 -I "$L" --root "$T" "$@" 2>&1); rc=$?
|
|
if [ "$want" = 0 ] && [ $rc -ne 0 ]; then echo "FAIL $what: exit $rc"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /'; fails=$((fails+1)); return; fi
|
|
if [ "$want" != 0 ] && [ $rc -eq 0 ]; then echo "FAIL $what: it passed"; fails=$((fails+1)); return; fi
|
|
if [ -n "$needle" ] && ! grep -qF -- "$needle" <<<"$out"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /'; fails=$((fails+1)); return; fi
|
|
echo "ok $what"
|
|
}
|
|
# The lists are printed sorted, so what else the tree holds (a step a pull request adds,
|
|
# wherever its name sorts) moves an entry along a line: these find the entry among the line's
|
|
# words instead of at a fixed place in it.
|
|
named() { # named <output> <lead> <entry>: a line `FAIL <lead> ...` names <entry> among its words
|
|
local lead="FAIL $2 " l w words
|
|
while IFS= read -r l; do
|
|
[[ $l == "$lead"* ]] || continue
|
|
read -ra words <<<"${l#"$lead"}"
|
|
for w in "${words[@]}"; do [ "$w" = "$3" ] && return 0; done
|
|
done <<<"$1"
|
|
return 1
|
|
}
|
|
expect_named() { # expect_named <what> <lead> <entry> [<lead> <entry>]... -- <lists.py args>: it fails, naming each
|
|
local what=$1 out rc miss="" pairs=() i
|
|
shift
|
|
while [ "$1" != -- ]; do pairs+=("$1" "$2"); shift 2; done
|
|
shift
|
|
out=$(python3 -I "$L" --root "$T" "$@" 2>&1); rc=$?
|
|
if [ $rc -eq 0 ]; then echo "FAIL $what: it passed"; fails=$((fails+1)); return; fi
|
|
for ((i = 0; i < ${#pairs[@]}; i += 2)); do
|
|
named "$out" "${pairs[i]}" "${pairs[i+1]}" || miss="$miss; no line \`${pairs[i]} ...\` names ${pairs[i+1]}"
|
|
done
|
|
if [ -n "$miss" ]; then echo "FAIL $what${miss/;/:}"; grep '^FAIL [A-Za-z_]*: ' <<<"$out" | head -5 | sed 's/^/ /'; fails=$((fails+1)); return; fi
|
|
echo "ok $what"
|
|
}
|
|
|
|
# The tree as committed.
|
|
mk
|
|
expect 0 "the committed files hold every entry of the hand lists" "steps of the hand list run" -- selftest
|
|
expect 0 "and the check finds nothing" "well formed" -- check
|
|
a=$(python3 -I "$L" --root "$T" dry-run); b=$(python3 -I "$L" --root "$T" dry-run)
|
|
if [ "$a" = "$b" ] && [ -n "$a" ]; then echo "ok dry-run prints the same twice"; else echo "FAIL dry-run is not stable"; fails=$((fails+1)); fi
|
|
python3 -I "$L" --root "$T" prebuild > "$tmp/prebuild.sh" && bash -n "$tmp/prebuild.sh" \
|
|
&& echo "ok the build step's script is shell" || { echo "FAIL the build step's script does not parse"; fails=$((fails+1)); }
|
|
|
|
# Each way coverage can leave a list fails.
|
|
mk; rm "$V/live-events.sh"
|
|
expect_named "a verifier step's file taken away" "WAI_FEATURES: lost" live_events "union_tests: lost" live_events -- selftest
|
|
expect 1 "and its step named" "step \`live events · wai-rs and live_events_verify.py\` no longer runs" -- selftest
|
|
mk; ed "$V/live-rooms.sh" 's/^# features: live_room$/# features: live_compose/'
|
|
expect_named "a feature taken out of a step's header" "WAI_FEATURES: lost" live_room -- selftest
|
|
mk; perl -ni -e 'print unless $. == 1' "$T/wai/wai-rs/tests/int_kernels_vectors.rs"
|
|
expect_named "a tag taken off a test file" "int_kernels_debug_tests: lost" int_kernels_vectors -- selftest
|
|
mk; rm "$T/.github/wai-conformance/corpora/worlds.txt"
|
|
expect 1 "a corpus file taken away" "corpora: lost" -- selftest
|
|
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/--paths-from .github\/wai-conformance\/verifiers //'
|
|
expect 1 "the gate given its wide glob without the files' paths" "it would run on every change" -- selftest
|
|
|
|
# A new step brings its own entries into every list, and edits no other file.
|
|
mk
|
|
# What `selftest --exact` says of the steps and of the order, for the tree as committed.
|
|
exact_steps() { python3 -I "$L" --root "$T" selftest --exact 2>&1 | grep -E '^FAIL (step |the steps |the corpora )' | LC_ALL=C sort; }
|
|
s0=$(exact_steps)
|
|
python3 - "$T/wai/wai-rs/Cargo.toml" <<'P'
|
|
import sys
|
|
p = sys.argv[1]
|
|
t = open(p).read().replace("[features]\n", "[features]\nfresh_step = []\n", 1)
|
|
open(p, "w").write(t)
|
|
P
|
|
printf '#[test]\nfn f() {}\n' > "$T/wai/wai-rs/tests/fresh_step.rs"
|
|
# Its order is below every step's in the tree, so it runs first whatever orders the tree holds.
|
|
lo=$(awk '/^# order: -?[0-9]+$/ { print $3 }' "$V"/*.sh | sort -n | head -1)
|
|
cat > "$V/fresh-step.sh" <<S
|
|
# step: fresh step · wai-rs and fresh_verify.py
|
|
# order: $((lo - 1))
|
|
# features: fresh_step
|
|
# bins: wai_claim_vectors
|
|
# paths: wai/extensions/fresh-step.md
|
|
#
|
|
# A step the self-test adds.
|
|
set -euo pipefail
|
|
echo "the step ran with \$WAI_FEATURES"
|
|
(cd wai-rs && cargo test --no-default-features --features "\$WAI_FEATURES" --test fresh_step)
|
|
S
|
|
expect 0 "a new step's file: the lists still hold the hand lists" "steps of the hand list run" -- selftest
|
|
expect_named "and are more than them" "WAI_FEATURES: more than the hand list:" fresh_step \
|
|
"union_tests: more than the hand list:" fresh_step "alone: more than the hand list:" fresh_step \
|
|
"paths: more than the hand list:" wai/extensions/fresh-step.md -- selftest --exact
|
|
expect 0 "its feature joins WAI_FEATURES, first by its order" "WAI_FEATURES=fresh_step," -- env
|
|
expect 0 "its test joins the build" "--test fresh_step" -- prebuild
|
|
expect 0 "its feature list is checked alone" "fresh_step" -- alone
|
|
expect 0 "its path joins the gate" "wai/extensions/fresh-step.md" -- paths
|
|
expect 0 "its step runs first, by its order" " - step: fresh step" -- dry-run
|
|
first=$(python3 -I "$L" --root "$T" dry-run | grep -m1 '^ - step:')
|
|
[ "$first" = " - step: fresh step · wai-rs and fresh_verify.py (working-directory: wai)" ] \
|
|
&& echo "ok and before every other" || { echo "FAIL the new step is not first: $first"; fails=$((fails+1)); }
|
|
# --exact compares each step by its name, not its place: the step planted first is one
|
|
# addition, and every step of the hand list keeps its body and its order.
|
|
s1=$(exact_steps)
|
|
more=$(LC_ALL=C comm -13 <(printf '%s\n' "$s0") <(printf '%s\n' "$s1"))
|
|
less=$(LC_ALL=C comm -23 <(printf '%s\n' "$s0") <(printf '%s\n' "$s1"))
|
|
[ "$more" = 'FAIL step `fresh step · wai-rs and fresh_verify.py`: more than the hand list' ] && [ -z "$less" ] \
|
|
&& echo "ok --exact names the step planted first as one addition, and no step of the hand list as changed or moved" \
|
|
|| { echo "FAIL --exact, a step planted first:"; printf '%s\n' "$more" | head -5 | sed 's/^/ + /'; printf '%s\n' "$less" | head -5 | sed 's/^/ - /'; fails=$((fails+1)); }
|
|
# And a corpus planted first is one addition, not a change of the corpora's order.
|
|
C="$T/.github/wai-conformance/corpora"
|
|
lo=$(awk '/^# order: -?[0-9]+$/ { print $3 }' "$C"/*.txt | sort -n | head -1)
|
|
printf '# corpus: fresh\n# order: %s\n# bin: wai_world_vectors\n# dir: ../fresh-conformance\n' "$((lo - 1))" > "$C/fresh.txt"
|
|
out=$(python3 -I "$L" --root "$T" selftest --exact 2>&1)
|
|
if grep -qE '^FAIL corpora: more than the hand list: (.* )?fresh wai_world_vectors \.\./fresh-conformance( |$)' <<<"$out" && [ "$(exact_steps)" = "$s1" ]; then
|
|
echo "ok --exact names a corpus planted first as an addition, and not as a change of order"
|
|
else
|
|
echo "FAIL --exact, a corpus planted first:"; grep '^FAIL' <<<"$out" | head -5 | sed 's/^/ /'; fails=$((fails+1))
|
|
fi
|
|
|
|
# The dry-run prints the command lines of the derived steps outside the verifiers' job, so a
|
|
# change to one of them shows in a diff of two dry-runs, and it refuses a step it cannot read.
|
|
mk
|
|
d0=$(python3 -I "$L" --root "$T" dry-run)
|
|
expect 0 "the dry-run prints the release kernel step's command" "$ cargo test --lib --no-default-features --release --features " -- dry-run
|
|
expect 0 "and int-kernels-amd's, both profiles" "$ nice -n 10 cargo test --lib --locked --no-default-features --release --features " -- dry-run
|
|
# Each of whatever corpora and feature lists the tree holds, by its own line.
|
|
tails=$(grep -o -- '--bin [^ ]* -- verify [^ ]*$' <<<"$d0")
|
|
cs=$(python3 -I "$L" --root "$T" corpora)
|
|
miss=$(while read -r c b dir; do [ -n "$c" ] || continue; grep -qxF -- "--bin $b -- verify $dir" <<<"$tails" || printf ' %s' "$c"; done <<<"$cs")
|
|
[ -n "$cs" ] && [ -z "$miss" ] && echo "ok and each corpus's" \
|
|
|| { echo "FAIL the dry-run prints no command for the corpora:${miss:- (none listed)}"; fails=$((fails+1)); }
|
|
checks=$(grep -o -- '\$ cargo check --no-default-features --tests --features [^ ]*$' <<<"$d0")
|
|
as=$(python3 -I "$L" --root "$T" alone)
|
|
miss=$(while read -r a; do
|
|
[ -n "$a" ] || continue
|
|
n=$(tr ',' '\n' <<<"$a" | LC_ALL=C sort | paste -sd, -)
|
|
grep -qxF -- "\$ cargo check --no-default-features --tests --features $n" <<<"$checks" || printf ' %s' "$a"
|
|
done <<<"$as")
|
|
[ -n "$as" ] && [ -z "$miss" ] && echo "ok and each list features-alone checks" \
|
|
|| { echo "FAIL the dry-run prints no features-alone check for the lists:${miss:- (none listed)}"; fails=$((fails+1)); }
|
|
ed "$T/.github/workflows/wai-conformance.yml" 's/(int-kernels int-kernels-release --extra-features i32_forms -- )--release /$1/'
|
|
d1=$(python3 -I "$L" --root "$T" dry-run)
|
|
dd=$(diff <(printf '%s\n' "$d0") <(printf '%s\n' "$d1"))
|
|
n=$(grep -c '^[<>]' <<<"$dd")
|
|
[ "$n" = 2 ] && grep -q '^< .*--release --features' <<<"$dd" \
|
|
&& echo "ok a flag taken off a kernel step is one changed line of the dry-run" \
|
|
|| { echo "FAIL taking --release off the release step changed $n line(s) of the dry-run"; fails=$((fails+1)); }
|
|
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/^(\s+for f in \$lists; do)$/$1\n cargo check \$NOT_SET/'
|
|
expect 1 "a command reading a value the dry-run cannot know fails it" "\`\$NOT_SET\` has no value the dry-run knows" -- dry-run
|
|
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/(run: python3 \.\.\/\.\.\/scripts\/wai-conformance-lists\.py corpora --verify.*)$/$1\n if: matrix.os != '"'"'windows'"'"'/'
|
|
expect 0 "a step one shell family skips prints each family's commands" "every cargo-class corpus (the Windows leg; working-directory: -)" -- dry-run
|
|
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/^(\s+- name: )wai-rs, each feature list alone$/$1wai-rs, every feature list alone/'
|
|
expect 1 "a derived step the dry-run cannot find fails it" "has no step \`wai-rs, each feature list alone...\`" -- dry-run
|
|
|
|
# Each malformed file fails the check, saying why.
|
|
mk; ed "$V/live-events.sh" 's/^# order: 100$/# order: 100\n# feature: live_events/'
|
|
expect 1 "a header key no step file has" "\`feature\` is not a key" -- check
|
|
mk; perl -ni -e 'print unless /^# step: /' "$V/live-events.sh"
|
|
expect 1 "a step file with no step name" "no \`step:\` line" -- check
|
|
mk; ed "$V/live-events.sh" 's/^# order: 100$/# order: soon/'
|
|
expect 1 "an order that is no integer" "is not an integer" -- check
|
|
mk; ed "$V/live-events.sh" 's#^\# paths: wai/extensions/live-events.md#\# paths: docs/live-events.md#'
|
|
expect 1 "a path the forge's filter does not cover" "so the forge would never start the workflow for it" -- check
|
|
mk; ed "$V/live-events.sh" 's/^# features: live_events$/# features: live_eventz/'
|
|
expect 1 "a feature Cargo.toml does not have" "\`live_eventz\` is not in" -- check
|
|
mk; ed "$V/live-events.sh" 's/--test live_events\)/--test live_eventz)/'
|
|
expect 1 "a test the crate does not have" "--test live_eventz: wai/wai-rs has no such test" -- check
|
|
mk; ed "$V/http-differential.sh" 's#wai-bins/union/wai_http_vectors#wai-bins/union/wai_http_vectorz#'
|
|
expect 1 "a binary the crate does not have" "\`wai_http_vectorz\` is not a [[bin]]" -- check
|
|
mk; ed "$T/wai/wai-rs/tests/int_kernels_vectors.rs" 's{^// wai-conformance: int-kernels }{// wai-conformance: int-kernel }'
|
|
expect 1 "a tag no step takes" "is no tag a step takes" -- check
|
|
mk; cp "$V/live-events.sh" "$V/live-events-again.sh"
|
|
expect 1 "two files with one step name" "is also" -- check
|
|
mk; printf '# step: empty\n# order: 1\n#\n# nothing to run\n' > "$V/empty.sh"
|
|
expect 1 "a step file with no body" "no body" -- check
|
|
mk; printf '# corpus: x\n# order: 1\n# bin: wai_world_vectors\n' > "$T/.github/wai-conformance/corpora/x.txt"
|
|
expect 1 "a corpus file with no directory" "no \`dir:\` line" -- check
|
|
|
|
# The independent verifiers: one `independent:` line an entry, run by the union step alone.
|
|
mk
|
|
expect 0 "a declaration, entries and no body, is no step" "a declaration; its verifiers run in the union step" -- run --only loc
|
|
expect 0 "the entries print in order, each with its command line" "loc live python3 -I live-conformance/loc_verify.py live-conformance/loc" -- independent
|
|
mk; ed "$V/loc.sh" 's/^# independent: loc live /# independent: loc nowhere /'
|
|
expect 1 "an entry of no area" "is none of trust, media, delivery, live" -- check
|
|
mk; ed "$V/loc.sh" 's/^# independent: loc live python3 /# independent: loc live /'
|
|
expect 1 "an entry with no command line" "before \`python3\`" -- check
|
|
mk; ed "$V/loc.sh" 's/^# independent: loc live /# independent: e2ee live /'
|
|
expect 1 "an entry's name another file gives" "is also" -- check
|
|
mk; printf 'python3 -I live-conformance/loc_verify.py live-conformance/loc\n' >> "$V/loc.sh"
|
|
expect 1 "a body that runs an entry's command line itself" "itself; the union step runs it" -- check
|
|
mk; perl -ni -e 'print unless /^# independent: /' "$V/loc.sh"
|
|
expect 1 "a declaration whose entries are taken away" "no body" -- check
|
|
|
|
# The corpus union's step fails when `wai conform --independent` does: `set -e` decides it.
|
|
W="$T/.github/workflows/wai-conformance.yml"
|
|
UNION=' "$w" conform --independent --root . --out "$d/records" --format junit > "$d/junit.xml"'
|
|
USTEP=" - name: Every corpus's independent verifier, through wai conform --independent"
|
|
sub() { # sub <from> <to>: replace the one occurrence in the workflow, which must exist
|
|
python3 - "$W" "$1" "$2" <<'P' || { echo "FAIL the fixture edit did not apply: $1"; fails=$((fails+1)); }
|
|
import sys
|
|
p, a, b = sys.argv[1:4]
|
|
s = open(p, encoding="utf-8").read()
|
|
if s.count(a) != 1:
|
|
sys.exit(1)
|
|
open(p, "w", encoding="utf-8").write(s.replace(a, b))
|
|
P
|
|
}
|
|
mk; expect 0 "the union's step as it is" "" -- check
|
|
mk; sub "$UNION" "$UNION || true"
|
|
expect 1 "the union's command with || true" "goes on after the command on its line" -- check
|
|
mk; sub "$UNION" ' status=0
|
|
'"$UNION"' || status=$?'
|
|
expect 1 "the union's status kept and not used" "can pass a failed union" -- check
|
|
mk; sub "$UNION" "$UNION; true"
|
|
expect 1 "the union's line ends in another command" "goes on after the command on its line" -- check
|
|
mk; sub "$UNION" " exit 0
|
|
$UNION"
|
|
expect 1 "the union's step ends before it" "can pass a failed union: \`exit 0\`" -- check
|
|
mk; sub "$UNION" " set +e
|
|
$UNION"
|
|
expect 1 "the union's step turns set -e off" "can pass a failed union: \`set +e\`" -- check
|
|
mk; sub "$UNION" " if ! true; then
|
|
$UNION
|
|
fi"
|
|
expect 1 "the union runs under a condition" "can pass a failed union: \`if ! true; then\`" -- check
|
|
mk; sub "$USTEP" "$USTEP
|
|
continue-on-error: true"
|
|
expect 1 "the union's step may fail without failing the job" "has \`continue-on-error:\`" -- check
|
|
mk; sub "$USTEP" "$USTEP
|
|
if: false"
|
|
expect 1 "the union's step is skipped" "has \`if:\`" -- check
|
|
mk; sub ' name: independent verifiers · claims' ' continue-on-error: true
|
|
name: independent verifiers · claims'
|
|
expect 1 "the union's job may fail" "the job \`independent-verifiers\` has \`continue-on-error:\`" -- check
|
|
mk; sub ' set -euo pipefail
|
|
(cd wai-rs && cargo build --locked --no-default-features --features quickstart --bin wai' ' (cd wai-rs && cargo build --locked --no-default-features --features quickstart --bin wai'
|
|
expect 1 "the union's step without set -e" "has no \`set -e\` before the union" -- check
|
|
mk; sub ' "$w" verify "$d/records"
|
|
' ''
|
|
expect 1 "the records unchecked" "does not check the records" -- check
|
|
mk; sub "$UNION" ' echo "$w" conform --independent'
|
|
expect 1 "the union only echoed" "in a line that is not its command" -- check
|
|
# set -e turned off in its other spellings.
|
|
mk; sub "$UNION" " set +o errexit
|
|
$UNION"
|
|
expect 1 "the union's step turns errexit off by name" "can pass a failed union: \`set +o errexit\`" -- check
|
|
mk; sub "$UNION" " set -u +e
|
|
$UNION"
|
|
expect 1 "the union's step turns set -e off among other options" "can pass a failed union: \`set -u +e\`" -- check
|
|
mk; sub "$UNION" " shopt -uo errexit
|
|
$UNION"
|
|
expect 1 "the union's step turns errexit off with shopt" "can pass a failed union: \`shopt -uo errexit\`" -- check
|
|
# A flag that narrows the run, stops it before it runs, or changes what fails or runs it.
|
|
for f in "--suite pin" "--dry-run" "--timeout 1" "--list" "--python true" "--all"; do
|
|
mk; sub "$UNION" "${UNION/--independent/--independent $f}"
|
|
expect 1 "the union given $f" "gives the union \`${f%% *}\`" -- check
|
|
done
|
|
mk; sub "$UNION" "${UNION/--root ./--root /tmp}"
|
|
expect 1 "the union reads another tree's corpora" "gives the union \`--root /tmp\`, not \`.\`" -- check
|
|
mk; sub "$UNION" "${UNION/--root ./}"
|
|
expect 1 "the union given no --root" "gives the union no \`--root .\`" -- check
|
|
mk; sub "$UNION" "${UNION/--independent/--independent stray}"
|
|
expect 1 "the union given a word that is no flag" "gives the union \`stray\`" -- check
|
|
mk; sub "$USTEP
|
|
working-directory: wai" "$USTEP
|
|
working-directory: wai/wai-rs"
|
|
expect 1 "the union run outside wai/" "runs in \`wai/wai-rs\`, not wai/" -- check
|
|
# The wai it runs, and checks the records with, is the one it built.
|
|
mk; sub ' w="$RUNNER_TEMP/wai-cli-target/debug/wai"' ' w=true'
|
|
expect 1 "the union run by a program that is not the wai built" "runs \`\$w\`, which is not the \`wai\` it built" -- check
|
|
mk; sub "$UNION" " w=true
|
|
$UNION"
|
|
expect 1 "the wai it runs set again before the union" "runs \`\$w\`, which is not the \`wai\` it built" -- check
|
|
mk; sub ' (cd wai-rs && cargo build --locked --no-default-features --features quickstart --bin wai --target-dir "$RUNNER_TEMP/wai-cli-target")' ' (cd wai-rs && cargo build --locked --no-default-features --features quickstart --bin wai_signer_vectors --target-dir "$RUNNER_TEMP/wai-cli-target")'
|
|
expect 1 "the step builds no wai" "builds no \`wai\` before the union" -- check
|
|
mk; sub ' "$w" verify "$d/records"' ' v=true
|
|
"$v" verify "$d/records"'
|
|
expect 1 "the records checked by a program that is not the wai built" "checks the records with \`\$v\`, which is not the \`wai\` it built" -- check
|
|
# The union, and the records' check, at the step's top level: not in a function never called,
|
|
# a string or a here-document, where the line is not run as written.
|
|
mk; sub "$UNION" " union() {
|
|
$UNION
|
|
}"
|
|
expect 1 "the union in a function never called" "has \`wai conform --independent\` inside a block" -- check
|
|
mk; sub "$UNION" " : '
|
|
$UNION
|
|
'"
|
|
expect 1 "the union in a quoted string" "has \`wai conform --independent\` inside a quoted string" -- check
|
|
mk; sub "$UNION" " cat > /dev/null <<'EOF'
|
|
$UNION
|
|
EOF"
|
|
expect 1 "the union in a here-document" "has \`wai conform --independent\` inside a here-document" -- check
|
|
mk; sub ' "$w" verify "$d/records"' ' check() {
|
|
"$w" verify "$d/records"
|
|
}'
|
|
expect 1 "the records' check in a function never called" "has \`wai verify\` inside a block" -- check
|
|
|
|
# wai-breaking.yml, the one workflow that runs the table's tests, starts on a change to what they
|
|
# read, by a push and by a pull request alike. Each trigger has a list of its own.
|
|
drop() { # drop <trigger> <glob>: take the glob out of that trigger's list, which must hold it
|
|
python3 - "$T/.github/workflows/wai-breaking.yml" "$1" "$2" <<'P' || { echo "FAIL the fixture edit did not apply: $1 $2"; fails=$((fails+1)); }
|
|
import sys
|
|
p, trigger, glob = sys.argv[1:4]
|
|
s = open(p, encoding="utf-8").read()
|
|
line = f" - '{glob}'\n"
|
|
i = s.index(line, s.index(f"\n {trigger}:\n"))
|
|
open(p, "w", encoding="utf-8").write(s[:i] + s[i + len(line):])
|
|
P
|
|
}
|
|
STEPS='.github/wai-conformance/verifiers/**'
|
|
CONF='.github/workflows/wai-conformance.yml'
|
|
mk; drop push "$STEPS"
|
|
expect 1 "a push to a step file runs no table test" "a push that changes $STEPS starts no run" -- check
|
|
mk; drop pull_request "$STEPS"
|
|
expect 1 "a pull request on a step file runs no table test" "a pull request that changes $STEPS starts no run" -- check
|
|
mk; drop push "$CONF"
|
|
expect 1 "a push to the conformance workflow runs no table test" "a push that changes $CONF starts no run" -- check
|
|
mk; drop pull_request "$CONF"
|
|
expect 1 "a pull request on the conformance workflow runs no table test" "a pull request that changes $CONF starts no run" -- check
|
|
# A commented-out glob is no glob.
|
|
mk; ed "$T/.github/workflows/wai-breaking.yml" "s{^ - '\Q$STEPS\E'}{ # - '$STEPS'}"
|
|
expect 1 "a glob commented out of both lists" "a push that changes $STEPS starts no run" -- check
|
|
|
|
# An entry's needs= is what a run without packages shows. A stand-in for `wai` answers as the
|
|
# tree's lines say, from rows written beforehand; the lines then change under it.
|
|
mk
|
|
rows="$tmp/rows.json"
|
|
python3 -I - "$V" > "$rows" <<'P'
|
|
import glob, json, re, sys
|
|
out = []
|
|
for f in sorted(glob.glob(sys.argv[1] + "/*.sh")):
|
|
for m in re.finditer(r"^# independent: (\S+) \S+ (.*)$", open(f, encoding="utf-8").read(), re.M):
|
|
out.append({"name": m.group(1), "pass": "needs=" not in m.group(2), "got": "exit 1: no such package"})
|
|
print(json.dumps({"data": {"independent": out}}))
|
|
P
|
|
stand_in="$tmp/wai-stand-in"
|
|
printf '#!/bin/sh\ncat "%s"\nexit 1\n' "$rows" > "$stand_in"
|
|
chmod +x "$stand_in"
|
|
expect 0 "every entry's needs= as a run shows it" "entries declare what a run without packages shows" -- needs "$stand_in"
|
|
mk; ed "$V/claims-jwp-signers.sh" 's/^(# independent: claim \S+) needs=cryptography /$1 /'
|
|
expect 1 "an entry that fails without packages and declares none" "independent \`claim\` fails without packages and declares no needs=" -- needs "$stand_in"
|
|
mk; ed "$V/loc.sh" 's/^(# independent: loc live) python3 /$1 needs=cryptography python3 /'
|
|
expect 1 "an entry that passes without the package it declares" "independent \`loc\` passes without cryptography" -- needs "$stand_in"
|
|
mk; printf '#!/bin/sh\necho nothing\nexit 5\n' > "$stand_in"
|
|
expect 1 "a wai that ran no verifier" "it did not run the verifiers" -- needs "$stand_in"
|
|
mk; expect 1 "--suite naming no entry" "--suite names no entry: nowhere" -- needs "$stand_in" --suite nowhere
|
|
|
|
# The commands refuse what they cannot answer, rather than answering with nothing.
|
|
mk
|
|
expect 1 "a tag no step takes, asked for" "a tag is one of" -- tests int-kernel
|
|
for f in "$T"/wai/wai-rs/tests/*.rs; do perl -ni -e 'print unless m{^// wai-conformance:}' "$f"; done
|
|
expect 1 "tests for a tag no file carries" "the step would run no test" -- tests int-kernels
|
|
expect 1 "and test-tagged with it" "the step would run no test" -- test-tagged int-kernels -- --lib
|
|
|
|
# A step that fails stops the run with its status, and names its file.
|
|
mk
|
|
printf '# step: passes\n# order: 1\nset -euo pipefail\necho "features: $WAI_FEATURES"\n' > "$V/aa-passes.sh"
|
|
printf '# step: fails\n# order: 2\nset -euo pipefail\nexit 3\n' > "$V/ab-fails.sh"
|
|
# The features it is given are the whole derived list, as `env` prints it for the tree.
|
|
wf=$(python3 -I "$L" --root "$T" env | sed -n 's/^WAI_FEATURES=//p')
|
|
out=$(python3 -I "$L" --root "$T" run --only aa-passes 2>&1); rc=$?
|
|
if [ $rc -eq 0 ] && [ -n "$wf" ] && grep -qxF -- "features: $wf" <<<"$out"; then echo "ok a step that passes runs, with the derived features"
|
|
else echo "FAIL a passing step: exit $rc, or not given WAI_FEATURES=$wf"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /'; fails=$((fails+1)); fi
|
|
out=$(python3 -I "$L" --root "$T" run --only aa-passes --only ab-fails 2>&1); rc=$?
|
|
if [ $rc -eq 3 ] && grep -qF "::error::fails failed with exit status 3 (.github/wai-conformance/verifiers/ab-fails.sh)" <<<"$out"; then
|
|
echo "ok a step that fails stops the run with its status, naming its file"
|
|
else
|
|
echo "FAIL a failing step: exit $rc"; printf '%s\n' "$out" | tail -5 | sed 's/^/ /'; fails=$((fails+1))
|
|
fi
|
|
printf '# step: pipe\n# order: 3\nfalse | true\n' > "$V/ac-pipe.sh"
|
|
expect 1 "a step whose pipe fails fails, as under a workflow's bash" "" -- run --only ac-pipe
|
|
expect 1 "a step --only names that no file holds" "names a step no file holds" -- run --only no-such-step
|
|
|
|
if [ $fails -ne 0 ]; then echo "wai-conformance-lists-selftest: $fails failure(s)"; exit 1; fi
|
|
echo "wai-conformance-lists-selftest: the derived lists hold the hand lists, and fail each way they can lose an entry."
|