open-standards/scripts/wai-conformance-lists-selftest.sh

#!/usr/bin/env bash
#
# wai-conformance-lists-selftest.sh — the lists wai-conformance.yml derives can fail.
#
# The workflow's feature lists, test lists, steps, corpora and gate paths are read from files
# (scripts/wai-conformance-lists.py). A derivation that silently loses an entry drops coverage
# with every job still green, so this copies the files it reads into a scratch tree and
# requires: the tree as committed holds every entry of the hand lists it replaced; taking a
# step, a feature, a tag or a corpus away fails that; a new step's file brings its feature,
# test, binary and path into every list; each malformed file fails the check, saying why; and
# a step that fails stops the run with its status. Each check holds whatever else the tree
# holds, since this runs on every pull request, one that adds a step too: no check looks for an
# entry at a place in a sorted list, a step at a place in the run, or a line made by a file the
# check does not change itself.
set -uo pipefail

here=$(cd "$(dirname "$0")" && pwd)
repo=$(cd "$here/.." && pwd)
L="$here/wai-conformance-lists.py"
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
T="$tmp/t"

mk() {
  rm -rf "$T"
  mkdir -p "$T/scripts" "$T/wai/wai-rs/tests" "$T/.github"
  cp -R "$repo/.github/workflows" "$repo/.github/wai-conformance" "$T/.github/"
  cp "$repo/scripts/pr-touches-paths.py" "$T/scripts/"
  cp "$repo/wai/wai-rs/Cargo.toml" "$T/wai/wai-rs/"
  cp "$repo"/wai/wai-rs/tests/*.rs "$T/wai/wai-rs/tests/"
}
V="$T/.github/wai-conformance/verifiers"
ed() { perl -pi -e "$2" "$1"; } # ed <file> <perl substitution>, the same on GNU and BSD systems

fails=0
expect() { # expect <0|1|n> <what> <text the output must hold or ""> -- <lists.py args>
  local want=$1 what=$2 needle=$3 out rc
  shift 4
  out=$(python3 -I "$L" --root "$T" "$@" 2>&1); rc=$?
  if [ "$want" = 0 ] && [ $rc -ne 0 ]; then echo "FAIL  $what: exit $rc"; printf '%s\n' "$out" | tail -5 | sed 's/^/      /'; fails=$((fails+1)); return; fi
  if [ "$want" != 0 ] && [ $rc -eq 0 ]; then echo "FAIL  $what: it passed"; fails=$((fails+1)); return; fi
  if [ -n "$needle" ] && ! grep -qF -- "$needle" <<<"$out"; then echo "FAIL  $what: the output does not say '$needle'"; printf '%s\n' "$out" | tail -5 | sed 's/^/      /'; fails=$((fails+1)); return; fi
  echo "ok    $what"
}
# The lists are printed sorted, so what else the tree holds (a step a pull request adds,
# wherever its name sorts) moves an entry along a line: these find the entry among the line's
# words instead of at a fixed place in it.
named() { # named <output> <lead> <entry>: a line `FAIL  <lead> ...` names <entry> among its words
  local lead="FAIL  $2 " l w words
  while IFS= read -r l; do
    [[ $l == "$lead"* ]] || continue
    read -ra words <<<"${l#"$lead"}"
    for w in "${words[@]}"; do [ "$w" = "$3" ] && return 0; done
  done <<<"$1"
  return 1
}
expect_named() { # expect_named <what> <lead> <entry> [<lead> <entry>]... -- <lists.py args>: it fails, naming each
  local what=$1 out rc miss="" pairs=() i
  shift
  while [ "$1" != -- ]; do pairs+=("$1" "$2"); shift 2; done
  shift
  out=$(python3 -I "$L" --root "$T" "$@" 2>&1); rc=$?
  if [ $rc -eq 0 ]; then echo "FAIL  $what: it passed"; fails=$((fails+1)); return; fi
  for ((i = 0; i < ${#pairs[@]}; i += 2)); do
    named "$out" "${pairs[i]}" "${pairs[i+1]}" || miss="$miss; no line \`${pairs[i]} ...\` names ${pairs[i+1]}"
  done
  if [ -n "$miss" ]; then echo "FAIL  $what${miss/;/:}"; grep '^FAIL  [A-Za-z_]*: ' <<<"$out" | head -5 | sed 's/^/      /'; fails=$((fails+1)); return; fi
  echo "ok    $what"
}

# The tree as committed.
mk
expect 0 "the committed files hold every entry of the hand lists" "steps of the hand list run" -- selftest
expect 0 "and the check finds nothing" "well formed" -- check
a=$(python3 -I "$L" --root "$T" dry-run); b=$(python3 -I "$L" --root "$T" dry-run)
if [ "$a" = "$b" ] && [ -n "$a" ]; then echo "ok    dry-run prints the same twice"; else echo "FAIL  dry-run is not stable"; fails=$((fails+1)); fi
python3 -I "$L" --root "$T" prebuild > "$tmp/prebuild.sh" && bash -n "$tmp/prebuild.sh" \
  && echo "ok    the build step's script is shell" || { echo "FAIL  the build step's script does not parse"; fails=$((fails+1)); }

# Each way coverage can leave a list fails.
mk; rm "$V/live-events.sh"
expect_named "a verifier step's file taken away" "WAI_FEATURES: lost" live_events "union_tests: lost" live_events -- selftest
expect 1 "and its step named" "step \`live events · wai-rs and live_events_verify.py\` no longer runs" -- selftest
mk; ed "$V/live-rooms.sh" 's/^# features: live_room$/# features: live_compose/'
expect_named "a feature taken out of a step's header" "WAI_FEATURES: lost" live_room -- selftest
mk; perl -ni -e 'print unless $. == 1' "$T/wai/wai-rs/tests/int_kernels_vectors.rs"
expect_named "a tag taken off a test file" "int_kernels_debug_tests: lost" int_kernels_vectors -- selftest
mk; rm "$T/.github/wai-conformance/corpora/worlds.txt"
expect 1 "a corpus file taken away" "corpora: lost" -- selftest
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/--paths-from .github\/wai-conformance\/verifiers //'
expect 1 "the gate given its wide glob without the files' paths" "it would run on every change" -- selftest

# A new step brings its own entries into every list, and edits no other file.
mk
# What `selftest --exact` says of the steps and of the order, for the tree as committed.
exact_steps() { python3 -I "$L" --root "$T" selftest --exact 2>&1 | grep -E '^FAIL  (step |the steps |the corpora )' | LC_ALL=C sort; }
s0=$(exact_steps)
python3 - "$T/wai/wai-rs/Cargo.toml" <<'P'
import sys
p = sys.argv[1]
t = open(p).read().replace("[features]\n", "[features]\nfresh_step = []\n", 1)
open(p, "w").write(t)
P
printf '#[test]\nfn f() {}\n' > "$T/wai/wai-rs/tests/fresh_step.rs"
# Its order is below every step's in the tree, so it runs first whatever orders the tree holds.
lo=$(awk '/^# order: -?[0-9]+$/ { print $3 }' "$V"/*.sh | sort -n | head -1)
cat > "$V/fresh-step.sh" <<S
# step: fresh step · wai-rs and fresh_verify.py
# order: $((lo - 1))
# features: fresh_step
# bins: wai_claim_vectors
# paths: wai/extensions/fresh-step.md
#
# A step the self-test adds.
set -euo pipefail
echo "the step ran with \$WAI_FEATURES"
(cd wai-rs && cargo test --no-default-features --features "\$WAI_FEATURES" --test fresh_step)
S
expect 0 "a new step's file: the lists still hold the hand lists" "steps of the hand list run" -- selftest
expect_named "and are more than them" "WAI_FEATURES: more than the hand list:" fresh_step \
  "union_tests: more than the hand list:" fresh_step "alone: more than the hand list:" fresh_step \
  "paths: more than the hand list:" wai/extensions/fresh-step.md -- selftest --exact
expect 0 "its feature joins WAI_FEATURES, first by its order" "WAI_FEATURES=fresh_step," -- env
expect 0 "its test joins the build" "--test fresh_step" -- prebuild
expect 0 "its feature list is checked alone" "fresh_step" -- alone
expect 0 "its path joins the gate" "wai/extensions/fresh-step.md" -- paths
expect 0 "its step runs first, by its order" "  - step: fresh step" -- dry-run
first=$(python3 -I "$L" --root "$T" dry-run | grep -m1 '^  - step:')
[ "$first" = "  - step: fresh step · wai-rs and fresh_verify.py  (working-directory: wai)" ] \
  && echo "ok    and before every other" || { echo "FAIL  the new step is not first: $first"; fails=$((fails+1)); }
# --exact compares each step by its name, not its place: the step planted first is one
# addition, and every step of the hand list keeps its body and its order.
s1=$(exact_steps)
more=$(LC_ALL=C comm -13 <(printf '%s\n' "$s0") <(printf '%s\n' "$s1"))
less=$(LC_ALL=C comm -23 <(printf '%s\n' "$s0") <(printf '%s\n' "$s1"))
[ "$more" = 'FAIL  step `fresh step · wai-rs and fresh_verify.py`: more than the hand list' ] && [ -z "$less" ] \
  && echo "ok    --exact names the step planted first as one addition, and no step of the hand list as changed or moved" \
  || { echo "FAIL  --exact, a step planted first:"; printf '%s\n' "$more" | head -5 | sed 's/^/      + /'; printf '%s\n' "$less" | head -5 | sed 's/^/      - /'; fails=$((fails+1)); }
# And a corpus planted first is one addition, not a change of the corpora's order.
C="$T/.github/wai-conformance/corpora"
lo=$(awk '/^# order: -?[0-9]+$/ { print $3 }' "$C"/*.txt | sort -n | head -1)
printf '# corpus: fresh\n# order: %s\n# bin: wai_world_vectors\n# dir: ../fresh-conformance\n' "$((lo - 1))" > "$C/fresh.txt"
out=$(python3 -I "$L" --root "$T" selftest --exact 2>&1)
if grep -qE '^FAIL  corpora: more than the hand list: (.* )?fresh wai_world_vectors \.\./fresh-conformance( |$)' <<<"$out" && [ "$(exact_steps)" = "$s1" ]; then
  echo "ok    --exact names a corpus planted first as an addition, and not as a change of order"
else
  echo "FAIL  --exact, a corpus planted first:"; grep '^FAIL' <<<"$out" | head -5 | sed 's/^/      /'; fails=$((fails+1))
fi

# The dry-run prints the command lines of the derived steps outside the verifiers' job, so a
# change to one of them shows in a diff of two dry-runs, and it refuses a step it cannot read.
mk
d0=$(python3 -I "$L" --root "$T" dry-run)
expect 0 "the dry-run prints the release kernel step's command" "$ cargo test --lib --no-default-features --release --features " -- dry-run
expect 0 "and int-kernels-amd's, both profiles" "$ nice -n 10 cargo test --lib --locked --no-default-features --release --features " -- dry-run
# Each of whatever corpora and feature lists the tree holds, by its own line.
tails=$(grep -o -- '--bin [^ ]* -- verify [^ ]*$' <<<"$d0")
cs=$(python3 -I "$L" --root "$T" corpora)
miss=$(while read -r c b dir; do [ -n "$c" ] || continue; grep -qxF -- "--bin $b -- verify $dir" <<<"$tails" || printf ' %s' "$c"; done <<<"$cs")
[ -n "$cs" ] && [ -z "$miss" ] && echo "ok    and each corpus's" \
  || { echo "FAIL  the dry-run prints no command for the corpora:${miss:- (none listed)}"; fails=$((fails+1)); }
checks=$(grep -o -- '\$ cargo check --no-default-features --tests --features [^ ]*$' <<<"$d0")
as=$(python3 -I "$L" --root "$T" alone)
miss=$(while read -r a; do
  [ -n "$a" ] || continue
  n=$(tr ',' '\n' <<<"$a" | LC_ALL=C sort | paste -sd, -)
  grep -qxF -- "\$ cargo check --no-default-features --tests --features $n" <<<"$checks" || printf ' %s' "$a"
done <<<"$as")
[ -n "$as" ] && [ -z "$miss" ] && echo "ok    and each list features-alone checks" \
  || { echo "FAIL  the dry-run prints no features-alone check for the lists:${miss:- (none listed)}"; fails=$((fails+1)); }
ed "$T/.github/workflows/wai-conformance.yml" 's/(int-kernels int-kernels-release --extra-features i32_forms -- )--release /$1/'
d1=$(python3 -I "$L" --root "$T" dry-run)
dd=$(diff <(printf '%s\n' "$d0") <(printf '%s\n' "$d1"))
n=$(grep -c '^[<>]' <<<"$dd")
[ "$n" = 2 ] && grep -q '^< .*--release --features' <<<"$dd" \
  && echo "ok    a flag taken off a kernel step is one changed line of the dry-run" \
  || { echo "FAIL  taking --release off the release step changed $n line(s) of the dry-run"; fails=$((fails+1)); }
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/^(\s+for f in \$lists; do)$/$1\n            cargo check \$NOT_SET/'
expect 1 "a command reading a value the dry-run cannot know fails it" "\`\$NOT_SET\` has no value the dry-run knows" -- dry-run
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/(run: python3 \.\.\/\.\.\/scripts\/wai-conformance-lists\.py corpora --verify.*)$/$1\n        if: matrix.os != '"'"'windows'"'"'/'
expect 0 "a step one shell family skips prints each family's commands" "every cargo-class corpus  (the Windows leg; working-directory: -)" -- dry-run
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/^(\s+- name: )wai-rs, each feature list alone$/$1wai-rs, every feature list alone/'
expect 1 "a derived step the dry-run cannot find fails it" "has no step \`wai-rs, each feature list alone...\`" -- dry-run

# Each malformed file fails the check, saying why.
mk; ed "$V/live-events.sh" 's/^# order: 100$/# order: 100\n# feature: live_events/'
expect 1 "a header key no step file has" "\`feature\` is not a key" -- check
mk; perl -ni -e 'print unless /^# step: /' "$V/live-events.sh"
expect 1 "a step file with no step name" "no \`step:\` line" -- check
mk; ed "$V/live-events.sh" 's/^# order: 100$/# order: soon/'
expect 1 "an order that is no integer" "is not an integer" -- check
mk; ed "$V/live-events.sh" 's#^\# paths: wai/extensions/live-events.md#\# paths: docs/live-events.md#'
expect 1 "a path the forge's filter does not cover" "so the forge would never start the workflow for it" -- check
mk; ed "$V/live-events.sh" 's/^# features: live_events$/# features: live_eventz/'
expect 1 "a feature Cargo.toml does not have" "\`live_eventz\` is not in" -- check
mk; ed "$V/live-events.sh" 's/--test live_events\)/--test live_eventz)/'
expect 1 "a test the crate does not have" "--test live_eventz: wai/wai-rs has no such test" -- check
mk; ed "$V/http-differential.sh" 's#wai-bins/union/wai_http_vectors#wai-bins/union/wai_http_vectorz#'
expect 1 "a binary the crate does not have" "\`wai_http_vectorz\` is not a [[bin]]" -- check
mk; ed "$T/wai/wai-rs/tests/int_kernels_vectors.rs" 's{^// wai-conformance: int-kernels }{// wai-conformance: int-kernel }'
expect 1 "a tag no step takes" "is no tag a step takes" -- check
mk; cp "$V/live-events.sh" "$V/live-events-again.sh"
expect 1 "two files with one step name" "is also" -- check
mk; printf '# step: empty\n# order: 1\n#\n# nothing to run\n' > "$V/empty.sh"
expect 1 "a step file with no body" "no body" -- check
mk; printf '# corpus: x\n# order: 1\n# bin: wai_world_vectors\n' > "$T/.github/wai-conformance/corpora/x.txt"
expect 1 "a corpus file with no directory" "no \`dir:\` line" -- check

# The commands refuse what they cannot answer, rather than answering with nothing.
mk
expect 1 "a tag no step takes, asked for" "a tag is one of" -- tests int-kernel
for f in "$T"/wai/wai-rs/tests/*.rs; do perl -ni -e 'print unless m{^// wai-conformance:}' "$f"; done
expect 1 "tests for a tag no file carries" "the step would run no test" -- tests int-kernels
expect 1 "and test-tagged with it" "the step would run no test" -- test-tagged int-kernels -- --lib

# A step that fails stops the run with its status, and names its file.
mk
printf '# step: passes\n# order: 1\nset -euo pipefail\necho "features: $WAI_FEATURES"\n' > "$V/aa-passes.sh"
printf '# step: fails\n# order: 2\nset -euo pipefail\nexit 3\n' > "$V/ab-fails.sh"
# The features it is given are the whole derived list, as `env` prints it for the tree.
wf=$(python3 -I "$L" --root "$T" env | sed -n 's/^WAI_FEATURES=//p')
out=$(python3 -I "$L" --root "$T" run --only aa-passes 2>&1); rc=$?
if [ $rc -eq 0 ] && [ -n "$wf" ] && grep -qxF -- "features: $wf" <<<"$out"; then echo "ok    a step that passes runs, with the derived features"
else echo "FAIL  a passing step: exit $rc, or not given WAI_FEATURES=$wf"; printf '%s\n' "$out" | tail -5 | sed 's/^/      /'; fails=$((fails+1)); fi
out=$(python3 -I "$L" --root "$T" run --only aa-passes --only ab-fails 2>&1); rc=$?
if [ $rc -eq 3 ] && grep -qF "::error::fails failed with exit status 3 (.github/wai-conformance/verifiers/ab-fails.sh)" <<<"$out"; then
  echo "ok    a step that fails stops the run with its status, naming its file"
else
  echo "FAIL  a failing step: exit $rc"; printf '%s\n' "$out" | tail -5 | sed 's/^/      /'; fails=$((fails+1))
fi
printf '# step: pipe\n# order: 3\nfalse | true\n' > "$V/ac-pipe.sh"
expect 1 "a step whose pipe fails fails, as under a workflow's bash" "" -- run --only ac-pipe
expect 1 "a step --only names that no file holds" "names a step no file holds" -- run --only no-such-step

if [ $fails -ne 0 ]; then echo "wai-conformance-lists-selftest: $fails failure(s)"; exit 1; fi
echo "wai-conformance-lists-selftest: the derived lists hold the hand lists, and fail each way they can lose an entry."