#!/usr/bin/env bash
#
# wai-conformance-lists-selftest.sh — the lists wai-conformance.yml derives can fail.
#
# The workflow's feature lists, test lists, steps, corpora and gate paths are read from files
# (scripts/wai-conformance-lists.py). A derivation that silently loses an entry drops coverage
# with every job still green, so this copies the files it reads into a scratch tree and
# requires: the tree as committed holds every entry of the hand lists it replaced; taking a
# step, a feature, a tag or a corpus away fails that; a new step's file brings its feature,
# test, binary and path into every list; each malformed file fails the check, saying why; a
# step that fails stops the run with its status; each way the corpus union's step has been
# planted to pass a failed run (set -e off in any spelling, a flag that narrows or stops the
# union or changes what runs it, a `wai` it did not build, the union or its check in a function
# never called, a string or a here-document) fails the check; a trigger of wai-breaking.yml
# that does not start on a change to what the table's tests read fails the check; and an entry
# whose needs= a run without packages does not bear out fails `needs`. Each check holds
# whatever else the tree holds, since this runs on every pull request, one that adds a step
# too: no check looks for an entry at a place in a sorted list, a step at a place in the run,
# or a line made by a file the check does not change itself.
set -uo pipefail

here=$(cd "$(dirname "$0")" && pwd)
repo=$(cd "$here/.." && pwd)
L="$here/wai-conformance-lists.py"
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
T="$tmp/t"

mk() {
  rm -rf "$T"
  mkdir -p "$T/scripts" "$T/wai/wai-rs/tests" "$T/.github"
  cp -R "$repo/.github/workflows" "$repo/.github/wai-conformance" "$T/.github/"
  cp "$repo/scripts/pr-touches-paths.py" "$T/scripts/"
  cp "$repo/wai/wai-rs/Cargo.toml" "$T/wai/wai-rs/"
  cp "$repo"/wai/wai-rs/tests/*.rs "$T/wai/wai-rs/tests/"
}
V="$T/.github/wai-conformance/verifiers"
ed() { perl -pi -e "$2" "$1"; } # ed <file> <perl substitution>, the same on GNU and BSD systems

fails=0
expect() { # expect <0|1|n> <what> <text the output must hold or ""> -- <lists.py args>
  local want=$1 what=$2 needle=$3 out rc
  shift 4
  out=$(python3 -I "$L" --root "$T" "$@" 2>&1); rc=$?
  if [ "$want" = 0 ] && [ $rc -ne 0 ]; then echo "FAIL  $what: exit $rc"; printf '%s\n' "$out" | tail -5 | sed 's/^/      /'; fails=$((fails+1)); return; fi
  if [ "$want" != 0 ] && [ $rc -eq 0 ]; then echo "FAIL  $what: it passed"; fails=$((fails+1)); return; fi
  if [ -n "$needle" ] && ! grep -qF -- "$needle" <<<"$out"; then echo "FAIL  $what: the output does not say '$needle'"; printf '%s\n' "$out" | tail -5 | sed 's/^/      /'; fails=$((fails+1)); return; fi
  echo "ok    $what"
}
# The lists are printed sorted, so what else the tree holds (a step a pull request adds,
# wherever its name sorts) moves an entry along a line: these find the entry among the line's
# words instead of at a fixed place in it.
named() { # named <output> <lead> <entry>: a line `FAIL  <lead> ...` names <entry> among its words
  local lead="FAIL  $2 " l w words
  while IFS= read -r l; do
    [[ $l == "$lead"* ]] || continue
    read -ra words <<<"${l#"$lead"}"
    for w in "${words[@]}"; do [ "$w" = "$3" ] && return 0; done
  done <<<"$1"
  return 1
}
expect_named() { # expect_named <what> <lead> <entry> [<lead> <entry>]... -- <lists.py args>: it fails, naming each
  local what=$1 out rc miss="" pairs=() i
  shift
  while [ "$1" != -- ]; do pairs+=("$1" "$2"); shift 2; done
  shift
  out=$(python3 -I "$L" --root "$T" "$@" 2>&1); rc=$?
  if [ $rc -eq 0 ]; then echo "FAIL  $what: it passed"; fails=$((fails+1)); return; fi
  for ((i = 0; i < ${#pairs[@]}; i += 2)); do
    named "$out" "${pairs[i]}" "${pairs[i+1]}" || miss="$miss; no line \`${pairs[i]} ...\` names ${pairs[i+1]}"
  done
  if [ -n "$miss" ]; then echo "FAIL  $what${miss/;/:}"; grep '^FAIL  [A-Za-z_]*: ' <<<"$out" | head -5 | sed 's/^/      /'; fails=$((fails+1)); return; fi
  echo "ok    $what"
}

# The tree as committed.
mk
expect 0 "the committed files hold every entry of the hand lists" "steps of the hand list run" -- selftest
expect 0 "and the check finds nothing" "well formed" -- check
a=$(python3 -I "$L" --root "$T" dry-run); b=$(python3 -I "$L" --root "$T" dry-run)
if [ "$a" = "$b" ] && [ -n "$a" ]; then echo "ok    dry-run prints the same twice"; else echo "FAIL  dry-run is not stable"; fails=$((fails+1)); fi
python3 -I "$L" --root "$T" prebuild > "$tmp/prebuild.sh" && bash -n "$tmp/prebuild.sh" \
  && echo "ok    the build step's script is shell" || { echo "FAIL  the build step's script does not parse"; fails=$((fails+1)); }

# Each way coverage can leave a list fails.
mk; rm "$V/live-events.sh"
expect_named "a verifier step's file taken away" "WAI_FEATURES: lost" live_events "union_tests: lost" live_events -- selftest
expect 1 "and its step named" "step \`live events · wai-rs and live_events_verify.py\` no longer runs" -- selftest
mk; ed "$V/live-rooms.sh" 's/^# features: live_room$/# features: live_compose/'
expect_named "a feature taken out of a step's header" "WAI_FEATURES: lost" live_room -- selftest
mk; perl -ni -e 'print unless $. == 1' "$T/wai/wai-rs/tests/int_kernels_vectors.rs"
expect_named "a tag taken off a test file" "int_kernels_debug_tests: lost" int_kernels_vectors -- selftest
mk; rm "$T/.github/wai-conformance/corpora/worlds.txt"
expect 1 "a corpus file taken away" "corpora: lost" -- selftest
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/--paths-from .github\/wai-conformance\/verifiers //'
expect 1 "the gate given its wide glob without the files' paths" "it would run on every change" -- selftest

# A new step brings its own entries into every list, and edits no other file.
mk
# What `selftest --exact` says of the steps and of the order, for the tree as committed.
exact_steps() { python3 -I "$L" --root "$T" selftest --exact 2>&1 | grep -E '^FAIL  (step |the steps |the corpora )' | LC_ALL=C sort; }
s0=$(exact_steps)
python3 - "$T/wai/wai-rs/Cargo.toml" <<'P'
import sys
p = sys.argv[1]
t = open(p).read().replace("[features]\n", "[features]\nfresh_step = []\n", 1)
open(p, "w").write(t)
P
printf '#[test]\nfn f() {}\n' > "$T/wai/wai-rs/tests/fresh_step.rs"
# Its order is below every step's in the tree, so it runs first whatever orders the tree holds.
lo=$(awk '/^# order: -?[0-9]+$/ { print $3 }' "$V"/*.sh | sort -n | head -1)
cat > "$V/fresh-step.sh" <<S
# step: fresh step · wai-rs and fresh_verify.py
# order: $((lo - 1))
# features: fresh_step
# bins: wai_claim_vectors
# paths: wai/extensions/fresh-step.md
#
# A step the self-test adds.
set -euo pipefail
echo "the step ran with \$WAI_FEATURES"
(cd wai-rs && cargo test --no-default-features --features "\$WAI_FEATURES" --test fresh_step)
S
expect 0 "a new step's file: the lists still hold the hand lists" "steps of the hand list run" -- selftest
expect_named "and are more than them" "WAI_FEATURES: more than the hand list:" fresh_step \
  "union_tests: more than the hand list:" fresh_step "alone: more than the hand list:" fresh_step \
  "paths: more than the hand list:" wai/extensions/fresh-step.md -- selftest --exact
expect 0 "its feature joins WAI_FEATURES, first by its order" "WAI_FEATURES=fresh_step," -- env
expect 0 "its test joins the build" "--test fresh_step" -- prebuild
expect 0 "its feature list is checked alone" "fresh_step" -- alone
expect 0 "its path joins the gate" "wai/extensions/fresh-step.md" -- paths
expect 0 "its step runs first, by its order" "  - step: fresh step" -- dry-run
first=$(python3 -I "$L" --root "$T" dry-run | grep -m1 '^  - step:')
[ "$first" = "  - step: fresh step · wai-rs and fresh_verify.py  (working-directory: wai)" ] \
  && echo "ok    and before every other" || { echo "FAIL  the new step is not first: $first"; fails=$((fails+1)); }
# --exact compares each step by its name, not its place: the step planted first is one
# addition, and every step of the hand list keeps its body and its order.
s1=$(exact_steps)
more=$(LC_ALL=C comm -13 <(printf '%s\n' "$s0") <(printf '%s\n' "$s1"))
less=$(LC_ALL=C comm -23 <(printf '%s\n' "$s0") <(printf '%s\n' "$s1"))
[ "$more" = 'FAIL  step `fresh step · wai-rs and fresh_verify.py`: more than the hand list' ] && [ -z "$less" ] \
  && echo "ok    --exact names the step planted first as one addition, and no step of the hand list as changed or moved" \
  || { echo "FAIL  --exact, a step planted first:"; printf '%s\n' "$more" | head -5 | sed 's/^/      + /'; printf '%s\n' "$less" | head -5 | sed 's/^/      - /'; fails=$((fails+1)); }
# And a corpus planted first is one addition, not a change of the corpora's order.
C="$T/.github/wai-conformance/corpora"
lo=$(awk '/^# order: -?[0-9]+$/ { print $3 }' "$C"/*.txt | sort -n | head -1)
printf '# corpus: fresh\n# order: %s\n# bin: wai_world_vectors\n# dir: ../fresh-conformance\n' "$((lo - 1))" > "$C/fresh.txt"
out=$(python3 -I "$L" --root "$T" selftest --exact 2>&1)
if grep -qE '^FAIL  corpora: more than the hand list: (.* )?fresh wai_world_vectors \.\./fresh-conformance( |$)' <<<"$out" && [ "$(exact_steps)" = "$s1" ]; then
  echo "ok    --exact names a corpus planted first as an addition, and not as a change of order"
else
  echo "FAIL  --exact, a corpus planted first:"; grep '^FAIL' <<<"$out" | head -5 | sed 's/^/      /'; fails=$((fails+1))
fi

# The dry-run prints the command lines of the derived steps outside the verifiers' job, so a
# change to one of them shows in a diff of two dry-runs, and it refuses a step it cannot read.
mk
d0=$(python3 -I "$L" --root "$T" dry-run)
expect 0 "the dry-run prints the release kernel step's command" "$ cargo test --lib --no-default-features --release --features " -- dry-run
expect 0 "and int-kernels-amd's, both profiles" "$ nice -n 10 cargo test --lib --locked --no-default-features --release --features " -- dry-run
# Each of whatever corpora and feature lists the tree holds, by its own line.
tails=$(grep -o -- '--bin [^ ]* -- verify [^ ]*$' <<<"$d0")
cs=$(python3 -I "$L" --root "$T" corpora)
miss=$(while read -r c b dir; do [ -n "$c" ] || continue; grep -qxF -- "--bin $b -- verify $dir" <<<"$tails" || printf ' %s' "$c"; done <<<"$cs")
[ -n "$cs" ] && [ -z "$miss" ] && echo "ok    and each corpus's" \
  || { echo "FAIL  the dry-run prints no command for the corpora:${miss:- (none listed)}"; fails=$((fails+1)); }
checks=$(grep -o -- '\$ cargo check --no-default-features --tests --features [^ ]*$' <<<"$d0")
as=$(python3 -I "$L" --root "$T" alone)
miss=$(while read -r a; do
  [ -n "$a" ] || continue
  n=$(tr ',' '\n' <<<"$a" | LC_ALL=C sort | paste -sd, -)
  grep -qxF -- "\$ cargo check --no-default-features --tests --features $n" <<<"$checks" || printf ' %s' "$a"
done <<<"$as")
[ -n "$as" ] && [ -z "$miss" ] && echo "ok    and each list features-alone checks" \
  || { echo "FAIL  the dry-run prints no features-alone check for the lists:${miss:- (none listed)}"; fails=$((fails+1)); }
ed "$T/.github/workflows/wai-conformance.yml" 's/(int-kernels int-kernels-release --extra-features i32_forms -- )--release /$1/'
d1=$(python3 -I "$L" --root "$T" dry-run)
dd=$(diff <(printf '%s\n' "$d0") <(printf '%s\n' "$d1"))
n=$(grep -c '^[<>]' <<<"$dd")
[ "$n" = 2 ] && grep -q '^< .*--release --features' <<<"$dd" \
  && echo "ok    a flag taken off a kernel step is one changed line of the dry-run" \
  || { echo "FAIL  taking --release off the release step changed $n line(s) of the dry-run"; fails=$((fails+1)); }
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/^(\s+for f in \$lists; do)$/$1\n            cargo check \$NOT_SET/'
expect 1 "a command reading a value the dry-run cannot know fails it" "\`\$NOT_SET\` has no value the dry-run knows" -- dry-run
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/(run: python3 \.\.\/\.\.\/scripts\/wai-conformance-lists\.py corpora --verify.*)$/$1\n        if: matrix.os != '"'"'windows'"'"'/'
expect 0 "a step one shell family skips prints each family's commands" "every cargo-class corpus  (the Windows leg; working-directory: -)" -- dry-run
mk; ed "$T/.github/workflows/wai-conformance.yml" 's/^(\s+- name: )wai-rs, each feature list alone$/$1wai-rs, every feature list alone/'
expect 1 "a derived step the dry-run cannot find fails it" "has no step \`wai-rs, each feature list alone...\`" -- dry-run

# Each malformed file fails the check, saying why.
mk; ed "$V/live-events.sh" 's/^# order: 100$/# order: 100\n# feature: live_events/'
expect 1 "a header key no step file has" "\`feature\` is not a key" -- check
mk; perl -ni -e 'print unless /^# step: /' "$V/live-events.sh"
expect 1 "a step file with no step name" "no \`step:\` line" -- check
mk; ed "$V/live-events.sh" 's/^# order: 100$/# order: soon/'
expect 1 "an order that is no integer" "is not an integer" -- check
mk; ed "$V/live-events.sh" 's#^\# paths: wai/extensions/live-events.md#\# paths: docs/live-events.md#'
expect 1 "a path the forge's filter does not cover" "so the forge would never start the workflow for it" -- check
mk; ed "$V/live-events.sh" 's/^# features: live_events$/# features: live_eventz/'
expect 1 "a feature Cargo.toml does not have" "\`live_eventz\` is not in" -- check
mk; ed "$V/live-events.sh" 's/--test live_events\)/--test live_eventz)/'
expect 1 "a test the crate does not have" "--test live_eventz: wai/wai-rs has no such test" -- check
mk; ed "$V/http-differential.sh" 's#wai-bins/union/wai_http_vectors#wai-bins/union/wai_http_vectorz#'
expect 1 "a binary the crate does not have" "\`wai_http_vectorz\` is not a [[bin]]" -- check
mk; ed "$T/wai/wai-rs/tests/int_kernels_vectors.rs" 's{^// wai-conformance: int-kernels }{// wai-conformance: int-kernel }'
expect 1 "a tag no step takes" "is no tag a step takes" -- check
mk; cp "$V/live-events.sh" "$V/live-events-again.sh"
expect 1 "two files with one step name" "is also" -- check
mk; printf '# step: empty\n# order: 1\n#\n# nothing to run\n' > "$V/empty.sh"
expect 1 "a step file with no body" "no body" -- check
mk; printf '# corpus: x\n# order: 1\n# bin: wai_world_vectors\n' > "$T/.github/wai-conformance/corpora/x.txt"
expect 1 "a corpus file with no directory" "no \`dir:\` line" -- check

# The independent verifiers: one `independent:` line an entry, run by the union step alone.
mk
expect 0 "a declaration, entries and no body, is no step" "a declaration; its verifiers run in the union step" -- run --only loc
expect 0 "the entries print in order, each with its command line" "loc live python3 -I live-conformance/loc_verify.py live-conformance/loc" -- independent
mk; ed "$V/loc.sh" 's/^# independent: loc live /# independent: loc nowhere /'
expect 1 "an entry of no area" "is none of trust, media, delivery, live" -- check
mk; ed "$V/loc.sh" 's/^# independent: loc live python3 /# independent: loc live /'
expect 1 "an entry with no command line" "before \`python3\`" -- check
mk; ed "$V/loc.sh" 's/^# independent: loc live /# independent: e2ee live /'
expect 1 "an entry's name another file gives" "is also" -- check
mk; printf 'python3 -I live-conformance/loc_verify.py live-conformance/loc\n' >> "$V/loc.sh"
expect 1 "a body that runs an entry's command line itself" "itself; the union step runs it" -- check
mk; perl -ni -e 'print unless /^# independent: /' "$V/loc.sh"
expect 1 "a declaration whose entries are taken away" "no body" -- check

# The corpus union's step fails when `wai conform --independent` does: `set -e` decides it.
W="$T/.github/workflows/wai-conformance.yml"
UNION='          "$w" conform --independent --root . --out "$d/records" --format junit > "$d/junit.xml"'
USTEP="      - name: Every corpus's independent verifier, through wai conform --independent"
sub() { # sub <from> <to>: replace the one occurrence in the workflow, which must exist
  python3 - "$W" "$1" "$2" <<'P' || { echo "FAIL  the fixture edit did not apply: $1"; fails=$((fails+1)); }
import sys
p, a, b = sys.argv[1:4]
s = open(p, encoding="utf-8").read()
if s.count(a) != 1:
    sys.exit(1)
open(p, "w", encoding="utf-8").write(s.replace(a, b))
P
}
mk; expect 0 "the union's step as it is" "" -- check
mk; sub "$UNION" "$UNION || true"
expect 1 "the union's command with || true" "goes on after the command on its line" -- check
mk; sub "$UNION" '          status=0
'"$UNION"' || status=$?'
expect 1 "the union's status kept and not used" "can pass a failed union" -- check
mk; sub "$UNION" "$UNION; true"
expect 1 "the union's line ends in another command" "goes on after the command on its line" -- check
mk; sub "$UNION" "          exit 0
$UNION"
expect 1 "the union's step ends before it" "can pass a failed union: \`exit 0\`" -- check
mk; sub "$UNION" "          set +e
$UNION"
expect 1 "the union's step turns set -e off" "can pass a failed union: \`set +e\`" -- check
mk; sub "$UNION" "          if ! true; then
$UNION
          fi"
expect 1 "the union runs under a condition" "can pass a failed union: \`if ! true; then\`" -- check
mk; sub "$USTEP" "$USTEP
        continue-on-error: true"
expect 1 "the union's step may fail without failing the job" "has \`continue-on-error:\`" -- check
mk; sub "$USTEP" "$USTEP
        if: false"
expect 1 "the union's step is skipped" "has \`if:\`" -- check
mk; sub '    name: independent verifiers · claims' '    continue-on-error: true
    name: independent verifiers · claims'
expect 1 "the union's job may fail" "the job \`independent-verifiers\` has \`continue-on-error:\`" -- check
mk; sub '          set -euo pipefail
          (cd wai-rs && cargo build --locked --no-default-features --features quickstart --bin wai' '          (cd wai-rs && cargo build --locked --no-default-features --features quickstart --bin wai'
expect 1 "the union's step without set -e" "has no \`set -e\` before the union" -- check
mk; sub '          "$w" verify "$d/records"
' ''
expect 1 "the records unchecked" "does not check the records" -- check
mk; sub "$UNION" '          echo "$w" conform --independent'
expect 1 "the union only echoed" "in a line that is not its command" -- check
# set -e turned off in its other spellings.
mk; sub "$UNION" "          set +o errexit
$UNION"
expect 1 "the union's step turns errexit off by name" "can pass a failed union: \`set +o errexit\`" -- check
mk; sub "$UNION" "          set -u +e
$UNION"
expect 1 "the union's step turns set -e off among other options" "can pass a failed union: \`set -u +e\`" -- check
mk; sub "$UNION" "          shopt -uo errexit
$UNION"
expect 1 "the union's step turns errexit off with shopt" "can pass a failed union: \`shopt -uo errexit\`" -- check
# A flag that narrows the run, stops it before it runs, or changes what fails or runs it.
for f in "--suite pin" "--dry-run" "--timeout 1" "--list" "--python true" "--all"; do
  mk; sub "$UNION" "${UNION/--independent/--independent $f}"
  expect 1 "the union given $f" "gives the union \`${f%% *}\`" -- check
done
mk; sub "$UNION" "${UNION/--root ./--root /tmp}"
expect 1 "the union reads another tree's corpora" "gives the union \`--root /tmp\`, not \`.\`" -- check
mk; sub "$UNION" "${UNION/--root ./}"
expect 1 "the union given no --root" "gives the union no \`--root .\`" -- check
mk; sub "$UNION" "${UNION/--independent/--independent stray}"
expect 1 "the union given a word that is no flag" "gives the union \`stray\`" -- check
mk; sub "$USTEP
        working-directory: wai" "$USTEP
        working-directory: wai/wai-rs"
expect 1 "the union run outside wai/" "runs in \`wai/wai-rs\`, not wai/" -- check
# The wai it runs, and checks the records with, is the one it built.
mk; sub '          w="$RUNNER_TEMP/wai-cli-target/debug/wai"' '          w=true'
expect 1 "the union run by a program that is not the wai built" "runs \`\$w\`, which is not the \`wai\` it built" -- check
mk; sub "$UNION" "          w=true
$UNION"
expect 1 "the wai it runs set again before the union" "runs \`\$w\`, which is not the \`wai\` it built" -- check
mk; sub '          (cd wai-rs && cargo build --locked --no-default-features --features quickstart --bin wai --target-dir "$RUNNER_TEMP/wai-cli-target")' '          (cd wai-rs && cargo build --locked --no-default-features --features quickstart --bin wai_signer_vectors --target-dir "$RUNNER_TEMP/wai-cli-target")'
expect 1 "the step builds no wai" "builds no \`wai\` before the union" -- check
mk; sub '          "$w" verify "$d/records"' '          v=true
          "$v" verify "$d/records"'
expect 1 "the records checked by a program that is not the wai built" "checks the records with \`\$v\`, which is not the \`wai\` it built" -- check
# The union, and the records' check, at the step's top level: not in a function never called,
# a string or a here-document, where the line is not run as written.
mk; sub "$UNION" "          union() {
$UNION
          }"
expect 1 "the union in a function never called" "has \`wai conform --independent\` inside a block" -- check
mk; sub "$UNION" "          : '
$UNION
          '"
expect 1 "the union in a quoted string" "has \`wai conform --independent\` inside a quoted string" -- check
mk; sub "$UNION" "          cat > /dev/null <<'EOF'
$UNION
          EOF"
expect 1 "the union in a here-document" "has \`wai conform --independent\` inside a here-document" -- check
mk; sub '          "$w" verify "$d/records"' '          check() {
          "$w" verify "$d/records"
          }'
expect 1 "the records' check in a function never called" "has \`wai verify\` inside a block" -- check

# wai-breaking.yml, the one workflow that runs the table's tests, starts on a change to what they
# read, by a push and by a pull request alike. Each trigger has a list of its own.
drop() { # drop <trigger> <glob>: take the glob out of that trigger's list, which must hold it
  python3 - "$T/.github/workflows/wai-breaking.yml" "$1" "$2" <<'P' || { echo "FAIL  the fixture edit did not apply: $1 $2"; fails=$((fails+1)); }
import sys
p, trigger, glob = sys.argv[1:4]
s = open(p, encoding="utf-8").read()
line = f"      - '{glob}'\n"
i = s.index(line, s.index(f"\n  {trigger}:\n"))
open(p, "w", encoding="utf-8").write(s[:i] + s[i + len(line):])
P
}
STEPS='.github/wai-conformance/verifiers/**'
CONF='.github/workflows/wai-conformance.yml'
mk; drop push "$STEPS"
expect 1 "a push to a step file runs no table test" "a push that changes $STEPS starts no run" -- check
mk; drop pull_request "$STEPS"
expect 1 "a pull request on a step file runs no table test" "a pull request that changes $STEPS starts no run" -- check
mk; drop push "$CONF"
expect 1 "a push to the conformance workflow runs no table test" "a push that changes $CONF starts no run" -- check
mk; drop pull_request "$CONF"
expect 1 "a pull request on the conformance workflow runs no table test" "a pull request that changes $CONF starts no run" -- check
# A commented-out glob is no glob.
mk; ed "$T/.github/workflows/wai-breaking.yml" "s{^      - '\Q$STEPS\E'}{      # - '$STEPS'}"
expect 1 "a glob commented out of both lists" "a push that changes $STEPS starts no run" -- check

# An entry's needs= is what a run without packages shows. A stand-in for `wai` answers as the
# tree's lines say, from rows written beforehand; the lines then change under it.
mk
rows="$tmp/rows.json"
python3 -I - "$V" > "$rows" <<'P'
import glob, json, re, sys
out = []
for f in sorted(glob.glob(sys.argv[1] + "/*.sh")):
    for m in re.finditer(r"^# independent: (\S+) \S+ (.*)$", open(f, encoding="utf-8").read(), re.M):
        out.append({"name": m.group(1), "pass": "needs=" not in m.group(2), "got": "exit 1: no such package"})
print(json.dumps({"data": {"independent": out}}))
P
stand_in="$tmp/wai-stand-in"
printf '#!/bin/sh\ncat "%s"\nexit 1\n' "$rows" > "$stand_in"
chmod +x "$stand_in"
expect 0 "every entry's needs= as a run shows it" "entries declare what a run without packages shows" -- needs "$stand_in"
mk; ed "$V/claims-jwp-signers.sh" 's/^(# independent: claim \S+) needs=cryptography /$1 /'
expect 1 "an entry that fails without packages and declares none" "independent \`claim\` fails without packages and declares no needs=" -- needs "$stand_in"
mk; ed "$V/loc.sh" 's/^(# independent: loc live) python3 /$1 needs=cryptography python3 /'
expect 1 "an entry that passes without the package it declares" "independent \`loc\` passes without cryptography" -- needs "$stand_in"
mk; printf '#!/bin/sh\necho nothing\nexit 5\n' > "$stand_in"
expect 1 "a wai that ran no verifier" "it did not run the verifiers" -- needs "$stand_in"
mk; expect 1 "--suite naming no entry" "--suite names no entry: nowhere" -- needs "$stand_in" --suite nowhere

# The commands refuse what they cannot answer, rather than answering with nothing.
mk
expect 1 "a tag no step takes, asked for" "a tag is one of" -- tests int-kernel
for f in "$T"/wai/wai-rs/tests/*.rs; do perl -ni -e 'print unless m{^// wai-conformance:}' "$f"; done
expect 1 "tests for a tag no file carries" "the step would run no test" -- tests int-kernels
expect 1 "and test-tagged with it" "the step would run no test" -- test-tagged int-kernels -- --lib

# A step that fails stops the run with its status, and names its file.
mk
printf '# step: passes\n# order: 1\nset -euo pipefail\necho "features: $WAI_FEATURES"\n' > "$V/aa-passes.sh"
printf '# step: fails\n# order: 2\nset -euo pipefail\nexit 3\n' > "$V/ab-fails.sh"
# The features it is given are the whole derived list, as `env` prints it for the tree.
wf=$(python3 -I "$L" --root "$T" env | sed -n 's/^WAI_FEATURES=//p')
out=$(python3 -I "$L" --root "$T" run --only aa-passes 2>&1); rc=$?
if [ $rc -eq 0 ] && [ -n "$wf" ] && grep -qxF -- "features: $wf" <<<"$out"; then echo "ok    a step that passes runs, with the derived features"
else echo "FAIL  a passing step: exit $rc, or not given WAI_FEATURES=$wf"; printf '%s\n' "$out" | tail -5 | sed 's/^/      /'; fails=$((fails+1)); fi
out=$(python3 -I "$L" --root "$T" run --only aa-passes --only ab-fails 2>&1); rc=$?
if [ $rc -eq 3 ] && grep -qF "::error::fails failed with exit status 3 (.github/wai-conformance/verifiers/ab-fails.sh)" <<<"$out"; then
  echo "ok    a step that fails stops the run with its status, naming its file"
else
  echo "FAIL  a failing step: exit $rc"; printf '%s\n' "$out" | tail -5 | sed 's/^/      /'; fails=$((fails+1))
fi
printf '# step: pipe\n# order: 3\nfalse | true\n' > "$V/ac-pipe.sh"
expect 1 "a step whose pipe fails fails, as under a workflow's bash" "" -- run --only ac-pipe
expect 1 "a step --only names that no file holds" "names a step no file holds" -- run --only no-such-step

if [ $fails -ne 0 ]; then echo "wai-conformance-lists-selftest: $fails failure(s)"; exit 1; fi
echo "wai-conformance-lists-selftest: the derived lists hold the hand lists, and fail each way they can lose an entry."