open-standards/scripts/check-tests-in-ci-selftest.sh

#!/usr/bin/env bash
#
# check-tests-in-ci-selftest.sh — check-tests-in-ci.py can fail.
#
# A check that cannot fail is decoration. This builds a scratch tree shaped like the
# repository (one workflow, one crate with a plain test, a feature-gated test and a
# test listed as not run) and asserts that the check passes it, and fails each way the
# property can break, saying which.
set -uo pipefail

here=$(cd "$(dirname "$0")" && pwd)
check="$here/check-tests-in-ci.py"
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
C="$tmp/wai/wai-rs"

mk() {
  rm -rf "$tmp/.github" "$tmp/wai"
  mkdir -p "$tmp/.github/workflows" "$C/tests"
  printf 'jobs:\n  t:\n    steps:\n      - run: cargo test --features f --test alpha --test gated\n' > "$tmp/.github/workflows/w.yml"
  printf '[package]\nname = "x"\n\n[features]\nf = []\ng = []\n\n[[test]]\nname = "gated"\npath = "tests/gated.rs"\nrequired-features = ["f"]\n' > "$C/Cargo.toml"
  printf '#[test]\nfn a() {}\n' > "$C/tests/alpha.rs"
  printf '#![cfg(feature = "f")]\n#[test]\nfn g() {}\n' > "$C/tests/gated.rs"
  printf '#[test]\nfn l() {}\n' > "$C/tests/local.rs"
  printf '# name, then why no job runs it\nlocal  needs a device no runner has\n' > "$C/tests/NOT-IN-CI.txt"
}

fails=0
expect() { # expect <pass|fail> <what> [text the output must hold]
  local want=$1 what=$2 needle=${3:-} out rc
  out=$(python3 -I "$check" --root "$tmp" 2>&1); rc=$?
  if [ "$want" = pass ] && [ $rc -ne 0 ]; then echo "FAIL  $what: the check failed"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  if [ "$want" = fail ] && [ $rc -eq 0 ]; then echo "FAIL  $what: the check passed"; fails=$((fails+1)); return; fi
  if [ -n "$needle" ] && ! grep -q -- "$needle" <<<"$out"; then echo "FAIL  $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  echo "ok    $what"
}

mk; expect pass "a tree where every test is run or listed passes"

mk; printf '#[test]\nfn b() {}\n' > "$C/tests/beta.rs"
expect fail "a test file no step names and nothing lists" "beta.rs: no workflow step names it"

mk; printf '[package]\nname = "x"\n\n[features]\nf = []\n' > "$C/Cargo.toml"
expect fail "a feature-gated test with no required-features" "runs it empty"

mk; sed -i.bak 's/required-features = \["f"\]/required-features = ["g"]/' "$C/Cargo.toml"
expect fail "required-features that are not the file's features" "is gated on"

mk; printf '#![cfg(any(feature = "f", feature = "g"))]\n#[test]\nfn g() {}\n' > "$C/tests/gated.rs"
expect fail "a gate the check cannot read" "cannot tell what it needs"

mk; printf 'alpha  no reason at all, a step runs it\n' >> "$C/tests/NOT-IN-CI.txt"
expect fail "a test listed as not run that a step names" "remove the line"

mk; printf 'ghost  it was deleted\n' >> "$C/tests/NOT-IN-CI.txt"
expect fail "a listed name that is no test file" "ghost is not a test file"

mk; printf 'local\n' > "$C/tests/NOT-IN-CI.txt"
expect fail "a listed test with no reason" "not .name  reason."

mk; printf '\n[[test]]\nname = "gone"\npath = "tests/gone.rs"\n' >> "$C/Cargo.toml"
expect fail "a [[test]] entry for a file that does not exist" "which does not exist"

# A verifier step's own file names a test as a workflow does.
mk; printf '#[test]\nfn b() {}\n' > "$C/tests/beta.rs"
mkdir -p "$tmp/.github/wai-conformance/verifiers"
printf '# step: beta\n# order: 10\ncargo test --no-default-features --test beta\n' > "$tmp/.github/wai-conformance/verifiers/beta.sh"
expect pass "a test a verifier step's file names"
rm "$tmp/.github/wai-conformance/verifiers/beta.sh"
expect fail "and the same test once the file is gone" "beta.rs: no workflow step names it"

# A tag a step takes names every test file that carries it, and only those.
mk; printf '// wai-conformance: int-kernels\n#[test]\nfn b() {}\n' > "$C/tests/beta.rs"
printf '      - run: python3 ../../scripts/wai-conformance-lists.py test-tagged int-kernels -- --lib\n' >> "$tmp/.github/workflows/w.yml"
expect pass "a test that carries a tag a step takes"
mk; printf '// wai-conformance: int-kernels-amd\n#[test]\nfn b() {}\n' > "$C/tests/beta.rs"
printf '      - run: python3 ../../scripts/wai-conformance-lists.py test-tagged int-kernels -- --lib\n' >> "$tmp/.github/workflows/w.yml"
expect fail "a test whose tag no step takes" "or takes a tag it carries"
mk; printf '// wai-conformance: int-kernels\n#[test]\nfn l() {}\n' > "$C/tests/local.rs"
printf '      - run: T=$(python3 ../../scripts/wai-conformance-lists.py tests int-kernels)\n' >> "$tmp/.github/workflows/w.yml"
expect fail "a test listed as not run that a step takes by its tag" "remove the line"

if [ $fails -ne 0 ]; then echo "check-tests-in-ci-selftest: $fails failure(s)"; exit 1; fi
echo "check-tests-in-ci-selftest: the check passes a covered tree and fails each uncovered one."