open-standards/scripts/check-wai-release-quickstart-selftest.sh

#!/usr/bin/env bash
#
# check-wai-release-quickstart-selftest.sh — check-wai-release-quickstart.py can fail.
#
# A check that cannot fail is decoration. This copies the release workflow and the two
# quickstart scripts into a scratch tree, asserts that the check passes them as they are, then
# breaks each rule in a fresh copy and asserts that the check fails, saying which: a job that
# does not run its script, or runs it only in a comment; a step that runs it gated by `if:` or
# allowed to fail by `continue-on-error:`, or a job that is; the archive attached before it is
# run; the twin's steps or commands not the script's; and either unpack step that no longer
# checks the archive's SHA-256, or checks it after unpacking.
set -uo pipefail

here=$(cd "$(dirname "$0")" && pwd)
repo=$(cd "$here/.." && pwd)
check="$here/check-wai-release-quickstart.py"
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
wf="$tmp/.github/workflows/wai-release.yml"
sh="$tmp/wai/quickstart/test/clean-machine.sh"
ps="$tmp/wai/quickstart/test/clean-machine.ps1"

mk() {
  rm -rf "$tmp/.github" "$tmp/wai"
  mkdir -p "$tmp/.github/workflows" "$tmp/wai/quickstart/test"
  cp "$repo/.github/workflows/wai-release.yml" "$wf"
  cp "$repo/wai/quickstart/test/clean-machine.sh" "$sh"
  cp "$repo/wai/quickstart/test/clean-machine.ps1" "$ps"
}

fails=0
expect() { # expect <pass|fail> <what> [text the output must hold]
  local want=$1 what=$2 needle=${3:-} out rc
  out=$(python3 -I "$check" --root "$tmp" 2>&1); rc=$?
  if [ "$want" = pass ] && [ $rc -ne 0 ]; then echo "FAIL  $what: the check failed"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  if [ "$want" = fail ] && [ $rc -eq 0 ]; then echo "FAIL  $what: the check passed"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  if [ -n "$needle" ] && ! grep -qF -- "$needle" <<<"$out"; then echo "FAIL  $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  echo "ok    $what"
}
# sub <file> <from> <to> [n]: replace the n-th (default the only) occurrence, which must exist.
sub() {
  python3 - "$@" <<'P' || { echo "FAIL  the fixture edit did not apply: $2"; fails=$((fails+1)); }
import sys
p, a, b = sys.argv[1:4]
n = int(sys.argv[4]) if len(sys.argv) > 4 else 0
s = open(p, encoding="utf-8").read()
if n == 0 and s.count(a) != 1 or s.count(a) < max(n, 1):
    sys.exit(1)
i = -1
for _ in range(max(n, 1)):
    i = s.index(a, i + 1)
open(p, "w", encoding="utf-8").write(s[:i] + b + s[i + len(a):])
P
}

mk; expect pass "the release and the two scripts as they are" "run the quickstart from the archive before attaching it"

# The jobs: each runs its script, in a step's command, ungated, before it attaches.
mk; sub "$wf" '        run: bash wai/quickstart/test/clean-machine.sh "$RUNNER_TEMP/dist/wai-${{ needs.version.outputs.version }}-${{ matrix.triple }}.tar.gz"' '        run: echo the quickstart is not run'
expect fail "the linux job runs no quickstart" "the linux job runs clean-machine.sh in no step's command"

mk; sub "$wf" '          bash wai/quickstart/test/clean-machine.sh "$RUNNER_TEMP/dist/wai-$v-${{ matrix.triple }}.tar.gz" $flag' '          # bash wai/quickstart/test/clean-machine.sh "$RUNNER_TEMP/dist/wai-$v-${{ matrix.triple }}.tar.gz" $flag'
expect fail "the macos job names the script in a comment alone" "the macos job runs clean-machine.sh in no step's command"

mk; sub "$wf" '          pwsh -NoProfile -File wai\quickstart\test\clean-machine.ps1' '          pwsh -NoProfile -File wai\quickstart\test\clean-machine-other.ps1'
expect fail "the windows job runs another script" "the windows job runs clean-machine.ps1 in no step's command"

mk; sub "$wf" '      - name: The quickstart from the archive, as a new user runs it
        shell: bash
        env:' '      - name: The quickstart from the archive, as a new user runs it
        continue-on-error: true
        shell: bash
        env:'
expect fail "the macos step may fail without failing the job" "has \`continue-on-error:\`: it can be skipped or fail"

mk; sub "$wf" '      - name: The quickstart from the archive, as a new user runs it
        shell: pwsh' '      - name: The quickstart from the archive, as a new user runs it
        if: false
        shell: pwsh'
expect fail "the windows step is skipped" "which runs clean-machine.ps1, has \`if:\`"

mk; sub "$wf" '    name: Linux · ${{ matrix.triple }}
' '    name: Linux · ${{ matrix.triple }}
    continue-on-error: true
'
expect fail "the linux job may fail without failing the release" "the linux job has \`continue-on-error:\`"

mk; sub "$wf" '      - name: The quickstart from the archive, as a new user runs it
        shell: bash
        run: bash' '      - name: Attach first
        shell: bash
        run: bash wai/quickstart/publish.sh "$GITHUB_REF_NAME" "wai" "$RUNNER_TEMP"/dist/*
      - name: The quickstart from the archive, as a new user runs it
        shell: bash
        run: bash'
expect fail "the linux job attaches the archive before running it" "comes after 'Attach first', which attaches the archive"

mk; sub "$wf" '      - name: Attach to the release
        shell: pwsh' '      - name: Upload it
        shell: pwsh
        run: Invoke-RestMethod -Method Post -Uri "$env:FORGE_API/repos/x/releases/1/assets?name=a.zip"
      - name: Attach to the release
        shell: pwsh'
expect pass "an upload after the quickstart is in order"
mk; python3 - "$wf" <<'P'
import sys
p = sys.argv[1]
s = open(p, encoding="utf-8").read()
q = s.index("      - name: The quickstart from the archive, as a new user runs it\n        shell: pwsh")
a = s.index("      - name: Attach to the release\n        shell: pwsh")
e = s.index("\n  formulae:", a)
open(p, "w", encoding="utf-8").write(s[:q] + s[a:e] + s[q:a] + s[e:])
P
expect fail "the windows job attaches its archive before it runs it" "which runs clean-machine.ps1, comes after 'Attach to the release'"

# The twin keeps the script's steps and commands.
mk; sub "$ps" "Step 'create' { wai create my-room --template video-room }" "Step 'create' { wai create my-room }"
expect fail "the twin runs another command" "the twin runs clean-machine.sh's"

mk; sub "$ps" "if (-not \$Failed) { Step 'verify' { wai verify wai-hello } }" ""
expect fail "the twin drops a step" "the twin runs clean-machine.sh's"

mk; sub "$sh" '[ -z "$failed" ] && step verify wai verify wai-hello' '[ -z "$failed" ] && step check wai verify wai-hello'
expect fail "the script's steps change" "its steps are"

# Each unpack step checks the archive's SHA-256, before it unpacks.
mk; sub "$sh" '  [ "$got" = "$want" ] || { echo "SHA-256 $got, not $want"; return 1; }' '  [ "$got" = "$want" ] || echo "SHA-256 $got, not $want"'
expect fail "the script's unpack step does not fail on a SHA-256 that differs" "clean-machine.sh: the unpack step does not fail when"

mk; sub "$sh" 'want=$(cut -d'"' '"' -f1 "$ARCHIVE.sha256")' 'want=$got'
expect fail "the script's unpack step does not read the SHA-256 beside the archive" 'clean-machine.sh: the unpack step does not read the SHA-256 beside the archive ("$ARCHIVE.sha256")'

mk; sub "$ps" '  if ($got -ne $want) { throw "SHA-256 $got, not $want" }' '  if ($got -ne $want) { Write-Output "SHA-256 $got, not $want" }'
expect fail "the twin's unpack step does not fail on a SHA-256 that differs" "clean-machine.ps1: the unpack step does not fail when"

mk; python3 - "$ps" <<'P'
import sys
p = sys.argv[1]
s = open(p, encoding="utf-8").read()
c = '  if ($got -ne $want) { throw "SHA-256 $got, not $want" }\n'
s = s.replace(c, "")
s = s.replace("    tar -xzf $Archive -C $Work\n  }\n", "    tar -xzf $Archive -C $Work\n  }\n" + c, 1)
open(p, "w", encoding="utf-8").write(s)
P
expect fail "the twin checks the SHA-256 after unpacking" "clean-machine.ps1: the unpack step unpacks the archive before it checks its SHA-256"

if [ "$fails" -ne 0 ]; then
  echo "check-wai-release-quickstart-selftest: $fails case(s) failed"
  exit 1
fi
echo "check-wai-release-quickstart-selftest: every case held"