open-standards/scripts/check-wai-release-quickstart-selftest.sh

#!/usr/bin/env bash
#
# check-wai-release-quickstart-selftest.sh — check-wai-release-quickstart.py can fail.
#
# A check that cannot fail is decoration. This copies the release workflow and the two
# quickstart scripts into a scratch tree, asserts that the check passes them as they are, then
# breaks each rule in a fresh copy and asserts that the check fails, saying which: a job that
# does not run its script, or runs it only in a comment or an echo; a step that runs it gated by
# `if:` or allowed to fail by `continue-on-error:`, or a job that is; a step that ignores its
# failure (`|| true`, `|| :`, `; exit 0`, no throw after it on Windows, another line after it),
# never reaches it or runs it in a block; the archive attached before it is run; a macOS job that
# runs npm with node off its PATH; the twin's steps or commands not the script's; either unpack
# step that no longer checks the archive's SHA-256, or checks it after unpacking; either
# contents step that looks for a binary and does not run it, or runs it and passes though it
# fails; a step that ends before the script but for the macos job's cross-built archive (the
# test inverted, its else, or the linux job's); and either script whose exit status is not
# whether a step failed (its last line gone or `|| true`, a step that does not record its
# failure, `failed` set again, a trap, an exit with 0 or with no status before its end).
set -uo pipefail

here=$(cd "$(dirname "$0")" && pwd)
repo=$(cd "$here/.." && pwd)
check="$here/check-wai-release-quickstart.py"
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
wf="$tmp/.github/workflows/wai-release.yml"
sh="$tmp/wai/quickstart/test/clean-machine.sh"
ps="$tmp/wai/quickstart/test/clean-machine.ps1"

mk() {
  rm -rf "$tmp/.github" "$tmp/wai"
  mkdir -p "$tmp/.github/workflows" "$tmp/wai/quickstart/test"
  cp "$repo/.github/workflows/wai-release.yml" "$wf"
  cp "$repo/wai/quickstart/test/clean-machine.sh" "$sh"
  cp "$repo/wai/quickstart/test/clean-machine.ps1" "$ps"
}

fails=0
expect() { # expect <pass|fail> <what> [text the output must hold]
  local want=$1 what=$2 needle=${3:-} out rc
  out=$(python3 -I "$check" --root "$tmp" 2>&1); rc=$?
  if [ "$want" = pass ] && [ $rc -ne 0 ]; then echo "FAIL  $what: the check failed"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  if [ "$want" = fail ] && [ $rc -eq 0 ]; then echo "FAIL  $what: the check passed"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  if [ -n "$needle" ] && ! grep -qF -- "$needle" <<<"$out"; then echo "FAIL  $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  echo "ok    $what"
}
# sub <file> <from> <to> [n]: replace the n-th (default the only) occurrence, which must exist.
sub() {
  python3 - "$@" <<'P' || { echo "FAIL  the fixture edit did not apply: $2"; fails=$((fails+1)); }
import sys
p, a, b = sys.argv[1:4]
n = int(sys.argv[4]) if len(sys.argv) > 4 else 0
s = open(p, encoding="utf-8").read()
if n == 0 and s.count(a) != 1 or s.count(a) < max(n, 1):
    sys.exit(1)
i = -1
for _ in range(max(n, 1)):
    i = s.index(a, i + 1)
open(p, "w", encoding="utf-8").write(s[:i] + b + s[i + len(a):])
P
}

mk; expect pass "the release and the two scripts as they are" "run the quickstart from the archive before attaching it"

# The jobs: each runs its script, in a step's command, ungated, before it attaches.
mk; sub "$wf" '        run: bash wai/quickstart/test/clean-machine.sh "$RUNNER_TEMP/dist/wai-${{ needs.version.outputs.version }}-${{ matrix.triple }}.tar.gz"' '        run: echo the quickstart is not run'
expect fail "the linux job runs no quickstart" "the linux job runs clean-machine.sh in no step's command"

mk; sub "$wf" '          bash wai/quickstart/test/clean-machine.sh "$RUNNER_TEMP/dist/wai-$v-${{ matrix.triple }}.tar.gz" $flag' '          # bash wai/quickstart/test/clean-machine.sh "$RUNNER_TEMP/dist/wai-$v-${{ matrix.triple }}.tar.gz" $flag'
expect fail "the macos job names the script in a comment alone" "the macos job runs clean-machine.sh in no step's command"

mk; sub "$wf" '          pwsh -NoProfile -File wai\quickstart\test\clean-machine.ps1' '          pwsh -NoProfile -File wai\quickstart\test\clean-machine-other.ps1'
expect fail "the windows job runs another script" "the windows job runs clean-machine.ps1 in no step's command"

# A step that names the script, or runs it, and passes though it fails.
LINUX_RUN='bash wai/quickstart/test/clean-machine.sh "$RUNNER_TEMP/dist/wai-${{ needs.version.outputs.version }}-${{ matrix.triple }}.tar.gz"'
MACOS_RUN='          bash wai/quickstart/test/clean-machine.sh "$RUNNER_TEMP/dist/wai-$v-${{ matrix.triple }}.tar.gz" $flag'
WINDOWS_THROW='          if ($LASTEXITCODE -ne 0) { throw "the quickstart failed from $name.zip" }
'
mk; sub "$wf" "$MACOS_RUN" '          echo skipped: wai/quickstart/test/clean-machine.sh'
expect fail "the macos job only echoes the script's path" "the macos job runs clean-machine.sh in no step's command"

mk; sub "$wf" "$LINUX_RUN" 'true || bash wai/quickstart/test/clean-machine.sh'
expect fail "the linux job's line runs something else first" "the linux job runs clean-machine.sh in no step's command"

mk; sub "$wf" "$LINUX_RUN" "$LINUX_RUN || true"
expect fail "the linux step ignores the script's failure with || true" "can pass though it fails: its line goes on after it ("

mk; sub "$wf" "$MACOS_RUN" "$MACOS_RUN || :"
expect fail "the macos step ignores the script's failure with || :" "can pass though it fails: its line goes on after it ("

mk; sub "$wf" "$LINUX_RUN" "$LINUX_RUN; exit 0"
expect fail "the linux step ends with ; exit 0" "can pass though it fails: its line goes on after it ("

mk; sub "$wf" "$MACOS_RUN" "$MACOS_RUN
          echo done"
expect fail "the macos step's last command is another" "\`echo done\` follows it, so the step's exit status is not its own"

mk; sub "$wf" "$MACOS_RUN" "          exit 0
$MACOS_RUN"
expect fail "the macos step ends before it reaches the script" "\`exit 0\` ends the step before it"

mk; sub "$wf" "$MACOS_RUN" "          if false; then
$MACOS_RUN
          fi"
expect fail "the macos step runs the script in a block" "it runs inside a block"

# The macos step ends before the script for a cross-built archive alone.
CROSS='          if [ "${{ matrix.triple }}" != "$(rustc -vV | sed -n '"'s/^host: //p'"')" ]; then'
mk; sub "$wf" "$CROSS" '          if [ "${{ matrix.triple }}" = "$(rustc -vV | sed -n '"'s/^host: //p'"')" ]; then'
expect fail "the macos step skips the host's own archive (the cross-built test inverted)" "\`exit 0\` ends the step before it, under \`if [ \"\${{ matrix.triple }}\" = "
mk; sub "$wf" "$CROSS
            echo \"cross-built for \${{ matrix.triple }}: this host cannot run it\"
            exit 0
          fi" "$CROSS
            echo \"cross-built for \${{ matrix.triple }}: this host cannot run it\"
          else
            exit 0
          fi"
expect fail "the macos step skips in the cross-built test's else" "\`exit 0\` ends the step before it, under \`else\`"
mk; sub "$wf" "$CROSS
            echo \"cross-built for \${{ matrix.triple }}: this host cannot run it\"
            exit 0
          fi" "$CROSS
            echo \"cross-built for \${{ matrix.triple }}: this host cannot run it\"
          else exit 0
          fi"
expect fail "the macos step skips in the cross-built test's else, on the else's line" "\`else exit 0\` ends the step before it, under \`else exit 0\`"
mk; sub "$wf" "$MACOS_RUN" '          if [ "\${{ matrix.triple }}" = "\$(rustc -vV | sed -n '"'s/^host: //p'"')" ]; then exit 0; fi
'"$MACOS_RUN"
expect fail "the macos step skips the host's own archive in one line" "ends the step before it"
mk; sub "$wf" "        run: $LINUX_RUN" "        run: |
          if [ \"\${{ matrix.triple }}\" != \"\$(rustc -vV | sed -n 's/^host: //p')\" ]; then
            exit 0
          fi
          $LINUX_RUN"
expect fail "the linux step skips a cross-built archive, which only the macos job may" "the linux job's step 'The quickstart from the archive, as a new user runs it', which runs clean-machine.sh, can pass though it fails: \`exit 0\` ends the step before it, under"

mk; sub "$wf" "$WINDOWS_THROW" ''
expect fail "the windows step does not throw on the script's exit status" "the line after it is not \`if (\$LASTEXITCODE -ne 0) { throw ... }\`"

# The macOS job has node on its PATH before the step that runs npm.
mk; sub "$wf" '          echo "/opt/homebrew/bin" >> "$GITHUB_PATH"
          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
      - name: Build, check and package' '          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
      - name: Build, check and package'
expect fail "the macos job runs npm and leaves node off its PATH" "the macos job's step 'The quickstart from the archive, as a new user runs it' runs node or npm, and no step before it puts /opt/homebrew/bin"

mk; python3 - "$wf" <<'P'
import sys
p = sys.argv[1]
s = open(p, encoding="utf-8").read()
a = '          echo "/opt/homebrew/bin" >> "$GITHUB_PATH"\n          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"\n      - name: Build, check and package\n'
assert s.count(a) == 1
s = s.replace(a, '          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"\n      - name: Build, check and package\n')
a = '      - name: Attach to the release\n        shell: bash\n        env:\n          FORGE_TOKEN: ${{ github.token }}\n        run: bash wai/quickstart/publish.sh "$GITHUB_REF_NAME" "wai ${{ needs.version.outputs.version }}" "$RUNNER_TEMP"/dist/*\n\n  linux:'
assert s.count(a) == 1
s = s.replace(a, '      - name: node, too late\n        run: echo "/opt/homebrew/bin" >> "$GITHUB_PATH"\n' + a)
open(p, "w", encoding="utf-8").write(s)
P
expect fail "the macos job puts node on its PATH only after the step that runs npm" "no step before it puts /opt/homebrew/bin"

mk; sub "$wf" '      - name: The quickstart from the archive, as a new user runs it
        shell: bash
        env:' '      - name: The quickstart from the archive, as a new user runs it
        continue-on-error: true
        shell: bash
        env:'
expect fail "the macos step may fail without failing the job" "has \`continue-on-error:\`: it can be skipped or fail"

mk; sub "$wf" '      - name: The quickstart from the archive, as a new user runs it
        shell: pwsh' '      - name: The quickstart from the archive, as a new user runs it
        if: false
        shell: pwsh'
expect fail "the windows step is skipped" "which runs clean-machine.ps1, has \`if:\`"

mk; sub "$wf" '    name: Linux · ${{ matrix.triple }}
' '    name: Linux · ${{ matrix.triple }}
    continue-on-error: true
'
expect fail "the linux job may fail without failing the release" "the linux job has \`continue-on-error:\`"

mk; sub "$wf" '      - name: The quickstart from the archive, as a new user runs it
        shell: bash
        run: bash' '      - name: Attach first
        shell: bash
        run: bash wai/quickstart/publish.sh "$GITHUB_REF_NAME" "wai" "$RUNNER_TEMP"/dist/*
      - name: The quickstart from the archive, as a new user runs it
        shell: bash
        run: bash'
expect fail "the linux job attaches the archive before running it" "comes after 'Attach first', which attaches the archive"

mk; sub "$wf" '      - name: Attach to the release
        shell: pwsh' '      - name: Upload it
        shell: pwsh
        run: Invoke-RestMethod -Method Post -Uri "$env:FORGE_API/repos/x/releases/1/assets?name=a.zip"
      - name: Attach to the release
        shell: pwsh'
expect pass "an upload after the quickstart is in order"
mk; python3 - "$wf" <<'P'
import sys
p = sys.argv[1]
s = open(p, encoding="utf-8").read()
q = s.index("      - name: The quickstart from the archive, as a new user runs it\n        shell: pwsh")
a = s.index("      - name: Attach to the release\n        shell: pwsh")
e = s.index("\n  formulae:", a)
open(p, "w", encoding="utf-8").write(s[:q] + s[a:e] + s[q:a] + s[e:])
P
expect fail "the windows job attaches its archive before it runs it" "which runs clean-machine.ps1, comes after 'Attach to the release'"

# The twin keeps the script's steps and commands.
mk; sub "$ps" "Step 'create' { wai create my-room --template video-room }" "Step 'create' { wai create my-room }"
expect fail "the twin runs another command" "the twin runs clean-machine.sh's"

mk; sub "$ps" "if (-not \$Failed) { Step 'verify' { wai verify wai-hello } }" ""
expect fail "the twin drops a step" "the twin runs clean-machine.sh's"

mk; sub "$sh" '[ -z "$failed" ] && step verify wai verify wai-hello' '[ -z "$failed" ] && step check wai verify wai-hello'
expect fail "the script's steps change" "its steps are"

# Each unpack step checks the archive's SHA-256, before it unpacks.
mk; sub "$sh" '  [ "$got" = "$want" ] || { echo "SHA-256 $got, not $want"; return 1; }' '  [ "$got" = "$want" ] || echo "SHA-256 $got, not $want"'
expect fail "the script's unpack step does not fail on a SHA-256 that differs" "clean-machine.sh: the unpack step does not fail when"

mk; sub "$sh" 'want=$(cut -d'"' '"' -f1 "$ARCHIVE.sha256")' 'want=$got'
expect fail "the script's unpack step does not read the SHA-256 beside the archive" 'clean-machine.sh: the unpack step does not read the SHA-256 beside the archive ("$ARCHIVE.sha256")'

mk; sub "$ps" '  if ($got -ne $want) { throw "SHA-256 $got, not $want" }' '  if ($got -ne $want) { Write-Output "SHA-256 $got, not $want" }'
expect fail "the twin's unpack step does not fail on a SHA-256 that differs" "clean-machine.ps1: the unpack step does not fail when"

mk; python3 - "$ps" <<'P'
import sys
p = sys.argv[1]
s = open(p, encoding="utf-8").read()
c = '  if ($got -ne $want) { throw "SHA-256 $got, not $want" }\n'
s = s.replace(c, "")
s = s.replace("    tar -xzf $Archive -C $Work\n  }\n", "    tar -xzf $Archive -C $Work\n  }\n" + c, 1)
open(p, "w", encoding="utf-8").write(s)
P
expect fail "the twin checks the SHA-256 after unpacking" "clean-machine.ps1: the unpack step unpacks the archive before it checks its SHA-256"

# Each contents step runs what it can, and does not only look for it.
mk; sub "$sh" '  says_usage wai-pack 2 || return 1
' ''
expect fail "the script's contents step only looks for wai-pack" "clean-machine.sh: the contents step does not run wai-pack"

mk; sub "$sh" '  says_usage wai-remote-host 1
' '  true
'
expect fail "the script's contents step only looks for wai-remote-host" "clean-machine.sh: the contents step does not run wai-remote-host"

mk; sub "$ps" "@{ n = 'wai-pack'; c = 2 }, " ''
expect fail "the twin's contents step only looks for wai-pack" "clean-machine.ps1: the contents step does not run wai-pack"

mk; sub "$ps" ", @{ n = 'wai-remote-host'; c = 1 }" ''
expect fail "the twin's contents step only looks for wai-remote-host" "clean-machine.ps1: the contents step does not run wai-remote-host"

# Each contents step fails when a binary it runs does.
mk; sub "$sh" '  says_usage wai-pack 2 || return 1' '  says_usage wai-pack 2 || true'
expect fail "the script's contents step passes though wai-pack fails" "clean-machine.sh: the contents step's line \`says_usage wai-pack 2 || true\` does not end in \`|| return 1\`"

mk; sub "$sh" '  wai-quantum --version || return 1' '  wai-quantum --version'
expect fail "the script's contents step passes though wai-quantum fails" "clean-machine.sh: the contents step's line \`wai-quantum --version\` does not end in"

mk; sub "$sh" '  says_usage wai-remote-host 1
' '  says_usage wai-remote-host 1 || true
'
expect fail "the script's contents step ends in || true" "clean-machine.sh: the contents step's last line, which runs wai-remote-host, goes on after it"

mk; sub "$ps" '    if ($LASTEXITCODE -ne 0) { throw "wai-quantum --version exits $LASTEXITCODE" }
' ''
expect fail "the twin's contents step passes though wai-quantum fails" "clean-machine.ps1: the contents step's line \`wai-quantum --version\` is not followed by"

mk; sub "$ps" '        throw "$($u.n) alone exits $($p.ExitCode), not $($u.c) with its usage"' '        Write-Output "$($u.n) alone exits $($p.ExitCode), not $($u.c) with its usage"'
expect fail "the twin's contents step passes though a program it starts fails" "clean-machine.ps1: the contents step does not throw when a program it starts"

# Each script's exit status is whether a step failed.
mk; python3 - "$sh" <<'Q'
import sys
p = sys.argv[1]
s = open(p, encoding="utf-8").read()
assert s.endswith('\n[ -z "$failed" ]\n')
open(p, "w", encoding="utf-8").write(s[:-len('[ -z "$failed" ]\n')])
Q
expect fail "the script's last line, its exit status, is gone" "clean-machine.sh: its last line is not \`[ -z \"\$failed\" ]\`"

mk; sub "$sh" '
[ -z "$failed" ]
' '
[ -z "$failed" ] || true
'
expect fail "the script's last line is || true" "clean-machine.sh: its last line is not \`[ -z \"\$failed\" ]\`"

mk; sub "$sh" '    failed="$name"
' ''
expect fail "the script's step() no longer records a failure" "clean-machine.sh: step() does not set \`failed=\"\$name\"\`"

mk; sub "$sh" 'rm -rf "$ROOT"
' 'rm -rf "$ROOT"
failed=""
'
expect fail "the script sets failed again before its end" "clean-machine.sh: \`failed=\"\"\` sets \`failed\` outside step()"

mk; sub "$sh" 'cd "$ROOT/work"
' 'cd "$ROOT/work"
trap '"'exit 0'"' EXIT
'
expect fail "the script's trap sets its exit status" "clean-machine.sh: \`trap 'exit 0' EXIT\`: a trap can change"

mk; sub "$ps" 'if ($Failed) { exit 1 }
' ''
expect fail "the twin's exit status is no longer whether a step failed" "clean-machine.ps1: its last lines are not \`if (\$Failed) { exit 1 }\` and \`exit 0\`"

mk; sub "$ps" '    $script:Failed = $Name
' ''
expect fail "the twin's Step no longer records a failure" "clean-machine.ps1: Step does not set \`\$script:Failed\`"

mk; sub "$ps" '    if ($LASTEXITCODE -ne 0) { $ok = $false }
' ''
expect fail "the twin's Step passes a program that exits non-zero" "clean-machine.ps1: Step does not set \`\$script:Failed\`"

mk; sub "$ps" '$Total = ' '$Failed = $null
$Total = '
expect fail "the twin sets Failed again before its end" "clean-machine.ps1: \`\$Failed = \$null\` sets \`\$Failed\` outside Step"

# An exit with 0 or with no status before the end: the script ends there, a failed or unrun
# step passing.
mk; sub "$sh" '[ -z "$failed" ] && step contents contents
' 'exit 0
[ -z "$failed" ] && step contents contents
'
expect fail "the script exits 0 before its steps" "clean-machine.sh: \`exit 0\` can end the script with status 0"
mk; sub "$sh" '  says_usage wai-pack 2 || return 1
' '  says_usage wai-pack 2 || exit
'
expect fail "the script's contents step exits with no status" "clean-machine.sh: \`says_usage wai-pack 2 || exit\` can end the script with status 0"
mk; sub "$ps" '$Total = ' 'if ($Failed) { exit }
$Total = '
expect fail "the twin exits with no status before its end" "clean-machine.ps1: \`if (\$Failed) { exit }\` can end the script with status 0"
mk; sub "$ps" 'if ($Failed) { exit 1 }
' 'if ($Failed) { exit 0 }
'
expect fail "the twin's failed run exits 0" "clean-machine.ps1: \`if (\$Failed) { exit 0 }\` can end the script with status 0"

if [ "$fails" -ne 0 ]; then
  echo "check-wai-release-quickstart-selftest: $fails case(s) failed"
  exit 1
fi
echo "check-wai-release-quickstart-selftest: every case held"