open-standards/scripts/check-wasmtime-pin-selftest.sh

#!/usr/bin/env bash
#
# check-wasmtime-pin-selftest.sh — check-wasmtime-pin.py can fail.
#
# A check that cannot fail is decoration. This builds a scratch tree with one workflow
# that installs the pinned wasmtime and runs it the ways the repository's workflows do,
# asserts that the check passes it, and then breaks it each way the property can break
# and asserts that the check fails, saying why.
set -uo pipefail

here=$(cd "$(dirname "$0")" && pwd)
check="$here/check-wasmtime-pin.py"
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
W="$tmp/.github/workflows/w.yml"

mk() {
  rm -rf "$tmp/.github"
  mkdir -p "$tmp/.github/workflows"
  cat > "$W" <<'YML'
jobs:
  t:
    steps:
      # Every run passes -W nan-canonicalization=y -W relaxed-simd-deterministic=y.
      - name: Install wasmtime 50.0.1
        run: |
          v=50.0.1
          f="wasmtime-v$v-x86_64-linux.tar.xz"
          curl -sSfL -o "$RUNNER_TEMP/$f" \
            "https://github.com/bytecodealliance/wasmtime/releases/download/v$v/$f"
          echo "685466455dd42a14774a86c36c8f2e45cbb9bd6d92a850139315df60a75498a5  $RUNNER_TEMP/$f" | sha256sum -c -
      - name: tests under wasmtime
        env:
          CARGO_TARGET_WASM32_WASIP2_RUNNER: wasmtime run -W nan-canonicalization=y -W relaxed-simd-deterministic=y
        run: cargo test --target wasm32-wasip2
      - name: the corpus under wasmtime
        run: |
          W=$(pwd)
          CARGO_TARGET_WASM32_WASIP2_RUNNER="wasmtime run -W nan-canonicalization=y -W relaxed-simd-deterministic=y --dir $W::$W" \
            cargo test --release --target wasm32-wasip2
          wasmtime --version
          wasmtime run -W nan-canonicalization=y -W relaxed-simd-deterministic=y --dir .::/w \
            x.wasm verify /w
          wasmtime run \
            -W nan-canonicalization=y -W relaxed-simd-deterministic=y x.wasm
          cmp a b || { echo "::error::the output differs under wasmtime"; exit 1; }
YML
}

rep() { # rep <old> <new>: replace the first occurrence in the scratch workflow
  python3 -I -c 'import sys; p, a, b = sys.argv[1:]; s = open(p).read(); assert a in s, a; open(p, "w").write(s.replace(a, b, 1))' "$W" "$1" "$2"
}

fails=0
expect() { # expect <pass|fail> <what> [text the output must hold]
  local want=$1 what=$2 needle=${3:-} out rc
  out=$(python3 -I "$check" --root "$tmp" 2>&1); rc=$?
  if [ "$want" = pass ] && [ $rc -ne 0 ]; then echo "FAIL  $what: the check failed"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  if [ "$want" = fail ] && [ $rc -eq 0 ]; then echo "FAIL  $what: the check passed"; fails=$((fails+1)); return; fi
  if [ -n "$needle" ] && ! grep -qF -- "$needle" <<<"$out"; then echo "FAIL  $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  echo "ok    $what"
}

mk; expect pass "the pinned release, run every way with both settings, passes"
mk; mkdir -p "$tmp/wai/c"; printf '[features]\nwasmtime = ["dep:wasmtime"]\n[target.x.dependencies]\nwasmtime = { version = "=50.0.1", default-features = false }\n' > "$tmp/wai/c/Cargo.toml"
  expect pass "a crate that pins the release exactly passes"
mk; printf '[dependencies]\nwasmtime = { version = "50", default-features = false }\n' > "$tmp/wai/c/Cargo.toml"
  expect fail "a crate on a caret range of the release fails" 'a crate pins the release the workflows run'
mk; printf '[dependencies]\nwasmtime = "=49.0.2"\n' > "$tmp/wai/c/Cargo.toml"
  expect fail "a crate on another release fails" 'depends on wasmtime =49.0.2'
rm -rf "$tmp/wai"

mk; rep "v=50.0.1" "v=49.0.2"
expect fail "another version" "downloads wasmtime 49.0.2"

mk; rep '/download/v$v/$f"' '/download/v38.0.3/wasmtime-v38.0.3-x86_64-linux.tar.xz"'; rep "v=50.0.1" ":"
expect fail "another version, named in the URL" "downloads wasmtime 38.0.3"

mk; rep "685466455dd4" "685466455dd5"
expect fail "another digest" "not its pinned digest"

mk; rep '| sha256sum -c -' '> /dev/null'
expect fail "no digest checked" "checks no SHA-256"

mk; rep "x86_64-linux.tar.xz" "aarch64-linux.tar.xz"
expect fail "an archive with no pinned digest" "has no pinned digest here"

mk; rep "    steps:" "    steps:
      - run: cargo install wasmtime-cli --locked"
expect fail "wasmtime built from source" "installs wasmtime some other way"

mk; rep "RUNNER: wasmtime run -W nan-canonicalization=y -W relaxed-simd-deterministic=y" "RUNNER: wasmtime"
expect fail "a cargo runner with neither setting" "runs wasmtime without -W nan-canonicalization=y and -W relaxed-simd-deterministic=y"

mk; rep '--dir $W::$W"' '--dir $W::$W -W nan-canonicalization=y"'; rep 'RUNNER="wasmtime run -W nan-canonicalization=y -W relaxed-simd-deterministic=y' 'RUNNER="wasmtime run'
expect fail "a runner with one setting" "runs wasmtime without -W relaxed-simd-deterministic=y"

mk; rep "-W relaxed-simd-deterministic=y --dir .::/w" "--dir .::/w"
expect fail "a run without deterministic relaxed SIMD" "runs wasmtime without -W relaxed-simd-deterministic=y"

mk; rep "            -W nan-canonicalization=y -W relaxed-simd-deterministic=y x.wasm" "            x.wasm"
expect fail "a run continued over two lines, with neither setting" "runs wasmtime without"

mk; rep "wasmtime --version" "wasmtime x.wasm"
expect fail "a module given to wasmtime with no subcommand" "runs wasmtime without"

mk; rep "wasmtime --version" "wasmtime serve -W nan-canonicalization=y app.wasm"
expect fail "a serve without deterministic relaxed SIMD" "runs wasmtime without -W relaxed-simd-deterministic=y"

mk; printf 'jobs:\n  t:\n    steps:\n      - run: true\n' > "$W"
expect fail "no workflow that runs wasmtime" "looked in the wrong place"

if [ $fails -ne 0 ]; then echo "check-wasmtime-pin-selftest: $fails failure(s)"; exit 1; fi
echo "check-wasmtime-pin-selftest: the check passes the pinned, flagged runs and fails each other."