open-standards/wai/wai-sink-swift/scripts/test-simulator.sh

#!/usr/bin/env bash
# Run the package's tests in a simulator, hosted in TestHost/ as on a device, with every step bounded.
#
#   scripts/test-simulator.sh ios|tvos
#
# A fresh simulator is created for the run (the newest installed runtime of the platform, its first
# device type of the family), booted, and shut down and deleted at the end, however the run ends.
# The test host is built for it (`build-for-testing`) and the tests run in it
# (`test-without-building`). Each step has a hard limit and fails, saying which, when it passes it:
# a step that does not finish is a failure, never a wait. Needs build/WaiFFI.xcframework with the
# platform's simulator slice (scripts/build-xcframework.sh).
#
# Limits, in seconds, from the environment: WAI_SIM_BOOT_S (300), WAI_SIM_BUILD_S (900),
# WAI_SIM_TEST_S (600).
set -uo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")/.."
plat="${1:-}"
case "$plat" in
  ios) family="iPhone"; runtime_prefix="com.apple.CoreSimulator.SimRuntime.iOS-"; dest="iOS Simulator"; slice="ios-arm64_x86_64-simulator" ;;
  tvos) family="Apple TV"; runtime_prefix="com.apple.CoreSimulator.SimRuntime.tvOS-"; dest="tvOS Simulator"; slice="tvos-arm64-simulator" ;; # allow-name: the simulator device family simctl lists, by its name
  *) echo "usage: test-simulator.sh ios|tvos" >&2; exit 2 ;;
esac
BOOT_S="${WAI_SIM_BOOT_S:-300}"; BUILD_S="${WAI_SIM_BUILD_S:-900}"; TEST_S="${WAI_SIM_TEST_S:-600}"
mkdir -p .build
LOGS="$(pwd)/.build/sim-$plat"; rm -rf "$LOGS"; mkdir -p "$LOGS"
DD="$(pwd)/.build/xc-host-$plat"

# bounded <seconds> <what> <command…>: the command's status, or 142 with an error when the limit
# passes. The alarm survives exec, so the command itself is ended at the limit.
bounded() {
  local secs="$1" what="$2"; shift 2
  perl -e 'alarm shift @ARGV; exec @ARGV or die "exec: $!"' "$secs" "$@"
  local rc=$?
  if [ $rc -eq 142 ]; then echo "::error::$what did not finish within ${secs} s; the run is failed, not waited on" >&2; fi
  return $rc
}

[ -d "build/WaiFFI.xcframework/$slice" ] || { echo "::error::build/WaiFFI.xcframework has no $slice slice: run scripts/build-xcframework.sh" >&2; exit 1; }
cp ../live-conformance/native-sink/vectors.json Tests/WaiSinkTests/Resources/vectors.json
# The integer decode's vectors (IntDecodeTests, native-sink §13), which scripts/build-xcframework.sh
# stages: the host's test bundle carries the folder, so the decode runs in the simulator as on a device.
[ -f Tests/WaiSinkTests/Resources/int/cases.json ] || { echo "::error::Tests/WaiSinkTests/Resources/int is not staged: run scripts/build-xcframework.sh" >&2; exit 1; }

# The newest installed runtime of the platform, and its first device type of the family.
read -r RUNTIME DEVTYPE < <(bounded 60 "listing simulator runtimes" xcrun simctl list -j runtimes available 2>/dev/null | python3 -c '
import json, sys
prefix, family = sys.argv[1], sys.argv[2]
rts = [r for r in json.load(sys.stdin)["runtimes"] if r.get("isAvailable") and r["identifier"].startswith(prefix)]
rts.sort(key=lambda r: [int(x) for x in r["version"].split(".")])
if rts:
    r = rts[-1]
    dts = [d for d in r.get("supportedDeviceTypes", []) if d.get("productFamily") == family]
    if dts:
        print(r["identifier"], dts[0]["identifier"])
' "$runtime_prefix" "$family")
[ -n "${RUNTIME:-}" ] && [ -n "${DEVTYPE:-}" ] || { echo "::error::no available $plat simulator runtime with a $family device type" >&2; exit 1; }

UDID=$(bounded 120 "creating the simulator" xcrun simctl create "wai-sink-test-$plat-$$" "$DEVTYPE" "$RUNTIME" 2>"$LOGS/create.err")
[ -n "$UDID" ] || { echo "::error::could not create a $plat simulator: $(head -2 "$LOGS/create.err")" >&2; exit 1; }
cleanup() {
  perl -e 'alarm 120; exec @ARGV' xcrun simctl shutdown "$UDID" >/dev/null 2>&1
  perl -e 'alarm 120; exec @ARGV' xcrun simctl delete "$UDID" >/dev/null 2>&1
  echo "simulator $UDID shut down and deleted"
}
trap cleanup EXIT
echo "simulator $UDID: $DEVTYPE on $RUNTIME"

t0=$(date +%s)
bounded "$BOOT_S" "booting the simulator" xcrun simctl boot "$UDID" > "$LOGS/boot.log" 2>&1
bounded "$BOOT_S" "waiting for the simulator to finish booting" xcrun simctl bootstatus "$UDID" -b > "$LOGS/bootstatus.log" 2>&1 \
  || { echo "::error::the simulator did not boot: $(tail -2 "$LOGS/bootstatus.log")" >&2; exit 1; }
t1=$(date +%s)
echo "booted in $((t1 - t0)) s"

rm -rf "$DD"
bounded "$BUILD_S" "building the test host for the $plat simulator" \
  xcodebuild build-for-testing -project TestHost/WaiSinkTestHost.xcodeproj -scheme WaiSinkTestHost \
  -destination "platform=$dest,id=$UDID" -derivedDataPath "$DD" > "$LOGS/build.log" 2>&1
rc=$?
grep -E "\*\* .* \*\*|error:" "$LOGS/build.log" | sort -u | cut -c1-300 | head -20
[ $rc -eq 0 ] || { echo "::error::the test host did not build for the $plat simulator (exit $rc)" >&2; exit 1; }
t2=$(date +%s)
echo "built in $((t2 - t1)) s"

XR=$(ls "$DD"/Build/Products/*.xctestrun | head -1)
bounded "$TEST_S" "the tests in the $plat simulator" \
  xcodebuild test-without-building -xctestrun "$XR" -destination "platform=$dest,id=$UDID" \
  -resultBundlePath "$LOGS/result.xcresult" > "$LOGS/test.log" 2>&1
rc=$?
t3=$(date +%s)
grep -E "WAI-SINK|Test Case .*(passed|failed)|error:|Executed [0-9]+ tests?|\*\* TEST" "$LOGS/test.log" | cut -c1-260
echo "tests ran in $((t3 - t2)) s (exit $rc); boot, build and tests $((t3 - t0)) s"
# A run passes when xcodebuild exits 0 and said every test passed.
[ $rc -eq 0 ] && grep -qE "Executed [1-9][0-9]* tests?, with 0 failures" "$LOGS/test.log" \
  || { echo "::error::the $plat simulator tests did not pass" >&2; exit 1; }