open-standards/wai/http-conformance/falsify/run.sh

#!/usr/bin/env bash
# Falsify the HTTP corpus tool and its independent verifier.
#
# Patches: each patch here breaks one function of wai-rs. In one worktree of HEAD,
# built once, each patch is applied, the tree rebuilt incrementally, and
# `wai_http_vectors verify` must exit non-zero naming every case the patch's first line
# lists; then the patch is reversed and the tree must be clean again. The builds use
# the `falsify` profile (wai-rs/Cargo.toml): overflow checks and debug assertions on, as
# in the tests, so a patch is judged by the code's own checks, and incremental, so each
# patch rebuilds only what it touched. The unpatched tree must pass first.
#
# Mutations: each `http_verify.py --mutate NAME` must exit non-zero naming, as FAIL lines,
# every case mutations.txt lists for it; a crash, or a non-zero exit that names none, is
# a failure of the harness.
#
# The differential: wai-rs writes random envelopes' covered prefixes and head proofs,
# http_verify.py verifies every one, and refuses exactly the ten it changes one
# chaining-value byte in.
#
#   bash wai/http-conformance/falsify/run.sh [work-dir] [--patches I/N | --mutations I/N |
#        --differential | --check-shards 'patches I/N …' 'mutations I/N …' | patch-name…]
#
# With no selector, all three run. `--patches I/N` runs the patches whose index in the
# sorted list, mod N, is I; `--mutations I/N` the same over the mutations. Each shard
# prints every patch or mutation it ran, and fails unless what it ran is exactly what was
# assigned to it. `--check-shards` takes the shards a workflow declares and fails unless
# their totals agree, their indices are 0..N-1, and their assignments cover every patch
# and every mutation exactly once. With patch names, only those patches run.
#
# Run from the repository root. CARGO_TARGET_DIR, if set, holds the builds, which need
# the features http, video_manifest (the playlist receipts) and key_discovery (the
# receipts, keys, grants and sets of §8.1–§8.5). The receipt and key cases of
# http_verify.py need the `cryptography` package.
set -uo pipefail
ROOT="$(git rev-parse --show-toplevel)"
FALSIFY="$ROOT/wai/http-conformance/falsify"
WORK="${RUNNER_TEMP:-/tmp}/wai-http-falsify"
if [ $# -gt 0 ] && [ "${1#--}" = "$1" ] && [ ! -f "$FALSIFY/$1.patch" ]; then
  WORK="$1"
  shift
fi
mkdir -p "$WORK"
FEATURES=http,video_manifest,key_discovery
fail=0

# The mutations and the cases each must fail, from mutations.txt (one a line: the name,
# then the cases).
MUTATIONS=()
while read -r m _; do
  case "$m" in ""|"#"*) continue ;; esac
  MUTATIONS+=("$m")
done < "$FALSIFY/mutations.txt"

# The cases the mutation $1 must fail, as mutations.txt lists them.
mutation_cases() {
  awk -v m="$1" '$1 == m { $1 = ""; print; exit }' "$FALSIFY/mutations.txt"
}

# Every patch, by name, in sorted order: the list shards index into.
all_patches() {
  local p
  for p in "$FALSIFY"/*.patch; do basename "$p" .patch; done | LC_ALL=C sort
}

# The members of a list (one per line on stdin) whose index mod N is I.
shard_of() {
  awk -v i="$1" -v n="$2" '(NR - 1) % n == i'
}

# Parse I/N into the globals SI and SN, refusing anything else.
parse_shard() {
  if ! [[ "$1" =~ ^([0-9]+)/([0-9]+)$ ]] || [ "${BASH_REMATCH[2]}" -eq 0 ] || [ "${BASH_REMATCH[1]}" -ge "${BASH_REMATCH[2]}" ]; then
    echo "::error::a shard is I/N with 0 <= I < N, not '$1'"
    exit 2
  fi
  SI="${BASH_REMATCH[1]}"
  SN="${BASH_REMATCH[2]}"
}

# The lines of stdin into the array NAMES (bash 3.2 has no mapfile).
read_names() {
  NAMES=()
  local line
  while IFS= read -r line; do [ -n "$line" ] && NAMES+=("$line"); done
}

# The patches stage over the names on stdin, in one worktree of HEAD.
run_patches() {
  local names wt bin name patch must rc missing ran=() t0 t1
  read_names
  names=("${NAMES[@]}")
  wt="$WORK/tree"
  git -C "$ROOT" worktree remove --force "$wt" > /dev/null 2>&1
  rm -rf "$wt"
  git -C "$ROOT" worktree add --detach --quiet "$wt" HEAD
  bin="${CARGO_TARGET_DIR:-$wt/wai/wai-rs/target}/falsify/wai_http_vectors"
  build() { (cd "$wt/wai/wai-rs" && cargo build --profile falsify --no-default-features --features "$FEATURES" --bin wai_http_vectors); }
  # The unpatched tree builds and passes, or no patch's failure means anything.
  t0=$(date +%s)
  if ! build > "$WORK/unpatched.build.log" 2>&1; then
    echo "::error::the unpatched tree does not build"; tail -20 "$WORK/unpatched.build.log"
    git -C "$ROOT" worktree remove --force "$wt"; return 1
  fi
  if ! "$bin" verify "$wt/wai/http-conformance" > "$WORK/unpatched.verify.log" 2>&1; then
    echo "::error::the unpatched tree does not pass"; tail -20 "$WORK/unpatched.verify.log"
    git -C "$ROOT" worktree remove --force "$wt"; return 1
  fi
  t1=$(date +%s)
  echo "the unpatched tree builds and passes ($((t1 - t0)) s)"
  for name in "${names[@]}"; do
    patch="$FALSIFY/$name.patch"
    t0=$(date +%s)
    must="$(head -1 "$patch" | sed 's/^# must fail: //')"
    if ! git -C "$wt" apply "$patch"; then
      echo "::error::$name: the patch does not apply"; fail=1; continue
    fi
    if ! build > "$WORK/$name.build.log" 2>&1; then
      echo "::error::$name: the patched tree does not build"; tail -20 "$WORK/$name.build.log"; fail=1
    else
      "$bin" verify "$wt/wai/http-conformance" > "$WORK/$name.verify.log" 2>&1
      rc=$?
      missing=""
      for c in $must; do grep -q "^FAIL $c:" "$WORK/$name.verify.log" || missing="$missing $c"; done
      t1=$(date +%s)
      if [ $rc -eq 0 ] || [ -n "$missing" ]; then
        echo "::error::$name: verify exited $rc; cases not failed:$missing"; fail=1
      else
        echo "ok  $name: verify exited $rc and failed$(printf ' %s' $must) ($(grep -c '^FAIL' "$WORK/$name.verify.log") failures, $((t1 - t0)) s)"
      fi
    fi
    ran+=("$name")
    # Back to HEAD exactly, or every later patch is judged on a changed tree.
    if ! git -C "$wt" apply -R "$patch" || [ -n "$(git -C "$wt" status --porcelain --untracked-files=no)" ]; then
      echo "::error::$name: the tree is not clean after reversing the patch"; git -C "$ROOT" worktree remove --force "$wt"; return 1
    fi
  done
  git -C "$ROOT" worktree remove --force "$wt"
  check_ran patches "$(printf '%s\n' "${names[@]}")" "$(printf '%s\n' "${ran[@]}")"
  return $fail
}

# The mutations stage over the names on stdin.
run_mutations() {
  local names m ran=()
  read_names
  names=("${NAMES[@]}")
  for m in "${names[@]}"; do
    local must log rc missing c
    must="$(mutation_cases "$m")"
    log="$WORK/py-$m.log"
    (cd "$ROOT/wai/http-conformance" && python3 http_verify.py . --mutate "$m") > "$log" 2>&1
    rc=$?
    missing=""
    for c in $must; do grep -q "^FAIL $c:" "$log" || missing="$missing $c"; done
    if [ -z "$must" ]; then
      echo "::error::$m: mutations.txt declares no case it must fail"; fail=1
    elif [ $rc -eq 0 ]; then
      echo "::error::http_verify.py --mutate $m passed"; fail=1
    elif grep -q '^Traceback' "$log" || ! grep -q '^FAIL ' "$log"; then
      # A crash, or a refusal to run, is not a catch: the harness failed.
      echo "::error::$m: a harness failure: http_verify.py exited $rc without naming the cases it failed"; tail -5 "$log"; fail=1
    elif [ -n "$missing" ]; then
      echo "::error::$m: http_verify.py exited $rc; cases not failed:$missing"; fail=1
    else
      echo "ok  http_verify.py --mutate $m: failed$(printf ' %s' $must) ($(grep -c '^FAIL' "$log") failures)"
    fi
    ran+=("$m")
  done
  check_ran mutations "$(printf '%s\n' "${names[@]}")" "$(printf '%s\n' "${ran[@]}")"
  return $fail
}

# A shard ran exactly what it was assigned (one name a line in each): printed, and
# compared.
check_ran() {
  local kind="$1" want got
  want="$(printf '%s\n' "$2" | grep -v '^$' | LC_ALL=C sort)"
  got="$(printf '%s\n' "$3" | grep -v '^$' | LC_ALL=C sort)"
  echo "$kind run in this shard ($(printf '%s\n' "$got" | grep -c .)): $(echo $got)"
  if [ "$want" != "$got" ]; then
    echo "::error::this shard did not run exactly the $kind assigned to it"
    diff <(printf '%s\n' "$want") <(printf '%s\n' "$got")
    fail=1
  fi
}

# The shards a workflow declares cover every patch and every mutation exactly once.
check_shards() {
  local kind list declared n total covered expected
  for kind in patches mutations; do
    if [ "$kind" = patches ]; then list="$(all_patches)"; else list="$(printf '%s\n' "${MUTATIONS[@]}")"; fi
    declared="$(printf '%s\n' "$@" | awk -v k="$kind" '$1 == k { print $2 }')"
    if [ -z "$declared" ]; then echo "::error::no $kind shard is declared"; fail=1; continue; fi
    total="$(printf '%s\n' "$declared" | cut -d/ -f2 | sort -u)"
    if [ "$(printf '%s\n' "$total" | wc -l)" -ne 1 ]; then echo "::error::$kind shards disagree on their total: $(echo $total)"; fail=1; continue; fi
    n="$total"
    if [ "$(printf '%s\n' "$declared" | cut -d/ -f1 | LC_ALL=C sort -n | tr '\n' ' ')" != "$(seq 0 $((n - 1)) | tr '\n' ' ')" ]; then
      echo "::error::$kind shards are not 0..$((n - 1)) of $n, each once: $(echo $declared)"; fail=1; continue
    fi
    covered="$(for s in $declared; do parse_shard "$s"; printf '%s\n' "$list" | shard_of "$SI" "$SN"; done | LC_ALL=C sort)"
    expected="$(printf '%s\n' "$list" | LC_ALL=C sort)"
    if [ "$covered" != "$expected" ]; then
      echo "::error::the $kind shards do not cover every one exactly once"
      diff <(printf '%s\n' "$expected") <(printf '%s\n' "$covered")
      fail=1
    else
      echo "the $n $kind shards cover all $(printf '%s\n' "$expected" | wc -l | tr -d ' ') $kind, each once"
    fi
  done
}

# The differential, with the unpatched tool.
run_differential() {
  local bin
  if ! (cd "$ROOT/wai/wai-rs" && cargo build --profile falsify --no-default-features --features "$FEATURES" --bin wai_http_vectors) > "$WORK/differential.build.log" 2>&1; then
    echo "::error::the unpatched tool does not build"; tail -20 "$WORK/differential.build.log"; fail=1; return
  fi
  bin="${CARGO_TARGET_DIR:-$ROOT/wai/wai-rs/target}/falsify/wai_http_vectors"
  cd "$ROOT/wai/http-conformance" || return 1
  "$bin" diff . --seed 7 --count 500 --out "$WORK/diff.jsonl" > /dev/null || fail=1
  python3 http_verify.py diff "$WORK/diff.jsonl" || fail=1
  python3 http_verify.py diff "$WORK/diff.jsonl" --flip 10 --seed 3 || fail=1
  cd "$ROOT" || return 1
}

case "${1:-}" in
  --patches)
    parse_shard "${2:?--patches I/N}"
    all_patches | shard_of "$SI" "$SN" | run_patches || fail=1
    ;;
  --mutations)
    parse_shard "${2:?--mutations I/N}"
    printf '%s\n' "${MUTATIONS[@]}" | shard_of "$SI" "$SN" | run_mutations || fail=1
    ;;
  --differential)
    run_differential
    ;;
  --check-shards)
    shift
    check_shards "$@"
    ;;
  "")
    all_patches | run_patches || fail=1
    printf '%s\n' "${MUTATIONS[@]}" | run_mutations || fail=1
    run_differential
    ;;
  --*)
    echo "::error::unknown option $1"; exit 2
    ;;
  *)
    for name in "$@"; do
      [ -f "$FALSIFY/$name.patch" ] || { echo "::error::no patch $name"; exit 2; }
    done
    printf '%s\n' "$@" | run_patches || fail=1
    ;;
esac
exit $fail