open-standards/wai/cache-conformance/run.sh

#!/usr/bin/env bash
# Cache conformance (extensions/http-delivery.md §9): stage the corpus's objects, start
# the reference origin, and run driver.py
#
#   1. against the origin itself, which must pass every MUST test (the positive control);
#   2. against each negative-control proxy in mutators.py, which must fail the tests its
#      defect reaches (the driver's own falsification);
#   3. against each labelled cache an environment provides: <dir>/labels lists the
#      labels, and <dir>/<label>/start.sh <port> <origin URL> starts one in front of the
#      origin (with version and config files beside it). <dir> is /opt/wai-cache, or
#      WAI_CACHE_DIR where that is set. The public repository holds no cache
#      configuration; without labels in /opt/wai-cache only 1 and 2 run, and the summary
#      says so. A WAI_CACHE_DIR that names no label is an error, since it was asked for.
#      The summary says how many repeat GETs each cache answered from its store, and marks
#      one that answered none as a relay.
#
#   WAI=<path to a `wai` built with http> [WAI_CACHE_DIR=<dir>] \
#     bash wai/cache-conformance/run.sh <out-dir>
#
# Reports go to <out-dir>/reports: report-<label>.json and requirements-<label>.md.
# Exit status 1 when a control does not behave as stated or a labelled cache fails a
# MUST test.
set -uo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
OUT="${1:?usage: run.sh <out-dir>}"
WAI="${WAI:?set WAI to a wai binary built with the http feature}"
CACHES="${WAI_CACHE_DIR:-/opt/wai-cache}"
mkdir -p "$OUT/reports"
ROOT="$OUT/root"
python3 "$HERE/driver.py" stage "$HERE/../http-conformance" "$ROOT" || exit 1

pids=()
cleanup() { for p in "${pids[@]}"; do kill "$p" 2>/dev/null; done; }
trap cleanup EXIT

free_port() { python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()'; }
wait_port() { for _ in $(seq 1 100); do python3 -c "import socket,sys; socket.create_connection(('127.0.0.1',$1),1)" 2>/dev/null && return 0; sleep 0.1; done; return 1; }

OP="$(free_port)"
"$WAI" http origin --root "$ROOT" --port "$OP" --count-hits --live-prefix live/ --live-max-age 30 > "$OUT/origin.log" 2>&1 &
pids+=($!)
wait_port "$OP" || { echo "::error::the origin did not start"; cat "$OUT/origin.log"; exit 1; }
ORIGIN="http://127.0.0.1:$OP"
fail=0
summary="| run | expectation | MUST passed | result |\n|---|---|---|---|\n"

run_driver() { # label base expect-pass [expected failing ids]
  local label="$1" base="$2" expect="$3"; shift 3
  python3 "$HERE/driver.py" run --base "$base" --root "$ROOT" --origin "$ORIGIN" --label "$label" \
    --report "$OUT/reports/report-$label.json" > "$OUT/reports/$label.log" 2>&1
  local rc=$?
  local passed; passed="$(tail -1 "$OUT/reports/$label.log" | sed 's/.*: //')"
  if [ "$expect" = pass ]; then
    if [ $rc -eq 0 ]; then summary+="| $label | every MUST passes | $passed | ok |\n"
    else summary+="| $label | every MUST passes | $passed | **failed** |\n"; echo "::error::$label failed a MUST test"; cat "$OUT/reports/$label.log"; fail=1; fi
  else
    local missing=""
    for id in "$@"; do grep -q "^FAIL $id:" "$OUT/reports/$label.log" || missing="$missing $id"; done
    if [ $rc -ne 0 ] && [ -z "$missing" ]; then summary+="| $label | fails $* | $passed | caught |\n"
    else summary+="| $label | fails $* | $passed | **not caught:$missing** |\n"; echo "::error::$label: the driver did not catch$missing (exit $rc)"; cat "$OUT/reports/$label.log"; fail=1; fi
  fi
}

# 1. The origin, directly.
run_driver origin-direct "$ORIGIN" pass

# 2. The negative controls.
expect_of() { case "$1" in flip) echo "C1 C3 C11 C14" ;; gzip) echo "C7" ;; truncate) echo "C1 C2 C14" ;; vary) echo "C13" ;; stored) echo "C14" ;; ranged) echo "C3 C5 C8" ;; reflip) echo "C1 C14" ;; esac; }
for m in flip gzip truncate vary stored ranged reflip; do
  MP="$(free_port)"
  python3 "$HERE/mutators.py" --mode "$m" --listen "$MP" --upstream "$ORIGIN" > "$OUT/mutator-$m.log" 2>&1 &
  pids+=($!)
  wait_port "$MP" || { echo "::error::mutator $m did not start"; fail=1; continue; }
  # shellcheck disable=SC2086
  run_driver "control-$m" "http://127.0.0.1:$MP" fail $(expect_of "$m")
done

# 3. Labelled caches, where the environment provides them.
labels="$(cat "$CACHES/labels" 2>/dev/null | xargs)"
if [ -n "$labels" ]; then
  for label in $labels; do
    CP="$(free_port)"
    "$CACHES/$label/start.sh" "$CP" "$ORIGIN" > "$OUT/cache-$label.log" 2>&1 &
    pids+=($!)
    wait_port "$CP" || { summary+="| $label | every MUST passes | — | **did not start** |\n"; echo "::error::cache $label did not start"; fail=1; continue; }
    python3 "$HERE/driver.py" run --base "http://127.0.0.1:$CP" --root "$ROOT" --origin "$ORIGIN" --label "$label" \
      --version "$(cat "$CACHES/$label/version" 2>/dev/null || echo unknown)" \
      --config "$CACHES/$label/config" --report "$OUT/reports/report-$label.json" > "$OUT/reports/$label.log" 2>&1
    rc=$?
    stored="$(sed -n 's/^info S3: //p' "$OUT/reports/$label.log")"
    case "$stored" in 0\ of*) stored="**a relay:** $stored" ;; esac
    summary+="| $label | every MUST passes | $(tail -1 "$OUT/reports/$label.log" | sed 's/.*: //') | $([ $rc -eq 0 ] && echo ok || echo '**failed**'); $stored |\n"
    [ $rc -eq 0 ] || fail=1
  done
elif [ -n "${WAI_CACHE_DIR:-}" ]; then
  summary+="| labelled caches | — | — | **no labels in $CACHES/labels** |\n"; echo "::error::WAI_CACHE_DIR names no label: $CACHES/labels"; fail=1
else
  summary+="| labelled caches | — | — | none in this environment (no labels in $CACHES/labels): controls only |\n"
fi

printf "%b" "$summary" | tee "$OUT/reports/summary.md"
[ -n "${GITHUB_STEP_SUMMARY:-}" ] && printf "%b" "$summary" >> "$GITHUB_STEP_SUMMARY"
exit $fail