39d9894e73
W3-7, public part. cache-conformance/: tests.json (C1–C13, R1–R5 as MUST tests of §9 and the sidecar rules, S1–S2 informative), driver.py (stages the corpus behind an origin and runs the tests against a cache; standard library and _blake3.py only), mutators.py (flip, gzip, truncate, vary: one §9 defect each), run.sh (the reference origin must pass every MUST test; each control must be caught by the tests named for it; labelled caches run where an environment provides /opt/wai-cache, and the summary says when none did) and a README. Reports name a cache by label, package version and configuration digest only. The private harness image is not in this repository. The reference origin gains what the tests need of it: If-Range by strong comparison, a live-edge prefix served with a shorter max-age, §8.4's Cache-Control for the key log and grants, and a request counter (--count-hits) for the informative tests. .github/workflows/wai-http.yml: the falsify job (every patch caught, every Python mutation caught, the differential) and the cache-conformance job (reports uploaded as an artifact). wai-conformance.yml: `http` joins the build, the library tests, the corpus loop (wai_http_vectors), the feature-alone and wasm32 checks; http_verify.py joins the independent verifiers. wai_http_vectors verify now names a case whose evaluation panics. http-delivery.md writes byte spans in words where a half-open interval read as link text to the section-reference check. Co-Authored-By: Claude Opus 5.5 <[email protected]>
93 lines
4.8 KiB
Shell
Executable file
93 lines
4.8 KiB
Shell
Executable file
#!/usr/bin/env bash
|
|
# Cache conformance (extensions/http-delivery.md §9): stage the corpus's objects, start
|
|
# the reference origin, and run driver.py
|
|
#
|
|
# 1. against the origin itself, which must pass every MUST test (the positive control);
|
|
# 2. against each negative-control proxy in mutators.py, which must fail the tests its
|
|
# defect reaches (the driver's own falsification);
|
|
# 3. against each labelled cache an environment provides: /opt/wai-cache/labels lists
|
|
# the labels, and /opt/wai-cache/<label>/start.sh <port> <origin URL> starts one in
|
|
# front of the origin (with version and config files beside it). The public
|
|
# repository holds no cache configuration; without that directory only 1 and 2 run,
|
|
# and the summary says so.
|
|
#
|
|
# WAI=<path to a `wai` built with http> bash wai/cache-conformance/run.sh <out-dir>
|
|
#
|
|
# Reports go to <out-dir>/reports: report-<label>.json and requirements-<label>.md.
|
|
# Exit status 1 when a control does not behave as stated or a labelled cache fails a
|
|
# MUST test.
|
|
set -uo pipefail
|
|
HERE="$(cd "$(dirname "$0")" && pwd)"
|
|
OUT="${1:?usage: run.sh <out-dir>}"
|
|
WAI="${WAI:?set WAI to a wai binary built with the http feature}"
|
|
mkdir -p "$OUT/reports"
|
|
ROOT="$OUT/root"
|
|
python3 "$HERE/driver.py" stage "$HERE/../http-conformance" "$ROOT" || exit 1
|
|
|
|
pids=()
|
|
cleanup() { for p in "${pids[@]}"; do kill "$p" 2>/dev/null; done; }
|
|
trap cleanup EXIT
|
|
|
|
free_port() { python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()'; }
|
|
wait_port() { for _ in $(seq 1 100); do python3 -c "import socket,sys; socket.create_connection(('127.0.0.1',$1),1)" 2>/dev/null && return 0; sleep 0.1; done; return 1; }
|
|
|
|
OP="$(free_port)"
|
|
"$WAI" http origin --root "$ROOT" --port "$OP" --count-hits --live-prefix live/ --live-max-age 30 > "$OUT/origin.log" 2>&1 &
|
|
pids+=($!)
|
|
wait_port "$OP" || { echo "::error::the origin did not start"; cat "$OUT/origin.log"; exit 1; }
|
|
ORIGIN="http://127.0.0.1:$OP"
|
|
fail=0
|
|
summary="| run | expectation | MUST passed | result |\n|---|---|---|---|\n"
|
|
|
|
run_driver() { # label base expect-pass [expected failing ids]
|
|
local label="$1" base="$2" expect="$3"; shift 3
|
|
python3 "$HERE/driver.py" run --base "$base" --root "$ROOT" --origin "$ORIGIN" --label "$label" \
|
|
--report "$OUT/reports/report-$label.json" > "$OUT/reports/$label.log" 2>&1
|
|
local rc=$?
|
|
local passed; passed="$(tail -1 "$OUT/reports/$label.log" | sed 's/.*: //')"
|
|
if [ "$expect" = pass ]; then
|
|
if [ $rc -eq 0 ]; then summary+="| $label | every MUST passes | $passed | ok |\n"
|
|
else summary+="| $label | every MUST passes | $passed | **failed** |\n"; echo "::error::$label failed a MUST test"; cat "$OUT/reports/$label.log"; fail=1; fi
|
|
else
|
|
local missing=""
|
|
for id in "$@"; do grep -q "^FAIL $id:" "$OUT/reports/$label.log" || missing="$missing $id"; done
|
|
if [ $rc -ne 0 ] && [ -z "$missing" ]; then summary+="| $label | fails $* | $passed | caught |\n"
|
|
else summary+="| $label | fails $* | $passed | **not caught:$missing** |\n"; echo "::error::$label: the driver did not catch$missing (exit $rc)"; cat "$OUT/reports/$label.log"; fail=1; fi
|
|
fi
|
|
}
|
|
|
|
# 1. The origin, directly.
|
|
run_driver origin-direct "$ORIGIN" pass
|
|
|
|
# 2. The negative controls.
|
|
expect_of() { case "$1" in flip) echo "C1 C3 C11" ;; gzip) echo "C7" ;; truncate) echo "C1 C2" ;; vary) echo "C13" ;; esac; }
|
|
for m in flip gzip truncate vary; do
|
|
MP="$(free_port)"
|
|
python3 "$HERE/mutators.py" --mode "$m" --listen "$MP" --upstream "$ORIGIN" > "$OUT/mutator-$m.log" 2>&1 &
|
|
pids+=($!)
|
|
wait_port "$MP" || { echo "::error::mutator $m did not start"; fail=1; continue; }
|
|
# shellcheck disable=SC2086
|
|
run_driver "control-$m" "http://127.0.0.1:$MP" fail $(expect_of "$m")
|
|
done
|
|
|
|
# 3. Labelled caches, where the environment provides them.
|
|
if [ -f /opt/wai-cache/labels ]; then
|
|
for label in $(cat /opt/wai-cache/labels); do
|
|
CP="$(free_port)"
|
|
/opt/wai-cache/"$label"/start.sh "$CP" "$ORIGIN" > "$OUT/cache-$label.log" 2>&1 &
|
|
pids+=($!)
|
|
wait_port "$CP" || { echo "::error::cache $label did not start"; fail=1; continue; }
|
|
python3 "$HERE/driver.py" run --base "http://127.0.0.1:$CP" --root "$ROOT" --origin "$ORIGIN" --label "$label" \
|
|
--version "$(cat /opt/wai-cache/"$label"/version 2>/dev/null || echo unknown)" \
|
|
--config /opt/wai-cache/"$label"/config --report "$OUT/reports/report-$label.json" > "$OUT/reports/$label.log" 2>&1
|
|
rc=$?
|
|
summary+="| $label | every MUST passes | $(tail -1 "$OUT/reports/$label.log" | sed 's/.*: //') | $([ $rc -eq 0 ] && echo ok || echo '**failed**') |\n"
|
|
[ $rc -eq 0 ] || fail=1
|
|
done
|
|
else
|
|
summary+="| labelled caches | — | — | none in this environment (/opt/wai-cache/labels absent): controls only |\n"
|
|
fi
|
|
|
|
printf "%b" "$summary" | tee "$OUT/reports/summary.md"
|
|
[ -n "${GITHUB_STEP_SUMMARY:-}" ] && printf "%b" "$summary" >> "$GITHUB_STEP_SUMMARY"
|
|
exit $fail
|