open-standards/wai/quickstart/conformance-records.sh

#!/usr/bin/env bash
# WAI's own implementations through the conformance runner, each run's record signed with WAI's
# key and packed for publication: a push to main keeps the pack as its run's artefact
# (wai-breaking.yml), and a wai-v* tag attaches it to the release (wai-release.yml), where the
# WAI site's list of conforming implementations fetches it by tag (its
# scripts/sync-conform.mjs --release <tag>) and admits each record.
#
#   WAI_CONFORMANCE_SEED=<64 hexadecimal digits> \
#     bash wai/quickstart/conformance-records.sh --bin <dir> --out <dir> [--commit <sha>] [--label <name>]
#
# --bin holds the programs it runs, built from this checkout: wai (with the features the
# conformance runner job of wai-breaking.yml builds), wai_native_sink_vectors,
# wai_browser_sink_vectors, wai_int_synth_vectors, wai-pack, wai-relay and joule-moq-interop.
# --commit is the commit the records name (default HEAD), --label the pack's name (default
# <wai version>+<the commit's first 12 digits>). It writes two files into --out:
#
#   wai-conformance-<label>.tar.gz         the folder wai-conformance-<label>/: one record a run
#                                          (<run>.wai-conform.json) and MANIFEST.json, which
#                                          names the commit, the standard, wai's version, the
#                                          signer's public key and each record's file, kind,
#                                          suite and SHA-256
#   wai-conformance-<label>.tar.gz.sha256  its SHA-256, as `shasum -a 256` writes it
#
# The runs are the conformance runner job's: the corpora's independent verifiers (claim, jwp,
# signer, pq and conform-result), the reference packager's policy reader, the native sink
# through its module and through its C interface, the score renderer, the browser sink, the
# integer-synthesis decoder, a cache in front of the reference origin, and the reference relay
# at each draft it speaks. Each names its implementation's version <wai version>+<commit>, is
# signed with the seed (`"key": "supplied"`), and must be admitted by `wai verify --conformant`
# against this checkout's corpus.
#
# A list publishes what a record says (conformance-results §8), so the runs are made from a
# folder holding bin/ (a copy of --bin) and wai (a link to this checkout's wai/), each program
# found through PATH's relative entry `bin`: a record names bin/<program> and wai/<path>, never
# a path of the machine that ran it. A python3 found under /Users/, /Volumes/, /home/, /private/
# or /tmp/, or a record holding such a path anywhere, stops it: the list refuses those records
# (its check f).
#
# The seed is read from WAI_CONFORMANCE_SEED, written to a file only this user can read, removed
# when this exits, and never printed; the variable is unset before anything runs. Without it:
# one notice line, nothing written, exit 0, since a record signed with a key made for its run
# says nothing of who ran it. A seed that is not 64 hexadecimal digits, or one whose key is a
# test key conform-result-conformance publishes (which attests nothing), stops it.
#
# Exit 0 when the pack was written, or when no seed was given; 1 when a seed, a run, a check or
# the pack failed, with nothing written to --out; 2 for a usage error.
set -euo pipefail

usage() {
  echo "usage: WAI_CONFORMANCE_SEED=<64 hexadecimal digits> bash wai/quickstart/conformance-records.sh --bin <dir> --out <dir> [--commit <sha>] [--label <name>]" >&2
  exit 2
}
fail() {
  echo "conformance-records: $*; nothing was written" >&2
  exit 1
}
BIN= OUT= COMMIT= LABEL=
while [ $# -gt 0 ]; do
  case "$1" in
    --bin|--out|--commit|--label)
      [ $# -ge 2 ] || usage
      case "$1" in --bin) BIN=$2 ;; --out) OUT=$2 ;; --commit) COMMIT=$2 ;; --label) LABEL=$2 ;; esac
      shift 2 ;;
    *) usage ;;
  esac
done
[ -n "$BIN" ] && [ -n "$OUT" ] || usage
case "$BIN" in /*) ;; *) BIN="$PWD/$BIN" ;; esac
case "$OUT" in /*) ;; *) OUT="$PWD/$OUT" ;; esac
ROOT=$(cd "$(dirname "$0")/../.." && pwd -P)

if [ -z "${WAI_CONFORMANCE_SEED:-}" ]; then
  echo "::notice::the repository secret WAI_CONFORMANCE_SEED is not set: WAI's own conformance records are not signed with WAI's key, and none is published (a key made for a run says nothing of who ran it)"
  exit 0
fi

W=$(mktemp -d "${TMPDIR:-/tmp}/wai-conformance.XXXXXX")
chmod 700 "$W"
relay=
cleanup() {
  if [ -n "$relay" ]; then kill "$relay" 2>/dev/null || true; fi
  rm -rf "$W"
}
trap cleanup EXIT
mkdir -m 700 "$W/key"
KEY="$W/key/seed"
( umask 077; printf '%s' "$WAI_CONFORMANCE_SEED" | tr -d ' \t\r\n' > "$KEY" )
unset WAI_CONFORMANCE_SEED
grep -Eq '^[0-9a-fA-F]{64}$' "$KEY" || fail "WAI_CONFORMANCE_SEED is not an Ed25519 seed (64 hexadecimal digits)"

# The seed's public key, which every record must name; and not a test key.
PUB=$(python3 -I - "$KEY" <<'PY'
import sys
try:
    from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
    from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
except ImportError:
    sys.exit("conformance-records: python3 has no cryptography package (python3 -m pip install cryptography); nothing was written")
with open(sys.argv[1]) as f:
    seed = bytearray(bytes.fromhex(f.read().strip()))
print(Ed25519PrivateKey.from_private_bytes(bytes(seed)).public_key().public_bytes(Encoding.Raw, PublicFormat.Raw).hex())
seed[:] = bytes(len(seed))
PY
)
[[ "$PUB" =~ ^[0-9a-f]{64}$ ]] || fail "the seed's public key could not be read"
if python3 -I -c 'import json,sys; sys.exit(0 if sys.argv[2] in json.load(open(sys.argv[1])).get("public_keys", []) else 1)' \
     "$ROOT/wai/conform-result-conformance/vectors.json" "$PUB"; then
  fail "WAI_CONFORMANCE_SEED holds a test key conform-result-conformance publishes ($PUB), which attests nothing"
fi

for p in wai wai_native_sink_vectors wai_browser_sink_vectors wai_int_synth_vectors wai-pack wai-relay joule-moq-interop; do
  [ -x "$BIN/$p" ] || fail "--bin $BIN holds no program $p"
done
if [ -e "$OUT" ] && [ -n "$(ls -A "$OUT" 2>/dev/null)" ]; then fail "--out $OUT is not empty"; fi
COMMIT=${COMMIT:-$(git -C "$ROOT" rev-parse HEAD)}
[[ "$COMMIT" =~ ^[0-9a-f]{40}$ ]] || fail "--commit $COMMIT: not a commit's 40 hexadecimal digits"
WV=$(sed -n 's/^version = "\(.*\)"$/\1/p' "$ROOT/wai/wai-rs/Cargo.toml" | head -1)
[ -n "$WV" ] || fail "wai/wai-rs/Cargo.toml names no version"
VERSION="$WV+${COMMIT:0:12}"
LABEL=${LABEL:-$VERSION}
[[ "$LABEL" =~ ^[A-Za-z0-9][A-Za-z0-9.+_-]*$ ]] || fail "--label $LABEL: letters, digits and . + _ - only"

# The path-neutral folder the runs are made from.
S="$W/run"
R="$W/records"
mkdir -p "$S/bin" "$R"
for p in wai wai_native_sink_vectors wai_browser_sink_vectors wai_int_synth_vectors wai-pack wai-relay joule-moq-interop; do
  cp "$BIN/$p" "$S/bin/"
done
ln -s "$ROOT/wai" "$S/wai"
cd "$S"
export PATH="bin:$PATH" WAI_MOQ_INTEROP_BIN=bin/joule-moq-interop
LOCAL='(^|[[:space:]"'"'"'`(=:,;[{<])/(Volumes|Users|private|tmp|home)/'
py=$(command -v python3) || fail "no python3 on PATH"
if [[ " $py" =~ $LOCAL ]]; then
  fail "python3 is $py, a local path the records would name; put a python3 outside /Users/, /Volumes/, /home/, /private/ and /tmp/ first on PATH"
fi
echo "conformance-records: open-standards ${COMMIT:0:12}, wai $WV, signed by $PUB"

# conform <run> <kind> <suite> <name> <impl> [option...]
conform() {
  local id=$1 kind=$2 suite=$3 name=$4 impl=$5
  shift 5
  if ! bin/wai conform "$kind" --suite "$suite" --text --root wai --impl "$impl" "$@" \
         --name "$name" --version "$VERSION" --key "$KEY" --out "$R/$id.wai-conform.json" --yes > "$W/run.txt" 2>&1; then
    cat "$W/run.txt"
    fail "WAI's own run $id ($kind $suite) did not pass"
  fi
  echo "conformance-records: $id: $(grep -E '^wai conform' "$W/run.txt" | tail -1)"
}
for s in claim jwp signer pq conform-result; do
  conform "$s" verifier "$s" "${s//-/_}_verify.py" "python3 -I wai/$s-conformance/${s//-/_}_verify.py verdicts {corpus}"
done
conform policy packager policy wai-pack "wai-pack policy-verdict {in}"
conform native-sink sink native-sink "wai native sink" "wai_native_sink_vectors answers {case}"
conform native-sink-c-abi sink native-sink "wai native sink, C interface" "wai_native_sink_vectors answers --c-abi {case}"
conform score decoder score "wai score render" "wai score render {in} --pcm {out}"
conform browser sink browser "wai browser sink" "wai_browser_sink_vectors answers {case}"
conform neural decoder neural "wai int_synth decoder" "wai_int_synth_vectors decode {model} {in} {out}"
conform cache cache cache "mutators.py, relay mode" "python3 -I wai/cache-conformance/mutators.py --mode relay --listen {port} --upstream {origin}"
# The reference relay on loopback, at each draft it speaks; another program already on its port
# stops the relay, and with it this, rather than having the client test that program.
bin/wai-relay --dev > "$W/relay.txt" 2>&1 &
relay=$!
for _ in $(seq 1 150); do
  grep -q 'on udp 127.0.0.1:4443' "$W/relay.txt" && break
  kill -0 "$relay" 2>/dev/null || break
  sleep 0.2
done
grep -q 'on udp 127.0.0.1:4443' "$W/relay.txt" || { cat "$W/relay.txt"; fail "the reference relay did not start on udp 127.0.0.1:4443"; }
for d in 22 18 16; do
  conform "moqt-$d" relay moqt wai-relay moqt://127.0.0.1:4443/moq --draft "$d" --tls-disable-verify
done
kill "$relay" 2>/dev/null || true
wait "$relay" 2>/dev/null || true
relay=

# Every record admitted against this checkout's corpus.
if ! bin/wai verify --conformant --root wai "$R" > "$W/admit.txt" 2>&1; then
  cat "$W/admit.txt"
  fail "wai verify --conformant refused a record"
fi
grep '^wai verify' "$W/admit.txt" | sed 's/^/conformance-records: /'

# Each record held to what the list admits, then the pack.
mkdir -p "$W/pack"
python3 -I - "$R" "$W/pack" "$LABEL" "$COMMIT" "$PUB" "$VERSION" "$WV" "$LOCAL" <<'PY'
import gzip, hashlib, io, json, os, re, sys, tarfile
rec_dir, pack, label, commit, pub, version, wv, local = sys.argv[1:9]
LOCAL = re.compile(local.replace("[:space:]", r"\s"))
def stop(why):
    sys.exit(f"conformance-records: {why}; nothing was written")
top = f"wai-conformance-{label}"
names = sorted(n for n in os.listdir(rec_dir) if n.endswith(".wai-conform.json"))
if len(names) != 15:
    stop(f"{len(names)} records, not the 15 runs")
files, entries, standards = [], [], set()
for n in names:
    b = open(os.path.join(rec_dir, n), "rb").read()
    r = json.loads(b)
    impl = r["implementation"]
    if r.get("key") != "supplied" or r.get("signer_pubkey") != pub:
        stop(f"{n}: signed by {r.get('signer_pubkey')} ({r.get('key')} key), not by the seed")
    if impl.get("version") != version:
        stop(f"{n}: names the version {impl.get('version')}, not {version}")
    if r.get("conformant") is not True:
        stop(f"{n}: not conformant")
    if (r.get("toolchain") or {}).get("version") != wv:
        stop(f"{n}: made by wai {(r.get('toolchain') or {}).get('version')}, not {wv}")
    text = b.decode("utf-8")
    m = LOCAL.search(text)
    if m:
        stop(f"{n}: holds a local path ({text[max(0, m.start() - 40):m.end() + 40]!r}), which a list publishing it would refuse")
    standards.add(r["standard"])
    files.append((n, b))
    entries.append({"file": n, "kind": r["kind"], "suite": r["suite"], "sha256": hashlib.sha256(b).hexdigest()})
if len(standards) != 1:
    stop(f"the records name the standards {sorted(standards)}")
manifest = {
    "about": "WAI's own implementations through the conformance runner (wai conform) at one commit of open-standards: one record (conform-result/1) a run, each signed with WAI's key and admitted by wai verify --conformant against that commit's corpus",
    "commit": commit,
    "standard": standards.pop(),
    "wai": wv,
    "signer_pubkey": pub,
    "records": entries,
}
files.append(("MANIFEST.json", (json.dumps(manifest, indent=2) + "\n").encode()))
buf = io.BytesIO()
with gzip.GzipFile(filename="", mode="wb", fileobj=buf, mtime=0) as gz:
    with tarfile.open(fileobj=gz, mode="w", format=tarfile.PAX_FORMAT) as tar:
        d = tarfile.TarInfo(top)
        d.type, d.mode = tarfile.DIRTYPE, 0o755
        tar.addfile(d)
        for n, b in files:
            t = tarfile.TarInfo(f"{top}/{n}")
            t.size, t.mode = len(b), 0o644
            tar.addfile(t, io.BytesIO(b))
tgz = f"{top}.tar.gz"
data = buf.getvalue()
open(os.path.join(pack, tgz), "wb").write(data)
open(os.path.join(pack, tgz + ".sha256"), "w").write(f"{hashlib.sha256(data).hexdigest()}  {tgz}\n")
print(f"conformance-records: {tgz}: {len(entries)} records, sha256 {hashlib.sha256(data).hexdigest()}")
PY
mkdir -p "$OUT"
mv "$W/pack/"* "$OUT/"
echo "conformance-records: written to $OUT"