open-standards/.github/workflows/wai-breaking.yml

name: WAI breaking changes and the conformance runner

# The gate WAI's libraries are published behind (toolchain v1, decision 7): every pull request
# that touches WAI runs `wai breaking --against origin/main`, and every push to main that touches
# WAI runs it against main as it was before the push; each fails on a breaking change that
# wai/CHANGELOG.md or a wai/changes/ entry does not name. Two jobs:
#
# - breaking: the check itself, against the merge base of the change and main, in all
#   three categories: wire (the registry export, the media and delivery corpora), record (the
#   signing domains, magics and versions, the record corpora, the published JSON Schemas) and
#   API (the Rust public API of the WAI library crates through rustdoc's JSON, the C header,
#   the room service's OpenAPI document, the operation catalogue and the MCP door's tools, the
#   browser packages' JavaScript bindings and the native sinks' Swift and Kotlin APIs).
# - runner: the tests of `wai breaking` (each kind of break reported and failing until a
#   changelog entry names it) and of `wai conform` (the reference implementations conform, a
#   changed verdict or output byte fails, a changed record is refused by `wai verify`, and
#   `wai verify --conformant` admits a passing run of the native sink and refuses a planted
#   failing one; the JUnit and HTML reports are the records'; `--independent` holds the
#   corpus's independent verifier to the same cases and fails on a disagreement, and alone
#   runs the verifiers it names); then `wai conform` runs the corpora's independent Python
#   verifiers and the reference packager, and `wai verify` checks every record those runs
#   wrote; it runs the native sink, through its module and its C interface, and the score
#   decoder, each record admitted against this checkout's corpus and a planted failing sink
#   refused (conformance-results §4.1); and it runs the browser sink, the integer-synthesis
#   decoder and a cache in front of the reference origin, each record admitted, and a planted
#   failing browser sink refused at rule 2; and it runs the reference relay through the MOQT
#   interop client at each draft (`wai conform relay --suite moqt`), each record admitted. The
#   corpus union itself, every independent verifier through `wai conform --independent`, runs in
#   wai-conformance.yml. On a push to main it runs WAI's own implementations once more for
#   publication (wai/quickstart/conformance-records.sh): each suite's record signed with WAI's
#   key, the repository secret WAI_CONFORMANCE_SEED, from paths a record may publish, and kept
#   as the run's artefact wai-conformance-records, which a list of conforming implementations
#   admits; without the secret, one notice and nothing kept.
#
# Push filters match the push's delta, and the `relevant` job gates the work from the pull
# request's own diff (scripts/pr-touches-paths.py, reading the list below), as wai-pack.yml
# explains. On a push to main it has no pull request to read and answers `true`, so the push's
# own filter decides.

# A pull request's newer push supersedes its older run. A push to main does not: each push is
# checked against main before it, so a cancelled run would leave that push's changes compared
# by nothing. Pushes to main therefore each take their own group.
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }}
  cancel-in-progress: true

on:
  push:
    branches: [main]
    paths:
      - 'wai/**'
      - 'joule-code/joule-code-rs/crates/joule-moq-room/openapi.json'
      - 'rust-toolchain.toml'
      - 'scripts/pr-touches-paths.py'
      # As below: the runner's tests read these.
      - '.github/wai-conformance/verifiers/**'
      - '.github/wai-conformance/corpora/**'
      - 'joule-code/joule-code-rs/crates/joule-moq-interop/**'
      - '.github/workflows/wai-conformance.yml'
      - '.github/workflows/wai-breaking.yml'
  pull_request:
    paths:
      - 'wai/**'
      # The room service's API is part of the API category.
      - 'joule-code/joule-code-rs/crates/joule-moq-room/openapi.json'
      # The toolchain pin decides which compiler documents both sides of the Rust API.
      - 'rust-toolchain.toml'
      - 'scripts/pr-touches-paths.py'
      # wai-rs/build.rs reads `wai conform`'s table of independent verifiers from the step files
      # of wai-conformance.yml, and the runner's tests below hold the table to them, and hold
      # that workflow to running the union and no entry by hand.
      - '.github/wai-conformance/verifiers/**'
      # wai-rs/build.rs reads `wai conformance gen`'s table of corpus generators from these, and
      # fails on one that names no corpus.
      - '.github/wai-conformance/corpora/**'
      # The relay suite's cases are this client's tests, by name and order (the runner's tests
      # read its source), and the relay step below runs it.
      - 'joule-code/joule-code-rs/crates/joule-moq-interop/**'
      - '.github/workflows/wai-conformance.yml'
      - '.github/workflows/wai-breaking.yml'

jobs:
  relevant:
    name: WAI breaking · does this change touch WAI
    runs-on: linux
    outputs:
      run: ${{ steps.gate.outputs.run }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0
      - id: gate
        run: |
          set -euo pipefail
          r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-breaking.yml \
                "${{ github.event.pull_request.base.sha }}" \
                "${{ github.event.pull_request.head.sha }}")
          echo "this change touches WAI: $r"
          echo "run=$r" >> "$GITHUB_OUTPUT"

  breaking:
    name: WAI breaking · wire, record and API against main · linux
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: linux
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0
      - name: Install Rust 1.99.1
        shell: bash
        run: |
          export PATH="$HOME/.cargo/bin:$PATH"
          command -v rustup >/dev/null 2>&1 || \
            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.99.1
          rustup toolchain install 1.99.1 --profile minimal
          rustup default 1.99.1
          SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo"
          command -v python3 >/dev/null || { $SUDO apt-get update -qq; $SUDO apt-get install -y -qq python3; }
          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
      - name: Build wai
        working-directory: wai/wai-rs
        shell: bash
        run: |
          set -euo pipefail
          cargo build --locked --no-default-features --bin wai
          out=$(cargo metadata --format-version 1 --no-deps --locked | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')
          mkdir -p "$RUNNER_TEMP/bin" && cp "$out/debug/wai" "$RUNNER_TEMP/bin/"
      # On a pull request the base is main's tip as fetched now; `wai breaking` compares
      # against its merge base with this change, so what main gained since the branch left it
      # is not read as this change removing it. On a push to main, main is this commit, so the
      # base is main before the push (`github.event.before`). That sha is all zeros on a
      # branch's first push or after a force-push, and a force-push can name a commit this
      # clone does not hold; either way the base falls back to the pushed commit's first
      # parent, so at least the newest commit is checked rather than nothing. Every finding is
      # printed: each break with its category, what changed and why it breaks, and the line
      # that acknowledges it if it is intended.
      - name: wai breaking against main
        shell: bash
        env:
          EVENT: ${{ github.event_name }}
          BEFORE: ${{ github.event.before }}
        run: |
          set -euo pipefail
          # The job's container user may not own the checkout; git then refuses to read it.
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git fetch -q --no-tags origin +refs/heads/main:refs/remotes/origin/main
          against=origin/main
          if [ "$EVENT" = push ]; then
            if [[ "$BEFORE" =~ ^[0-9a-f]{40}$ ]] && [[ ! "$BEFORE" =~ ^0+$ ]] && git cat-file -e "$BEFORE^{commit}" 2>/dev/null; then
              against=$BEFORE
            else
              echo "the push's previous main (${BEFORE:-none}) is not a commit here: comparing with the first parent"
              against=$(git rev-parse HEAD^1)
            fi
          fi
          echo "wai breaking --against $against"
          "$RUNNER_TEMP/bin/wai" breaking --against "$against" --text --all

  runner:
    name: WAI breaking and conform · their tests, and the independent verifiers through wai conform · linux
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: linux
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v5
      - name: Install Rust 1.99.1
        shell: bash
        run: |
          export PATH="$HOME/.cargo/bin:$PATH"
          command -v rustup >/dev/null 2>&1 || \
            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.99.1
          rustup toolchain install 1.99.1 --profile minimal
          rustup default 1.99.1
          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      # claim_verify.py, jwp_verify.py and signer_verify.py check Ed25519's equation with it.
      - name: Install the Ed25519 library
        shell: bash
        run: python -m pip install cryptography
      # The tests build fixture repositories with git and document a fixture crate with rustdoc.
      - name: The tests of wai breaking and wai conform
        working-directory: wai/wai-rs
        shell: bash
        run: |
          set -euo pipefail
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git config --global user.email [email protected]
          git config --global user.name ci
          cargo test --locked --no-default-features --features quickstart,score,key_discovery,native_sink,browser_sink,neural_int,pq_signatures \
            --bin wai --test cli_breaking --test cli_conform --test cli_conform_sinks --test cli_conform_relay
      - name: Build wai and the reference packager
        shell: bash
        run: |
          set -euo pipefail
          out() { (cd "$1" && cargo metadata --format-version 1 --no-deps --locked | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])'); }
          mkdir -p "$RUNNER_TEMP/bin"
          # The test step's features, so cargo reuses what it built.
          (cd wai/wai-rs && cargo build --locked --no-default-features --features quickstart,score,key_discovery,native_sink,browser_sink,neural_int,pq_signatures \
            --bin wai --bin wai_native_sink_vectors --bin wai_browser_sink_vectors --bin wai_int_synth_vectors)
          for b in wai wai_native_sink_vectors wai_browser_sink_vectors wai_int_synth_vectors; do cp "$(out wai/wai-rs)/debug/$b" "$RUNNER_TEMP/bin/"; done
          (cd wai/wai-pack && cargo build --locked --bin wai-pack) && cp "$(out wai/wai-pack)/debug/wai-pack" "$RUNNER_TEMP/bin/"
      # The second implementations of the claim, JWP, key-discovery and hybrid-signature corpora,
      # written from the extension texts and sharing no code with wai-rs, run through the
      # conformance runner as any other implementation would be; then the reference packager's
      # policy reader. Each run must be conformant, and every record it signs must verify.
      - name: wai conform with the independent verifiers and the reference packager
        shell: bash
        run: |
          set -euo pipefail
          w="$RUNNER_TEMP/bin/wai"; d="$RUNNER_TEMP/conform"; mkdir -p "$d"
          for s in claim jwp signer pq; do
            "$w" conform verifier --suite "$s" --text --root wai \
              --impl "python3 -I wai/$s-conformance/${s}_verify.py verdicts {corpus}" --out "$d/$s.wai-conform.json"
          done
          "$w" conform verifier --suite conform-result --text --root wai \
            --impl "python3 -I wai/conform-result-conformance/conform_result_verify.py verdicts {corpus}" --out "$d/conform-result.wai-conform.json"
          "$w" conform packager --suite policy --text --root wai \
            --impl "$RUNNER_TEMP/bin/wai-pack policy-verdict {in}" --out "$d/policy.wai-conform.json"
          "$w" verify "$d"
      # The native sink through the conformance runner, as another implementation would be run:
      # through its Rust module and through the C interface its platform integrations link; and
      # the score decoder. Each run must be conformant, and `wai verify
      # --conformant` must admit each record against this checkout's corpus. A planted sink that
      # presents an object whose digest does not match must be refused: its run fails, its record
      # still verifies, and admission refuses it (not_conformant).
      - name: wai conform with the native sink and the score decoder, each record admitted, a planted failing sink refused
        shell: bash
        run: |
          set -euo pipefail
          w="$RUNNER_TEMP/bin/wai"; s="$RUNNER_TEMP/bin/wai_native_sink_vectors"; d="$RUNNER_TEMP/sinks"; mkdir -p "$d"
          "$w" conform sink --suite native-sink --text --root wai --impl "$s answers {case}" --out "$d/native-sink.wai-conform.json"
          "$w" conform sink --suite native-sink --text --root wai --impl "$s answers --c-abi {case}" --out "$d/native-sink-c-abi.wai-conform.json"
          "$w" conform decoder --suite score --text --root wai --impl "$w score render {in} --pcm {out}" --out "$d/score.wai-conform.json"
          "$w" verify --conformant --root wai "$d"
          # The planted sink's run must fail as a run (exit 1), not as a usage error, a missing
          # program or an adapter that does not answer, and on the case it tampers with; its
          # record must be refused at rule 2, not at another.
          p="$RUNNER_TEMP/planted"; mkdir -p "$p"
          planted="$s answers {case} | sed '/object_digest_mismatch/s/\"present\":false/\"present\":true/'"
          set +e
          "$w" conform sink --suite native-sink --text --root wai --impl "$planted" --out "$p/planted.wai-conform.json" > "$RUNNER_TEMP/planted-run.txt" 2>&1
          c=$?
          set -e
          cat "$RUNNER_TEMP/planted-run.txt"
          [ "$c" = 1 ] || { echo "the planted failing sink: wai conform exited $c, not 1"; exit 1; }
          grep -q 'tampered-fragment' "$RUNNER_TEMP/planted-run.txt" || { echo "the planted failing sink: tampered-fragment is not a failed case"; exit 1; }
          "$w" verify "$p"
          set +e
          "$w" verify --conformant --root wai "$p" > "$RUNNER_TEMP/planted-admit.txt" 2>&1
          c=$?
          set -e
          cat "$RUNNER_TEMP/planted-admit.txt"
          [ "$c" = 1 ] || { echo "the planted failing sink: wai verify --conformant exited $c, not 1"; exit 1; }
          grep -q 'FAIL admission rule 2 not_conformant' "$RUNNER_TEMP/planted-admit.txt" || { echo "the planted failing sink was not refused at rule 2"; exit 1; }
      # The browser sink, the integer-synthesis learned decoder and a cache through the conformance
      # runner, each signing its record: the browser sink's answer to every call of its corpus,
      # byte for byte; the decoder's RGB8 for every vector of neural-conformance, byte for byte;
      # and a relay (cache-conformance/mutators.py --mode relay, which §9 rule 7 allows) in front
      # of the reference origin, held to every MUST test of http-delivery §9, its informative
      # tests reported and not counted. Each run must be conformant, and `wai verify --conformant`
      # must admit each record and give the live, media and delivery areas their verdicts. A
      # planted browser sink that shows a frame whose object was refused as verified must be
      # refused: its run fails on those frames, its record still verifies, and admission refuses
      # it at rule 2.
      - name: wai conform with the browser sink, the learned decoder and a cache, each record admitted, a planted failing browser sink refused
        shell: bash
        run: |
          set -euo pipefail
          w="$RUNNER_TEMP/bin/wai"; b="$RUNNER_TEMP/bin/wai_browser_sink_vectors"; d="$RUNNER_TEMP/records"; mkdir -p "$d"
          "$w" conform sink --suite browser --text --root wai --impl "$b answers {case}" --out "$d/browser.wai-conform.json"
          "$w" conform decoder --suite neural --text --root wai --impl "$RUNNER_TEMP/bin/wai_int_synth_vectors decode {model} {in} {out}" --out "$d/neural.wai-conform.json"
          "$w" conform cache --suite cache --text --root wai --impl "python3 -I wai/cache-conformance/mutators.py --mode relay --listen {port} --upstream {origin}" --out "$d/cache.wai-conform.json"
          "$w" verify --conformant --root wai "$d" | tee "$RUNNER_TEMP/records-admit.txt"
          for a in live media delivery; do
            grep -q "^wai verify: area $a: conformant" "$RUNNER_TEMP/records-admit.txt" || { echo "the $a area has no conformant verdict"; exit 1; }
          done
          p="$RUNNER_TEMP/planted-browser"; mkdir -p "$p"
          planted="$b answers {case} | sed 's/\"state\":\"refused\"/\"state\":\"verified\"/'"
          set +e
          "$w" conform sink --suite browser --text --root wai --impl "$planted" --out "$p/planted.wai-conform.json" > "$RUNNER_TEMP/planted-browser-run.txt" 2>&1
          c=$?
          set -e
          cat "$RUNNER_TEMP/planted-browser-run.txt"
          [ "$c" = 1 ] || { echo "the planted failing browser sink: wai conform exited $c, not 1"; exit 1; }
          grep -q '^FAIL frames/object_refused$' "$RUNNER_TEMP/planted-browser-run.txt" || { echo "the planted failing browser sink: frames/object_refused is not a failed case"; exit 1; }
          "$w" verify "$p"
          set +e
          "$w" verify --conformant --root wai "$p" > "$RUNNER_TEMP/planted-browser-admit.txt" 2>&1
          c=$?
          set -e
          cat "$RUNNER_TEMP/planted-browser-admit.txt"
          [ "$c" = 1 ] || { echo "the planted failing browser sink: wai verify --conformant exited $c, not 1"; exit 1; }
          grep -q 'FAIL admission rule 2 not_conformant' "$RUNNER_TEMP/planted-browser-admit.txt" || { echo "the planted failing browser sink was not refused at rule 2"; exit 1; }
      # A relay through the conformance runner: the reference relay (wai-relay --dev, on loopback)
      # held to moqt-conformance by the interop client, joule-moq-interop, at each draft it
      # speaks. Each run must be conformant, and `wai verify --conformant` must admit every record
      # and give the live area its verdict. A relay that fails a case fails its record
      # (tests/cli_conform_relay.rs plants one).
      - name: wai conform relay with the reference relay at each draft, each record admitted
        shell: bash
        run: |
          set -euo pipefail
          jc=joule-code/joule-code-rs
          (cd "$jc" && cargo build --locked -p joule-moq-relay --bin wai-relay -p joule-moq-interop --bin joule-moq-interop)
          jt=$(cd "$jc" && cargo metadata --format-version 1 --no-deps --locked | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')
          w="$RUNNER_TEMP/bin/wai"; d="$RUNNER_TEMP/relay"; mkdir -p "$d"
          "$jt/debug/wai-relay" --dev > "$RUNNER_TEMP/relay.log" 2>&1 &
          relay=$!
          trap 'kill $relay 2>/dev/null || true' EXIT
          for _ in $(seq 1 100); do grep -q 'on udp 127.0.0.1:4443' "$RUNNER_TEMP/relay.log" && break; sleep 0.2; done
          grep -q 'on udp 127.0.0.1:4443' "$RUNNER_TEMP/relay.log" || { cat "$RUNNER_TEMP/relay.log"; echo "the reference relay did not start"; exit 1; }
          export WAI_MOQ_INTEROP_BIN="$jt/debug/joule-moq-interop"
          v=$("$jt/debug/wai-relay" --version | awk '{print $2}' | tr -d ,)
          for draft in 22 18 16; do
            "$w" conform relay --suite moqt --text --root wai --impl moqt://127.0.0.1:4443/moq --draft "$draft" --tls-disable-verify \
              --name wai-relay --version "$v" --out "$d/moqt-$draft.wai-conform.json"
          done
          "$w" verify --conformant --root wai "$d" | tee "$RUNNER_TEMP/relay-admit.txt"
          grep -q '^wai verify: area live: conformant' "$RUNNER_TEMP/relay-admit.txt" || { echo "the live area has no conformant verdict"; exit 1; }
      # The records above are signed with keys made for their runs and name this runner's paths,
      # so a list of conforming implementations admits none of them. The publication below runs
      # the same implementations through wai/quickstart/conformance-records.sh, which signs each
      # record with WAI's key, the repository secret WAI_CONFORMANCE_SEED, from a folder whose
      # paths a record may publish. Planted here first, on every change: with no seed it writes
      # nothing and says so in one notice; with the test key conform-result-conformance publishes
      # (32 bytes of 0x44), or a seed that is not one, it stops and writes nothing.
      - name: WAI's records for publication · planted, no seed and a test key write nothing
        shell: bash
        run: |
          set -euo pipefail
          s=wai/quickstart/conformance-records.sh; t="$RUNNER_TEMP/publish-planted"; mkdir -p "$t"
          WAI_CONFORMANCE_SEED='' bash "$s" --bin "$RUNNER_TEMP/bin" --out "$t/none" > "$t/none.txt" 2>&1
          grep -q '^::notice::the repository secret WAI_CONFORMANCE_SEED is not set' "$t/none.txt" || { cat "$t/none.txt"; echo "no seed: no notice"; exit 1; }
          [ ! -e "$t/none" ] || { echo "no seed: something was written"; exit 1; }
          echo "no seed: one notice, nothing written"
          for planted in test-key not-a-seed; do
            case "$planted" in test-key) seed=$(printf '44%.0s' $(seq 32)); want='a test key conform-result-conformance publishes' ;; not-a-seed) seed=not-hexadecimal; want='is not an Ed25519 seed' ;; esac
            set +e
            WAI_CONFORMANCE_SEED="$seed" bash "$s" --bin "$RUNNER_TEMP/bin" --out "$t/$planted" > "$t/$planted.txt" 2>&1
            c=$?
            set -e
            [ "$c" = 1 ] && grep -q "$want" "$t/$planted.txt" && [ ! -e "$t/$planted" ] || { cat "$t/$planted.txt"; echo "$planted: exit $c, not refused as planted"; exit 1; }
            echo "$planted: refused, nothing written: $(grep -m1 "$want" "$t/$planted.txt")"
          done
      # On a push to main, WAI's own records at this commit, signed with WAI's key, kept as the
      # run's artefact wai-conformance-records: wai-conformance-<wai version>+<commit>.tar.gz and
      # its SHA-256 (the release attaches the same pack to a wai-v* tag, wai-release.yml). Without
      # the secret the step says so in one notice, writes nothing and succeeds, and nothing is
      # uploaded. The seed is never printed: the script writes it to a file only it reads.
      - name: WAI's own records, signed with WAI's key, for publication
        id: records
        if: github.event_name == 'push'
        shell: bash
        env:
          WAI_CONFORMANCE_SEED: ${{ secrets.WAI_CONFORMANCE_SEED }}
        run: |
          set -euo pipefail
          jt=$(cd joule-code/joule-code-rs && cargo metadata --format-version 1 --no-deps --locked | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')
          cp "$jt/debug/wai-relay" "$jt/debug/joule-moq-interop" "$RUNNER_TEMP/bin/"
          out="$RUNNER_TEMP/wai-conformance"
          bash wai/quickstart/conformance-records.sh --bin "$RUNNER_TEMP/bin" --commit "$GITHUB_SHA" --out "$out"
          if [ -n "$(ls -A "$out" 2>/dev/null)" ]; then echo "made=true" >> "$GITHUB_OUTPUT"; else echo "made=false" >> "$GITHUB_OUTPUT"; fi
      - name: Keep them as the run's artefact
        if: github.event_name == 'push' && steps.records.outputs.made == 'true'
        uses: actions/upload-artifact@v3
        with:
          name: wai-conformance-records
          path: ${{ runner.temp }}/wai-conformance