open-standards/.github/workflows/wai-breaking.yml

name: WAI breaking changes and the conformance runner

# The gate WAI's libraries are published behind (toolchain v1, decision 7): every pull request
# that touches WAI runs `wai breaking --against origin/main`, and fails on a breaking change that
# wai/CHANGELOG.md does not name. Two jobs:
#
# - breaking: the check itself, against the merge base of the pull request and main, in all
#   three categories: wire (the registry export, the media and delivery corpora), record (the
#   signing domains, magics and versions, the record corpora, the published JSON Schemas) and
#   API (the Rust public API of the WAI library crates through rustdoc's JSON, the C header,
#   the room service's OpenAPI document).
# - runner: the tests of `wai breaking` (each kind of break reported and failing until a
#   changelog entry names it) and of `wai conform` (the reference implementations conform, a
#   changed verdict or output byte fails, a changed record is refused by `wai verify`, and
#   `wai verify --conformant` admits a passing run of the native sink and refuses a planted
#   failing one; the JUnit and HTML reports are the records'; `--independent` holds the
#   corpus's independent verifier to the same cases and fails on a disagreement, and alone
#   runs the verifiers it names); then `wai conform` runs the corpora's independent Python
#   verifiers and the reference packager, and `wai verify` checks every record those runs
#   wrote; and it runs the native sink, through its module and its C interface, and the score
#   decoder, each record admitted against this checkout's corpus and a planted failing sink
#   refused (conformance-results §4.1). The corpus union itself, every independent verifier
#   through `wai conform --independent`, runs in wai-conformance.yml.
#
# Push filters match the push's delta, and the `relevant` job gates the work from the pull
# request's own diff (scripts/pr-touches-paths.py, reading the list below), as wai-pack.yml
# explains.

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
    paths:
      - 'wai/**'
      # The room service's API is part of the API category.
      - 'joule-code/joule-code-rs/crates/joule-moq-room/openapi.json'
      # The toolchain pin decides which compiler documents both sides of the Rust API.
      - 'rust-toolchain.toml'
      - 'scripts/pr-touches-paths.py'
      # wai-rs/build.rs reads `wai conform`'s table of independent verifiers from the step files
      # of wai-conformance.yml, and the runner's tests below hold the table to them.
      - '.github/wai-conformance/verifiers/**'
      - '.github/workflows/wai-breaking.yml'

jobs:
  relevant:
    name: WAI breaking · does this change touch WAI
    runs-on: linux
    outputs:
      run: ${{ steps.gate.outputs.run }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0
      - id: gate
        run: |
          set -euo pipefail
          r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-breaking.yml \
                "${{ github.event.pull_request.base.sha }}" \
                "${{ github.event.pull_request.head.sha }}")
          echo "this change touches WAI: $r"
          echo "run=$r" >> "$GITHUB_OUTPUT"

  breaking:
    name: WAI breaking · wire, record and API against main · linux
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: linux
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0
      - name: Install Rust 1.99.1
        shell: bash
        run: |
          export PATH="$HOME/.cargo/bin:$PATH"
          command -v rustup >/dev/null 2>&1 || \
            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.99.1
          rustup toolchain install 1.99.1 --profile minimal
          rustup default 1.99.1
          SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo"
          command -v python3 >/dev/null || { $SUDO apt-get update -qq; $SUDO apt-get install -y -qq python3; }
          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
      - name: Build wai
        working-directory: wai/wai-rs
        shell: bash
        run: |
          set -euo pipefail
          cargo build --locked --no-default-features --bin wai
          out=$(cargo metadata --format-version 1 --no-deps --locked | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')
          mkdir -p "$RUNNER_TEMP/bin" && cp "$out/debug/wai" "$RUNNER_TEMP/bin/"
      # The base is main's tip as fetched now; `wai breaking` compares against its merge base
      # with this pull request, so what main gained since the branch left it is not read as
      # this change removing it. Every finding is printed: each break with its category, what
      # changed and why it breaks, and the line that acknowledges it if it is intended.
      - name: wai breaking --against origin/main
        shell: bash
        run: |
          set -euo pipefail
          # The job's container user may not own the checkout; git then refuses to read it.
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git fetch -q --no-tags origin +refs/heads/main:refs/remotes/origin/main
          "$RUNNER_TEMP/bin/wai" breaking --against origin/main --text --all

  runner:
    name: WAI breaking and conform · their tests, and the independent verifiers through wai conform · linux
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: linux
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v5
      - name: Install Rust 1.99.1
        shell: bash
        run: |
          export PATH="$HOME/.cargo/bin:$PATH"
          command -v rustup >/dev/null 2>&1 || \
            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.99.1
          rustup toolchain install 1.99.1 --profile minimal
          rustup default 1.99.1
          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      # claim_verify.py, jwp_verify.py and signer_verify.py check Ed25519's equation with it.
      - name: Install the Ed25519 library
        shell: bash
        run: python -m pip install cryptography
      # The tests build fixture repositories with git and document a fixture crate with rustdoc.
      - name: The tests of wai breaking and wai conform
        working-directory: wai/wai-rs
        shell: bash
        run: |
          set -euo pipefail
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git config --global user.email [email protected]
          git config --global user.name ci
          cargo test --locked --no-default-features --features quickstart,score,key_discovery,native_sink \
            --bin wai --test cli_breaking --test cli_conform --test cli_conform_sinks
      - name: Build wai and the reference packager
        shell: bash
        run: |
          set -euo pipefail
          out() { (cd "$1" && cargo metadata --format-version 1 --no-deps --locked | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])'); }
          mkdir -p "$RUNNER_TEMP/bin"
          # The test step's features, so cargo reuses what it built.
          (cd wai/wai-rs && cargo build --locked --no-default-features --features quickstart,score,key_discovery,native_sink \
            --bin wai --bin wai_native_sink_vectors)
          cp "$(out wai/wai-rs)/debug/wai" "$(out wai/wai-rs)/debug/wai_native_sink_vectors" "$RUNNER_TEMP/bin/"
          (cd wai/wai-pack && cargo build --locked --bin wai-pack) && cp "$(out wai/wai-pack)/debug/wai-pack" "$RUNNER_TEMP/bin/"
      # The second implementations of the claim, JWP and key-discovery corpora, written from the
      # extension texts and sharing no code with wai-rs, run through the conformance runner as
      # any other implementation would be; then the reference packager's policy reader. Each
      # run must be conformant, and every record it signs must verify.
      - name: wai conform with the independent verifiers and the reference packager
        shell: bash
        run: |
          set -euo pipefail
          w="$RUNNER_TEMP/bin/wai"; d="$RUNNER_TEMP/conform"; mkdir -p "$d"
          for s in claim jwp signer; do
            "$w" conform verifier --suite "$s" --text --root wai \
              --impl "python3 -I wai/$s-conformance/${s}_verify.py verdicts {corpus}" --out "$d/$s.wai-conform.json"
          done
          "$w" conform verifier --suite conform-result --text --root wai \
            --impl "python3 -I wai/conform-result-conformance/conform_result_verify.py verdicts {corpus}" --out "$d/conform-result.wai-conform.json"
          "$w" conform packager --suite policy --text --root wai \
            --impl "$RUNNER_TEMP/bin/wai-pack policy-verdict {in}" --out "$d/policy.wai-conform.json"
          "$w" verify "$d"
      # The native sink through the conformance runner, as another implementation would be run:
      # through its Rust module and through the C interface its platform integrations link; and
      # the score decoder. Each run must be conformant, and `wai verify
      # --conformant` must admit each record against this checkout's corpus. A planted sink that
      # presents an object whose digest does not match must be refused: its run fails, its record
      # still verifies, and admission refuses it (not_conformant).
      - name: wai conform with the native sink and the score decoder, each record admitted, a planted failing sink refused
        shell: bash
        run: |
          set -euo pipefail
          w="$RUNNER_TEMP/bin/wai"; s="$RUNNER_TEMP/bin/wai_native_sink_vectors"; d="$RUNNER_TEMP/sinks"; mkdir -p "$d"
          "$w" conform sink --suite native-sink --text --root wai --impl "$s answers {case}" --out "$d/native-sink.wai-conform.json"
          "$w" conform sink --suite native-sink --text --root wai --impl "$s answers --c-abi {case}" --out "$d/native-sink-c-abi.wai-conform.json"
          "$w" conform decoder --suite score --text --root wai --impl "$w score render {in} --pcm {out}" --out "$d/score.wai-conform.json"
          "$w" verify --conformant --root wai "$d"
          # The planted sink's run must fail as a run (exit 1), not as a usage error, a missing
          # program or an adapter that does not answer, and on the case it tampers with; its
          # record must be refused at rule 2, not at another.
          p="$RUNNER_TEMP/planted"; mkdir -p "$p"
          planted="$s answers {case} | sed '/object_digest_mismatch/s/\"present\":false/\"present\":true/'"
          set +e
          "$w" conform sink --suite native-sink --text --root wai --impl "$planted" --out "$p/planted.wai-conform.json" > "$RUNNER_TEMP/planted-run.txt" 2>&1
          c=$?
          set -e
          cat "$RUNNER_TEMP/planted-run.txt"
          [ "$c" = 1 ] || { echo "the planted failing sink: wai conform exited $c, not 1"; exit 1; }
          grep -q 'tampered-fragment' "$RUNNER_TEMP/planted-run.txt" || { echo "the planted failing sink: tampered-fragment is not a failed case"; exit 1; }
          "$w" verify "$p"
          set +e
          "$w" verify --conformant --root wai "$p" > "$RUNNER_TEMP/planted-admit.txt" 2>&1
          c=$?
          set -e
          cat "$RUNNER_TEMP/planted-admit.txt"
          [ "$c" = 1 ] || { echo "the planted failing sink: wai verify --conformant exited $c, not 1"; exit 1; }
          grep -q 'FAIL admission rule 2 not_conformant' "$RUNNER_TEMP/planted-admit.txt" || { echo "the planted failing sink was not refused at rule 2"; exit 1; }