open-standards/.github/workflows/wai-breaking.yml

name: WAI breaking changes and the conformance runner

# The gate WAI's libraries are published behind (toolchain v1, decision 7): every pull request
# that touches WAI runs `wai breaking --against origin/main`, and every push to main that touches
# WAI runs it against main as it was before the push; each fails on a breaking change that
# wai/CHANGELOG.md or a wai/changes/ entry does not name. Two jobs:
#
# - breaking: the check itself, against the merge base of the change and main, in all
#   three categories: wire (the registry export, the media and delivery corpora), record (the
#   signing domains, magics and versions, the record corpora, the published JSON Schemas) and
#   API (the Rust public API of the WAI library crates through rustdoc's JSON, the C header,
#   the room service's OpenAPI document).
# - runner: the tests of `wai breaking` (each kind of break reported and failing until a
#   changelog entry names it) and of `wai conform` (the reference implementations conform, a
#   changed verdict or output byte fails, a changed record is refused by `wai verify`, and
#   `wai verify --conformant` admits a passing run of the native sink and refuses a planted
#   failing one); then `wai conform` runs the corpora's independent Python verifiers and the
#   reference packager, and `wai verify` checks every record those runs wrote; and it runs the
#   native sink, through its module and its C interface, and the score decoder, each record
#   admitted against this checkout's corpus and a planted failing sink refused
#   (conformance-results §4.1).
#
# Push filters match the push's delta, and the `relevant` job gates the work from the pull
# request's own diff (scripts/pr-touches-paths.py, reading the list below), as wai-pack.yml
# explains. On a push to main it has no pull request to read and answers `true`, so the push's
# own filter decides.

# A pull request's newer push supersedes its older run. A push to main does not: each push is
# checked against main before it, so a cancelled run would leave that push's changes compared
# by nothing. Pushes to main therefore each take their own group.
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.sha || github.ref }}
  cancel-in-progress: true

on:
  push:
    branches: [main]
    paths:
      - 'wai/**'
      - 'joule-code/joule-code-rs/crates/joule-moq-room/openapi.json'
      - 'rust-toolchain.toml'
      - 'scripts/pr-touches-paths.py'
      - '.github/workflows/wai-breaking.yml'
  pull_request:
    paths:
      - 'wai/**'
      # The room service's API is part of the API category.
      - 'joule-code/joule-code-rs/crates/joule-moq-room/openapi.json'
      # The toolchain pin decides which compiler documents both sides of the Rust API.
      - 'rust-toolchain.toml'
      - 'scripts/pr-touches-paths.py'
      - '.github/workflows/wai-breaking.yml'

jobs:
  relevant:
    name: WAI breaking · does this change touch WAI
    runs-on: linux
    outputs:
      run: ${{ steps.gate.outputs.run }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0
      - id: gate
        run: |
          set -euo pipefail
          r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-breaking.yml \
                "${{ github.event.pull_request.base.sha }}" \
                "${{ github.event.pull_request.head.sha }}")
          echo "this change touches WAI: $r"
          echo "run=$r" >> "$GITHUB_OUTPUT"

  breaking:
    name: WAI breaking · wire, record and API against main · linux
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: linux
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0
      - name: Install Rust 1.99.1
        shell: bash
        run: |
          export PATH="$HOME/.cargo/bin:$PATH"
          command -v rustup >/dev/null 2>&1 || \
            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.99.1
          rustup toolchain install 1.99.1 --profile minimal
          rustup default 1.99.1
          SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo"
          command -v python3 >/dev/null || { $SUDO apt-get update -qq; $SUDO apt-get install -y -qq python3; }
          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
      - name: Build wai
        working-directory: wai/wai-rs
        shell: bash
        run: |
          set -euo pipefail
          cargo build --locked --no-default-features --bin wai
          out=$(cargo metadata --format-version 1 --no-deps --locked | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])')
          mkdir -p "$RUNNER_TEMP/bin" && cp "$out/debug/wai" "$RUNNER_TEMP/bin/"
      # On a pull request the base is main's tip as fetched now; `wai breaking` compares
      # against its merge base with this change, so what main gained since the branch left it
      # is not read as this change removing it. On a push to main, main is this commit, so the
      # base is main before the push (`github.event.before`). That sha is all zeros on a
      # branch's first push or after a force-push, and a force-push can name a commit this
      # clone does not hold; either way the base falls back to the pushed commit's first
      # parent, so at least the newest commit is checked rather than nothing. Every finding is
      # printed: each break with its category, what changed and why it breaks, and the line
      # that acknowledges it if it is intended.
      - name: wai breaking against main
        shell: bash
        env:
          EVENT: ${{ github.event_name }}
          BEFORE: ${{ github.event.before }}
        run: |
          set -euo pipefail
          # The job's container user may not own the checkout; git then refuses to read it.
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git fetch -q --no-tags origin +refs/heads/main:refs/remotes/origin/main
          against=origin/main
          if [ "$EVENT" = push ]; then
            if [[ "$BEFORE" =~ ^[0-9a-f]{40}$ ]] && [[ ! "$BEFORE" =~ ^0+$ ]] && git cat-file -e "$BEFORE^{commit}" 2>/dev/null; then
              against=$BEFORE
            else
              echo "the push's previous main (${BEFORE:-none}) is not a commit here: comparing with the first parent"
              against=$(git rev-parse HEAD^1)
            fi
          fi
          echo "wai breaking --against $against"
          "$RUNNER_TEMP/bin/wai" breaking --against "$against" --text --all

  runner:
    name: WAI breaking and conform · their tests, and the independent verifiers through wai conform · linux
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: linux
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v5
      - name: Install Rust 1.99.1
        shell: bash
        run: |
          export PATH="$HOME/.cargo/bin:$PATH"
          command -v rustup >/dev/null 2>&1 || \
            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.99.1
          rustup toolchain install 1.99.1 --profile minimal
          rustup default 1.99.1
          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      # claim_verify.py, jwp_verify.py and signer_verify.py check Ed25519's equation with it.
      - name: Install the Ed25519 library
        shell: bash
        run: python -m pip install cryptography
      # The tests build fixture repositories with git and document a fixture crate with rustdoc.
      - name: The tests of wai breaking and wai conform
        working-directory: wai/wai-rs
        shell: bash
        run: |
          set -euo pipefail
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git config --global user.email [email protected]
          git config --global user.name ci
          cargo test --locked --no-default-features --features quickstart,score,key_discovery,native_sink \
            --bin wai --test cli_breaking --test cli_conform --test cli_conform_sinks
      - name: Build wai and the reference packager
        shell: bash
        run: |
          set -euo pipefail
          out() { (cd "$1" && cargo metadata --format-version 1 --no-deps --locked | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])'); }
          mkdir -p "$RUNNER_TEMP/bin"
          # The test step's features, so cargo reuses what it built.
          (cd wai/wai-rs && cargo build --locked --no-default-features --features quickstart,score,key_discovery,native_sink \
            --bin wai --bin wai_native_sink_vectors)
          cp "$(out wai/wai-rs)/debug/wai" "$(out wai/wai-rs)/debug/wai_native_sink_vectors" "$RUNNER_TEMP/bin/"
          (cd wai/wai-pack && cargo build --locked --bin wai-pack) && cp "$(out wai/wai-pack)/debug/wai-pack" "$RUNNER_TEMP/bin/"
      # The second implementations of the claim, JWP and key-discovery corpora, written from the
      # extension texts and sharing no code with wai-rs, run through the conformance runner as
      # any other implementation would be; then the reference packager's policy reader. Each
      # run must be conformant, and every record it signs must verify.
      - name: wai conform with the independent verifiers and the reference packager
        shell: bash
        run: |
          set -euo pipefail
          w="$RUNNER_TEMP/bin/wai"; d="$RUNNER_TEMP/conform"; mkdir -p "$d"
          for s in claim jwp signer; do
            "$w" conform verifier --suite "$s" --text --root wai \
              --impl "python3 -I wai/$s-conformance/${s}_verify.py verdicts {corpus}" --out "$d/$s.wai-conform.json"
          done
          "$w" conform verifier --suite conform-result --text --root wai \
            --impl "python3 -I wai/conform-result-conformance/conform_result_verify.py verdicts {corpus}" --out "$d/conform-result.wai-conform.json"
          "$w" conform packager --suite policy --text --root wai \
            --impl "$RUNNER_TEMP/bin/wai-pack policy-verdict {in}" --out "$d/policy.wai-conform.json"
          "$w" verify "$d"
      # The native sink through the conformance runner, as another implementation would be run:
      # through its Rust module and through the C interface its platform integrations link; and
      # the score decoder. Each run must be conformant, and `wai verify
      # --conformant` must admit each record against this checkout's corpus. A planted sink that
      # presents an object whose digest does not match must be refused: its run fails, its record
      # still verifies, and admission refuses it (not_conformant).
      - name: wai conform with the native sink and the score decoder, each record admitted, a planted failing sink refused
        shell: bash
        run: |
          set -euo pipefail
          w="$RUNNER_TEMP/bin/wai"; s="$RUNNER_TEMP/bin/wai_native_sink_vectors"; d="$RUNNER_TEMP/sinks"; mkdir -p "$d"
          "$w" conform sink --suite native-sink --text --root wai --impl "$s answers {case}" --out "$d/native-sink.wai-conform.json"
          "$w" conform sink --suite native-sink --text --root wai --impl "$s answers --c-abi {case}" --out "$d/native-sink-c-abi.wai-conform.json"
          "$w" conform decoder --suite score --text --root wai --impl "$w score render {in} --pcm {out}" --out "$d/score.wai-conform.json"
          "$w" verify --conformant --root wai "$d"
          # The planted sink's run must fail as a run (exit 1), not as a usage error, a missing
          # program or an adapter that does not answer, and on the case it tampers with; its
          # record must be refused at rule 2, not at another.
          p="$RUNNER_TEMP/planted"; mkdir -p "$p"
          planted="$s answers {case} | sed '/object_digest_mismatch/s/\"present\":false/\"present\":true/'"
          set +e
          "$w" conform sink --suite native-sink --text --root wai --impl "$planted" --out "$p/planted.wai-conform.json" > "$RUNNER_TEMP/planted-run.txt" 2>&1
          c=$?
          set -e
          cat "$RUNNER_TEMP/planted-run.txt"
          [ "$c" = 1 ] || { echo "the planted failing sink: wai conform exited $c, not 1"; exit 1; }
          grep -q 'tampered-fragment' "$RUNNER_TEMP/planted-run.txt" || { echo "the planted failing sink: tampered-fragment is not a failed case"; exit 1; }
          "$w" verify "$p"
          set +e
          "$w" verify --conformant --root wai "$p" > "$RUNNER_TEMP/planted-admit.txt" 2>&1
          c=$?
          set -e
          cat "$RUNNER_TEMP/planted-admit.txt"
          [ "$c" = 1 ] || { echo "the planted failing sink: wai verify --conformant exited $c, not 1"; exit 1; }
          grep -q 'FAIL admission rule 2 not_conformant' "$RUNNER_TEMP/planted-admit.txt" || { echo "the planted failing sink was not refused at rule 2"; exit 1; }