open-standards/.github/workflows/wai-pack.yml

name: wai-pack

# The reference packager of WAI's packaging extension (wai/extensions/packaging.md) and
# its conformance corpus (wai/pack-conformance). Four jobs:
#
# - legs: on each architecture of the fleet, clippy with float arithmetic denied (no
#   float decides anything in a package), the tests in release with a floor on how many
#   each target lists, and the corpus recomputed by the crate: the golden packages
#   packed again byte for byte and verified, every mutation's reason.
# - independent: the corpus recomputed by its second implementation, written in Python
#   from the extension text and sharing no code with wai-pack, the golden packages
#   verified and re-derived from their sources among it (every refinement layer and
#   still byte for byte, every delivery case through the staged-measurement simulator's
#   Python twin); and the differentials: source readers, policies, scene cuts, the
#   staged packer (layers, stills, piece plans, deliveries), and a still's reader.
# - media: the golden's floor, its CMAF initialization segment and fragments, read and
#   decoded by a media prober (test 14).
# - licences: every crate in wai-pack's dependency graph is under a licence deny.toml
#   allows; a source-available licence fails it.
#
# Push filters match the push's delta, so every path a build or a verifier reads is
# listed: the crate, the wai crate it builds on, the corpus, the BLAKE3, integer payload,
# staged-delivery and staged-measurement verifiers, the claim verifier (a metered
# package's claim) and the OER verifier it imports, the registry export pack_verify.py
# reads, and this file.

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
    paths:
      - 'wai/wai-pack/**'
      - 'wai/wai-rs/**'
      - 'wai/pack-conformance/**'
      - 'wai/claim-conformance/_blake3.py'
      - 'wai/claim-conformance/_ed25519.py'
      - 'wai/claim-conformance/claim_verify.py'
      - 'wai/oer-conformance/oer2_verify.py'
      - 'wai/int-payload-conformance/int_payload_verify.py'
      - 'wai/staged-conformance/staged_verify.py'
      - 'wai/staged-measure-conformance/sim_verify.py'
      - 'wai/staged-measure-conformance/measure_verify.py'
      - 'wai/http-conformance/http_verify.py'
      - '.github/workflows/wai-pack.yml'
  schedule:
    - cron: '38 6 * * *'
jobs:
  # Forgejo matches the `paths:` filters above against the **push delta**, not
  # the pull request's diff. A rebase moves a branch across every commit main
  # has gained, so its delta touches nearly every path and this workflow fires
  # on changes that cannot affect it. Measured on this repo: a jcp-only change
  # pushed to a fresh branch ran 2 workflows; the same change force-pushed after
  # a rebase ran 15, this one among them, putting three platforms of wai-pack and
  # the JPEG-AI whole decode in a queue they had no business being in.
  #
  # So the filters above still gate the *trigger*, and this job gates the
  # *work*, from the pull request's own diff. It reads the path list out of this
  # file, so there is one list and not a third copy to drift. On a push to main
  # there is no pull-request context, the shas are empty, and the gate answers
  # `true` — a push to main runs everything, as before.
  relevant:
    name: wai-pack · does this change touch it
    runs-on: linux-arm64
    outputs:
      run: ${{ steps.gate.outputs.run }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0
      - id: gate
        run: |
          set -euo pipefail
          r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-pack.yml \
                "${{ github.event.pull_request.base.sha }}" \
                "${{ github.event.pull_request.head.sha }}")
          echo "this change touches wai-pack: $r"
          echo "run=$r" >> "$GITHUB_OUTPUT"

  legs:
    name: wai-pack · ${{ matrix.os }}
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        # Self-hosted fleet labels: x86_64 by linux, arm64 by linux-arm64 and macos.
        # A Windows leg waits for its runner (see wai-byte-equality.yml).
        os: [linux, linux-arm64, macos]
    steps:
      - uses: actions/checkout@v5

      # The docker runners ship no Rust, so rustup is bootstrapped; the macOS host has
      # it, but launchd's PATH omits ~/.cargo/bin.
      - name: Install Rust 1.98.0
        shell: bash
        run: |
          export PATH="$HOME/.cargo/bin:$PATH"
          command -v rustup >/dev/null 2>&1 || \
            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.98.0
          rustup toolchain install 1.98.0 --profile minimal
          rustup default 1.98.0
          rustup component add --toolchain 1.98.0 clippy
          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"

      # clippy::float_arithmetic is denied for the library and the CLI (Cargo.toml and
      # the crate attributes); -D warnings makes every other lint fail the step too.
      - name: clippy, float arithmetic denied
        working-directory: wai/wai-pack
        run: cargo clippy --locked --all-targets -- -D warnings

      - name: Tests (release)
        working-directory: wai/wai-pack
        run: cargo test --release --locked

      # A target that lost its tests (a module dropped, a cfg that compiles them out)
      # would still report ok. Each target must list at least as many tests as it had
      # when the floor was set; raise a floor when tests are added.
      - name: Test-count floor per target
        shell: bash
        working-directory: wai/wai-pack
        run: |
          set -euo pipefail
          fail=0
          while read -r target floor; do
            n=$(cargo test --release --locked $target -- --list 2>/dev/null | grep -c ': test$' || true)
            echo "$target: $n tests (floor $floor)"
            [ "$n" -ge "$floor" ] || { echo "::error::$target lists $n tests, under its floor of $floor"; fail=1; }
          done <<'EOF'
          --lib 34
          --test=cmaf 4
          --test=conformance 1
          --test=floor_only 2
          --test=ladder 3
          --test=pack_alloc 1
          --test=packs 8
          --test=review 9
          --test=refusals 6
          --test=repack 3
          --test=scenecut 3
          --test=source_alloc 1
          --test=staged 10
          EOF
          exit $fail

      - name: The corpus, recomputed by wai-pack
        working-directory: wai/wai-pack
        run: cargo run --release --locked --bin wai-pack -- conformance check ../pack-conformance

  independent:
    name: wai-pack · independent verifier (Python)
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: linux-arm64
    steps:
      - uses: actions/checkout@v5
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      # The standard library (tomllib, hashlib); BLAKE3 from the shared pure-Python module
      # beside the claim corpus; the package receipts' Ed25519 equation from the
      # `cryptography` package, pinned with its dependencies by SHA-256 so pip installs only
      # the files the requirements name.
      - name: Install the Ed25519 library (hashes required)
        shell: bash
        working-directory: wai
        run: python -m pip install --require-hashes -r pack-conformance/requirements-verify.txt
      - name: pack_verify.py
        working-directory: wai
        shell: bash
        run: python3 pack-conformance/pack_verify.py pack-conformance
      # The same checks with the hashes and decodes run ahead in worker processes
      # (--jobs): every check and its reason are the serial run's.
      - name: pack_verify.py --jobs 2
        working-directory: wai
        shell: bash
        run: python3 pack-conformance/pack_verify.py --jobs 2 pack-conformance
      - name: Install Rust 1.98.0
        shell: bash
        run: |
          export PATH="$HOME/.cargo/bin:$PATH"
          command -v rustup >/dev/null 2>&1 || \
            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.98.0
          rustup toolchain install 1.98.0 --profile minimal
          rustup default 1.98.0
          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
      # The source readers differentially: seeded Y4M and WAV mutations, every verdict of
      # wai-pack (debug and release, both with overflow checks) equal to pack_verify.py's,
      # and no reader panicking.
      - name: source_diff.py, wai-pack against pack_verify.py
        shell: bash
        working-directory: wai/wai-pack
        run: |
          set -euo pipefail
          cargo build --locked --bin wai-pack
          cargo build --release --locked --bin wai-pack
          for profile in debug release; do
            python3 ../pack-conformance/source_diff.py "target/$profile/wai-pack" 1 3000
          done
      # The policy reader differentially: byte-level cases and seeded random policies,
      # every verdict and digest of wai-pack equal to pack_verify.py's.
      - name: policy_diff.py, wai-pack against pack_verify.py
        shell: bash
        working-directory: wai/wai-pack
        run: python3 ../pack-conformance/policy_diff.py target/release/wai-pack 1 3000
      # wai.pack.scenecut/1 differentially: the streaming segmenter against
      # pack_verify.py's on-demand form on seeded random clips.
      - name: segment_diff.py, wai-pack against pack_verify.py
        shell: bash
        working-directory: wai/wai-pack
        run: python3 ../pack-conformance/segment_diff.py target/release/wai-pack 1 2000
      # The staged packer differentially: refinement layers, stills, piece plans and
      # deliveries on seeded random inputs, every result of wai-pack equal to
      # pack_verify.py's (debug and release, both with overflow checks).
      - name: stage_diff.py, wai-pack against pack_verify.py
        shell: bash
        working-directory: wai/wai-pack
        run: |
          set -euo pipefail
          python3 ../pack-conformance/stage_diff.py target/debug/wai-pack 1 1000
          python3 ../pack-conformance/stage_diff.py target/release/wai-pack 2 3000
      # The supplied CMAF reader differentially (packaging §7.7): seeded alterations of a
      # media muxer's segments and of §7.6's writer's, every verdict and accepted segment's
      # samples of wai-pack's reader equal to pack_verify.py's.
      - name: cmaf_diff.py, wai-pack against pack_verify.py
        shell: bash
        working-directory: wai/wai-pack
        run: python3 ../pack-conformance/cmaf_diff.py target/release/wai-pack 1 3000
      # A still's reader differentially: the zlib library's own streams, dynamic blocks
      # built on each of RFC 1951's and zlib's code-set rules, and their alterations, every
      # verdict of wai-pack's reader equal to pack_verify.py's, which inflates with
      # Python's zlib (debug and release).
      - name: png_diff.py, wai-pack against pack_verify.py
        shell: bash
        working-directory: wai/wai-pack
        run: |
          set -euo pipefail
          python3 ../pack-conformance/png_diff.py target/debug/wai-pack 1 2000
          python3 ../pack-conformance/png_diff.py target/release/wai-pack 2 4000

  media:
    name: wai-pack · the floor through a media prober
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: linux-arm64
    steps:
      - uses: actions/checkout@v5
      - name: Install a media prober
        shell: bash
        run: |
          SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo"
          $SUDO apt-get update -qq
          $SUDO apt-get install -y -qq ffmpeg
      # Test 14: each golden floor's init.mp4 and its fragments, in its floor playlist's
      # order, read as one AV1 track of every frame of the source, with a key frame at each
      # piece start of the plan and nowhere else, and decode without an error; a ladder
      # golden has one floor playlist per floor rung (packaging §7.1), each read so.
      - name: The goldens' floors read and decode
        shell: bash
        working-directory: wai/pack-conformance/packs
        run: |
          set -euo pipefail
          for g in video-base-01 video-staged-01 video-ladder-01 video-cmaf-01 video-floor-only-02; do
            ( cd "$g/package"
              for pl in hls/classic/floor-*.m3u8; do
                f=$(basename "$pl" .m3u8); f=${f#floor-}
                tmp=$(mktemp -d)
                cat "floor/$f/init.mp4" $(grep -v '^#' "$pl" | sed 's#^\.\./\.\./##') > "$tmp/floor.mp4"
                ffprobe -v error -count_frames -select_streams v:0 -show_entries stream=codec_name,width,height,nb_read_frames -of default=nw=1 "$tmp/floor.mp4" | tee "$tmp/probe.txt"
                grep -qx 'codec_name=av1' "$tmp/probe.txt"
                grep -qx 'nb_read_frames=200' "$tmp/probe.txt"
                keys=$(ffprobe -v error -select_streams v:0 -show_entries packet=pts,flags -of csv=p=0 "$tmp/floor.mp4" | awk -F, '$2 ~ /K/ {print $1}' | tr '\n' ' ')
                want=$(tr '\n' ' ' < plan-v.txt)
                echo "$g $f key frames: $keys"
                [ "$keys" = "$want" ] || { echo "::error::$g $f: key frames at $keys; the plan says $want"; exit 1; }
                ffmpeg -v error -xerror -i "$tmp/floor.mp4" -f null -
                echo "$g $f: the floor reads as one AV1 track of 200 frames, a key frame at each piece start"
              done )
          done

  licences:
    name: wai-pack · dependency licences
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: linux-arm64
    steps:
      - uses: actions/checkout@v5
      - name: Install Rust 1.98.0
        shell: bash
        run: |
          export PATH="$HOME/.cargo/bin:$PATH"
          command -v rustup >/dev/null 2>&1 || \
            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.98.0
          rustup toolchain install 1.98.0 --profile minimal
          rustup default 1.98.0
          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
      - name: Install cargo-deny
        shell: bash
        run: cargo install --locked cargo-deny --version 0.19.8
      - name: cargo deny check licenses
        working-directory: wai/wai-pack
        run: cargo deny --locked check licenses
      # deny.toml's MPL-2.0 exception covers cbindgen as wai's build-time tool: it must
      # reach wai-pack through no normal (linked) edge. The second query sees the build
      # edge, so an empty answer from the first is an answer.
      - name: cbindgen is a build dependency only
        shell: bash
        working-directory: wai/wai-pack
        run: |
          set -euo pipefail
          err=$(mktemp)
          if ! linked=$(cargo tree --locked -e normal --target all -i cbindgen 2>"$err"); then
            grep -q "did not match any packages" "$err" || { cat "$err"; exit 1; }
            linked=""
          fi
          if [ -n "$linked" ]; then
            echo "::error::cbindgen is a linked dependency of wai-pack; its MPL-2.0 exception covers a build tool only"
            echo "$linked"
            exit 1
          fi
          cargo tree --locked -e normal,build --target all -i cbindgen | grep -q '^cbindgen v' || { echo "::error::cbindgen not found as wai's build dependency"; exit 1; }
          echo "cbindgen is a build dependency only"
      # The `wai` binary's HTTP commands with their TLS client (the cli_tls feature:
      # rustls and the operating system's trust store), under wai-rs's own deny.toml: the
      # same allow-list, over the binary's features on the hosts it is built for.
      - name: cargo deny check licenses (the wai binary's TLS client)
        working-directory: wai/wai-rs
        run: cargo deny --locked check licenses