open-standards/.github/workflows/wai-quantum-landscape-devices.yml

name: wai-quantum energy-landscape GPU members

# The energy-landscape kernels on GPUs (wai_quantum::quantum_landscape::{webgpu, cuda, metal};
# extensions/quantum-landscape.md §5), each device held to the reference as a host holds a member:
# certified through member::certify on the conformance set, bit-exact for the settle kernel and
# the sampling kernel; every worked example at three temperatures over 50 seeds, and jobs past
# every window and workgroup the devices split them into, giving the reference's bits; each
# fault a port of the kernels can have (a dropped carry, a high-word compare, fields held in 16
# or 32 bits, a draw counter that starts again each window, cells last to first, a settle that
# stops at one workgroup), put into the kernels on the device, costing them their grade; the
# coloured WebGPU kernel earning statistical and no more; a WebGPU wait covering one window, and
# a job a member cannot finish going on to the reference; a workload a device declines placed
# on the reference, the answer naming the refusal. Each host declares the members it must have
# (WAI_LANDSCAPE_EXPECT): one missing, or whose kernels no longer compile, fails its leg. The
# timings that follow are wall-clock on that shared host, with its load printed beside them;
# their bits are asserted and their times are not.
#
# - jetson-hub (linux-hw): its laptop GPU through CUDA and through Vulkan, its integrated GPU and
#   the software Vulkan driver through WebGPU. This is the only leg with a CUDA device: while its
#   laptop GPU is off its bus, the leg fails, naming the members it is missing, and nothing here
#   runs the CUDA kernels. The CUDA kernels' compile test still runs there.
# - framework-128gb (linux-amd): its integrated GPU and the software Vulkan driver through WebGPU.
# - the Mac Studio (macos-m3-ultra): its GPU through WebGPU on Metal, and through Metal itself.
# No leg runs WebGPU on DX12, and no parity is claimed for it.

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
    paths:
      - 'wai/wai-quantum/src/quantum_landscape/**'
      - 'wai/wai-quantum/src/quantum_landscape.rs'
      - 'wai/wai-quantum/src/quantum_fabric.rs'
      - 'wai/wai-quantum/src/quantum_fabric/**'
      - 'wai/wai-quantum/src/quantum_fabric_webgpu.rs'
      - 'wai/wai-quantum/src/quantum_fabric_cuda.rs'
      - 'wai/wai-quantum/src/quantum_fabric_metal.rs'
      - 'wai/wai-quantum/src/quantum_energy.rs'
      - 'wai/wai-quantum/src/lib.rs'
      - 'wai/wai-quantum/tests/landscape_devices.rs'
      - 'wai/wai-quantum/Cargo.toml'
      - 'wai/wai-quantum/Cargo.lock'
      - 'wai/quantum-conformance/landscape/vectors.json'
      - 'rust-toolchain.toml'
      - '.github/workflows/wai-quantum-landscape-devices.yml'
  schedule:
    - cron: '17 8 * * *'

jobs:
  relevant:
    name: does this change touch it
    runs-on: linux
    outputs:
      run: ${{ steps.gate.outputs.run }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0
      - id: gate
        run: |
          set -euo pipefail
          r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-landscape-devices.yml \
                "${{ github.event.pull_request.base.sha }}" \
                "${{ github.event.pull_request.head.sha }}")
          echo "this change touches the landscape GPU members: $r"
          echo "run=$r" >> "$GITHUB_OUTPUT"

  devices:
    name: members · ${{ matrix.host }}
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        include:
          # Each host's members, declared: a prefix of each member id it must have. A member
          # missing, for any reason, fails the leg with why.
          - host: jetson-hub
            runner: linux-hw
            features: quantum_landscape_webgpu,quantum_landscape_cuda
            expect: gpu:cuda.nvidia,gpu:wgpu-vulkan.nvidia,gpu:wgpu-vulkan.intel,cpu:wgpu-vulkan.llvmpipe
            units: 9
            target: hw-target
          - host: framework-128gb
            runner: linux-amd
            features: quantum_landscape_webgpu
            expect: gpu:wgpu-vulkan.radeon,cpu:wgpu-vulkan.llvmpipe
            units: 8
            target: hw-target
          - host: mac-studio
            runner: macos-m3-ultra
            features: quantum_landscape_webgpu,quantum_landscape_metal
            expect: gpu:metal.apple-m3-ultra,gpu:wgpu-metal.apple-m3-ultra
            units: 8
            # The job's own workspace, on the Studio's external volume: never its internal disk.
            target: ''
    env:
      WAI_REQUIRE_GPU: '1'
      WAI_LANDSCAPE_EXPECT: ${{ matrix.expect }}
      WAI_LANDSCAPE_NVRTC: ${{ contains(matrix.features, 'cuda') && '1' || '' }}
      CARGO_PROFILE_DEV_DEBUG: '0'
      CARGO_BUILD_JOBS: '8'
    steps:
      - name: Fetch the source
        shell: bash
        run: |
          set -euo pipefail
          want="${{ github.event.pull_request.head.sha || github.sha }}"
          rm -rf src && mkdir src && cd src
          git init -q
          # Fetch the commit itself, not the ref. The host names its own route to the forge.
          git fetch -q --depth 1 "${WAI_FORGE_URL:-http://100.64.224.0:3400}/Transaction-Science/open-standards.git" "$want"
          git checkout -q FETCH_HEAD
          got="$(git rev-parse HEAD)"
          [ "$got" = "$want" ] || { echo "::error::fetched $got, not $want"; exit 1; }
      - name: The members' own unit tests (ids, labels, the meter's slot, colouring, bindings, the CUDA kernels compiling)
        shell: bash
        working-directory: src/wai/wai-quantum
        run: |
          set -euo pipefail
          export PATH="$HOME/.cargo/bin:$PATH"
          if [ -n "${{ matrix.target }}" ]; then export CARGO_TARGET_DIR="$HOME/${{ matrix.target }}/wai-quantum-landscape"; fi
          nice -n 10 cargo test --locked --release --features "${{ matrix.features }}" --lib -- \
            quantum_landscape::device:: quantum_landscape::webgpu:: quantum_landscape::cuda:: quantum_landscape::metal:: 2>&1 | tee "$RUNNER_TEMP/units.log"
          grep -qE '^test result: ok\. ${{ matrix.units }} passed; 0 failed' "$RUNNER_TEMP/units.log" || { echo "::error::not every unit test of the members ran and passed (${{ matrix.units }} expected)"; exit 1; }
      # Each test that certifies a member writes its certificates to a report, which is read
      # here rather than the test's output, so that no test's printing can hide a line.
      - name: Every member, certified, held to the reference past every window, and every fault losing its grade
        shell: bash
        working-directory: src/wai/wai-quantum
        run: |
          set -euo pipefail
          export PATH="$HOME/.cargo/bin:$PATH"
          if [ -n "${{ matrix.target }}" ]; then export CARGO_TARGET_DIR="$HOME/${{ matrix.target }}/wai-quantum-landscape"; fi
          report="$RUNNER_TEMP/landscape-report.txt"
          rm -f "$report"
          WAI_LANDSCAPE_REPORT="$report" nice -n 10 \
            cargo test --locked --release --features "${{ matrix.features }}" --test landscape_devices -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/devices.log"
          grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/devices.log" || { echo "::error::not every device test ran and passed (9 expected)"; exit 1; }
          cat "$report"
          IFS=',' read -ra want <<< "$WAI_LANDSCAPE_EXPECT"
          for p in "${want[@]}"; do
            grep -qE "^certified ${p}[^ ]*: settle bit-exact, sample bit-exact$" "$report" || { echo "::error::$p was not certified bit-exact for both kernels"; exit 1; }
            n=$(grep -cE "^fault .* in ${p}[^ ]*: " "$report" || true)
            [ "$n" -ge 3 ] || { echo "::error::$p lost its grade under $n faults, and 3 or more are put into every backend's kernels"; exit 1; }
          done
          grep -qE "^waits [^ ]+: 40 windows" "$report" || { echo "::error::no WebGPU member showed a wait covering one window"; exit 1; }
      - name: Timings, wall-clock on this host, with its load (the bits asserted, the times printed)
        shell: bash
        working-directory: src/wai/wai-quantum
        run: |
          set -euo pipefail
          export PATH="$HOME/.cargo/bin:$PATH"
          if [ -n "${{ matrix.target }}" ]; then export CARGO_TARGET_DIR="$HOME/${{ matrix.target }}/wai-quantum-landscape"; fi
          nice -n 10 cargo test --locked --release --features "${{ matrix.features }}" --test landscape_devices -- --ignored --nocapture --test-threads=1 timings 2>&1 | tee "$RUNNER_TEMP/timings.log"
          grep -qE '^test result: ok\. 1 passed; 0 failed' "$RUNNER_TEMP/timings.log" || { echo "::error::the timings' bits differ from the reference's"; exit 1; }