open-standards/.github/workflows/wai-quantum-dataflow.yml

name: wai-quantum dataflow member

# The fabric's dataflow member: the gate's fixed-point arithmetic on the first
# RISC-V core of a dataflow accelerator's compute tile, driven through the
# device's library at the level of configuration space, memory windows and
# clocks. The library here is the device's open-source simulator (Apache-2.0),
# built from source at a pinned commit, so every certificate is labelled
# simulator. Two device generations, each its own leg.
#
# Before the device runs anything, the RV32IM encoder that builds its programs
# is checked against an independent assembler: every listing, conforming and
# mutant, must assemble to the encoder's words exactly.

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
    paths:
      - 'wai/wai-quantum/**'
      - 'rust-toolchain.toml'
      - '.github/workflows/wai-quantum-dataflow.yml'
  schedule:
    - cron: '4 8 * * *'

jobs:
  # Does this pull request's own diff touch the paths below? This forge matches a
  # workflow's `paths:` against the push, so a merge of main into a branch fires the
  # workflow whatever the pull request changes; every job here waits on this answer.
  # A push to main, a schedule, or anything the script cannot determine answers
  # `true` (scripts/pr-touches-paths.py).
  relevant:
    name: does this change touch it
    runs-on: linux
    outputs:
      run: ${{ steps.gate.outputs.run }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0
      - id: gate
        run: |
          set -euo pipefail
          r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-dataflow.yml \
                "${{ github.event.pull_request.base.sha }}" \
                "${{ github.event.pull_request.head.sha }}")
          echo "this change touches wai-quantum-dataflow: $r"
          echo "run=$r" >> "$GITHUB_OUTPUT"

  dataflow:
    name: member · dataflow device, simulated (pci ${{ matrix.pci }})
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: linux
    timeout-minutes: 60
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        include:
          # `vector`: the generation whose vector engine the vector programs
          # drive, and whose instruction table their encoder is checked against.
          - pci: '1e52:b140'
            build: bh
            key: dataflow-1e52-b140,dataflow-1e52-b140-vector
            vector: 'true'
          - pci: '1e52:401e'
            build: wh
            key: dataflow-1e52-401e
            vector: ''

    container:
      image: debian@sha256:913f6706df59a68922d1dd08f78c2476560a8d367897200a6005b00e5f67c2d5
      options: --memory=6g --memory-swap=6g
    env:
      SIM_REPO: https://github.com/tenstorrent/ttsim.git
      SIM_COMMIT: 283f621e7384d86f01e4ac5a9c83b60704f77d92
      WAI_QUANTUM_DATAFLOW_EXPECT: ${{ matrix.key }}
      WAI_QUANTUM_DATAFLOW_SIMULATOR: device simulator 283f621e (built from source, ${{ matrix.build }})
    steps:
      - name: Tools
        shell: bash
        run: |
          set -euo pipefail
          apt-get update -qq
          apt-get install -y -qq --no-install-recommends ca-certificates curl git build-essential python3 binutils-riscv64-linux-gnu >/dev/null
      # No Node for a checkout action: the forge's archive of the exact commit,
      # its embedded commit id checked.
      - name: Fetch the source
        shell: bash
        run: |
          set -euo pipefail
          want="${{ github.event.pull_request.head.sha || github.sha }}"
          rm -rf src src.tar.gz src.tar && mkdir src
          curl -sSf -o src.tar.gz "http://100.64.224.0:3400/Transaction-Science/open-standards/archive/$want.tar.gz"
          gzip -dc src.tar.gz > src.tar
          got="$(git get-tar-commit-id < src.tar)"
          [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; }
          # -m: every file gets the time it was extracted, not its commit's. The builds
          # reuse a target directory kept between runs, and cargo takes a crate as
          # built when no source file is newer than its last build: with the commit's
          # times, an older commit ran the binary another commit had built there.
          tar -xmf src.tar -C src --strip-components=1
          rm src.tar.gz src.tar
      - name: The simulator, built from source at the pinned commit (cached), and Rust
        shell: bash
        working-directory: src/wai/wai-quantum
        run: |
          set -euo pipefail
          cache=/ci-cache/wai-quantum-dataflow
          mkdir -p "$cache"
          exec 9>"$cache/.lock"
          flock 9
          lib="$cache/sim-$SIM_COMMIT/${{ matrix.build }}/libttsim.so"
          # The table is part of the cache entry: an entry built before it was
          # kept is rebuilt, not used without it.
          if [ ! -f "$lib" ] || [ ! -f "$(dirname "$lib")/tensix_isa.json" ]; then
            b="$cache/sim-build"
            rm -rf "$b" && git init -q "$b"
            git -C "$b" fetch -q --depth 1 "$SIM_REPO" "$SIM_COMMIT"
            git -C "$b" checkout -q FETCH_HEAD
            [ "$(git -C "$b" rev-parse HEAD)" = "$SIM_COMMIT" ] || { echo "::error::the simulator is not at the pinned commit"; exit 1; }
            (cd "$b" && ./make.py "src/_out/release_${{ matrix.build }}/libttsim.so" > build.log 2>&1) || { tail -30 "$b/build.log"; exit 1; }
            mkdir -p "$(dirname "$lib")"
            # The instruction table too: the vector programs' encoder is checked against it.
            cp "$b/data/${{ matrix.build }}/tensix_isa.json" "$(dirname "$lib")/tensix_isa.json"
            cp "$b/src/_out/release_${{ matrix.build }}/libttsim.so" "$lib.tmp" && mv "$lib.tmp" "$lib"
            rm -rf "$b"
          fi
          export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo"
          export PATH="$CARGO_HOME/bin:$PATH"
          if ! command -v rustup >/dev/null; then
            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain none
          fi
          rustup toolchain install
      - name: The encoder against an independent assembler
        shell: bash
        working-directory: src/wai/wai-quantum
        run: |
          set -euo pipefail
          cache=/ci-cache/wai-quantum-dataflow
          exec 9>"$cache/.lock"
          flock 9
          # Every source file newer than every build in the cache, now that the lock is
          # held: another run may have built here while this one waited for it, after
          # this one's extraction. (../.. is the extracted tree.) The next step takes the
          # lock afresh, and refreshes again.
          find ../.. -type f -exec touch {} +
          export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target"
          export PATH="$CARGO_HOME/bin:$PATH"
          out="$RUNNER_TEMP/rv32"
          # Not -q: the build's own lines show it compiled wai-quantum; one that compiled
          # none ran a binary another commit left in the cache.
          cargo run --locked --release --example rv32_listing --features quantum_fabric_dataflow -- "$out" 2>&1 | tee "$RUNNER_TEMP/rv32-build.log"
          grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/rv32-build.log" || { echo "::error::the build compiled no wai-quantum: the encoder that ran is one already in the cache"; exit 1; }
          riscv64-linux-gnu-as --version | head -1
          n=0
          for s in "$out"/*.s; do
            p="${s%.s}"
            riscv64-linux-gnu-as -march=rv32im -mabi=ilp32 "$s" -o "$p.o"
            riscv64-linux-gnu-objcopy -O binary -j .text "$p.o" "$p.as.bin"
            cmp "$p.bin" "$p.as.bin" || { echo "::error::$(basename "$p"): the encoder's words differ from the assembler's"; exit 1; }
            n=$((n + 1))
          done
          [ "$n" = 14 ] || { echo "::error::$n of 14 programs were compared"; exit 1; }
          echo "all $n programs: the encoder's words are the assembler's"
      - name: The member's tests
        shell: bash
        working-directory: src/wai/wai-quantum
        run: |
          set -euo pipefail
          cache=/ci-cache/wai-quantum-dataflow
          exec 9>"$cache/.lock"
          flock 9
          find ../.. -type f -exec touch {} +
          export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target"
          export PATH="$CARGO_HOME/bin:$PATH"
          export WAI_QUANTUM_DATAFLOW_LIB="$cache/sim-$SIM_COMMIT/${{ matrix.build }}/libttsim.so"
          # The vector programs' encoder is a transcription of this generation's
          # table; another generation lays some instructions out differently.
          if [ -n "${{ matrix.vector }}" ]; then export WAI_QUANTUM_TENSIX_ISA="$cache/sim-$SIM_COMMIT/${{ matrix.build }}/tensix_isa.json"; fi
          cargo test --locked --release -j 6 --features quantum_fabric_dataflow --lib -- quantum_fabric_dataflow:: quantum_fabric::rv32:: quantum_fabric::tensix:: quantum_fabric::vector_engine:: --nocapture 2>&1 | tee "$RUNNER_TEMP/unit.log"
          grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; }
          grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; }
          # The vector engine's encoder against the simulator's own table (the
          # test returns early without the table, so require its line). Tests
          # print from parallel threads, so the line may follow another test's
          # "ok" on the same line: it is matched anywhere in a line.
          if [ -n "${{ matrix.vector }}" ]; then
            grep -qE 'tensix layouts: [0-9]+ instructions match the table' "$RUNNER_TEMP/unit.log" || { echo "::error::the instruction table was not checked"; exit 1; }
          fi
          report="$RUNNER_TEMP/dataflow-report.txt"; : > "$report"
          WAI_QUANTUM_DATAFLOW_REPORT="$report" cargo test --locked --release -j 6 --features quantum_fabric_dataflow --test dataflow_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log"
          grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
          cat "$report"
          grep -qx "dataflow devices: as declared ($WAI_QUANTUM_DATAFLOW_EXPECT)" "$report" || { echo "::error::the devices found are not the declared ones"; exit 1; }
          # Each declared member (a generation with a vector engine has two).
          for key in ${WAI_QUANTUM_DATAFLOW_EXPECT//,/ }; do
            grep -q "^dataflow member: key=$key .*simulator:device simulator 283f621e" "$report" || { echo "::error::$key: the member does not say it ran on the pinned simulator"; exit 1; }
            n="$(grep -c "^dataflow certified: key=$key .* parity=bit-exact " "$report")"
            [ "$n" = 4 ] || { echo "::error::$key: $n of 4 certificates were earned"; exit 1; }
            grep -qx "dataflow mutants: key=$key killed 3 of 3" "$report" || { echo "::error::$key: a kernel mutant survived"; exit 1; }
          done

  dataflow-absent:
    name: member · dataflow device absent (macos)
    needs: relevant
    if: needs.relevant.outputs.run == 'true'
    runs-on: macos
    timeout-minutes: 60
    env:
      WAI_QUANTUM_DATAFLOW_EXPECT: none
    steps:
      - uses: actions/checkout@v5
      # launchd's PATH omits cargo on the macOS host runner.
      - name: Put cargo on PATH
        shell: bash
        run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
      - name: No device library, and the member says so
        shell: bash
        working-directory: wai/wai-quantum
        run: |
          set -euo pipefail
          report="$RUNNER_TEMP/dataflow-report.txt"; : > "$report"
          nice -n 10 cargo test --locked --features quantum_fabric_dataflow --lib -- quantum_fabric_dataflow:: quantum_fabric::rv32:: 2>&1 | tee "$RUNNER_TEMP/unit.log"
          grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; }
          WAI_QUANTUM_DATAFLOW_REPORT="$report" nice -n 10 cargo test --locked --features quantum_fabric_dataflow --test dataflow_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log"
          grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
          grep -qx 'dataflow devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; }