Transaction-Science/open-standards
Watch
1
0
Fork
You've already forked open-standards
 0
Code Issues Pull requests Projects Releases 17 Packages Wiki Activity Actions 71

wai-quantum 0.5.1: neural-processor, matrix-extension, vector-unit, signal-processor and dataflow members #263

Merged
dcharlot merged 15 commits from wai/quantum-npu into main 2026-10-15 10:08:35 -04:00
Conversation 0 Commits 15 Files changed 39 +8225 -20
39 changed files with 8225 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

198
.github/workflows/wai-quantum-dataflow.yml vendored Normal file
View file

@ -0,0 +1,198 @@
name: wai-quantum dataflow member
# The fabric's dataflow member: the gate's fixed-point arithmetic on the first
# RISC-V core of a dataflow accelerator's compute tile, driven through the
# device's library at the level of configuration space, memory windows and
# clocks. The library here is the device's open-source simulator (Apache-2.0),
# built from source at a pinned commit, so every certificate is labelled
# simulator. Two device generations, each its own leg.
#
# Before the device runs anything, the RV32IM encoder that builds its programs
# is checked against an independent assembler: every listing, conforming and
# mutant, must assemble to the encoder's words exactly.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
push:
branches: [main]
paths:
- 'wai/wai-quantum/**'
- 'rust-toolchain.toml'
- '.github/workflows/wai-quantum-dataflow.yml'
pull_request:
paths:
- 'wai/wai-quantum/**'
- 'rust-toolchain.toml'
- '.github/workflows/wai-quantum-dataflow.yml'
jobs:
# Does this pull request's own diff touch the paths below? This forge matches a
# workflow's `paths:` against the push, so a merge of main into a branch fires the
# workflow whatever the pull request changes; every job here waits on this answer.
# A push to main, a schedule, or anything the script cannot determine answers
# `true` (scripts/pr-touches-paths.py).
relevant:
name: does this change touch it
runs-on: linux-arm64
outputs:
run: ${{ steps.gate.outputs.run }}
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0
- id: gate
run: |
set -euo pipefail
r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-dataflow.yml \
"${{ github.event.pull_request.base.sha }}" \
"${{ github.event.pull_request.head.sha }}")
echo "this change touches wai-quantum-dataflow: $r"
echo "run=$r" >> "$GITHUB_OUTPUT"
dataflow:
name: member · dataflow device, simulated (pci ${{ matrix.pci }})
needs: relevant
if: needs.relevant.outputs.run == 'true'
runs-on: linux
timeout-minutes: 60
strategy:
fail-fast: false
max-parallel: 1
matrix:
include:
- pci: '1e52:b140'
build: bh
key: dataflow-1e52-b140
- pci: '1e52:401e'
build: wh
key: dataflow-1e52-401e
container:
image: debian@sha256:913f6706df59a68922d1dd08f78c2476560a8d367897200a6005b00e5f67c2d5
options: --memory=6g --memory-swap=6g
env:
SIM_REPO: https://github.com/tenstorrent/ttsim.git
SIM_COMMIT: 283f621e7384d86f01e4ac5a9c83b60704f77d92
WAI_QUANTUM_DATAFLOW_EXPECT: ${{ matrix.key }}
WAI_QUANTUM_DATAFLOW_SIMULATOR: device simulator 283f621e (built from source, ${{ matrix.build }})
steps:
- name: Tools
shell: bash
run: |
set -euo pipefail
apt-get update -qq
apt-get install -y -qq --no-install-recommends ca-certificates curl git build-essential python3 binutils-riscv64-linux-gnu >/dev/null
# No Node for a checkout action: the forge's archive of the exact commit,
# its embedded commit id checked.
- name: Fetch the source
shell: bash
run: |
set -euo pipefail
want="${{ github.event.pull_request.head.sha || github.sha }}"
rm -rf src src.tar.gz src.tar && mkdir src
curl -sSf -o src.tar.gz "http://100.64.224.0:3400/Transaction-Science/open-standards/archive/$want.tar.gz"
gzip -dc src.tar.gz > src.tar
got="$(git get-tar-commit-id < src.tar)"
[ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; }
tar -xf src.tar -C src --strip-components=1
rm src.tar.gz src.tar
- name: The simulator, built from source at the pinned commit (cached), and Rust
shell: bash
working-directory: src/wai/wai-quantum
run: |
set -euo pipefail
cache=/ci-cache/wai-quantum-dataflow
mkdir -p "$cache"
exec 9>"$cache/.lock"
flock 9
lib="$cache/sim-$SIM_COMMIT/${{ matrix.build }}/libttsim.so"
if [ ! -f "$lib" ]; then
b="$cache/sim-build"
rm -rf "$b" && git init -q "$b"
git -C "$b" fetch -q --depth 1 "$SIM_REPO" "$SIM_COMMIT"
git -C "$b" checkout -q FETCH_HEAD
[ "$(git -C "$b" rev-parse HEAD)" = "$SIM_COMMIT" ] || { echo "::error::the simulator is not at the pinned commit"; exit 1; }
(cd "$b" && ./make.py "src/_out/release_${{ matrix.build }}/libttsim.so" > build.log 2>&1) || { tail -30 "$b/build.log"; exit 1; }
mkdir -p "$(dirname "$lib")" && cp "$b/src/_out/release_${{ matrix.build }}/libttsim.so" "$lib.tmp" && mv "$lib.tmp" "$lib"
rm -rf "$b"
fi
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo"
export PATH="$CARGO_HOME/bin:$PATH"
if ! command -v rustup >/dev/null; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain none
fi
rustup toolchain install
- name: The encoder against an independent assembler
shell: bash
working-directory: src/wai/wai-quantum
run: |
set -euo pipefail
cache=/ci-cache/wai-quantum-dataflow
exec 9>"$cache/.lock"
flock 9
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target"
export PATH="$CARGO_HOME/bin:$PATH"
out="$RUNNER_TEMP/rv32"
cargo run --locked --release -q --example rv32_listing --features quantum_fabric_dataflow -- "$out"
riscv64-linux-gnu-as --version | head -1
n=0
for s in "$out"/*.s; do
p="${s%.s}"
riscv64-linux-gnu-as -march=rv32im -mabi=ilp32 "$s" -o "$p.o"
riscv64-linux-gnu-objcopy -O binary -j .text "$p.o" "$p.as.bin"
cmp "$p.bin" "$p.as.bin" || { echo "::error::$(basename "$p"): the encoder's words differ from the assembler's"; exit 1; }
n=$((n + 1))
done
[ "$n" = 7 ] || { echo "::error::$n of 7 programs were compared"; exit 1; }
echo "all $n programs: the encoder's words are the assembler's"
- name: The member's tests
shell: bash
working-directory: src/wai/wai-quantum
run: |
set -euo pipefail
cache=/ci-cache/wai-quantum-dataflow
exec 9>"$cache/.lock"
flock 9
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target"
export PATH="$CARGO_HOME/bin:$PATH"
export WAI_QUANTUM_DATAFLOW_LIB="$cache/sim-$SIM_COMMIT/${{ matrix.build }}/libttsim.so"
cargo test --locked --release -j 6 --features quantum_fabric_dataflow --lib -- quantum_fabric_dataflow:: quantum_fabric::rv32:: 2>&1 | tee "$RUNNER_TEMP/unit.log"
grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; }
report="$RUNNER_TEMP/dataflow-report.txt"; : > "$report"
WAI_QUANTUM_DATAFLOW_REPORT="$report" cargo test --locked --release -j 6 --features quantum_fabric_dataflow --test dataflow_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log"
grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
cat "$report"
key="$WAI_QUANTUM_DATAFLOW_EXPECT"
grep -qx "dataflow devices: as declared ($key)" "$report" || { echo "::error::the device found is not the declared one"; exit 1; }
grep -q "^dataflow member: key=$key .*simulator:device simulator 283f621e" "$report" || { echo "::error::the member does not say it ran on the pinned simulator"; exit 1; }
n="$(grep -c "^dataflow certified: key=$key .* parity=bit-exact " "$report")"
[ "$n" = 4 ] || { echo "::error::$n of 4 certificates were earned"; exit 1; }
grep -qx "dataflow mutants: key=$key killed 3 of 3" "$report" || { echo "::error::a kernel mutant survived"; exit 1; }
dataflow-absent:
name: member · dataflow device absent (macos)
needs: relevant
if: needs.relevant.outputs.run == 'true'
runs-on: macos
timeout-minutes: 60
env:
WAI_QUANTUM_DATAFLOW_EXPECT: none
steps:
- uses: actions/checkout@v5
# launchd's PATH omits cargo on the macOS host runner.
- name: Put cargo on PATH
shell: bash
run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
- name: No device library, and the member says so
shell: bash
working-directory: wai/wai-quantum
run: |
set -euo pipefail
report="$RUNNER_TEMP/dataflow-report.txt"; : > "$report"
nice -n 10 cargo test --locked --features quantum_fabric_dataflow --lib -- quantum_fabric_dataflow:: quantum_fabric::rv32:: 2>&1 | tee "$RUNNER_TEMP/unit.log"
grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; }
WAI_QUANTUM_DATAFLOW_REPORT="$report" nice -n 10 cargo test --locked --features quantum_fabric_dataflow --test dataflow_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log"
grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
grep -qx 'dataflow devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; }

239
.github/workflows/wai-quantum-dsp.yml vendored Normal file
View file

@ -0,0 +1,239 @@
name: wai-quantum signal processor
# The signal processor (DSP) as a processor and as a fabric member, under its
# user-mode emulator:
#
# - the processor: the byte-pinned bit-identity digests, the frame fixtures and
# the byte-pinning library tests, built for the processor's Linux target (a
# tier-3 target: the standard library is built from source);
# - the member: the vector unit (128-byte vectors) runs the gate's
# arithmetic, and its device tests must find it, agree with the CPU, reproduce
# all four pinned certificates (each naming the emulator) and catch three
# kernel mutants;
# - the same build on x86_64, where the member is absent and says so.
#
# The emulator is QEMU 11.1.1, built here from its release tarball (SHA-256
# pinned; its signature was checked against the release key when it was pinned).
# Releases before 11.0 test a whole predicate register, not its low bit, when a
# branch reads a predicate written in the same packet, and the processor
# branches on the low bit. Compiled code relies on that (a predicate built from
# a general register's 1 and xored with a comparison's 0xff, so 0xfe means
# "equal"), and a library test fails under such an emulator though the code is
# right. A probe shows which emulator does what before any suite runs on it.
#
# The cross toolchain (clang, lld and the musl sysroot for the target) is the
# open-source release for this processor, SHA-256 pinned.
#
# RUSTC_BOOTSTRAP: building the standard library for a tier-3 target, and the
# vector unit's intrinsics, are unstable; this workflow's builds for this
# processor are the only ones that use it.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
push:
branches: [main]
paths:
- 'wai/wai-quantum/**'
- 'rust-toolchain.toml'
- '.github/workflows/wai-quantum-dsp.yml'
pull_request:
paths:
- 'wai/wai-quantum/**'
- 'rust-toolchain.toml'
- '.github/workflows/wai-quantum-dsp.yml'
jobs:
# Does this pull request's own diff touch the paths below? This forge matches a
# workflow's `paths:` against the push, so a merge of main into a branch fires the
# workflow whatever the pull request changes; every job here waits on this answer.
# A push to main, a schedule, or anything the script cannot determine answers
# `true` (scripts/pr-touches-paths.py).
relevant:
name: does this change touch it
runs-on: linux-arm64
outputs:
run: ${{ steps.gate.outputs.run }}
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0
- id: gate
run: |
set -euo pipefail
r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-dsp.yml \
"${{ github.event.pull_request.base.sha }}" \
"${{ github.event.pull_request.head.sha }}")
echo "this change touches wai-quantum-dsp: $r"
echo "run=$r" >> "$GITHUB_OUTPUT"
dsp-emulated:
name: processor and member · signal processor, emulated
needs: relevant
if: needs.relevant.outputs.run == 'true'
runs-on: linux
timeout-minutes: 180
container:
image: debian@sha256:913f6706df59a68922d1dd08f78c2476560a8d367897200a6005b00e5f67c2d5
options: --memory=6g --memory-swap=6g
env:
TOOLCHAIN_VERSION: 22.1.8
TOOLCHAIN_SHA256: bc122fe6473ead9693613a6ab3a177c5583653ff76b937e7578cf7de0273315b
# Not QEMU_*: qemu-user reads every QEMU_<option> variable as that
# option, and QEMU_VERSION is -version (print it and exit).
HEXAGON_EMULATOR_VERSION: 11.1.1
HEXAGON_EMULATOR_SHA256: 079ffbff8a7111bbc89022107cbabf3bbfd614d5fc9d7cc675991196aca12482
WAI_QUANTUM_DSP_EXPECT: dsp-vector
WAI_QUANTUM_DSP_EMULATOR: qemu-user 11.1.1 (built from the signed release)
steps:
- name: Tools
shell: bash
run: |
set -euo pipefail
apt-get update -qq
apt-get install -y -qq --no-install-recommends ca-certificates curl git xz-utils zstd build-essential \
ninja-build meson python3 python3-venv pkg-config libglib2.0-dev flex bison >/dev/null
# No Node for a checkout action: the forge's archive of the exact commit,
# its embedded commit id checked.
- name: Fetch the source
shell: bash
run: |
set -euo pipefail
want="${{ github.event.pull_request.head.sha || github.sha }}"
rm -rf src src.tar.gz src.tar && mkdir src
curl -sSf -o src.tar.gz "http://100.64.224.0:3400/Transaction-Science/open-standards/archive/$want.tar.gz"
gzip -dc src.tar.gz > src.tar
got="$(git get-tar-commit-id < src.tar)"
[ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; }
tar -xf src.tar -C src --strip-components=1
rm src.tar.gz src.tar
- name: The cross toolchain, the emulator and Rust (cached, pinned)
shell: bash
working-directory: src/wai/wai-quantum
run: |
set -euo pipefail
cache=/ci-cache/wai-quantum-hexagon
mkdir -p "$cache"
exec 9>"$cache/.lock"
flock 9
tc="$cache/toolchain-$TOOLCHAIN_VERSION"
if [ ! -x "$tc/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu/bin/hexagon-linux-musl-clang" ]; then
f="clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl_x86_64-linux-gnu.tar.zst"
# The release directory's name carries trailing underscores.
curl -sSfL -o "$cache/$f" "https://artifacts.codelinaro.org/artifactory/codelinaro-toolchain-for-hexagon/${TOOLCHAIN_VERSION}________/$f"
echo "$TOOLCHAIN_SHA256 $cache/$f" | sha256sum -c -
rm -rf "$tc.tmp" && mkdir "$tc.tmp"
tar --zstd -xf "$cache/$f" -C "$tc.tmp"
rm -rf "$tc" && mv "$tc.tmp" "$tc" && rm "$cache/$f"
fi
q="$cache/qemu-hexagon-$HEXAGON_EMULATOR_VERSION"
if [ ! -x "$q" ]; then
curl -sSfL -o "$cache/qemu.tar.xz" "https://download.qemu.org/qemu-$HEXAGON_EMULATOR_VERSION.tar.xz"
echo "$HEXAGON_EMULATOR_SHA256 $cache/qemu.tar.xz" | sha256sum -c -
b="$cache/qemu-build"
rm -rf "$b" && mkdir "$b" && tar -xJf "$cache/qemu.tar.xz" -C "$b" --strip-components=1
(cd "$b" && ./configure --target-list=hexagon-linux-user --static --disable-docs --disable-tools --disable-werror > configure.log 2>&1 && ninja -C build -j 4 qemu-hexagon > ninja.log 2>&1) || { tail -30 "$b/configure.log" "$b"/ninja.log 2>/dev/null; exit 1; }
cp "$b/build/qemu-hexagon" "$q.tmp" && mv "$q.tmp" "$q"
rm -rf "$b" "$cache/qemu.tar.xz"
fi
"$q" --version | head -1
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo"
export PATH="$CARGO_HOME/bin:$PATH"
if ! command -v rustup >/dev/null; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain none
fi
rustup toolchain install
rustup component add rust-src
- name: Which emulator branches on a predicate's low bit
shell: bash
working-directory: src/wai/wai-quantum
run: |
set -euo pipefail
cache=/ci-cache/wai-quantum-hexagon
bin="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu/bin"
"$bin/hexagon-linux-musl-clang" -O1 -mv68 -static -o "$RUNNER_TEMP/predicate_branch" ci/dsp/predicate_branch.c
echo "== the pinned emulator"
# The probe must say what it found: an emulator that exits without
# running the program (as qemu-user does for an environment variable
# it reads as -version) would otherwise pass this check.
"$cache/qemu-hexagon-$HEXAGON_EMULATOR_VERSION" "$RUNNER_TEMP/predicate_branch" > "$RUNNER_TEMP/predicate.log" 2>&1 || true
cat "$RUNNER_TEMP/predicate.log"
grep -qx "this emulator branches on the low bit" "$RUNNER_TEMP/predicate.log" || { echo "::error::the pinned emulator does not branch on the low bit, or did not run the probe"; exit 1; }
# The host's own emulator, where it has one registered: shown, not required.
if [ -e /proc/sys/fs/binfmt_misc/qemu-hexagon ]; then
echo "== the host's emulator"
"$RUNNER_TEMP/predicate_branch" || echo "(the host's emulator differs; the suites below run on the pinned one)"
fi
- name: The processor's suites
shell: bash
working-directory: src/wai/wai-quantum
run: |
set -euo pipefail
cache=/ci-cache/wai-quantum-hexagon
exec 9>"$cache/.lock"
flock 9
tc="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu"
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target"
export PATH="$CARGO_HOME/bin:$tc/bin:$PATH" RUSTC_BOOTSTRAP=1
export CARGO_TARGET_HEXAGON_UNKNOWN_LINUX_MUSL_LINKER=hexagon-linux-musl-clang
export CARGO_TARGET_HEXAGON_UNKNOWN_LINUX_MUSL_RUNNER="$cache/qemu-hexagon-$HEXAGON_EMULATOR_VERSION -L $tc/target/hexagon-unknown-linux-musl"
export CC_hexagon_unknown_linux_musl=hexagon-linux-musl-clang AR_hexagon_unknown_linux_musl=hexagon-linux-musl-ar
logs="$cache/logs"; mkdir -p "$logs"
t() {
name="$1"; shift
timeout 2700 cargo test --locked --release -j 6 -Zbuild-std -Zbuild-std-features=system-llvm-libunwind \
--target hexagon-unknown-linux-musl --features full "$@" 2>&1 | tee "$logs/$name.log"
}
t bit_identity --test bit_identity -- --test-threads=1
grep -qE '^test result: ok\. 23 passed; 0 failed' "$logs/bit_identity.log" || { echo "::error::the 23 bit-identity digests did not all pass"; exit 1; }
t frame --test frame_reference -- --test-threads=1
grep -qE '^test result: ok\. 2 passed; 0 failed' "$logs/frame.log" || { echo "::error::the fixtures did not pass"; exit 1; }
t lib --lib -- quantum:: quantum_fabric:: ijson:: quantum_receipt quantum_energy quantum_qbom merkle
grep -qE '^test result: ok\. [1-9][0-9]+ passed; 0 failed' "$logs/lib.log" || { echo "::error::the byte-pinning library tests did not pass"; exit 1; }
- name: The vector-unit member's tests
shell: bash
working-directory: src/wai/wai-quantum
run: |
set -euo pipefail
cache=/ci-cache/wai-quantum-hexagon
exec 9>"$cache/.lock"
flock 9
tc="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu"
# The vector unit in 128-byte mode, for the whole build (the standard
# library included): a separate target directory.
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target-vector"
export PATH="$CARGO_HOME/bin:$tc/bin:$PATH" RUSTC_BOOTSTRAP=1 RUSTFLAGS="-C target-feature=+hvxv68,+hvx-length128b"
export CARGO_TARGET_HEXAGON_UNKNOWN_LINUX_MUSL_LINKER=hexagon-linux-musl-clang
export CARGO_TARGET_HEXAGON_UNKNOWN_LINUX_MUSL_RUNNER="$cache/qemu-hexagon-$HEXAGON_EMULATOR_VERSION -L $tc/target/hexagon-unknown-linux-musl"
export CC_hexagon_unknown_linux_musl=hexagon-linux-musl-clang AR_hexagon_unknown_linux_musl=hexagon-linux-musl-ar
b="-Zbuild-std -Zbuild-std-features=system-llvm-libunwind --target hexagon-unknown-linux-musl --features quantum_fabric_dsp"
timeout 1800 cargo test --locked --release -j 6 $b --lib -- quantum_fabric_dsp:: 2>&1 | tee "$RUNNER_TEMP/unit.log"
grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; }
report="$RUNNER_TEMP/dsp-report.txt"; : > "$report"
WAI_QUANTUM_DSP_REPORT="$report" timeout 5400 cargo test --locked --release -j 6 $b --test dsp_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log"
grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
cat "$report"
key="$WAI_QUANTUM_DSP_EXPECT"
grep -qx "dsp devices: as declared ($key)" "$report" || { echo "::error::the member found is not the declared one"; exit 1; }
grep -q "^dsp member: key=$key .*emulator:qemu-user 11.1.1" "$report" || { echo "::error::the member does not say it ran under the pinned emulator"; exit 1; }
n="$(grep -c "^dsp certified: key=$key .* parity=bit-exact " "$report")"
[ "$n" = 4 ] || { echo "::error::$n of 4 certificates were earned"; exit 1; }
grep -qx "dsp mutants: key=$key killed 3 of 3" "$report" || { echo "::error::a kernel mutant survived"; exit 1; }
- name: Elsewhere the member is absent and says so (x86_64)
shell: bash
working-directory: src/wai/wai-quantum
run: |
set -euo pipefail
cache=/ci-cache/wai-quantum-hexagon
exec 9>"$cache/.lock"
flock 9
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target-host"
export PATH="$CARGO_HOME/bin:$PATH"
cargo test --locked -j 6 --features quantum_fabric_dsp --lib -- quantum_fabric_dsp:: 2>&1 | tee "$RUNNER_TEMP/host-unit.log"
grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/host-unit.log" || { echo "::error::no unit test ran"; exit 1; }
report="$RUNNER_TEMP/dsp-absent.txt"; : > "$report"
WAI_QUANTUM_DSP_EXPECT=none WAI_QUANTUM_DSP_REPORT="$report" cargo test --locked -j 6 --features quantum_fabric_dsp --test dsp_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/host-device.log"
grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/host-device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
grep -qx 'dsp devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; }

123
.github/workflows/wai-quantum-npu.yml vendored Normal file
View file

@ -0,0 +1,123 @@
name: wai-quantum neural-processor member
# The fabric's neural-processor member: exact 32×32 products from 4-bit limbs,
# every limb product and limb sum formed in fp16 on the neural processor (each
# value an integer fp16 holds exactly), the host assembling; certified
# bit-exact against the CPU reference's pinned corpus digests.
#
# - npu-mac (the macOS runner): the platform's model runtime must place every
# operation of the model on the neural processor (its compute plan says so,
# or the member is refused); the device tests must find the declared
# processor, agree with the CPU on the probe and on random states, reproduce
# all four pinned certificates, and catch both model mutants.
# - npu-absent (a Linux container): the same build finds no member and says
# why; the device tests run over no member with the declaration `none`.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
push:
branches: [main]
paths:
- 'wai/wai-quantum/**'
- 'rust-toolchain.toml'
- '.github/workflows/wai-quantum-npu.yml'
pull_request:
paths:
- 'wai/wai-quantum/**'
- 'rust-toolchain.toml'
- '.github/workflows/wai-quantum-npu.yml'
jobs:
# Does this pull request's own diff touch the paths below? This forge matches a
# workflow's `paths:` against the push, so a merge of main into a branch fires the
# workflow whatever the pull request changes; every job here waits on this answer.
# A push to main, a schedule, or anything the script cannot determine answers
# `true` (scripts/pr-touches-paths.py).
relevant:
name: does this change touch it
runs-on: linux-arm64
outputs:
run: ${{ steps.gate.outputs.run }}
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0
- id: gate
run: |
set -euo pipefail
r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-npu.yml \
"${{ github.event.pull_request.base.sha }}" \
"${{ github.event.pull_request.head.sha }}")
echo "this change touches wai-quantum-npu: $r"
echo "run=$r" >> "$GITHUB_OUTPUT"
npu-mac:
name: member · neural processor (macos)
needs: relevant
if: needs.relevant.outputs.run == 'true'
runs-on: macos
timeout-minutes: 120
env:
WAI_QUANTUM_NPU_EXPECT: npu-0
steps:
- uses: actions/checkout@v5
# launchd's PATH omits cargo on the macOS host runner.
- name: Put cargo on PATH
shell: bash
run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
- name: The member's unit tests
shell: bash
working-directory: wai/wai-quantum
run: |
set -euo pipefail
nice -n 10 cargo test --locked --features quantum_fabric_npu --lib -- quantum_fabric_npu:: quantum_fabric::npu:: 2>&1 | tee "$RUNNER_TEMP/unit.log"
grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; }
- name: The member's device tests
shell: bash
working-directory: wai/wai-quantum
run: |
set -euo pipefail
report="$RUNNER_TEMP/npu-report.txt"; : > "$report"
export WAI_QUANTUM_NPU_CACHE="$RUNNER_TEMP/npu-cache"
WAI_QUANTUM_NPU_REPORT="$report" nice -n 10 \
cargo test --locked --release --features quantum_fabric_npu --test npu_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log"
grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
cat "$report"
key="$WAI_QUANTUM_NPU_EXPECT"
grep -qx "npu devices: as declared ($key)" "$report" || { echo "::error::the processor found is not the declared one"; exit 1; }
grep -qE "^npu placement: key=$key( [a-z0-9_.]+=MLNeuralEngineComputeDevice)+$" "$report" || { echo "::error::an operation was not placed on the neural processor"; exit 1; }
n="$(grep -c "^npu certified: key=$key .* parity=bit-exact " "$report")"
[ "$n" = 4 ] || { echo "::error::$n of 4 certificates were earned"; exit 1; }
grep -qx "npu mutants: key=$key killed 2 of 2" "$report" || { echo "::error::a model mutant survived"; exit 1; }
npu-absent:
name: member · neural processor absent (linux)
needs: relevant
if: needs.relevant.outputs.run == 'true'
runs-on: linux
timeout-minutes: 60
env:
WAI_QUANTUM_NPU_EXPECT: none
steps:
- uses: actions/checkout@v5
- name: Install Rust 1.98.0
shell: bash
run: |
export PATH="$HOME/.cargo/bin:$PATH"
command -v rustup >/dev/null 2>&1 || \
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.98.0
rustup toolchain install 1.98.0 --profile minimal
rustup default 1.98.0
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
- name: No neural processor here, and the member says so
shell: bash
working-directory: wai/wai-quantum
run: |
set -euo pipefail
report="$RUNNER_TEMP/npu-report.txt"; : > "$report"
WAI_QUANTUM_NPU_REPORT="$report" cargo test --locked --features quantum_fabric_npu --test npu_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log"
grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
grep -qx 'npu devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; }

140
.github/workflows/wai-quantum-sme.yml vendored Normal file
View file

@ -0,0 +1,140 @@
name: wai-quantum matrix-extension member
# The fabric's matrix-extension member: the gate's fixed-point arithmetic in the
# processor's streaming vector mode (SME), 64-bit lanes, certified bit-exact
# against the CPU reference's pinned corpus digests.
#
# - sme-emulated (the container runner): the member's device tests for aarch64
# Linux, run through the host's user-mode emulator (its binfmt handler, which
# emulates SME); the runner declares the emulator, and every certificate says
# it was earned there, never on silicon.
# - sme-absent (the macOS runner, whose processor has no SME): the same build
# finds no member and says why.
#
# Silicon: a processor with SME runs the same tests with the declaration
# `cpu-matrix-sme` and no emulator.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
push:
branches: [main]
paths:
- 'wai/wai-quantum/**'
- 'rust-toolchain.toml'
- '.github/workflows/wai-quantum-sme.yml'
pull_request:
paths:
- 'wai/wai-quantum/**'
- 'rust-toolchain.toml'
- '.github/workflows/wai-quantum-sme.yml'
jobs:
# Does this pull request's own diff touch the paths below? This forge matches a
# workflow's `paths:` against the push, so a merge of main into a branch fires the
# workflow whatever the pull request changes; every job here waits on this answer.
# A push to main, a schedule, or anything the script cannot determine answers
# `true` (scripts/pr-touches-paths.py).
relevant:
name: does this change touch it
runs-on: linux-arm64
outputs:
run: ${{ steps.gate.outputs.run }}
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0
- id: gate
run: |
set -euo pipefail
r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-sme.yml \
"${{ github.event.pull_request.base.sha }}" \
"${{ github.event.pull_request.head.sha }}")
echo "this change touches wai-quantum-sme: $r"
echo "run=$r" >> "$GITHUB_OUTPUT"
sme-emulated:
name: member · matrix extension, emulated (aarch64 Linux)
needs: relevant
if: needs.relevant.outputs.run == 'true'
runs-on: linux
timeout-minutes: 90
container:
image: ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:7f8308a8734d9fcd2ebbe9a3e4bdea74af293f0799d80c3cc341e340cda49a4c
options: --memory=6g --memory-swap=6g
env:
WAI_QUANTUM_SME_EXPECT: cpu-matrix-sme
# The host's binfmt handler runs every aarch64 binary; its emulator is
# what the certificates name.
WAI_QUANTUM_SME_EMULATOR: qemu-user 8.2.2 (the host's binfmt handler)
# Run the test binary directly, so the host's handler (which emulates
# SME) runs it, not the image's older emulator.
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER: env
steps:
# The image has no Node for a checkout action: the forge's archive of the
# exact commit, its embedded commit id checked.
- name: Fetch the source
shell: bash
run: |
set -euo pipefail
want="${{ github.event.pull_request.head.sha || github.sha }}"
rm -rf src src.tar.gz src.tar && mkdir src
curl -sSf -o src.tar.gz "http://100.64.224.0:3400/Transaction-Science/open-standards/archive/$want.tar.gz"
gzip -dc src.tar.gz > src.tar
got="$(git get-tar-commit-id < src.tar)"
[ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; }
tar -xf src.tar -C src --strip-components=1
rm src.tar.gz src.tar
- name: The member's device tests under the emulator
shell: bash
working-directory: src/wai/wai-quantum
run: |
set -euo pipefail
cache=/ci-cache/wai-quantum-sme
mkdir -p "$cache"
exec 9>"$cache/.lock"
flock 9
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target"
export PATH="$CARGO_HOME/bin:$PATH"
if ! command -v rustup >/dev/null; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain none
fi
rustup toolchain install
rustup target add aarch64-unknown-linux-gnu
report="$(mktemp)"
WAI_QUANTUM_SME_REPORT="$report" cargo test --locked --release -j 6 --target aarch64-unknown-linux-gnu --features quantum_fabric_sme --test sme_member -- --test-threads=1 2>&1 | tee "$cache/device.log"
grep -qE '^test result: ok\. 9 passed; 0 failed' "$cache/device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
cat "$report"
key="$WAI_QUANTUM_SME_EXPECT"
grep -qx "sme devices: as declared ($key)" "$report" || { echo "::error::the member found is not the declared one"; exit 1; }
grep -q "^sme member: key=$key .*emulator:qemu-user" "$report" || { echo "::error::the member does not say it ran under the emulator"; exit 1; }
n="$(grep -c "^sme certified: key=$key .* parity=bit-exact " "$report")"
[ "$n" = 4 ] || { echo "::error::$n of 4 certificates were earned"; exit 1; }
grep -qx "sme mutants: key=$key killed 3 of 3" "$report" || { echo "::error::a kernel mutant survived"; exit 1; }
sme-absent:
name: member · matrix extension absent (macos)
needs: relevant
if: needs.relevant.outputs.run == 'true'
runs-on: macos
timeout-minutes: 60
env:
WAI_QUANTUM_SME_EXPECT: none
steps:
- uses: actions/checkout@v5
# launchd's PATH omits cargo on the macOS host runner.
- name: Put cargo on PATH
shell: bash
run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
- name: No matrix extension here, and the member says so
shell: bash
working-directory: wai/wai-quantum
run: |
set -euo pipefail
[ "$(sysctl -n hw.optional.arm.FEAT_SME 2>/dev/null || echo 0)" = 0 ] || { echo "::error::this runner has SME; declare it and run the silicon tests"; exit 1; }
report="$RUNNER_TEMP/sme-report.txt"; : > "$report"
WAI_QUANTUM_SME_REPORT="$report" nice -n 10 cargo test --locked --features quantum_fabric_sme --test sme_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log"
grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
grep -qx 'sme devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; }

198
.github/workflows/wai-quantum-vector.yml vendored Normal file
View file

@ -0,0 +1,198 @@
name: wai-quantum CPU vector units
# The fabric's CPU vector-unit members: the gate's fixed-point arithmetic in
# the processor's own vector extension, certified bit-exact against the CPU
# reference's pinned corpus digests.
#
# - avx2 on silicon: the container runner's x86-64 processor;
# - sve and rvv under the host's user-mode emulator (its binfmt handler), at
# several vector lengths each: the scalable kernels set their length per
# pass, and every leg checks the length the member read from the unit is the
# one the emulator was given, so a leg cannot pass at the wrong length;
# - the macOS runner, whose processor has no vector extension these kernels
# use outside streaming mode, finding no member.
#
# Silicon for sve and rvv: a processor with them runs the same tests with no
# emulator declared.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
push:
branches: [main]
paths:
- 'wai/wai-quantum/**'
- 'rust-toolchain.toml'
- '.github/workflows/wai-quantum-vector.yml'
pull_request:
paths:
- 'wai/wai-quantum/**'
- 'rust-toolchain.toml'
- '.github/workflows/wai-quantum-vector.yml'
jobs:
# Does this pull request's own diff touch the paths below? This forge matches a
# workflow's `paths:` against the push, so a merge of main into a branch fires the
# workflow whatever the pull request changes; every job here waits on this answer.
# A push to main, a schedule, or anything the script cannot determine answers
# `true` (scripts/pr-touches-paths.py).
relevant:
name: does this change touch it
runs-on: linux-arm64
outputs:
run: ${{ steps.gate.outputs.run }}
steps:
- uses: actions/checkout@v5
with:
fetch-depth: 0
- id: gate
run: |
set -euo pipefail
r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-vector.yml \
"${{ github.event.pull_request.base.sha }}" \
"${{ github.event.pull_request.head.sha }}")
echo "this change touches wai-quantum-vector: $r"
echo "run=$r" >> "$GITHUB_OUTPUT"
vector:
name: member · vector unit, ${{ matrix.name }}
needs: relevant
if: needs.relevant.outputs.run == 'true'
runs-on: linux
timeout-minutes: 120
strategy:
fail-fast: false
max-parallel: 1
matrix:
include:
- name: avx2 (x86_64 silicon)
target: x86_64-unknown-linux-gnu
image: ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:7f8308a8734d9fcd2ebbe9a3e4bdea74af293f0799d80c3cc341e340cda49a4c
key: cpu-vector-avx2
bits: 256
emulator: ''
qemu_cpu: ''
- name: sve 128-bit (aarch64, emulated)
target: aarch64-unknown-linux-gnu
image: ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:7f8308a8734d9fcd2ebbe9a3e4bdea74af293f0799d80c3cc341e340cda49a4c
key: cpu-vector-sve
bits: 128
emulator: qemu-user 8.2.2 (the host's binfmt handler)
qemu_cpu: max,sve-default-vector-length=16
- name: sve 512-bit (aarch64, emulated)
target: aarch64-unknown-linux-gnu
image: ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:7f8308a8734d9fcd2ebbe9a3e4bdea74af293f0799d80c3cc341e340cda49a4c
key: cpu-vector-sve
bits: 512
emulator: qemu-user 8.2.2 (the host's binfmt handler)
qemu_cpu: max,sve-default-vector-length=64
- name: sve 2048-bit (aarch64, emulated)
target: aarch64-unknown-linux-gnu
image: ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:7f8308a8734d9fcd2ebbe9a3e4bdea74af293f0799d80c3cc341e340cda49a4c
key: cpu-vector-sve
bits: 2048
emulator: qemu-user 8.2.2 (the host's binfmt handler)
qemu_cpu: max,sve-default-vector-length=256
- name: rvv 128-bit (riscv64, emulated)
target: riscv64gc-unknown-linux-gnu
image: ghcr.io/cross-rs/riscv64gc-unknown-linux-gnu@sha256:67ee97dd4719390d299edfcf39f226360ccfb1539e08b078ec187d4848415794
key: cpu-vector-rvv
bits: 128
emulator: qemu-user 8.2.2 (the host's binfmt handler)
qemu_cpu: rv64,v=true,vlen=128,elen=64
- name: rvv 1024-bit (riscv64, emulated)
target: riscv64gc-unknown-linux-gnu
image: ghcr.io/cross-rs/riscv64gc-unknown-linux-gnu@sha256:67ee97dd4719390d299edfcf39f226360ccfb1539e08b078ec187d4848415794
key: cpu-vector-rvv
bits: 1024
emulator: qemu-user 8.2.2 (the host's binfmt handler)
qemu_cpu: rv64,v=true,vlen=1024,elen=64
container:
image: ${{ matrix.image }}
options: --memory=6g --memory-swap=6g
env:
WAI_QUANTUM_VECTOR_EXPECT: ${{ matrix.key }}
WAI_QUANTUM_VECTOR_EMULATOR: ${{ matrix.emulator }}
# Run test binaries directly, so the host's handler runs them with the
# processor given here, not the image's older emulator.
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER: env
CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER: env
steps:
# The image has no Node for a checkout action: the forge's archive of the
# exact commit, its embedded commit id checked.
- name: Fetch the source
shell: bash
run: |
set -euo pipefail
want="${{ github.event.pull_request.head.sha || github.sha }}"
rm -rf src src.tar.gz src.tar && mkdir src
curl -sSf -o src.tar.gz "http://100.64.224.0:3400/Transaction-Science/open-standards/archive/$want.tar.gz"
gzip -dc src.tar.gz > src.tar
got="$(git get-tar-commit-id < src.tar)"
[ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; }
tar -xf src.tar -C src --strip-components=1
rm src.tar.gz src.tar
- name: The member's tests
shell: bash
working-directory: src/wai/wai-quantum
run: |
set -euo pipefail
cache=/ci-cache/wai-quantum-vector
mkdir -p "$cache"
exec 9>"$cache/.lock"
flock 9
export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target-${{ matrix.target }}"
export PATH="$CARGO_HOME/bin:$PATH"
if ! command -v rustup >/dev/null; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain none
fi
rustup toolchain install
rustup target add ${{ matrix.target }}
# The emulated processor, for the test binaries only.
if [ -n "${{ matrix.qemu_cpu }}" ]; then export QEMU_CPU="${{ matrix.qemu_cpu }}"; fi
t="--target ${{ matrix.target }} --features quantum_fabric_vector"
cargo test --locked --release -j 6 $t --lib -- quantum_fabric_vector:: 2>&1 | tee "$cache/unit.log"
grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$cache/unit.log" || { echo "::error::no unit test ran"; exit 1; }
report="$(mktemp)"
WAI_QUANTUM_VECTOR_REPORT="$report" cargo test --locked --release -j 6 $t --test vector_member -- --test-threads=1 2>&1 | tee "$cache/device.log"
grep -qE '^test result: ok\. 9 passed; 0 failed' "$cache/device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
cat "$report"
key="$WAI_QUANTUM_VECTOR_EXPECT"
grep -qx "vector devices: as declared ($key)" "$report" || { echo "::error::the unit found is not the declared one"; exit 1; }
grep -q "^vector member: key=$key .* bits=${{ matrix.bits }} " "$report" || { echo "::error::the unit's vector length is not ${{ matrix.bits }} bits"; exit 1; }
if [ -n "$WAI_QUANTUM_VECTOR_EMULATOR" ]; then
grep -q "^vector member: key=$key .*emulator:qemu-user" "$report" || { echo "::error::the member does not say it ran under the emulator"; exit 1; }
else
grep -q "^vector member: key=$key .*; silicon;" "$report" || { echo "::error::the member does not say it ran on silicon"; exit 1; }
fi
n="$(grep -c "^vector certified: key=$key .* parity=bit-exact " "$report")"
[ "$n" = 4 ] || { echo "::error::$n of 4 certificates were earned"; exit 1; }
grep -qx "vector mutants: key=$key killed 3 of 3" "$report" || { echo "::error::a kernel mutant survived"; exit 1; }
vector-absent:
name: member · vector unit absent (macos)
needs: relevant
if: needs.relevant.outputs.run == 'true'
runs-on: macos
timeout-minutes: 60
env:
WAI_QUANTUM_VECTOR_EXPECT: none
steps:
- uses: actions/checkout@v5
# launchd's PATH omits cargo on the macOS host runner.
- name: Put cargo on PATH
shell: bash
run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
- name: No vector unit these kernels use, and the member says so
shell: bash
working-directory: wai/wai-quantum
run: |
set -euo pipefail
report="$RUNNER_TEMP/vector-report.txt"; : > "$report"
nice -n 10 cargo test --locked --features quantum_fabric_vector --lib -- quantum_fabric_vector:: 2>&1 | tee "$RUNNER_TEMP/unit.log"
grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; }
WAI_QUANTUM_VECTOR_REPORT="$report" nice -n 10 cargo test --locked --features quantum_fabric_vector --test vector_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log"
grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; }
grep -qx 'vector devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; }

2
scripts/check-lockfiles-selftest.sh
View file

@ -36,7 +36,7 @@ expect() { # expect <pass|fail> <what> [text the output must hold]
out=$("$check" "$tmp/ws" 2>&1); rc=$?
if [ "$want" = pass ] && [ $rc -ne 0 ]; then echo "FAIL $what: the check failed"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi
if [ "$want" = fail ] && [ $rc -eq 0 ]; then echo "FAIL $what: the check passed"; fails=$((fails+1)); return; fi
if [ -n "$needle" ] && ! printf '%s' "$out" | grep -q -- "$needle"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi
if [ -n "$needle" ] && ! grep -q -- "$needle" <<<"$out"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi
echo "ok $what"
}

2
scripts/check-lockfiles.sh
View file

@ -64,7 +64,7 @@ for d in "${dirs[@]}"; do
out=$( (cd "$d" && cargo update --workspace --locked) 2>&1 ); rc=$?
if [ $rc -eq 0 ]; then
continue
elif printf '%s' "$out" | grep -qE -- '--locked was passed|lock file .* (is missing|needs to be updated)'; then
elif grep -qE -- '--locked was passed|lock file .* (is missing|needs to be updated)' <<<"$out"; then
stale+=("$d")
echo "STALE $d"
printf '%s\n' "$out" | grep -E '^error|^Caused by|^ ' | head -6 | sed 's/^/ /'

2
scripts/check-tests-in-ci-selftest.sh
View file

@ -31,7 +31,7 @@ expect() { # expect <pass|fail> <what> [text the output must hold]
out=$(python3 -I "$check" --root "$tmp" 2>&1); rc=$?
if [ "$want" = pass ] && [ $rc -ne 0 ]; then echo "FAIL $what: the check failed"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi
if [ "$want" = fail ] && [ $rc -eq 0 ]; then echo "FAIL $what: the check passed"; fails=$((fails+1)); return; fi
if [ -n "$needle" ] && ! printf '%s' "$out" | grep -q -- "$needle"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi
if [ -n "$needle" ] && ! grep -q -- "$needle" <<<"$out"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi
echo "ok $what"
}

2
scripts/standards-touched-selftest.sh
View file

@ -49,7 +49,7 @@ got=$(python3 -I scripts/standards-touched.py "" "" | python3 -c 'import json,sy
got=$(python3 -I scripts/standards-touched.py "$base" "0000000000000000000000000000000000000bad" 2>/dev/null | python3 -c 'import json,sys; print(",".join(sorted(json.load(sys.stdin)["run"])))')
[ "$got" = "a-rs,b-rs,c-rs" ] && echo "ok a diff it cannot compute: runs everything" || { echo "FAIL an unknown head ran [$got]"; fails=$((fails+1)); }
out=$(GITHUB_OUTPUT="$tmp/out" python3 -I scripts/standards-touched.py --github-output "" "" >/dev/null; cat "$tmp/out")
printf '%s\n' "$out" | grep -q '"z3":true' && printf '%s\n' "$out" | grep -q '"feature_tests":"-p x --features y,z"' && printf '%s\n' "$out" | grep -q '^energy=false$' \
grep -q '"z3":true' <<<"$out" && grep -q '"feature_tests":"-p x --features y,z"' <<<"$out" && grep -q '^energy=false$' <<<"$out" \
&& echo "ok the matrix entries keep their booleans and quoted values" || { echo "FAIL the emitted matrix: $out"; fails=$((fails+1)); }
if [ $fails -ne 0 ]; then echo "standards-touched-selftest: $fails failure(s)"; exit 1; fi
echo "standards-touched-selftest: the plan builds what a change can affect, and everything when unsure."

70
wai/CHANGELOG.md
View file

@ -6,6 +6,76 @@ wire format may change and such changes are recorded here.
## v0.4.0 — current
### wai-quantum 0.5.1: neural-processor, matrix-extension, vector-unit, signal-processor and dataflow members
Additive. The default features, `full`, and every pinned digest are unchanged.
- **The neural-processor member** (`quantum_fabric_npu`, opt-in, macOS only). A neural
processor computes in fp16 and does not document how it accumulates, so the member
never relies on it: each 32-bit operand is split into eight 4-bit limbs, and the
processor computes the fifteen coefficients `Σ_{i+j=s} a_i·b_j` of each product. Every
limb product is at most 225 in size and every coefficient at most 1800, so every value
it forms is an integer fp16 holds exactly, in any order. The host assembles each
product exactly and applies `lanes::apply_words`'s arithmetic.
- **Credited by placement.** The member writes its model (one multiply, one sum) as the
platform runtime's model description, compiles it at run time, and reads the
runtime's compute plan: every operation must be placed on the neural processor, or
the member is refused with the placement reported, so no CPU result is credited to the
processor. It is then probed and certified against the pinned corpus digests, and
the two model mutants (the multiply made an add, the sum a maximum) are caught.
- **The same program, portable.** `npu::onnx_limb_model` writes it in the portable model
format (ONNX, opset 17: one `Mul`, one `ReduceSum`), with fp16 or int32 tensors, and
`npu::expand_i32` lays out the int32 inputs; `examples/npu_onnx.rs` writes both models,
a batch of inputs and the exact products they must give, so a neural processor
reached through a runtime that reads that format can be checked against them.
- **Fixed** before release: `npu::f16_of(±2048)` subtracted past zero in a debug build.
- **The matrix-extension member** (`quantum_fabric_sme`, opt-in, aarch64 only): the
gate's fixed-point arithmetic in the processor's streaming vector mode (SME), 64-bit
lanes, by inline assembly; the host gathers each gate's pairs and checks each row's
range. Its instructions exist only in that mode, so a kernel that ran, ran there.
Detection reads the processor's feature flags (on Linux, `AT_HWCAP2`, which a
user-mode emulator supplies for its guest); a runner under an emulator declares it,
and every certificate carries that provenance. Three kernel mutants are compiled in.
- **The DSP vector-unit member** (`quantum_fabric_dsp`, opt-in, in a build for the signal
processor with its 128-byte vector extension). The gate's arithmetic in 32-bit lanes,
each 64-bit value a pair of words: the product is the vector unit's exact 32×32
multiply (the 32-bit operand times the other's low half, unsigned, plus its high
half, signed, sixteen bits up); the rounding constant's carry and each row's carries
and borrows go into the high word; the shift is arithmetic over all 64 bits. The
fabric gains the class `dsp`. Three kernel mutants are compiled in.
- **The CPU vector-unit members** (`quantum_fabric_vector`, opt-in): the gate's arithmetic
in the processor's own vector extension — `avx2` on x86-64 (the 64-bit arithmetic
shift, which the unit lacks, built from a logical shift and the sign bit), `sve` on
aarch64 and `rvv` on riscv64, both written once for every vector length (the length
set per pass and read back from the unit into the member's id). Silicon for `avx2`;
`sve` at 128, 512 and 2048 bits and `rvv` at 128 and 1024 bits under the user-mode
emulator. Three kernel mutants per instruction set.
- **The dataflow member** (`quantum_fabric_dataflow`, opt-in, Linux): the gate's arithmetic on
the first RISC-V core of each of 32 compute tiles of a dataflow accelerator, each over a
contiguous slice of a gate's pairs, two device generations (PCI 1e52:b140 and 1e52:401e),
driven through the device's own library at the level of configuration space, memory
windows, reset registers and clocks. The program comes from a new RV32IM
encoder (`quantum_fabric::rv32`), whose listings an independent assembler turns into
the same words; every 64-bit value is a pair of words. The library is the device's
open-source simulator, built from source, and every certificate is labelled simulator.
- **The signal processor as a processor.** The bit-identity digests, the frame fixtures and
the byte-pinning library tests pass for the processor's Linux target under its
user-mode emulator. Emulator releases before 11.0 test a whole predicate register
where the processor tests its low bit, in a branch on a predicate written in the same
packet; compiled code relies on the low bit, and one library test fails under such an
emulator. `ci/dsp/predicate_branch.c` shows which emulator does which, and the
suites run on 11.1.1, built from its release.
- **CI.** `wai-quantum-npu.yml`: the device tests on the macOS runner's neural processor,
and the same build on Linux, where the member is absent and says so.
`wai-quantum-sme.yml`: the matrix-extension member's device tests for aarch64 Linux
under the host's user-mode emulator, certificates labelled emulator; and the macOS
runner, whose processor has no SME, finding no member. `wai-quantum-dsp.yml`: the
signal processor's suites and the vector-unit member's device tests under the pinned
emulator, after the emulator's own branch probe, and the same build on x86_64, where
the member is absent and says so. `wai-quantum-vector.yml`: `avx2` on the container
runner's processor, `sve` and `rvv` under the host's emulator at the lengths above (each
leg checks the length the member read), and the macOS runner finding no member.
### `wai.audio.int_codec` through the integer operator set
- **The decode.** `int_audio::decode_typed` (so the `WIA1` dispatch, the staged sink and

2
wai/tools/quantum-meter/Cargo.lock generated
View file

@ -1017,7 +1017,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "wai-quantum"
version = "0.5.0"
version = "0.5.1"
dependencies = [
"blake3",
"ed25519-dalek",

2
wai/tools/quantum-wgsl-check/Cargo.lock generated
View file

@ -599,7 +599,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "wai-quantum"
version = "0.5.0"
version = "0.5.1"
dependencies = [
"blake3",
"ed25519-dalek",

2
wai/wai-pack/Cargo.lock generated
View file

@ -703,7 +703,7 @@ dependencies = [
[[package]]
name = "wai-quantum"
version = "0.5.0"
version = "0.5.1"
dependencies = [
"blake3",
"ed25519-dalek",

2
wai/wai-quantum-qpu/Cargo.lock generated
View file

@ -422,7 +422,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "wai-quantum"
version = "0.5.0"
version = "0.5.1"
dependencies = [
"blake3",
"ed25519-dalek",

2
wai/wai-quantum-serve/Cargo.lock generated
View file

@ -589,7 +589,7 @@ dependencies = [
[[package]]
name = "wai-quantum"
version = "0.5.0"
version = "0.5.1"
dependencies = [
"blake3",
"ed25519-dalek",

17
wai/wai-quantum/Cargo.lock generated
View file

@ -616,6 +616,17 @@ dependencies = [
"objc2-io-surface",
]
[[package]]
name = "objc2-core-ml"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "201b055e6acfa0f9f15568255d3f03ce2b54bc86d1814442dc69138e36813e18"
dependencies = [
"block2",
"objc2",
"objc2-foundation",
]
[[package]]
name = "objc2-encode"
version = "4.1.0"
@ -1068,11 +1079,15 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "wai-quantum"
version = "0.5.0"
version = "0.5.1"
dependencies = [
"blake3",
"block2",
"cudarc",
"ed25519-dalek",
"objc2",
"objc2-core-ml",
"objc2-foundation",
"opencl3",
"serde_json",
"wgpu",

53
wai/wai-quantum/Cargo.toml
View file

@ -1,6 +1,6 @@
[package]
name = "wai-quantum"
version = "0.5.0"
version = "0.5.1"
edition = "2024"
rust-version = "1.97"
license = "Apache-2.0"
@ -42,6 +42,19 @@ cudarc = { version = "0.19.10", optional = true, default-features = false, featu
# feature needs no OpenCL installed and finds no member where there is none.
opencl3 = { version = "0.12.3", optional = true, default-features = false, features = ["dynamic"] }
# The neural-processor member: the platform's model runtime on macOS, its
# framework loaded by the system; absent on every other target.
[target.'cfg(target_os = "macos")'.dependencies]
objc2 = { version = "0.6.5", optional = true }
block2 = { version = "0.6.2", optional = true }
objc2-foundation = { version = "0.3.2", optional = true, default-features = false, features = ["std", "NSArray", "NSDictionary", "NSError", "NSString", "NSURL", "NSValue"] }
objc2-core-ml = { version = "0.3.2", optional = true, default-features = false, features = [
"std", "block2", "MLModel", "MLModel_MLModelCompilation", "MLModelConfiguration", "MLMultiArray", "MLDictionaryFeatureProvider",
"MLFeatureProvider", "MLFeatureValue", "MLComputePlan", "MLComputePlanDeviceUsage", "MLComputeDeviceProtocol",
"MLNeuralEngineComputeDevice", "MLModelStructure", "MLModelStructureProgram", "MLModelStructureProgramFunction",
"MLModelStructureProgramBlock", "MLModelStructureProgramOperation",
] }
[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies]
wgpu = { version = "30.0.1", optional = true, default-features = false, features = ["std", "wgsl", "webgpu"] }
@ -152,6 +165,16 @@ quantum_fabric_cuda = ["quantum_fabric", "dep:cudarc"]
quantum_fabric_opencl = ["quantum_fabric", "dep:opencl3"]
# The simulated FPGA member: Verilator, invoked at run time, compiles the datapath.
quantum_fabric_fpga = ["quantum_fabric"]
# The neural-processor member: exact fp16 limb products on the platform's neural processor.
quantum_fabric_npu = ["quantum_fabric", "dep:objc2", "dep:block2", "dep:objc2-foundation", "dep:objc2-core-ml"]
# The matrix-extension member: the gate's arithmetic in the processor's streaming vector mode (SME).
quantum_fabric_sme = ["quantum_fabric"]
# The DSP vector-unit member: the gate's arithmetic in the signal processor's vector unit.
quantum_fabric_dsp = ["quantum_fabric"]
# The CPU vector-unit members: the gate's arithmetic in the processor's vector extension (avx2, sve, rvv).
quantum_fabric_vector = ["quantum_fabric"]
# The dataflow member: the gate's arithmetic on a dataflow accelerator's RISC-V cores, through a device library loaded at run time.
quantum_fabric_dataflow = ["quantum_fabric"]
# --- interference / ML layer ----------------------------------------------
quantum_vml = []
@ -194,10 +217,38 @@ required-features = ["quantum_fabric_opencl", "quantum_sv"]
name = "fpga_member"
required-features = ["quantum_fabric_fpga"]
[[test]]
name = "npu_member"
required-features = ["quantum_fabric_npu"]
[[test]]
name = "sme_member"
required-features = ["quantum_fabric_sme"]
[[test]]
name = "dsp_member"
required-features = ["quantum_fabric_dsp"]
[[test]]
name = "vector_member"
required-features = ["quantum_fabric_vector"]
[[test]]
name = "dataflow_member"
required-features = ["quantum_fabric_dataflow"]
[[test]]
name = "fabric_expected"
required-features = ["quantum_fabric"]
[[example]]
name = "rv32_listing"
required-features = ["quantum_fabric_dataflow"]
[[example]]
name = "npu_onnx"
required-features = ["quantum_fabric"]
[[example]]
name = "qio_gset"
required-features = ["quantum_qio"]

17
wai/wai-quantum/README.md
View file

@ -6,12 +6,18 @@ every stage.
No QPU, no cloud service, no vendor SDK, and **no system libraries** — so the
same source runs natively, in the browser via wasm, and as a WASI component at
the edge, producing **byte-identical results and receipts on all three**. Four
the edge, producing **byte-identical results and receipts on all three**. Nine
opt-in features reach outside the crate: `quantum_fabric_webgpu` (the
platform's GPU interface), `quantum_fabric_cuda` (the CUDA driver and runtime
compiler) and `quantum_fabric_opencl` (the OpenCL ICD loader), the last two
loaded when first used, and `quantum_fabric_fpga` (Verilator, run as a program
when first used); no default feature reaches a system library.
loaded when first used, `quantum_fabric_fpga` (Verilator, run as a program
when first used), `quantum_fabric_npu` (the platform's model runtime, on
macOS), `quantum_fabric_sme` (the processor's streaming vector mode, by
inline assembly), `quantum_fabric_vector` (the processor's vector extension),
`quantum_fabric_dsp` (the signal processor's vector
unit, in a build for that processor) and `quantum_fabric_dataflow` (a dataflow
device's library, loaded when first used); no default feature reaches a system
library.
```toml
[dependencies]
@ -102,6 +108,11 @@ never creates information.
| `quantum_fabric_cuda` | **the fabric's CUDA member**, opt-in and not in `full`, native targets only: the gate kernel in CUDA C with exact 64-bit integers, compiled at run time for each device's own architecture, one launch per gate. The CUDA driver and runtime compiler are loaded when first used, so a build with the feature runs on a host without CUDA and finds no member there, saying why. Each device is probed against the CPU's arithmetic in the process that uses it and certified bit-exact by reproducing the CPU reference's pinned corpus digests; a result that would leave 32 bits is refused, never returned. With `quantum_sv` too, the **dense f64 state vector** runs on the device bit for bit as on the CPU: the CPU's own launches (diagonal, one-qubit and grouped formulas), each an IEEE binary64 operation with no fused multiply-add, certified against a pinned corpus digest |
| `quantum_fabric_opencl` | **the fabric's OpenCL member**, opt-in and not in `full`, native targets only: the same exact 64-bit kernel in OpenCL C, built at run time by each device's own compiler, one launch per gate, on any OpenCL GPU or CPU (an OpenCL accelerator is refused: no fabric class names it). The ICD loader is opened when first used, so a build with the feature runs on a host without OpenCL and finds no member there, saying why. Each device is probed and certified as the CUDA member's are. With `quantum_sv` too, the dense f64 state vector runs bit for bit as on the CPU on every device with binary64 (contraction off); a device without it refuses, saying so |
| `quantum_fabric_fpga` | **the fabric's FPGA member, simulated**, opt-in and not in `full`, native targets only: the gate datapath in Verilog (each 32×32 product from four 16-bit partial products, the size of an FPGA's multiplier blocks; a state RAM; one pair per two cycles), compiled at run time by Verilator into a cycle-accurate simulation driven over a pipe. Probed and certified as the other members are, and every certificate says it was earned in a simulator, so placement ranks it after silicon. A host without Verilator finds no member and says why |
| `quantum_fabric_npu` | **the fabric's neural-processor member**, opt-in and not in `full`, macOS only: each 32-bit operand split into eight 4-bit limbs, every limb product and limb sum formed in fp16 on the platform's neural processor (each value an integer fp16 holds exactly, whatever order it is added in), the host assembling each product and applying the fixed-point arithmetic. The member exists only where the platform model runtime's compute plan places every operation of the model on the neural processor; a model it would run elsewhere is refused with the placement it reported. Probed and certified as every member is |
| `quantum_fabric_sme` | **the fabric's matrix-extension member**, opt-in and not in `full`, aarch64 only: the gate's fixed-point arithmetic in the processor's streaming vector mode (SME), 64-bit lanes, the host gathering each gate's pairs and checking each row's range. Every instruction it uses exists only in that mode, so a kernel that ran, ran there. Detected by the processor's feature flags (on Linux, the kernel's `AT_HWCAP2`, which a user-mode emulator supplies); under an emulator the runner says so (`WAI_QUANTUM_SME_EMULATOR`) and every certificate carries it. Three kernel mutants are compiled in and must be caught |
| `quantum_fabric_dsp` | **the fabric's DSP vector-unit member**, opt-in and not in `full`, in a build for the signal processor with its vector unit in 128-byte mode: the gate's fixed-point arithmetic in 32-bit lanes, each 64-bit value a pair of words, every product the vector unit's exact 32×32 multiply and every carry and borrow kept; the host gathers each gate's pairs and checks each row's range. Its instructions exist only on that unit, so a kernel that ran, ran there. Under an emulator the runner says so (`WAI_QUANTUM_DSP_EMULATOR`) and every certificate carries it. Three kernel mutants are compiled in and must be caught. Its class is `dsp` |
| `quantum_fabric_vector` | **the fabric's CPU vector-unit members**, opt-in and not in `full`: the gate's fixed-point arithmetic in the processor's own vector extension, one kernel per instruction set — `avx2` (x86-64, 256-bit), `sve` (aarch64, the scalable vector extension, any length) and `rvv` (riscv64, the vector extension 1.0, any length) — each operand sign-extended to 64-bit lanes, the host gathering each gate's pairs and checking each row's range. The scalable kernels set their length per pass; the member reads the length from the unit and names it in its id and detail. Detected by the processor's feature flags (on Linux, `AT_HWCAP` for the vector extension, which a user-mode emulator supplies); under an emulator the runner says so (`WAI_QUANTUM_VECTOR_EMULATOR`). Its id extends the CPU reference's, so placement with no energy figures keeps the reference. Three kernel mutants per instruction set are compiled in and must be caught |
| `quantum_fabric_dataflow` | **the fabric's dataflow member**, opt-in and not in `full`, Linux: the gate's fixed-point arithmetic on the first RISC-V core of each of 32 compute tiles of a dataflow accelerator, each over a contiguous slice of a gate's pairs. The program is built in this crate (`quantum_fabric::rv32`, an RV32IM encoder whose listings an independent assembler reproduces word for word): every 64-bit value a pair of words, `mul`/`mulh` for products, `sltu` for carries and borrows. The device's library (`WAI_QUANTUM_DATAFLOW_LIB`, loaded at run time) presents the device as a PCI endpoint; the host opens two memory windows on each tile, holds its cores in reset, writes the program at the reset address and that tile's parameters and slice beside it, releases the first cores and clocks the device until every tile reports done. The device is named by its PCI ids (`dataflow-1e52-<device>`); the library is a simulator, and every certificate says so (`WAI_QUANTUM_DATAFLOW_SIMULATOR`). Three kernel mutants are compiled in |
| `full` | everything |
Default features are `quantum`, `quantum_toolchain`, `quantum_receipt`.

55
wai/wai-quantum/ci/dsp/predicate_branch.c Normal file
View file

@ -0,0 +1,55 @@
/* Does this emulator branch on a predicate's low bit?
*
* The signal processor's conditional instructions test the least significant
* bit of a predicate register. A compiler may build a predicate from a general
* register (`p = r`, so `1` gives 0x01) and combine it with a comparison's
* (0xff) by `xor`, leaving 0xfe for "equal": its low bit is clear, so a branch
* on it is not taken. An emulator that tests the whole register takes it.
*
* Each case branches on the same predicate twice: written in an earlier packet
* (`if (p1) jump`) and in the same packet (`if (p1.new) jump`). Both must test
* the low bit. Exit status 0 when they do, 1 when the emulator differs.
*/
#include <stdio.h>
static int same_packet(int a, int b) {
int r;
__asm__ volatile(
"{ p2 = %1 }\n"
"{ p0 = %2 }\n"
"{ p1 = xor(p2,p0)\n if (p1.new) jump:nt 1f }\n"
"{ %0 = #0 ; jump 2f }\n"
"1: { %0 = #1 }\n"
"2:\n"
: "=&r"(r) : "r"(a), "r"(b) : "p0", "p1", "p2");
return r;
}
static int earlier_packet(int a, int b) {
int r;
__asm__ volatile(
"{ p2 = %1 }\n"
"{ p0 = %2 }\n"
"{ p1 = xor(p2,p0) }\n"
"{ if (p1) jump:nt 1f }\n"
"{ %0 = #0 ; jump 2f }\n"
"1: { %0 = #1 }\n"
"2:\n"
: "=&r"(r) : "r"(a), "r"(b) : "p0", "p1", "p2");
return r;
}
int main(void) {
/* (p2, p0): the predicate is their xor; taken iff its low bit is set. */
static const int cases[][2] = {{1, 0xff}, {0xff, 0xff}, {1, 1}, {0, 0xff}, {0, 1}, {0xfe, 0}, {2, 0}};
int bad = 0;
for (unsigned i = 0; i < sizeof cases / sizeof cases[0]; i++) {
int a = cases[i][0], b = cases[i][1], want = (a ^ b) & 1;
int n = same_packet(a, b), o = earlier_packet(a, b);
printf("predicate %#04x: same packet %s, earlier packet %s, low bit %s\n", (a ^ b) & 0xff, n ? "taken" : "not taken", o ? "taken" : "not taken",
want ? "set" : "clear");
bad += (n != want) + (o != want);
}
printf(bad ? "this emulator does not branch on the low bit\n" : "this emulator branches on the low bit\n");
return bad != 0;
}

42
wai/wai-quantum/examples/npu_onnx.rs Normal file
View file

@ -0,0 +1,42 @@
//! Writes the limb-product model in the portable format (fp16 and int32), and a
//! batch of inputs with the products they must give, for a neural-processor
//! service that runs such models:
//!
//! ```text
//! cargo run --example npu_onnx --features quantum_fabric -- <dir>
//! ```
//!
//! In `<dir>`: `limbs_f16.onnx`, `limbs_i32.onnx`; `x_f16.bin`, `y_f16.bin`
//! (fp16 bits, little-endian), `x_i32.bin`, `y_i32.bin` (int32, little-endian),
//! each `[batch, 15, 8]`; `pairs.bin` (the operands, int32 pairs) and
//! `products.bin` (each exact `a·b`, int64). A service's output `z`
//! (`[batch, 15]`) is right when `Σ_s z[p, s]·16^s` is `products[p]` for every
//! `p`.
use std::io::Write;
use wai_quantum::quantum_fabric::lanes;
use wai_quantum::quantum_fabric::npu::{BATCH, Elem, LIMBS, SHIFTS, expand, expand_i32, onnx_limb_model};
fn write(path: &std::path::Path, bytes: &[u8]) {
std::fs::File::create(path).and_then(|mut f| f.write_all(bytes)).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
}
fn main() {
let dir = std::path::PathBuf::from(std::env::args().nth(1).expect("an output directory"));
std::fs::create_dir_all(&dir).expect("the directory");
write(&dir.join("limbs_f16.onnx"), &onnx_limb_model(BATCH as u64, SHIFTS as u64, LIMBS as u64, Elem::F16));
write(&dir.join("limbs_i32.onnx"), &onnx_limb_model(BATCH as u64, SHIFTS as u64, LIMBS as u64, Elem::I32));
let mut pairs = lanes::probe_pairs(BATCH, 0x0a1b);
pairs.truncate(BATCH);
let n = BATCH * SHIFTS * LIMBS;
let (mut x, mut y) = (vec![0u16; n], vec![0u16; n]);
expand(&pairs, &mut x, &mut y);
write(&dir.join("x_f16.bin"), &x.iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<u8>>());
write(&dir.join("y_f16.bin"), &y.iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<u8>>());
let (mut xi, mut yi) = (vec![0i32; n], vec![0i32; n]);
expand_i32(&pairs, &mut xi, &mut yi);
write(&dir.join("x_i32.bin"), &xi.iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<u8>>());
write(&dir.join("y_i32.bin"), &yi.iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<u8>>());
write(&dir.join("pairs.bin"), &pairs.iter().flat_map(|&(a, b)| [a.to_le_bytes(), b.to_le_bytes()].concat()).collect::<Vec<u8>>());
write(&dir.join("products.bin"), &pairs.iter().flat_map(|&(a, b)| (i64::from(a) * i64::from(b)).to_le_bytes()).collect::<Vec<u8>>());
println!("wrote {} pairs of a batch of {BATCH} to {}", pairs.len(), dir.display());
}

32
wai/wai-quantum/examples/rv32_listing.rs Normal file
View file

@ -0,0 +1,32 @@
//! Writes the dataflow member's RV32IM programs, exactly as the device runs
//! them (conforming and mutants), as assembly listings and as the machine words
//! this crate encodes, so an independent assembler can be checked against the
//! encoder:
//!
//! ```text
//! cargo run --example rv32_listing --features quantum_fabric_dataflow -- <dir>
//! ```
//!
//! In `<dir>`, for each program `<name>`: `<name>.s` (the listing, with a
//! `.text` header) and `<name>.bin` (the words, little-endian).
use wai_quantum::quantum_fabric_dataflow::{Variant, programs};
fn main() {
let dir = std::path::PathBuf::from(std::env::args().nth(1).expect("an output directory"));
std::fs::create_dir_all(&dir).expect("the directory");
// The programs the device runs: the rows and probe programs of the
// conforming kernel and of each mutant (M3's probe is the conforming one).
let mut all = Vec::new();
for (tag, v) in [("", Variant::Conforming), ("_m1", Variant::M1), ("_m2", Variant::M2), ("_m3", Variant::M3)] {
let (rows, probe) = programs(v);
all.push((format!("rows{tag}"), rows));
if v != Variant::M3 {
all.push((format!("probe{tag}"), probe));
}
}
for (name, (words, text)) in all {
std::fs::write(dir.join(format!("{name}.s")), format!(".text\n.option norvc\n{text}")).expect("the listing");
std::fs::write(dir.join(format!("{name}.bin")), words.iter().flat_map(|w| w.to_le_bytes()).collect::<Vec<u8>>()).expect("the words");
println!("{name}: {} words", words.len());
}
}

23
wai/wai-quantum/src/lib.rs
View file

@ -10,15 +10,24 @@
//! Rust, so the same source runs natively, in the browser via wasm, and as a
//! WASI component at the edge, producing **byte-identical** results and receipts
//! on all three. A fabric receipt is the exception by design: it names the
//! member that ran the work. Four opt-in features reach outside the crate:
//! member that ran the work. Nine opt-in features reach outside the crate:
//! `quantum_fabric_webgpu` (the platform's GPU interface),
//! `quantum_fabric_cuda` (the CUDA driver and runtime compiler) and
//! `quantum_fabric_opencl` (the OpenCL ICD loader), the last two loaded when
//! first used, and `quantum_fabric_fpga` (Verilator, run as a program when
//! first used, `quantum_fabric_fpga` (Verilator, run as a program when first
//! used), `quantum_fabric_npu` (the platform's model runtime, on macOS),
//! `quantum_fabric_sme` (the processor's streaming vector mode),
//! `quantum_fabric_vector` (the processor's vector extension),
//! `quantum_fabric_dsp` (the signal processor's vector unit, in a build for
//! it) and `quantum_fabric_dataflow` (a dataflow device's library, loaded when
//! first used); no default feature reaches a system library.
//!
//! Everything is behind a feature; enable only what you use.
// The signal processor's vector intrinsics are unstable; only a build for that
// processor, which needs an unstable standard-library build anyway, reaches them.
#![cfg_attr(all(target_arch = "hexagon", feature = "quantum_fabric_dsp"), feature(stdarch_hexagon))] // allow-name: the toolchain's architecture name and feature gate
// Every receipt seals over a Merkle root; nothing else needs one.
#[cfg(feature = "_signing")]
mod merkle;
@ -85,6 +94,16 @@ pub mod quantum_fabric_cuda;
pub mod quantum_fabric_opencl;
#[cfg(all(feature = "quantum_fabric_fpga", not(target_arch = "wasm32")))]
pub mod quantum_fabric_fpga;
#[cfg(all(feature = "quantum_fabric_npu", not(target_arch = "wasm32")))]
pub mod quantum_fabric_npu;
#[cfg(all(feature = "quantum_fabric_sme", not(target_arch = "wasm32")))]
pub mod quantum_fabric_sme;
#[cfg(all(feature = "quantum_fabric_dsp", not(target_arch = "wasm32")))]
pub mod quantum_fabric_dsp;
#[cfg(all(feature = "quantum_fabric_vector", not(target_arch = "wasm32")))]
pub mod quantum_fabric_vector;
#[cfg(all(feature = "quantum_fabric_dataflow", not(target_arch = "wasm32")))]
pub mod quantum_fabric_dataflow;
#[cfg(feature = "quantum_control")]
pub mod quantum_control;

20
wai/wai-quantum/src/quantum_fabric.rs
View file

@ -259,11 +259,23 @@ pub enum FabricClass {
Qpu,
/// An analog Ising machine: coupled oscillators, optical or electronic.
AnalogIsing,
/// A digital signal processor and its vector unit.
Dsp,
}
impl FabricClass {
pub const ALL: [FabricClass; 9] =
[FabricClass::Cpu, FabricClass::CpuMatrix, FabricClass::Gpu, FabricClass::Npu, FabricClass::Tensor, FabricClass::Dataflow, FabricClass::Fpga, FabricClass::Qpu, FabricClass::AnalogIsing];
pub const ALL: [FabricClass; 10] = [
FabricClass::Cpu,
FabricClass::CpuMatrix,
FabricClass::Gpu,
FabricClass::Npu,
FabricClass::Tensor,
FabricClass::Dataflow,
FabricClass::Fpga,
FabricClass::Qpu,
FabricClass::AnalogIsing,
FabricClass::Dsp,
];
pub fn label(self) -> &'static str {
match self {
@ -276,6 +288,7 @@ impl FabricClass {
FabricClass::Fpga => "fpga",
FabricClass::Qpu => "qpu",
FabricClass::AnalogIsing => "analog-ising",
FabricClass::Dsp => "dsp",
}
}
@ -2036,8 +2049,11 @@ pub mod lanes {
pub mod cuda;
pub mod fpga;
pub mod npu;
pub mod opencl;
pub mod plan;
#[cfg(feature = "quantum_fabric_dataflow")]
pub mod rv32;
pub mod sampling;
#[cfg(feature = "quantum_sv")]
pub mod sv64;

142
wai/wai-quantum/src/quantum_fabric/dsp.rs Normal file
View file

@ -0,0 +1,142 @@
//! The vector unit's kernels: the gate's fixed-point arithmetic in its
//! 128-byte vectors, thirty-two lanes of 32 bits. Each lane's 64-bit value is a
//! pair of words, high and low; every step on it is exact.
//!
//! This file's text is the implementation id's input. `RND` is the rounding
//! constant, `SH` the shift and `LAST_ADD` whether row 0's last real term is
//! added (a mutant) instead of subtracted.
use core::arch::hexagon::v128::*; // allow-name: the toolchain's module of the vector unit's intrinsics
/// One 128-byte vector.
type V = HvxVector; // allow-name: the toolchain's vector type
/// Lanes per vector.
pub const LANES: usize = 32;
/// Each lane's 64-bit value as its high and low words.
#[derive(Clone, Copy)]
struct Wide {
hi: V,
lo: V,
}
// Carries and borrows are bit logic on the words, not predicates: the carry out
// of `s = a + b` is bit 31 of `(a & b) | ((a | b) & !s)`, and the borrow out of
// `d = a − b` is bit 31 of `(!a & b) | ((!a | b) & d)`.
/// `a + b` in each lane: the low words' carry goes into the high word.
#[inline(always)]
unsafe fn add(a: Wide, b: Wide) -> Wide {
unsafe {
let lo = Q6_Vw_vadd_VwVw(a.lo, b.lo);
let carry = Q6_Vuw_vlsr_VuwR(Q6_V_vor_VV(Q6_V_vand_VV(a.lo, b.lo), Q6_V_vand_VV(Q6_V_vor_VV(a.lo, b.lo), Q6_V_vnot_V(lo))), 31);
Wide { hi: Q6_Vw_vadd_VwVw(Q6_Vw_vadd_VwVw(a.hi, b.hi), carry), lo }
}
}
/// `a − b` in each lane: the low words' borrow comes out of the high word.
#[inline(always)]
unsafe fn sub(a: Wide, b: Wide) -> Wide {
unsafe {
let lo = Q6_Vw_vsub_VwVw(a.lo, b.lo);
let na = Q6_V_vnot_V(a.lo);
let borrow = Q6_Vuw_vlsr_VuwR(Q6_V_vor_VV(Q6_V_vand_VV(na, b.lo), Q6_V_vand_VV(Q6_V_vor_VV(na, b.lo), lo)), 31);
Wide { hi: Q6_Vw_vsub_VwVw(Q6_Vw_vsub_VwVw(a.hi, b.hi), borrow), lo }
}
}
/// `a·b` in each lane, exactly: `a` times `b`'s low half (unsigned), plus `a`
/// times `b`'s high half (signed) sixteen bits up.
#[inline(always)]
unsafe fn product(a: V, b: V) -> Wide {
unsafe {
let p = Q6_W_vmpyoacc_WVwVh(Q6_W_vmpye_VwVuh(a, b), a, b);
Wide { hi: Q6_V_hi_W(p), lo: Q6_V_lo_W(p) }
}
}
/// `(e·a + RND) >> SH` in each lane, the shift arithmetic over all 64 bits.
#[inline(always)]
unsafe fn fx<const RND: i32, const SH: i32>(e: V, a: V) -> Wide {
unsafe {
let r = add(product(a, e), Wide { hi: Q6_V_vzero(), lo: Q6_V_vsplat_R(RND) });
Wide { hi: Q6_Vw_vasr_VwR(r.hi, SH), lo: Q6_V_vor_VV(Q6_Vuw_vlsr_VuwR(r.lo, SH), Q6_Vw_vasl_VwR(r.hi, 32 - SH)) }
}
}
/// The vector of `s` from lane `k`.
#[inline(always)]
unsafe fn load(s: &[i32], k: usize) -> V {
debug_assert!(k + LANES <= s.len());
// SAFETY: the caller's slices hold whole vectors; the load is unaligned.
unsafe { core::ptr::read_unaligned(s.as_ptr().add(k).cast::<V>()) }
}
/// Writes `v` to `d` from lane `k`.
#[inline(always)]
unsafe fn store(d: &mut [i32], k: usize, v: V) {
debug_assert!(k + LANES <= d.len());
// SAFETY: as for `load`.
unsafe { core::ptr::write_unaligned(d.as_mut_ptr().add(k).cast::<V>(), v) }
}
/// A gate's four rows for every pair: `m` the gate's entries, `cols` the pairs'
/// columns (re a0, im a0, re a1, im a1); row `r` (0r, 0i, 1r, 1i) goes to
/// `hi[r]` and `lo[r]`.
///
/// # Safety
///
/// The processor has the vector unit in 128-byte mode; every slice has the same
/// length, a whole number of vectors.
pub unsafe fn rows<const RND: i32, const SH: i32, const LAST_ADD: bool>(m: &[i32; 8], cols: [&[i32]; 4], hi: [&mut [i32]; 4], lo: [&mut [i32]; 4]) {
let n = cols[0].len();
debug_assert!(n.is_multiple_of(LANES) && cols.iter().all(|c| c.len() == n));
unsafe {
let mut e = [Q6_V_vzero(); 8];
for (v, &x) in e.iter_mut().zip(m) {
*v = Q6_V_vsplat_R(x);
}
let [hi0r, hi0i, hi1r, hi1i] = hi;
let [lo0r, lo0i, lo1r, lo1i] = lo;
let f = fx::<RND, SH>;
let mut k = 0;
while k < n {
let (a0r, a0i, a1r, a1i) = (load(cols[0], k), load(cols[1], k), load(cols[2], k), load(cols[3], k));
let t = add(sub(f(e[0], a0r), f(e[1], a0i)), f(e[2], a1r));
let o0r = if LAST_ADD { add(t, f(e[3], a1i)) } else { sub(t, f(e[3], a1i)) };
let o0i = add(add(add(f(e[0], a0i), f(e[1], a0r)), f(e[2], a1i)), f(e[3], a1r));
let o1r = sub(add(sub(f(e[4], a0r), f(e[5], a0i)), f(e[6], a1r)), f(e[7], a1i));
let o1i = add(add(add(f(e[4], a0i), f(e[5], a0r)), f(e[6], a1i)), f(e[7], a1r));
for (h, l, o) in [(&mut *hi0r, &mut *lo0r, o0r), (&mut *hi0i, &mut *lo0i, o0i), (&mut *hi1r, &mut *lo1r, o1r), (&mut *hi1i, &mut *lo1i, o1i)] {
store(h, k, o.hi);
store(l, k, o.lo);
}
k += LANES;
}
}
}
/// Each pair's `a·b` and `(a·b + RND) >> SH`, as high and low words.
///
/// # Safety
///
/// As for [`rows`].
pub unsafe fn probe<const RND: i32, const SH: i32>(a: &[i32], b: &[i32], prod: [&mut [i32]; 2], fixed: [&mut [i32]; 2]) {
let n = a.len();
debug_assert!(n.is_multiple_of(LANES) && b.len() == n);
let [ph, pl] = prod;
let [fh, fl] = fixed;
unsafe {
let mut k = 0;
while k < n {
let (va, vb) = (load(a, k), load(b, k));
let p = product(va, vb);
let f = fx::<RND, SH>(vb, va);
store(ph, k, p.hi);
store(pl, k, p.lo);
store(fh, k, f.hi);
store(fl, k, f.lo);
k += LANES;
}
}
}

546
wai/wai-quantum/src/quantum_fabric/npu.rs Normal file
View file

@ -0,0 +1,546 @@
//! The fabric's arithmetic on a neural processor, and what names that member:
//! the limb-product model, the layout of a batch, the exact assembly of each
//! product, and the implementation id. Nothing here touches a device, so all of
//! it is testable on any host; the member itself is `quantum_fabric_npu`.
//!
//! **Why the products are exact.** A neural processor computes in fp16, and does
//! not document how it accumulates. So each 32-bit operand is split into eight
//! 4-bit limbs (the top one signed, `−8..=7`, the rest `0..=15`), and the
//! processor computes, for each of the 15 shifts `s`, the limb sum
//! `Σ_{i+j=s} a_i·b_j`: the coefficients of the product of two polynomials in
//! 16. Every limb product is at most 225 in size and every sum of at most eight
//! of them at most 1800, below 2048, so every value the processor forms is an
//! integer fp16 holds exactly, whatever order it adds in. The host then
//! assembles `a·b = Σ_s z_s·16^s` exactly in 64 bits, and the gate's arithmetic
//! (`fx`, the row sums, the 32-bit lane and the range report) is
//! [`lanes::apply_words`](super::lanes::apply_words)'s.
//!
//! The model is one multiply and one sum over the last axis
//! ([`limb_product_model`]); the layout puts operand `a`'s limbs in `x[p, s, j]`
//! and `b`'s limb `s − j` in `y[p, s, j]` (zero outside), so
//! `z[p, s] = Σ_j x[p, s, j]·y[p, s, j]` is the coefficient `s`.
//!
//! **What the processor is credited with.** The member exists only where the
//! runtime's compute plan places every operation of the model on the neural
//! processor; a run the runtime would place on the CPU is refused, not counted.
use super::FabricClass;
/// Products per batch: the model's fixed first dimension.
pub const BATCH: usize = 16384;
/// Limbs per operand.
pub const LIMBS: usize = 8;
/// Coefficients of a product: `2·LIMBS − 1`.
pub const SHIFTS: usize = 2 * LIMBS - 1;
/// Version of the host-side layout: the limb split, the `x`/`y` expansion, the
/// assembly, and the batch. Bumped on any change; it is part of the
/// implementation id.
pub const HOST_LAYOUT: u32 = 1;
/// The implementation name the id starts with.
pub const IMPLEMENTATION: &str = "npu-limbs4";
const DOMAIN_IMPLEMENTATION: &[u8] = b"wai:quantum-fabric-npu\x01";
/// The class a neural-processor member has.
pub const CLASS: FabricClass = FabricClass::Npu;
/// Protobuf wire types.
const VARINT: u32 = 0;
const LEN: u32 = 2;
fn varint(o: &mut Vec<u8>, mut v: u64) {
loop {
let b = (v & 0x7f) as u8;
v >>= 7;
if v == 0 {
o.push(b);
return;
}
o.push(b | 0x80);
}
}
fn key(o: &mut Vec<u8>, field: u32, wire: u32) {
varint(o, u64::from(field << 3 | wire));
}
fn uint(o: &mut Vec<u8>, field: u32, v: u64) {
key(o, field, VARINT);
varint(o, v);
}
/// A signed int64 field (two's complement, as protobuf's int64 is written).
fn int(o: &mut Vec<u8>, field: u32, v: i64) {
key(o, field, VARINT);
varint(o, v as u64);
}
fn bytes(o: &mut Vec<u8>, field: u32, b: &[u8]) {
key(o, field, LEN);
varint(o, b.len() as u64);
o.extend_from_slice(b);
}
fn msg(o: &mut Vec<u8>, field: u32, f: impl FnOnce(&mut Vec<u8>)) {
let mut inner = Vec::new();
f(&mut inner);
bytes(o, field, &inner);
}
// The model description's field numbers (its Model, FeatureTypes and MIL schemas).
const ARRAY_FLOAT16: u64 = 65552;
/// The operation set the program is written in.
const OPSET: &[u8] = b"CoreML8"; // allow-name: the runtime's operation-set identifier
const MIL_BOOL: u64 = 1;
const MIL_STRING: u64 = 2;
const MIL_FLOAT16: u64 = 10;
const MIL_INT32: u64 = 23;
/// A MIL tensor type of `dtype` and `shape` (rank 0 for a scalar).
fn tensor_type(o: &mut Vec<u8>, dtype: u64, shape: &[u64]) {
// ValueType.tensorType = 1
msg(o, 1, |t| {
uint(t, 1, dtype); // dataType
if !shape.is_empty() {
int(t, 2, shape.len() as i64); // rank
}
for &d in shape {
// dimensions = 3 { constant = 1 { size = 1 } }
msg(t, 3, |dim| msg(dim, 1, |c| uint(c, 1, d)));
}
});
}
/// A feature description: `name`, a multi-array of fp16 and `shape`.
fn feature(o: &mut Vec<u8>, field: u32, name: &str, shape: &[u64]) {
msg(o, field, |f| {
bytes(f, 1, name.as_bytes());
// type = 3 { multiArrayType = 5 { shape = 1 (repeated), dataType = 2 } }
msg(f, 3, |t| {
msg(t, 5, |a| {
for &d in shape {
int(a, 1, d as i64);
}
uint(a, 2, ARRAY_FLOAT16);
})
});
});
}
/// A function input `name`: a MIL fp16 tensor of `shape`.
fn named_input(o: &mut Vec<u8>, name: &str, shape: &[u64]) {
// Function.inputs = 1 { name = 1, type = 2 }
msg(o, 1, |n| {
bytes(n, 1, name.as_bytes());
msg(n, 2, |v| tensor_type(v, MIL_FLOAT16, shape));
});
}
/// An operation's `name` attribute.
fn name_attribute(o: &mut Vec<u8>, name: &str) {
// attributes = 5 (map<string, Value>) { key = 1, value = 2 }
msg(o, 5, |e| {
bytes(e, 1, b"name");
msg(e, 2, |v| {
msg(v, 2, |t| tensor_type(t, MIL_STRING, &[])); // Value.type
// immediateValue = 3 { tensor = 1 { strings = 4 { values = 1 } } }
msg(v, 3, |iv| msg(iv, 1, |tv| msg(tv, 4, |s| bytes(s, 1, name.as_bytes()))));
});
});
}
/// An operation input binding: parameter `param` reads variable `var`.
fn input_binding(o: &mut Vec<u8>, param: &str, var: &str) {
// inputs = 2 (map<string, Argument>) { key, value = Argument { arguments = 1 { name = 1 } } }
msg(o, 2, |e| {
bytes(e, 1, param.as_bytes());
msg(e, 2, |arg| msg(arg, 1, |b| bytes(b, 1, var.as_bytes())));
});
}
/// An operation output `name` of `dtype` and `shape`.
fn op_output(o: &mut Vec<u8>, name: &str, dtype: u64, shape: &[u64]) {
// outputs = 3 { name = 1, type = 2 }
msg(o, 3, |n| {
bytes(n, 1, name.as_bytes());
msg(n, 2, |v| tensor_type(v, dtype, shape));
});
}
/// A `const` operation `name` holding `value` (written by `write`, a
/// TensorValue) of `dtype` and `shape`.
fn const_op(o: &mut Vec<u8>, name: &str, dtype: u64, shape: &[u64], write: impl FnOnce(&mut Vec<u8>)) {
// Block.operations = 3
msg(o, 3, |op| {
bytes(op, 1, b"const");
op_output(op, name, dtype, shape);
msg(op, 5, |e| {
bytes(e, 1, b"val");
msg(e, 2, |v| {
msg(v, 2, |t| tensor_type(t, dtype, shape));
msg(v, 3, |iv| msg(iv, 1, write));
});
});
name_attribute(op, name);
});
}
/// The model: `z[b, s] = Σ_j x[b, s, j] · y[b, s, j]`, all fp16, for
/// `batch` products of `shifts` limb sums of `limbs` terms.
pub fn limb_product_model(batch: u64, shifts: u64, limbs: u64) -> Vec<u8> {
let full = [batch, shifts, limbs];
let out = [batch, shifts];
let mut m = Vec::new();
uint(&mut m, 1, 9); // specificationVersion
msg(&mut m, 2, |d| {
feature(d, 1, "x", &full);
feature(d, 1, "y", &full);
feature(d, 10, "z", &out);
});
// mlProgram = 502 { version = 1, functions = 2 (map<string, Function>) }
msg(&mut m, 502, |p| {
int(p, 1, 1);
msg(p, 2, |e| {
bytes(e, 1, b"main");
msg(e, 2, |f| {
named_input(f, "x", &full);
named_input(f, "y", &full);
bytes(f, 2, OPSET); // opset
// block_specializations = 3 (map<string, Block>)
msg(f, 3, |e| {
bytes(e, 1, OPSET);
msg(e, 2, |b| {
bytes(b, 2, b"z"); // Block.outputs
// mul
msg(b, 3, |op| {
bytes(op, 1, b"mul");
input_binding(op, "x", "x");
input_binding(op, "y", "y");
op_output(op, "mul_0", MIL_FLOAT16, &full);
name_attribute(op, "mul_0");
});
const_op(b, "z_axes_0", MIL_INT32, &[1], |tv| msg(tv, 2, |ints| int(ints, 1, -1)));
const_op(b, "z_keep_dims_0", MIL_BOOL, &[], |tv| msg(tv, 3, |bools| uint(bools, 1, 0)));
// reduce_sum
msg(b, 3, |op| {
bytes(op, 1, b"reduce_sum");
input_binding(op, "x", "mul_0");
input_binding(op, "axes", "z_axes_0");
input_binding(op, "keep_dims", "z_keep_dims_0");
op_output(op, "z", MIL_FLOAT16, &out);
name_attribute(op, "z");
});
});
});
});
});
});
m
}
/// The element type of a portable model's tensors.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Elem {
/// fp16: every value the model forms is an integer of at most 1800 in size,
/// which fp16 holds exactly.
F16,
/// int32: exact on any processor that computes it in integers without
/// wrapping; a coefficient is at most 1800 in size.
I32,
}
// ONNX's field numbers (onnx.proto) and element types.
const ONNX_FLOAT16: u64 = 10;
const ONNX_INT32: u64 = 6;
const ONNX_INT64: u64 = 7;
const ONNX_ATTR_INT: u64 = 2;
/// An ONNX value description: `name`, a tensor of `elem` and `shape`.
fn onnx_value(o: &mut Vec<u8>, field: u32, name: &str, elem: u64, shape: &[u64]) {
msg(o, field, |v| {
bytes(v, 1, name.as_bytes());
// type = 2 { tensor_type = 1 { elem_type = 1, shape = 2 { dim = 1 { dim_value = 1 } } } }
msg(v, 2, |t| {
msg(t, 1, |tt| {
uint(tt, 1, elem);
msg(tt, 2, |sh| {
for &d in shape {
msg(sh, 1, |dim| int(dim, 1, d as i64));
}
});
})
});
});
}
/// The same program in the portable model format (ONNX, opset 17):
/// `z = ReduceSum(Mul(x, y), axes = [-1], keepdims = 0)`, `x` and `y`
/// `[batch, shifts, limbs]`, all of `elem`. Neural-processor runtimes that
/// read this format take it as it is; the inputs are [`expand`]'s (as fp16
/// bits) or [`expand_i32`]'s.
pub fn onnx_limb_model(batch: u64, shifts: u64, limbs: u64, elem: Elem) -> Vec<u8> {
let e = match elem {
Elem::F16 => ONNX_FLOAT16,
Elem::I32 => ONNX_INT32,
};
let full = [batch, shifts, limbs];
let out = [batch, shifts];
let mut m = Vec::new();
int(&mut m, 1, 8); // ir_version
bytes(&mut m, 2, b"wai-quantum"); // producer_name
// graph = 7
msg(&mut m, 7, |g| {
// node = 1: Mul
msg(g, 1, |n| {
bytes(n, 1, b"x");
bytes(n, 1, b"y");
bytes(n, 2, b"p");
bytes(n, 3, b"mul");
bytes(n, 4, b"Mul");
});
// node = 1: ReduceSum over the last axis, the axes as an input
msg(g, 1, |n| {
bytes(n, 1, b"p");
bytes(n, 1, b"axes");
bytes(n, 2, b"z");
bytes(n, 3, b"sum");
bytes(n, 4, b"ReduceSum");
msg(n, 5, |a| {
bytes(a, 1, b"keepdims");
int(a, 3, 0);
uint(a, 20, ONNX_ATTR_INT);
});
});
bytes(g, 2, b"limb_products");
// initializer = 5: axes = [-1], int64
msg(g, 5, |t| {
int(t, 1, 1); // dims
uint(t, 2, ONNX_INT64); // data_type
int(t, 7, -1); // int64_data
bytes(t, 8, b"axes"); // name
});
onnx_value(g, 11, "x", e, &full);
onnx_value(g, 11, "y", e, &full);
onnx_value(g, 12, "z", e, &out);
});
// opset_import = 8 { domain = 1 "", version = 2 }
msg(&mut m, 8, |op| {
bytes(op, 1, b"");
int(op, 2, 17);
});
m
}
/// [`expand`] with int32 values: `x[p, s, j]` and `y[p, s, j]` as the limbs
/// themselves.
pub fn expand_i32(pairs: &[(i32, i32)], x: &mut [i32], y: &mut [i32]) {
assert!(pairs.len() <= BATCH && x.len() == BATCH * SHIFTS * LIMBS && y.len() == x.len());
x.fill(0);
y.fill(0);
for (p, &(a, b)) in pairs.iter().enumerate() {
let (la, lb) = (limbs(a), limbs(b));
for s in 0..SHIFTS {
for j in 0..LIMBS {
if s >= j && s - j < LIMBS {
let at = (p * SHIFTS + s) * LIMBS + j;
x[at] = la[j];
y[at] = lb[s - j];
}
}
}
}
}
/// `"npu-limbs4/<16 hex>"` over the model's bytes and the host layout.
pub fn implementation_id_of(model: &[u8]) -> String {
let mut o = DOMAIN_IMPLEMENTATION.to_vec();
o.extend_from_slice(&(model.len() as u64).to_be_bytes());
o.extend_from_slice(model);
o.extend_from_slice(&HOST_LAYOUT.to_be_bytes());
let h: String = blake3::hash(&o).as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect();
format!("{IMPLEMENTATION}/{h}")
}
/// The model this member runs.
pub fn model() -> Vec<u8> {
limb_product_model(BATCH as u64, SHIFTS as u64, LIMBS as u64)
}
pub fn implementation_id() -> String {
implementation_id_of(&model())
}
/// The eight 4-bit limbs of `v`, least significant first, the top one signed.
pub fn limbs(v: i32) -> [i32; LIMBS] {
let u = v as u32;
let mut out = [0i32; LIMBS];
for (i, l) in out.iter_mut().enumerate() {
*l = ((u >> (4 * i)) & 15) as i32;
}
if out[LIMBS - 1] >= 8 {
out[LIMBS - 1] -= 16;
}
out
}
/// The fp16 bits of an integer `v` with `|v| ≤ 2048`, exactly.
pub fn f16_of(v: i32) -> u16 {
debug_assert!(v.abs() <= 2048);
if v == 0 {
return 0;
}
let sign: u16 = if v < 0 { 0x8000 } else { 0 };
let m = v.unsigned_abs();
let e = 31 - m.leading_zeros(); // floor(log2 m), 0..=11
// Significand bits below the leading one, left-aligned in 10 bits (2048,
// the one value with e = 11, has none).
let below = m - (1 << e);
let frac = if e <= 10 { below << (10 - e) } else { below >> (e - 10) } as u16;
sign | (((e + 15) as u16) << 10) | frac
}
/// The integer an fp16 value holds, or `None` if it is not an integer of at most
/// 2048 in size.
pub fn int_of_f16(bits: u16) -> Option<i64> {
let sign = if bits & 0x8000 != 0 { -1 } else { 1 };
let e = i32::from((bits >> 10) & 0x1f);
let frac = i64::from(bits & 0x3ff);
if e == 0 {
return (frac == 0).then_some(0);
}
if e == 31 {
return None;
}
let p = e - 15; // the exponent
if !(0..=11).contains(&p) {
return None;
}
let m = (1024 + frac) << p; // value · 2^10
(m & 1023 == 0).then_some(sign * (m >> 10))
}
/// Writes the inputs of one batch for `pairs` (at most [`BATCH`]; the rest is
/// zero): `x[p, s, j] = limb j of a`, `y[p, s, j] = limb s − j of b`, as fp16
/// bits, row-major.
pub fn expand(pairs: &[(i32, i32)], x: &mut [u16], y: &mut [u16]) {
assert!(pairs.len() <= BATCH && x.len() == BATCH * SHIFTS * LIMBS && y.len() == x.len());
x.fill(0);
y.fill(0);
for (p, &(a, b)) in pairs.iter().enumerate() {
let (la, lb) = (limbs(a), limbs(b));
for s in 0..SHIFTS {
for j in 0..LIMBS {
if s >= j && s - j < LIMBS {
let at = (p * SHIFTS + s) * LIMBS + j;
x[at] = f16_of(la[j]);
y[at] = f16_of(lb[s - j]);
}
}
}
}
}
/// The product `a·b` from its 15 coefficients: `Σ_s z_s·16^s`, exact in 64 bits.
pub fn assemble(z: &[i64]) -> i64 {
z.iter().rev().fold(0i64, |acc, &c| acc.wrapping_mul(16).wrapping_add(c))
}
/// The model's arithmetic in integers, on the host: what a processor that
/// computes it exactly returns. For the tests, and for nothing else.
pub fn replay(x: &[u16], y: &[u16]) -> Vec<i64> {
x.chunks(LIMBS).zip(y.chunks(LIMBS)).map(|(a, b)| a.iter().zip(b).map(|(&p, &q)| int_of_f16(p).unwrap_or(0) * int_of_f16(q).unwrap_or(0)).sum()).collect()
}
/// The model with one operation's type replaced: for the mutation tests.
pub fn mutate_model(op_from: &str, op_to: &str) -> Vec<u8> {
let m = model();
let (from, to) = (op_from.as_bytes(), op_to.as_bytes());
assert_eq!(from.len(), to.len(), "a mutant keeps the operation name's length");
let at = m.windows(from.len()).position(|w| w == from).expect("the operation is in the model");
let mut out = m;
out[at..at + to.len()].copy_from_slice(to);
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::quantum_fabric::lanes;
#[test]
fn small_integers_round_trip_through_fp16_exactly() {
for v in -2048..=2048 {
assert_eq!(int_of_f16(f16_of(v)), Some(i64::from(v)), "{v}");
}
assert_eq!(f16_of(1), 0x3c00);
assert_eq!(f16_of(-2), 0xc000);
assert_eq!(f16_of(1800), 0x6708);
assert_eq!(int_of_f16(0x3800), None, "one half is no integer");
assert_eq!(int_of_f16(0x7c00), None, "infinity is no integer");
}
#[test]
fn limbs_reassemble_and_every_value_stays_exact_in_fp16() {
for v in [0, 1, -1, 7, -8, i32::MAX, i32::MIN, 0x7654_3210, -0x0f0f_0f0f] {
let l = limbs(v);
assert_eq!(l.iter().rev().fold(0i64, |a, &x| a * 16 + i64::from(x)), i64::from(v), "{v}");
assert!(l[..LIMBS - 1].iter().all(|x| (0..=15).contains(x)) && (-8..=7).contains(&l[LIMBS - 1]));
}
// The largest coefficient a batch can form.
assert!(LIMBS as i64 * 15 * 15 <= 2048);
}
#[test]
fn a_batch_replayed_assembles_every_product_exactly() {
let mut pairs = lanes::probe_pairs(4096, 0x11);
pairs.truncate(BATCH);
let (mut x, mut y) = (vec![0u16; BATCH * SHIFTS * LIMBS], vec![0u16; BATCH * SHIFTS * LIMBS]);
expand(&pairs, &mut x, &mut y);
let z = replay(&x, &y);
assert!(z.iter().all(|c| c.abs() <= 1800));
for (p, &(a, b)) in pairs.iter().enumerate() {
let got = assemble(&z[p * SHIFTS..(p + 1) * SHIFTS]);
assert_eq!(got, i64::from(a) * i64::from(b), "({a}, {b})");
}
}
#[test]
fn the_portable_model_names_its_program_and_its_element_type() {
for (elem, code) in [(Elem::F16, ONNX_FLOAT16), (Elem::I32, ONNX_INT32)] {
let m = onnx_limb_model(BATCH as u64, SHIFTS as u64, LIMBS as u64, elem);
assert_eq!(&m[..2], &[0x08, 8], "ir_version first");
for op in [&b"Mul"[..], b"ReduceSum", b"keepdims", b"axes"] {
assert!(m.windows(op.len()).any(|w| w == op));
}
// elem_type = code, inside each value description.
assert!(m.windows(2).any(|w| w == [0x08, code as u8]));
}
assert_ne!(onnx_limb_model(4, 15, 8, Elem::F16), onnx_limb_model(4, 15, 8, Elem::I32));
// The int32 layout carries the limbs themselves.
let pairs = [(i32::MIN, -1), (0x7654_3210, 0x0123_4567)];
let n = BATCH * SHIFTS * LIMBS;
let (mut x, mut y) = (vec![0i32; n], vec![0i32; n]);
expand_i32(&pairs, &mut x, &mut y);
for (p, &(a, b)) in pairs.iter().enumerate() {
let z: Vec<i64> = (0..SHIFTS).map(|s| (0..LIMBS).map(|j| i64::from(x[(p * SHIFTS + s) * LIMBS + j]) * i64::from(y[(p * SHIFTS + s) * LIMBS + j])).sum()).collect();
assert_eq!(assemble(&z), i64::from(a) * i64::from(b));
}
}
#[test]
fn the_model_is_the_program_it_names_and_its_mutants_differ() {
let m = model();
assert_eq!(m[0], 0x08, "field 1, specificationVersion, comes first");
assert_eq!(m[1], 9);
for op in [&b"mul"[..], b"reduce_sum", OPSET] {
assert!(m.windows(op.len()).any(|w| w == op));
}
let id = implementation_id();
assert!(id.starts_with("npu-limbs4/") && id.len() == "npu-limbs4/".len() + 16);
assert_ne!(implementation_id_of(&mutate_model("mul", "add")), id);
assert_ne!(implementation_id_of(&mutate_model("reduce_sum", "reduce_max")), id);
}
}

380
wai/wai-quantum/src/quantum_fabric/rv32.rs Normal file
View file

@ -0,0 +1,380 @@
//! A small RV32IM encoder and the gate's kernels for a 32-bit RISC-V core that
//! runs with no operating system: each program is built here, as machine words
//! and as the assembly listing they encode (the implementation id's input).
//!
//! The kernels read a parameter block at a base address in the core's memory and
//! write their results beside it ([`Layout`]); a done word tells the host they
//! finished. Every 64-bit value is a pair of words: the product is `mul` (low)
//! and `mulh` (high, signed); the rounding constant's carry is `sltu`; the
//! arithmetic shift is built from word shifts; a row's carries and borrows are
//! `sltu` too.
use std::collections::BTreeMap;
use std::fmt::Write as _;
/// A register, `x0` to `x31`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct R(pub u8);
/// The ABI names, for the listing.
const NAMES: [&str; 32] =
["zero", "ra", "sp", "gp", "tp", "t0", "t1", "t2", "s0", "s1", "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10", "s11", "t3", "t4", "t5", "t6"];
impl std::fmt::Display for R {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(NAMES[self.0 as usize])
}
}
pub const ZERO: R = R(0);
pub const T0: R = R(5);
pub const T1: R = R(6);
pub const T2: R = R(7);
pub const S0: R = R(8);
pub const S1: R = R(9);
pub const A0: R = R(10);
pub const A1: R = R(11);
pub const A2: R = R(12);
pub const A3: R = R(13);
pub const A4: R = R(14);
pub const A5: R = R(15);
pub const A6: R = R(16);
pub const A7: R = R(17);
pub const S2: R = R(18);
pub const T3: R = R(28);
pub const T4: R = R(29);
pub const T5: R = R(30);
pub const T6: R = R(31);
fn r_type(f7: u32, rs2: R, rs1: R, f3: u32, rd: R, op: u32) -> u32 {
(f7 << 25) | (u32::from(rs2.0) << 20) | (u32::from(rs1.0) << 15) | (f3 << 12) | (u32::from(rd.0) << 7) | op
}
fn i_type(imm: i32, rs1: R, f3: u32, rd: R, op: u32) -> u32 {
assert!((-2048..2048).contains(&imm), "an I-type immediate is 12 bits: {imm}");
(((imm as u32) & 0xfff) << 20) | (u32::from(rs1.0) << 15) | (f3 << 12) | (u32::from(rd.0) << 7) | op
}
/// One branch waiting for its label.
struct Fixup {
at: usize,
label: &'static str,
f3: u32,
rs1: R,
rs2: R,
}
/// A program being built: machine words and their listing.
#[derive(Default)]
pub struct Asm {
words: Vec<u32>,
lines: Vec<String>,
labels: BTreeMap<&'static str, usize>,
fixups: Vec<Fixup>,
}
macro_rules! r_ops {
($($name:ident = $f7:literal, $f3:literal;)*) => {$(
pub fn $name(&mut self, rd: R, rs1: R, rs2: R) -> &mut Self {
self.emit(r_type($f7, rs2, rs1, $f3, rd, 0x33), format!("{} {rd}, {rs1}, {rs2}", stringify!($name)))
}
)*};
}
impl Asm {
fn emit(&mut self, word: u32, line: String) -> &mut Self {
self.words.push(word);
self.lines.push(line);
self
}
r_ops! {
add = 0x00, 0;
sub = 0x20, 0;
sltu = 0x00, 3;
or = 0x00, 6;
mul = 0x01, 0;
mulh = 0x01, 1;
}
pub fn addi(&mut self, rd: R, rs1: R, imm: i32) -> &mut Self {
self.emit(i_type(imm, rs1, 0, rd, 0x13), format!("addi {rd}, {rs1}, {imm}"))
}
pub fn slli(&mut self, rd: R, rs1: R, sh: u32) -> &mut Self {
assert!(sh < 32);
self.emit(i_type(sh as i32, rs1, 1, rd, 0x13), format!("slli {rd}, {rs1}, {sh}"))
}
pub fn srli(&mut self, rd: R, rs1: R, sh: u32) -> &mut Self {
assert!(sh < 32);
self.emit(i_type(sh as i32, rs1, 5, rd, 0x13), format!("srli {rd}, {rs1}, {sh}"))
}
pub fn srai(&mut self, rd: R, rs1: R, sh: u32) -> &mut Self {
assert!(sh < 32);
self.emit(i_type(0x400 | sh as i32, rs1, 5, rd, 0x13), format!("srai {rd}, {rs1}, {sh}"))
}
pub fn lw(&mut self, rd: R, off: i32, rs1: R) -> &mut Self {
self.emit(i_type(off, rs1, 2, rd, 0x03), format!("lw {rd}, {off}({rs1})"))
}
pub fn sw(&mut self, rs2: R, off: i32, rs1: R) -> &mut Self {
assert!((-2048..2048).contains(&off), "an S-type offset is 12 bits: {off}");
let imm = off as u32 & 0xfff;
let word = ((imm >> 5) << 25) | (u32::from(rs2.0) << 20) | (u32::from(rs1.0) << 15) | (2 << 12) | ((imm & 0x1f) << 7) | 0x23;
self.emit(word, format!("sw {rs2}, {off}({rs1})"))
}
pub fn lui(&mut self, rd: R, upper: u32) -> &mut Self {
assert!(upper < 1 << 20);
self.emit((upper << 12) | (u32::from(rd.0) << 7) | 0x37, format!("lui {rd}, {upper:#x}"))
}
/// `rd = value`, by `lui` and `addi` (or `addi` alone).
pub fn li(&mut self, rd: R, value: u32) -> &mut Self {
let low = ((value << 20) as i32) >> 20; // the low 12 bits, sign-extended
let upper = value.wrapping_sub(low as u32) >> 12;
if upper != 0 {
self.lui(rd, upper);
if low != 0 {
self.addi(rd, rd, low);
}
self
} else {
self.addi(rd, ZERO, low)
}
}
pub fn label(&mut self, name: &'static str) -> &mut Self {
assert!(self.labels.insert(name, self.words.len()).is_none(), "label {name} twice");
self.lines.push(format!("{name}:"));
self
}
fn branch(&mut self, f3: u32, mnemonic: &str, rs1: R, rs2: R, label: &'static str) -> &mut Self {
self.fixups.push(Fixup { at: self.words.len(), label, f3, rs1, rs2 });
self.emit(0, format!("{mnemonic} {rs1}, {rs2}, {label}"))
}
pub fn beq(&mut self, rs1: R, rs2: R, label: &'static str) -> &mut Self {
self.branch(0, "beq", rs1, rs2, label)
}
/// `jal zero, label`.
pub fn j(&mut self, label: &'static str) -> &mut Self {
self.fixups.push(Fixup { at: self.words.len(), label, f3: u32::MAX, rs1: ZERO, rs2: ZERO });
self.emit(0, format!("j {label}"))
}
/// The machine words, every branch resolved, and the listing.
pub fn finish(mut self) -> (Vec<u32>, String) {
for f in std::mem::take(&mut self.fixups) {
let target = *self.labels.get(f.label).unwrap_or_else(|| panic!("no label {}", f.label));
let off = (target as i64 - f.at as i64) * 4;
let o = off as u32;
self.words[f.at] = if f.f3 == u32::MAX {
assert!((-(1 << 20)..(1 << 20)).contains(&off), "a jump reaches 1 MiB");
(((o >> 20) & 1) << 31) | (((o >> 1) & 0x3ff) << 21) | (((o >> 11) & 1) << 20) | (((o >> 12) & 0xff) << 12) | 0x6f
} else {
assert!((-(1 << 12)..(1 << 12)).contains(&off), "a branch reaches 4 KiB");
(((o >> 12) & 1) << 31)
| (((o >> 5) & 0x3f) << 25)
| (u32::from(f.rs2.0) << 20)
| (u32::from(f.rs1.0) << 15)
| (f.f3 << 12)
| (((o >> 1) & 0xf) << 8)
| (((o >> 11) & 1) << 7)
| 0x63
};
}
let mut text = String::new();
for l in &self.lines {
let indent = if l.ends_with(':') { "" } else { " " };
let _ = writeln!(text, "{indent}{l}");
}
(self.words, text)
}
}
/// Where a kernel's parameters and results live in the core's memory, from
/// `base`: the pair count, the gate's eight entries, the done word, four
/// columns of `capacity` operands, and the results.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Layout {
pub base: u32,
pub capacity: u32,
}
impl Layout {
pub const COUNT: u32 = 0;
pub const ENTRIES: u32 = 4;
pub const DONE: u32 = 36;
const COLUMNS: u32 = 0x40;
/// The address of column `k` (0 to 3).
pub fn column(&self, k: u32) -> u32 {
self.base + Self::COLUMNS + k * self.capacity * 4
}
/// The address of the results: for a gate, eight words per pair (each
/// row's low and high words, rows 0r, 0i, 1r, 1i); for the probe, four
/// (the product's low and high words, then `fx`'s).
pub fn results(&self) -> u32 {
self.column(4)
}
/// The bytes from `base` the layout spans.
pub fn span(&self) -> u32 {
Self::COLUMNS + 4 * self.capacity * 4 + 8 * self.capacity * 4
}
}
/// `fx(m, a)` into (`T4` low, `T5` high): `A5` holds the rounding constant.
fn fx(p: &mut Asm, m: R, a: R, sh: u32) {
p.mul(T4, m, a).mulh(T5, m, a);
p.add(T4, T4, A5).sltu(T6, T4, A5).add(T5, T5, T6);
p.srli(T4, T4, sh).slli(T6, T5, 32 - sh).or(T4, T4, T6).srai(T5, T5, sh);
}
/// The accumulator (`A6` low, `A7` high) plus or minus (`T4`, `T5`).
fn acc(p: &mut Asm, add: bool) {
if add {
p.add(A6, A6, T4).sltu(T6, A6, T4).add(A7, A7, T5).add(A7, A7, T6);
} else {
p.sltu(T6, A6, T4).sub(A6, A6, T4).sub(A7, A7, T5).sub(A7, A7, T6);
}
}
/// The program that applies one gate to every pair: `rnd` the rounding
/// constant, `sh` the shift, `last_add` whether row 0's last real term is
/// added (a mutant).
pub fn rows_program(l: Layout, rnd: u32, sh: u32, last_add: bool) -> (Vec<u32>, String) {
// s0 base, s1 pairs left, s2..s9 the entries, a0..a3 the columns, a4 the
// results, a5 the rounding constant, t0..t3 one pair's operands.
const M: [R; 8] = [R(18), R(19), R(20), R(21), R(22), R(23), R(24), R(25)];
let mut p = Asm::default();
p.li(S0, l.base).lw(S1, Layout::COUNT as i32, S0);
for (k, &m) in M.iter().enumerate() {
p.lw(m, (Layout::ENTRIES + 4 * k as u32) as i32, S0);
}
p.li(A0, l.column(0)).li(A1, l.column(1)).li(A2, l.column(2)).li(A3, l.column(3)).li(A4, l.results()).li(A5, rnd);
p.label("next").beq(S1, ZERO, "done");
p.lw(T0, 0, A0).lw(T1, 0, A1).lw(T2, 0, A2).lw(T3, 0, A3);
// Rows 0r, 0i, 1r, 1i: their terms (entry, operand, added) in order.
let rows: [[(usize, R, bool); 4]; 4] = [
[(0, T0, true), (1, T1, false), (2, T2, true), (3, T3, last_add)],
[(0, T1, true), (1, T0, true), (2, T3, true), (3, T2, true)],
[(4, T0, true), (5, T1, false), (6, T2, true), (7, T3, false)],
[(4, T1, true), (5, T0, true), (6, T3, true), (7, T2, true)],
];
for (r, terms) in rows.iter().enumerate() {
for (i, &(e, a, add)) in terms.iter().enumerate() {
fx(&mut p, M[e], a, sh);
if i == 0 {
p.addi(A6, T4, 0).addi(A7, T5, 0);
} else {
acc(&mut p, add);
}
}
p.sw(A6, 8 * r as i32, A4).sw(A7, 8 * r as i32 + 4, A4);
}
p.addi(A0, A0, 4).addi(A1, A1, 4).addi(A2, A2, 4).addi(A3, A3, 4).addi(A4, A4, 32).addi(S1, S1, -1).j("next");
p.label("done").addi(T6, ZERO, 1).sw(T6, Layout::DONE as i32, S0);
p.label("idle").j("idle");
p.finish()
}
/// The program that forms each pair's `a·b` and `fx(a, b)`: `a` in column 0,
/// `b` in column 1.
pub fn probe_program(l: Layout, rnd: u32, sh: u32) -> (Vec<u32>, String) {
let mut p = Asm::default();
p.li(S0, l.base).lw(S1, Layout::COUNT as i32, S0);
p.li(A0, l.column(0)).li(A1, l.column(1)).li(A4, l.results()).li(A5, rnd);
p.label("next").beq(S1, ZERO, "done");
p.lw(T0, 0, A0).lw(T1, 0, A1);
p.mul(T2, T0, T1).mulh(T3, T0, T1).sw(T2, 0, A4).sw(T3, 4, A4);
fx(&mut p, T1, T0, sh);
p.sw(T4, 8, A4).sw(T5, 12, A4);
p.addi(A0, A0, 4).addi(A1, A1, 4).addi(A4, A4, 16).addi(S1, S1, -1).j("next");
p.label("done").addi(T6, ZERO, 1).sw(T6, Layout::DONE as i32, S0);
p.label("idle").j("idle");
p.finish()
}
#[cfg(test)]
mod tests {
use super::*;
fn one(f: impl FnOnce(&mut Asm)) -> u32 {
let mut p = Asm::default();
f(&mut p);
p.finish().0[0]
}
/// Encodings from the instruction set manual's formats, worked by hand.
#[test]
fn encodings_match_the_manual() {
assert_eq!(one(|p| { p.addi(R(1), ZERO, 1); }), 0x0010_0093);
assert_eq!(one(|p| { p.add(R(3), R(1), R(2)); }), 0x0020_81b3);
assert_eq!(one(|p| { p.sub(R(3), R(1), R(2)); }), 0x4020_81b3);
assert_eq!(one(|p| { p.mul(R(3), R(1), R(2)); }), 0x0220_81b3);
assert_eq!(one(|p| { p.mulh(R(3), R(1), R(2)); }), 0x0220_91b3);
assert_eq!(one(|p| { p.sltu(T1, T0, A5); }), 0x00f2_b333);
assert_eq!(one(|p| { p.or(R(3), R(1), R(2)); }), 0x0020_e1b3);
assert_eq!(one(|p| { p.lw(T0, 8, A0); }), 0x0085_2283);
assert_eq!(one(|p| { p.sw(T0, 8, A0); }), 0x0055_2423);
assert_eq!(one(|p| { p.lui(T0, 0x12345); }), 0x1234_52b7);
assert_eq!(one(|p| { p.slli(T0, T0, 2); }), 0x0022_9293);
assert_eq!(one(|p| { p.srli(T0, T0, 30); }), 0x01e2_d293);
assert_eq!(one(|p| { p.srai(T0, T0, 30); }), 0x41e2_d293);
assert_eq!(one(|p| { p.addi(T0, T0, -1); }), 0xfff2_8293);
assert_eq!(one(|p| { p.label("x").j("x"); }), 0x0000_006f);
assert_eq!(one(|p| { p.label("x").beq(ZERO, ZERO, "x"); }), 0x0000_0063);
}
#[test]
fn branches_reach_forward_and_back() {
let mut p = Asm::default();
p.label("top").beq(A0, A1, "end").addi(A0, A0, 1).j("top").label("end");
let (w, text) = p.finish();
// beq a0, a1, +12: imm[4:1] = 6.
assert_eq!(w[0], (6 << 8) | (11 << 20) | (10 << 15) | 0x63);
// j -8: imm = -8, 21 bits.
assert_eq!(w[2], 0xff9f_f06f);
assert!(text.contains("beq a0, a1, end") && text.contains("j top"));
}
#[test]
fn li_builds_every_kind_of_constant() {
for v in [0u32, 1, 0x7ff, 0x800, 0xfff, 0x1000, 0x2000_0000, 0x1234_5678, 0x8000_0800, 0xffff_ffff, 0x1_0040] {
let mut p = Asm::default();
p.li(T0, v);
let (w, _) = p.finish();
// Evaluate: lui sets the upper bits, addi adds the sign-extended low.
let mut x = 0u32;
for word in w {
match word & 0x7f {
0x37 => x = word & 0xffff_f000,
0x13 => x = (if (word >> 15) & 0x1f == 0 { 0 } else { x }).wrapping_add(((word as i32) >> 20) as u32),
_ => unreachable!(),
}
}
assert_eq!(x, v, "{v:#x}");
}
}
#[test]
fn the_programs_and_their_mutants_differ_and_fit() {
let l = Layout { base: 0x1_0000, capacity: 1024 };
let (w, text) = rows_program(l, 1 << 29, 30, false);
assert!(text.contains("mulh") && text.contains("srai") && w.len() > 200);
assert_ne!(rows_program(l, 1 << 28, 30, false).1, text);
assert_ne!(rows_program(l, 1 << 29, 29, false).1, text);
assert_ne!(rows_program(l, 1 << 29, 30, true).1, text);
assert_ne!(probe_program(l, 1 << 29, 30).1, probe_program(l, 1 << 28, 30).1);
assert_eq!(l.results(), l.base + 0x40 + 16 * 1024);
assert_eq!(l.span(), 0x40 + 48 * 1024);
}
}

102
wai/wai-quantum/src/quantum_fabric/vector_avx2.rs Normal file
View file

@ -0,0 +1,102 @@
//! The x86-64 vector unit's kernels (AVX2): the gate's fixed-point arithmetic
//! in 256-bit vectors, four 64-bit lanes, every step exact.
//!
//! This file's text is the implementation id's input. `RND` is the rounding
//! constant, `SH` the shift and `LAST_ADD` whether row 0's last real term is
//! added (a mutant) instead of subtracted.
use std::arch::x86_64::*;
/// Lanes per vector.
pub const LANES: usize = 4;
/// Four operands of `s` from `k`, sign-extended to 64 bits.
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn load(s: &[i32], k: usize) -> __m256i {
debug_assert!(k + LANES <= s.len());
// SAFETY: four values from `k` are in bounds; the load is unaligned.
_mm256_cvtepi32_epi64(unsafe { _mm_loadu_si128(s.as_ptr().add(k).cast()) })
}
/// Writes the four lanes of `v` to `d` from `k`.
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn store(d: &mut [i64], k: usize, v: __m256i) {
debug_assert!(k + LANES <= d.len());
// SAFETY: as for `load`.
unsafe { _mm256_storeu_si256(d.as_mut_ptr().add(k).cast(), v) }
}
/// `(e·a + RND) >> SH` in each lane. The multiply takes each lane's low 32
/// bits as signed and forms the exact 64-bit product; the unit has no 64-bit
/// arithmetic shift, so the shift is a logical one with the sign bit's
/// position flipped and subtracted back.
#[inline]
#[target_feature(enable = "avx2")]
fn fx<const RND: i64, const SH: i32>(e: __m256i, a: __m256i) -> __m256i {
let r = _mm256_add_epi64(_mm256_mul_epi32(a, e), _mm256_set1_epi64x(RND));
let sign = _mm256_set1_epi64x(1i64 << (63 - SH));
_mm256_sub_epi64(_mm256_xor_si256(_mm256_srli_epi64::<SH>(r), sign), sign)
}
/// A gate's four rows for every pair: `m` the gate's entries, `cols` the pairs'
/// columns (re a0, im a0, re a1, im a1), `out` the rows (0r, 0i, 1r, 1i).
///
/// # Safety
///
/// The processor has AVX2; every slice has the same length, a multiple of
/// [`LANES`].
#[target_feature(enable = "avx2")]
pub unsafe fn rows<const RND: i64, const SH: i32, const LAST_ADD: bool>(m: &[i32; 8], cols: [&[i32]; 4], out: [&mut [i64]; 4]) {
let n = cols[0].len();
debug_assert!(n.is_multiple_of(LANES) && cols.iter().all(|c| c.len() == n) && out.iter().all(|o| o.len() == n));
let mut e = [_mm256_setzero_si256(); 8];
for (v, &x) in e.iter_mut().zip(m) {
*v = _mm256_set1_epi64x(i64::from(x));
}
let [o0r, o0i, o1r, o1i] = out;
macro_rules! f {
($e:expr, $a:expr) => {
fx::<RND, SH>($e, $a)
};
}
let mut k = 0;
while k < n {
// SAFETY: the caller's slices hold whole vectors.
let (a0r, a0i, a1r, a1i) = unsafe { (load(cols[0], k), load(cols[1], k), load(cols[2], k), load(cols[3], k)) };
let t = _mm256_add_epi64(_mm256_sub_epi64(f!(e[0], a0r), f!(e[1], a0i)), f!(e[2], a1r));
let r0r = if LAST_ADD { _mm256_add_epi64(t, f!(e[3], a1i)) } else { _mm256_sub_epi64(t, f!(e[3], a1i)) };
let r0i = _mm256_add_epi64(_mm256_add_epi64(_mm256_add_epi64(f!(e[0], a0i), f!(e[1], a0r)), f!(e[2], a1i)), f!(e[3], a1r));
let r1r = _mm256_sub_epi64(_mm256_add_epi64(_mm256_sub_epi64(f!(e[4], a0r), f!(e[5], a0i)), f!(e[6], a1r)), f!(e[7], a1i));
let r1i = _mm256_add_epi64(_mm256_add_epi64(_mm256_add_epi64(f!(e[4], a0i), f!(e[5], a0r)), f!(e[6], a1i)), f!(e[7], a1r));
// SAFETY: as for the loads.
unsafe {
store(o0r, k, r0r);
store(o0i, k, r0i);
store(o1r, k, r1r);
store(o1i, k, r1i);
}
k += LANES;
}
}
/// Each pair's `a·b` and `(a·b + RND) >> SH`.
///
/// # Safety
///
/// As for [`rows`].
#[target_feature(enable = "avx2")]
pub unsafe fn probe<const RND: i64, const SH: i32>(a: &[i32], b: &[i32], prod: &mut [i64], fixed: &mut [i64]) {
let n = a.len();
debug_assert!(n.is_multiple_of(LANES) && b.len() == n && prod.len() == n && fixed.len() == n);
let mut k = 0;
while k < n {
// SAFETY: the caller's slices hold whole vectors.
unsafe {
let (va, vb) = (load(a, k), load(b, k));
store(prod, k, _mm256_mul_epi32(va, vb));
store(fixed, k, fx::<RND, SH>(vb, va));
}
k += LANES;
}
}

776
wai/wai-quantum/src/quantum_fabric_dataflow.rs Normal file
View file

@ -0,0 +1,776 @@
//! The fabric's dataflow member: the gate's fixed-point arithmetic on a RISC-V
//! core of a dataflow accelerator's compute tile, reached through the device's
//! own library (loaded at run time from `WAI_QUANTUM_DATAFLOW_LIB`), which
//! presents the device as a PCI endpoint: configuration space, memory windows
//! and clocks.
//!
//! **What runs where.** The program is built in this crate ([`rv32`]): plain
//! RV32IM, with every 64-bit value a pair of words (`mul` and `mulh` for the
//! product, `sltu` for carries and borrows, the arithmetic shift from word
//! shifts). It runs on the first core of each of [`TILES`] compute tiles, each
//! over a contiguous slice of a gate's pairs: the host holds every tile's cores
//! in reset, writes the program at the core's reset address and each tile's
//! parameters and slice beside it through that tile's memory window, releases
//! the first cores, advances the clock until each program writes its done word,
//! and reads the rows back in order. It checks each row's range and keeps
//! its low 32 bits, as [`lanes::apply_words`] does.
//!
//! **Provenance.** The library this member drives is a simulator of the device,
//! so every certificate says so (`WAI_QUANTUM_DATAFLOW_SIMULATOR` names it).
//!
//! The device is named by its PCI vendor and device ids. Three mutants are
//! compiled in and must each be caught.
use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector};
use crate::quantum_fabric::lanes::{self, GateWords, Lane};
use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step};
use crate::quantum_fabric::rv32::{self, Layout};
use crate::quantum_fabric::{Certificate, Certification, Corpus, FabricClass, KernelEntry, Member, MemberError, Presence, Provenance, RunFuture, RunMeter, Runner, apply_kernel};
use std::sync::{Mutex, OnceLock};
/// One probe case: [`lanes::mul_wide`] and [`lanes::fxmul_wide`] as
/// `(high, low)` words.
pub type ProbeOutput = ((u32, u32), (u32, u32));
/// Why the device is not a member, or a run has no state.
#[derive(Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum DataflowError {
Unavailable(String),
Circuit(QuantumError),
TooWide { n_qubits: u8, max: u8 },
OutOfRange { gate: u32 },
Internal(&'static str),
Probe { a: i32, b: i32 },
/// The core did not finish within the clocks its program needs.
Stalled { clocks: u64 },
}
impl std::fmt::Display for DataflowError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
DataflowError::Unavailable(why) => write!(f, "no dataflow device: {why}"),
DataflowError::Circuit(e) => write!(f, "{e}"),
DataflowError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"),
DataflowError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"),
DataflowError::Internal(why) => write!(f, "host defect: {why}"),
DataflowError::Probe { a, b } => write!(f, "the kernel's arithmetic differs from the CPU's at ({a}, {b})"),
DataflowError::Stalled { clocks } => write!(f, "the core did not finish in {clocks} clocks"),
}
}
}
impl std::error::Error for DataflowError {}
impl From<DataflowError> for MemberError {
fn from(e: DataflowError) -> MemberError {
match e {
DataflowError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max },
DataflowError::OutOfRange { gate } => MemberError::OutOfRange { gate },
DataflowError::Circuit(e) => MemberError::Circuit(e),
// A core that ran past its clock budget: a wait past its limit.
DataflowError::Stalled { .. } => MemberError::Timeout,
other => MemberError::Other(other.to_string()),
}
}
}
/// The kernels a member runs: the conforming one, or a mutant.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Variant {
Conforming,
/// The rounding constant is half what it should be.
M1,
/// The product is shifted by one bit too few.
M2,
/// Row 0's real part adds its last term instead of subtracting it.
M3,
}
impl Variant {
pub const MUTANTS: [Variant; 3] = [Variant::M1, Variant::M2, Variant::M3];
/// The kernel's constants: the rounding constant, the shift, and whether
/// row 0's last real term is added.
pub fn constants(self) -> (u32, u32, bool) {
match self {
Variant::Conforming => (1 << 29, 30, false),
Variant::M1 => (1 << 28, 30, false),
Variant::M2 => (1 << 29, 29, false),
Variant::M3 => (1 << 29, 30, true),
}
}
}
/// Where the programs put their parameters and results in the tile's memory:
/// past the programs, a batch of up to `capacity` pairs.
pub const LAYOUT: Layout = Layout { base: 0x1_0000, capacity: 8192 };
/// The rows program and the probe program of a variant, as words and listing:
/// exactly what the device runs.
pub fn programs(v: Variant) -> ((Vec<u32>, String), (Vec<u32>, String)) {
let (rnd, sh, last_add) = v.constants();
(rv32::rows_program(LAYOUT, rnd, sh, last_add), rv32::probe_program(LAYOUT, rnd, sh))
}
/// The implementation name the id starts with.
pub const IMPLEMENTATION: &str = "rv32-lanes64";
/// Version of the host layout: the gather of a gate's pairs into four columns,
/// the batch of [`LAYOUT`], the rows as word pairs, and the host's range check.
/// Part of the implementation id.
pub const HOST_LAYOUT: u32 = 1;
/// `"rv32-lanes64/<16 hex>"` over a variant's two listings and the layout.
pub fn implementation_id_of(v: Variant) -> String {
let ((_, rows), (_, probe)) = programs(v);
let mut o = b"wai:quantum-fabric-dataflow\x01".to_vec();
for text in [&rows, &probe] {
o.extend_from_slice(&(text.len() as u64).to_be_bytes());
o.extend_from_slice(text.as_bytes());
}
o.extend_from_slice(&HOST_LAYOUT.to_be_bytes());
let h: String = blake3::hash(&o).as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect();
format!("{IMPLEMENTATION}/{h}")
}
pub fn implementation_id() -> String {
implementation_id_of(Variant::Conforming)
}
/// The compute tiles a run spreads its pairs over, the first core of each.
pub const TILES: usize = 32;
/// The device generations this member knows: the PCI device id, the compute
/// tiles it uses, and how a memory window is set up.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Chip {
pub device: u16,
/// The compute tiles' (x, y) on the device's network, as the host
/// addresses them.
pub tiles: &'static [(u32, u32); TILES],
/// A window's size, and the stride of its configuration registers.
window: u64,
cfg_stride: u64,
}
/// The PCI vendor id of the devices this member knows.
pub const VENDOR: u16 = 0x1e52;
/// Compute tiles in columns 1–4 and 6–9 of rows 1–4.
const TILES_401E: [(u32, u32); TILES] = {
let xs = [1, 2, 3, 4, 6, 7, 8, 9];
let mut t = [(0, 0); TILES];
let mut i = 0;
while i < TILES {
t[i] = (xs[i % 8], 1 + (i / 8) as u32);
i += 1;
}
t
};
/// Compute tiles in rows 2 and 3 (columns 1–7 and 10–14), then eight of row 4.
const TILES_B140: [(u32, u32); TILES] = {
let xs = [1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14];
let mut t = [(0, 0); TILES];
let mut i = 0;
while i < TILES {
t[i] = (xs[i % 12], 2 + (i / 12) as u32);
i += 1;
}
t
};
pub const CHIPS: [Chip; 2] = [
// One 64-bit configuration register per 1 MiB window.
Chip { device: 0x401e, tiles: &TILES_401E, window: 1 << 20, cfg_stride: 8 },
// Three 32-bit configuration registers per 2 MiB window.
Chip { device: 0xb140, tiles: &TILES_B140, window: 1 << 21, cfg_stride: 12 },
];
/// The tile's soft-reset register: bit 11 holds the first core, 12–14 and 18
/// the others; all five set holds every core.
const SOFT_RESET: u64 = 0xffb1_21b0;
const HOLD_ALL: u32 = 0x4_7800;
const RUN_FIRST: u32 = 0x4_7000;
/// Where the window configuration registers begin in the first memory window.
const TLB_CFG: u64 = 0x1fc0_0000;
#[cfg(target_os = "linux")]
mod dl {
use std::ffi::{c_char, c_int, c_void};
#[link(name = "dl")]
unsafe extern "C" {
pub fn dlopen(filename: *const c_char, flag: c_int) -> *mut c_void;
pub fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void;
pub fn dlerror() -> *const c_char;
}
pub const RTLD_NOW: c_int = 2;
}
/// The device library's entry points.
struct Lib {
init: unsafe extern "C" fn(),
cfg_rd: unsafe extern "C" fn(u32, u32) -> u32,
mem_rd: unsafe extern "C" fn(u64, *mut std::ffi::c_void, u32),
mem_wr: unsafe extern "C" fn(u64, *const std::ffi::c_void, u32),
clock: unsafe extern "C" fn(u32),
}
/// The device, open in this process: the library is one per process and runs
/// on the calling thread, so every access holds the lock.
struct Device {
lib: Lib,
chip: Chip,
bar0: u64,
/// The program now in every tile's memory.
loaded: Option<Vec<u32>>,
/// Clocks advanced so far.
clocks: u64,
}
static DEVICE: OnceLock<Result<Mutex<Device>, String>> = OnceLock::new();
#[cfg(target_os = "linux")]
fn open_library(path: &str) -> Result<Lib, String> {
use std::ffi::CString;
let c = CString::new(path).map_err(|_| format!("{path:?} holds a NUL"))?;
// SAFETY: dlopen and dlsym with valid C strings; each symbol is cast to the
// signature the library's interface documents.
unsafe {
let h = dl::dlopen(c.as_ptr(), dl::RTLD_NOW);
if h.is_null() {
let e = dl::dlerror();
return Err(if e.is_null() { format!("{path}: cannot load") } else { format!("{path}: {}", std::ffi::CStr::from_ptr(e).to_string_lossy()) });
}
let sym = |name: &str| -> Result<*mut std::ffi::c_void, String> {
let n = CString::new(name).expect("no NUL");
let p = dl::dlsym(h, n.as_ptr());
if p.is_null() { Err(format!("{path} has no {name}")) } else { Ok(p) }
};
Ok(Lib {
init: std::mem::transmute::<*mut std::ffi::c_void, unsafe extern "C" fn()>(sym("libttsim_init")?), // allow-name: the device library's entry point
cfg_rd: std::mem::transmute::<*mut std::ffi::c_void, unsafe extern "C" fn(u32, u32) -> u32>(sym("libttsim_pci_config_rd32")?), // allow-name: the device library's entry point
mem_rd: std::mem::transmute::<*mut std::ffi::c_void, unsafe extern "C" fn(u64, *mut std::ffi::c_void, u32)>(sym("libttsim_pci_mem_rd_bytes")?), // allow-name: the device library's entry point
mem_wr: std::mem::transmute::<*mut std::ffi::c_void, unsafe extern "C" fn(u64, *const std::ffi::c_void, u32)>(sym("libttsim_pci_mem_wr_bytes")?), // allow-name: the device library's entry point
clock: std::mem::transmute::<*mut std::ffi::c_void, unsafe extern "C" fn(u32)>(sym("libttsim_clock")?), // allow-name: the device library's entry point
})
}
}
#[cfg(not(target_os = "linux"))]
fn open_library(_path: &str) -> Result<Lib, String> {
Err(format!("the device library is loaded on Linux; this is {}", std::env::consts::OS))
}
impl Device {
fn open() -> Result<Device, String> {
let path = std::env::var("WAI_QUANTUM_DATAFLOW_LIB").ok().filter(|s| !s.trim().is_empty()).ok_or("no device library named (WAI_QUANTUM_DATAFLOW_LIB)")?;
let lib = open_library(path.trim())?;
// SAFETY: the library's documented sequence: initialise once per
// process (this runs once, under the OnceLock), then configuration
// reads of the one endpoint at bus 0, device 0, function 0.
let (id, lo, hi) = unsafe {
(lib.init)();
((lib.cfg_rd)(0, 0), (lib.cfg_rd)(0, 0x10), (lib.cfg_rd)(0, 0x14))
};
let (vendor, device) = (id as u16, (id >> 16) as u16);
if vendor != VENDOR {
return Err(format!("the endpoint is {vendor:04x}:{device:04x}, not vendor {VENDOR:04x}"));
}
let chip = CHIPS.into_iter().find(|c| c.device == device).ok_or(format!("device {vendor:04x}:{device:04x} is not one this member knows"))?;
let mut d = Device { lib, chip, bar0: (u64::from(hi) << 32) | u64::from(lo & !0xf), loaded: None, clocks: 0 };
// Two windows per tile: its memory from 0, and its registers.
for (j, &tile) in chip.tiles.iter().enumerate() {
d.window(2 * j as u64, tile, 0);
d.window(2 * j as u64 + 1, tile, SOFT_RESET & !(chip.window - 1));
d.reg_write(j, HOLD_ALL);
}
Ok(d)
}
/// Points window `i` at `addr` (window-aligned) on `tile`.
fn window(&mut self, i: u64, tile: (u32, u32), addr: u64) {
let (x, y) = tile;
let at = self.bar0 + TLB_CFG + i * self.chip.cfg_stride;
if self.chip.window == 1 << 20 {
let cfg = (addr >> 20) | (u64::from(x) << 16) | (u64::from(y) << 22);
self.write(at, &(cfg as u32).to_le_bytes());
self.write(at + 4, &((cfg >> 32) as u32).to_le_bytes());
} else {
let coord = u64::from(x) | (u64::from(y) << 6);
self.write(at, &((addr >> 21) as u32).to_le_bytes());
self.write(at + 4, &((((addr >> 53) & 0x7ff) | (coord << 11)) as u32).to_le_bytes());
self.write(at + 8, &0u32.to_le_bytes());
}
}
fn write(&mut self, paddr: u64, bytes: &[u8]) {
// SAFETY: a write within a window the device decodes; the library
// copies `bytes.len()` bytes from the buffer.
unsafe { (self.lib.mem_wr)(paddr, bytes.as_ptr().cast(), bytes.len() as u32) }
}
fn read(&mut self, paddr: u64, out: &mut [u8]) {
// SAFETY: as for `write`, into the buffer.
unsafe { (self.lib.mem_rd)(paddr, out.as_mut_ptr().cast(), out.len() as u32) }
}
/// Writes tile `j`'s soft-reset register.
fn reg_write(&mut self, j: usize, v: u32) {
let at = self.bar0 + (2 * j as u64 + 1) * self.chip.window + (SOFT_RESET & (self.chip.window - 1));
self.write(at, &v.to_le_bytes());
}
fn l1_write_words(&mut self, j: usize, addr: u32, words: &[u32]) {
let bytes: Vec<u8> = words.iter().flat_map(|w| w.to_le_bytes()).collect();
self.write(self.bar0 + 2 * j as u64 * self.chip.window + u64::from(addr), &bytes);
}
fn l1_read_words(&mut self, j: usize, addr: u32, n: usize) -> Vec<u32> {
let mut bytes = vec![0u8; 4 * n];
self.read(self.bar0 + 2 * j as u64 * self.chip.window + u64::from(addr), &mut bytes);
bytes.as_chunks::<4>().0.iter().map(|&c| u32::from_le_bytes(c)).collect()
}
/// Runs `program` on the first `counts.len()` tiles, tile `j` over
/// `counts[j]` pairs already written to it, until each writes its done
/// word; at most `budget` clocks.
fn run(&mut self, program: &[u32], counts: &[u32], budget: u64) -> Result<(), DataflowError> {
for j in 0..TILES {
self.reg_write(j, HOLD_ALL);
}
if self.loaded.as_deref() != Some(program) {
for j in 0..TILES {
self.l1_write_words(j, 0, program);
}
self.loaded = Some(program.to_vec());
}
for (j, &count) in counts.iter().enumerate() {
self.l1_write_words(j, LAYOUT.base + Layout::COUNT, &[count]);
self.l1_write_words(j, LAYOUT.base + Layout::DONE, &[0]);
}
for j in 0..counts.len() {
self.reg_write(j, RUN_FIRST);
}
let mut spent = 0u64;
// A first stretch near what the program needs, then short ones.
let mut step = budget.div_ceil(2).clamp(1, u64::from(u32::MAX));
let mut waiting: Vec<usize> = (0..counts.len()).collect();
loop {
// SAFETY: advances the device's clock; nothing else is touched.
unsafe { (self.lib.clock)(step as u32) };
spent += step;
self.clocks += step;
waiting.retain(|&j| self.l1_read_words(j, LAYOUT.base + Layout::DONE, 1)[0] != 1);
if waiting.is_empty() {
break;
}
if spent >= budget {
for j in 0..TILES {
self.reg_write(j, HOLD_ALL);
}
return Err(DataflowError::Stalled { clocks: spent });
}
step = 4096.min(budget - spent).max(1);
}
for j in 0..counts.len() {
self.reg_write(j, HOLD_ALL);
}
Ok(())
}
}
fn device() -> Result<&'static Mutex<Device>, String> {
DEVICE.get_or_init(|| Device::open().map(Mutex::new)).as_ref().map_err(|e| e.clone())
}
/// What [`discover`] found.
#[derive(Debug)]
pub struct Discovery {
pub found: Vec<DataflowMember>,
pub refused: Vec<(String, DataflowError)>,
pub unavailable: Option<String>,
}
/// The device, opened and probed in this process, or why there is none.
pub fn discover() -> Discovery {
let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None };
let chip = match device() {
Ok(dev) => dev.lock().unwrap_or_else(|p| p.into_inner()).chip,
Err(why) => {
d.unavailable = Some(why);
return d;
}
};
match DataflowMember::open(chip, Variant::Conforming) {
Ok(m) => d.found.push(m),
Err(e) => d.refused.push((key_of(chip), e)),
}
d
}
/// The key a runner declares: `dataflow-<vendor>-<device>`.
pub fn key_of(chip: Chip) -> String {
format!("dataflow-{VENDOR:04x}-{:04x}", chip.device)
}
/// The most qubits a run holds.
pub const MAX_DATAFLOW_QUBITS: u8 = if 20 < MAX_QUBITS { 20 } else { MAX_QUBITS };
/// Clocks a program needs for `count` pairs, with room to spare: the rows
/// program executes about 220 instructions a pair, the probe about 25.
fn budget(count: u32, per_pair: u64) -> u64 {
1_000 + 2 * per_pair * u64::from(count)
}
/// A compute tile's core as a fabric member.
#[derive(Clone, Debug)]
pub struct DataflowMember {
chip: Chip,
variant: Variant,
key: String,
id: String,
implementation: String,
detail: String,
provenance: Provenance,
max_qubits: u8,
}
impl DataflowMember {
fn open(chip: Chip, variant: Variant) -> Result<DataflowMember, DataflowError> {
let label = std::env::var("WAI_QUANTUM_DATAFLOW_SIMULATOR").ok().filter(|s| !s.trim().is_empty()).unwrap_or_else(|| "the device library".into());
let provenance = Provenance::Simulator(label.trim().to_string());
let key = key_of(chip);
let mut o = b"wai:quantum-fabric-dataflow-member\x01".to_vec();
o.extend_from_slice(key.as_bytes());
for &(x, y) in chip.tiles {
o.extend_from_slice(&x.to_be_bytes());
o.extend_from_slice(&y.to_be_bytes());
}
o.extend_from_slice(provenance.label().as_bytes());
let id = format!("local/dataflow-{}", blake3::hash(&o).as_bytes()[..6].iter().map(|b| format!("{b:02x}")).collect::<String>());
let member = DataflowMember {
chip,
variant,
detail: format!(
"dataflow; pci {VENDOR:04x}:{:04x}, the first RISC-V core (RV32IM) of {TILES} compute tiles; {}; up to {MAX_DATAFLOW_QUBITS} qubits; unmetered",
chip.device,
provenance.label()
),
key,
id,
implementation: implementation_id_of(variant),
provenance,
max_qubits: MAX_DATAFLOW_QUBITS,
};
if variant == Variant::Conforming {
let pairs = lanes::probe_pairs(1024, 0x5eed);
let out = member.probe(&pairs)?;
if let Some((a, b)) = lanes::first_mismatch(&pairs, &out) {
return Err(DataflowError::Probe { a, b });
}
}
Ok(member)
}
/// The same device running a mutant: for the mutation tests.
#[doc(hidden)]
pub fn with_variant(&self, v: Variant) -> DataflowMember {
DataflowMember { variant: v, implementation: implementation_id_of(v), ..self.clone() }
}
#[doc(hidden)]
pub fn with_max_qubits(&self, max_qubits: u8) -> DataflowMember {
DataflowMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() }
}
/// The kernel this member runs: the conforming one, or a mutant.
pub fn variant(&self) -> Variant {
self.variant
}
pub fn chip(&self) -> Chip {
self.chip
}
/// The clocks the device has advanced in this process.
pub fn clocks(&self) -> u64 {
device().map(|d| d.lock().unwrap_or_else(|p| p.into_inner()).clocks).unwrap_or(0)
}
pub fn id(&self) -> &str {
&self.id
}
pub fn key(&self) -> &str {
&self.key
}
pub fn class(&self) -> FabricClass {
FabricClass::Dataflow
}
pub fn detail(&self) -> &str {
&self.detail
}
pub fn implementation(&self) -> &str {
&self.implementation
}
pub fn provenance(&self) -> &Provenance {
&self.provenance
}
pub fn max_qubits(&self) -> u8 {
self.max_qubits
}
pub fn config(&self) -> Config {
Config::PER_GATE
}
fn device(&self) -> Result<std::sync::MutexGuard<'static, Device>, DataflowError> {
Ok(device().map_err(DataflowError::Unavailable)?.lock().unwrap_or_else(|p| p.into_inner()))
}
/// The kernel's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each pair.
pub fn probe(&self, pairs: &[(i32, i32)]) -> Result<Vec<ProbeOutput>, DataflowError> {
let (_, (program, _)) = programs(self.variant);
let mut dev = self.device()?;
let mut out = Vec::with_capacity(pairs.len());
for round in pairs.chunks(TILES * LAYOUT.capacity as usize) {
let slices: Vec<&[(i32, i32)]> = round.chunks(round.len().div_ceil(TILES)).collect();
for (j, slice) in slices.iter().enumerate() {
dev.l1_write_words(j, LAYOUT.column(0), &slice.iter().map(|p| p.0 as u32).collect::<Vec<u32>>());
dev.l1_write_words(j, LAYOUT.column(1), &slice.iter().map(|p| p.1 as u32).collect::<Vec<u32>>());
}
let counts: Vec<u32> = slices.iter().map(|s| s.len() as u32).collect();
dev.run(&program, &counts, budget(counts[0], 25))?;
for (j, slice) in slices.iter().enumerate() {
let w = dev.l1_read_words(j, LAYOUT.results(), 4 * slice.len());
out.extend(w.as_chunks::<4>().0.iter().map(|c| ((c[1], c[0]), (c[3], c[2]))));
}
}
Ok(out)
}
/// The four rows of each pair, from the kernel.
fn rows(&self, m: &[i32; 8], cols: &[Vec<i32>; 4]) -> Result<[Vec<i64>; 4], DataflowError> {
let ((program, _), _) = programs(self.variant);
let mut dev = self.device()?;
let n = cols[0].len();
let mut o: [Vec<i64>; 4] = Default::default();
for start in (0..n).step_by(TILES * LAYOUT.capacity as usize) {
let end = (start + TILES * LAYOUT.capacity as usize).min(n);
// Contiguous slices, one per tile, in order.
let share = (end - start).div_ceil(TILES);
let slices: Vec<(usize, usize)> = (start..end).step_by(share).map(|s| (s, (s + share).min(end))).collect();
for (j, &(s, e)) in slices.iter().enumerate() {
dev.l1_write_words(j, LAYOUT.base + Layout::ENTRIES, &m.map(|x| x as u32));
for (k, col) in cols.iter().enumerate() {
dev.l1_write_words(j, LAYOUT.column(k as u32), &col[s..e].iter().map(|&v| v as u32).collect::<Vec<u32>>());
}
}
let counts: Vec<u32> = slices.iter().map(|&(s, e)| (e - s) as u32).collect();
dev.run(&program, &counts, budget(counts[0], 220))?;
for (j, &(s, e)) in slices.iter().enumerate() {
let w = dev.l1_read_words(j, LAYOUT.results(), 8 * (e - s));
for pair in w.as_chunks::<8>().0 {
for (r, row) in o.iter_mut().enumerate() {
row.push((i64::from(pair[2 * r + 1] as i32) << 32) | i64::from(pair[2 * r]));
}
}
}
}
Ok(o)
}
/// Gates in order from `state`, as [`lanes::run_words`] runs them.
pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result<LaneRun, DataflowError> {
let len = state.len();
if len < 2 || !len.is_power_of_two() {
return Err(DataflowError::Internal("a state that is not 2^n lanes, n at least 1"));
}
let n = len.trailing_zeros() as u8;
if n > self.max_qubits {
return Err(DataflowError::TooWide { n_qubits: n, max: self.max_qubits });
}
let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect();
self.execute(state.to_vec(), &planned)
}
fn execute(&self, mut amps: Vec<Lane>, gates: &[(&GateWords, u32)]) -> Result<LaneRun, DataflowError> {
let len = amps.len();
for (w, _) in gates {
let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 {
return Err(DataflowError::Internal("malformed gate words for this state"));
}
}
let mut report: Option<u32> = None;
for &(w, index) in gates {
let m: [i32; 8] = w.m.map(|x| x as i32);
let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
let mut at = Vec::new();
let mut cols: [Vec<i32>; 4] = Default::default();
for p in 0..len / 2 {
let low = p & (tbit - 1);
let i = ((p - low) << 1) | low;
if i & ctrl != ctrl {
continue;
}
at.push(i);
let (a0, a1) = (amps[i], amps[i | tbit]);
cols[0].push(a0[0]);
cols[1].push(a0[1]);
cols[2].push(a1[0]);
cols[3].push(a1[1]);
}
if at.is_empty() {
continue;
}
let [o0r, o0i, o1r, o1i] = self.rows(&m, &cols)?;
for (k, &i) in at.iter().enumerate() {
let rows = [o0r[k], o0i[k], o1r[k], o1i[k]];
if rows.iter().any(|&v| i32::try_from(v).is_err()) {
report = Some(report.map_or(index, |r| r.min(index)));
}
let low = |v: i64| v as u64 as u32 as i32;
amps[i] = [low(rows[0]), low(rows[1])];
amps[i | tbit] = [low(rows[2]), low(rows[3])];
}
}
Ok(LaneRun { amps, out_of_range: report })
}
pub fn run_program(&self, p: &Program) -> Result<LaneRun, DataflowError> {
let n = p.n_qubits;
if n == 0 {
return Ok(LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None });
}
if n > self.max_qubits {
return Err(DataflowError::TooWide { n_qubits: n, max: self.max_qubits });
}
if p.start >= 1usize << n {
return Err(DataflowError::Internal("the start is outside the state"));
}
let mut gates = Vec::with_capacity(p.steps.len());
for step in &p.steps {
let Step::Gate { gate } = *step else { return Err(DataflowError::Internal("a tile step; this member runs gate steps only")) };
let g = p.gates.get(gate as usize).ok_or(DataflowError::Internal("a step names a gate the program does not hold"))?;
if !g.is_well_formed() {
return Err(DataflowError::Internal("malformed gate words for this state"));
}
gates.push((&g.words, g.index));
}
let mut amps = vec![[0i32, 0]; 1usize << n];
amps[p.start] = [ONE as i32, 0];
self.execute(amps, &gates)
}
pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result<StateVector, DataflowError> {
let p = plan::compile_with(c, start, &self.config()).map_err(DataflowError::Circuit)?;
if p.n_qubits > self.max_qubits {
return Err(DataflowError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits });
}
let run = self.run_program(&p)?;
if let Some(gate) = run.out_of_range {
return Err(DataflowError::OutOfRange { gate });
}
Ok(StateVector { n_qubits: p.n_qubits, amps: run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }).collect() })
}
pub fn simulate(&self, c: &Circuit) -> Result<StateVector, DataflowError> {
self.simulate_from(c, 0)
}
/// The certificate of [`apply_kernel`]`(gs)` on this member, with its
/// provenance.
pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result<Certificate, DataflowError> {
let mut cert = Certification::new();
for c in corpus.circuits(gs) {
let reference = lanes::reference_output(&c).map_err(DataflowError::Circuit)?;
let run = self.run_program(&plan::compile_with(&c, 0, &self.config()).map_err(DataflowError::Circuit)?)?;
if let Some(gate) = run.out_of_range {
return Err(DataflowError::OutOfRange { gate });
}
cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps));
}
Ok(cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)).with_provenance(self.provenance.clone()))
}
pub fn member(&self, corpus: Corpus) -> Result<Member, DataflowError> {
let mut kernels = Vec::new();
for gs in Gateset::ALL {
kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate: self.certify(gs, corpus)?, energy: None });
}
Ok(Member { id: self.id.clone(), class: FabricClass::Dataflow, presence: Presence::Detected, kernels, detail: self.detail.clone() })
}
}
impl Runner for DataflowMember {
fn member_id(&self) -> String {
self.id.clone()
}
fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, _meter: &'r mut dyn RunMeter) -> RunFuture<'r> {
Box::pin(async move {
if implementation != self.implementation {
return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation)));
}
Ok((self.simulate_from(c, start)?, 0, None))
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn every_variant_has_its_own_implementation_id() {
let ids: std::collections::BTreeSet<String> = [Variant::Conforming, Variant::M1, Variant::M2, Variant::M3].into_iter().map(implementation_id_of).collect();
assert_eq!(ids.len(), 4);
assert!(implementation_id().starts_with("rv32-lanes64/"));
}
#[test]
fn the_layout_fits_one_window_past_the_programs() {
let ((rows, _), (probe, _)) = programs(Variant::Conforming);
assert!(4 * rows.len().max(probe.len()) as u32 <= LAYOUT.base);
for chip in CHIPS {
assert!(u64::from(LAYOUT.base + LAYOUT.span()) <= chip.window);
}
}
#[test]
fn each_chip_names_distinct_tiles_whose_windows_it_has() {
// Window registers: 156 of 1 MiB on the first, 202 of 2 MiB on the
// second; two per tile.
for (chip, windows) in CHIPS.into_iter().zip([156usize, 202]) {
let set: std::collections::BTreeSet<(u32, u32)> = chip.tiles.iter().copied().collect();
assert_eq!(set.len(), TILES);
assert!(2 * TILES <= windows);
assert!(2 * TILES as u64 * chip.window <= TLB_CFG, "the windows end before their configuration registers");
}
assert!(TILES_B140.iter().all(|&(x, y)| (2..=11).contains(&y) && !(8..=9).contains(&x) && x <= 14));
assert!(TILES_401E.iter().all(|&(x, y)| x != 5 && x != 0 && (1..=5).contains(&y)));
}
#[test]
fn the_keys_name_the_pci_ids() {
assert_eq!(key_of(CHIPS[0]), "dataflow-1e52-401e");
assert_eq!(key_of(CHIPS[1]), "dataflow-1e52-b140");
}
#[test]
fn with_no_library_named_the_member_is_absent_and_says_why() {
if std::env::var_os("WAI_QUANTUM_DATAFLOW_LIB").is_none() {
let d = discover();
assert!(d.found.is_empty() && d.refused.is_empty());
assert!(d.unavailable.as_deref().is_some_and(|w| w.contains("WAI_QUANTUM_DATAFLOW_LIB") || w.contains("loaded on Linux")), "{:?}", d.unavailable);
}
}
}

512
wai/wai-quantum/src/quantum_fabric_dsp.rs Normal file
View file

@ -0,0 +1,512 @@
//! The fabric's DSP vector-unit member: the gate's fixed-point arithmetic in the
//! digital signal processor's vector unit, in 128-byte vectors.
//!
//! **Why it is bit-exact.** The kernel ([`KERNEL_TEXT`]) is the same
//! transcription of [`lanes::apply_words`] as the other members', in 32-bit
//! lanes holding each 64-bit value as two words: each product is the vector
//! unit's exact 32×32 multiply (the 32-bit operand times the other's low half,
//! unsigned, plus its high half, signed, sixteen bits up), the rounding constant
//! is added with its carry into the high word, the shift is arithmetic over all
//! 64 bits, and each row is the sum of four, every carry and borrow kept. The
//! host gathers the pairs a gate selects, the kernel forms the four rows of
//! thirty-two pairs at a time, and the host checks each row's range and keeps
//! its low 32 bits, as [`lanes::apply_words`] does.
//!
//! **Where it runs.** The kernels are compiled only in a build for the signal
//! processor with its vector unit in 128-byte mode ([`BUILT_FOR_UNIT`]); their
//! instructions exist nowhere else, so a kernel that ran, ran there. Any other
//! build finds no member and says why. Under an emulator, the runner declares it
//! (`WAI_QUANTUM_DSP_EMULATOR`) and every certificate says so.
//!
//! Three mutants, compiled in from the same kernel text with other constants,
//! must each be caught.
use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector};
use crate::quantum_fabric::lanes::{self, GateWords, Lane};
use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step};
use crate::quantum_fabric::{Certificate, Certification, Corpus, FabricClass, KernelEntry, Member, MemberError, Presence, Provenance, RunFuture, RunMeter, Runner, apply_kernel};
/// Whether this build is for the signal processor with its vector unit in
/// 128-byte mode.
pub const BUILT_FOR_UNIT: bool = cfg!(all(target_arch = "hexagon", target_feature = "hvx-length128b")); // allow-name: the toolchain's architecture and target-feature names
#[cfg(all(target_arch = "hexagon", target_feature = "hvx-length128b"))] // allow-name: the toolchain's architecture and target-feature names
#[path = "quantum_fabric/dsp.rs"]
mod kernel;
/// Any other build: no member is ever opened ([`discover`] finds none), so
/// nothing here runs.
#[cfg(not(all(target_arch = "hexagon", target_feature = "hvx-length128b")))] // allow-name: the toolchain's architecture and target-feature names
mod kernel {
pub const LANES: usize = 32;
/// # Safety
///
/// Never called: no member exists in this build.
pub unsafe fn rows<const RND: i32, const SH: i32, const LAST_ADD: bool>(_: &[i32; 8], _: [&[i32]; 4], _: [&mut [i32]; 4], _: [&mut [i32]; 4]) {
unreachable!("no vector unit in this build")
}
/// # Safety
///
/// Never called: no member exists in this build.
pub unsafe fn probe<const RND: i32, const SH: i32>(_: &[i32], _: &[i32], _: [&mut [i32]; 2], _: [&mut [i32]; 2]) {
unreachable!("no vector unit in this build")
}
}
/// One probe case: [`lanes::mul_wide`] and [`lanes::fxmul_wide`] as
/// `(high, low)` words.
pub type ProbeOutput = ((u32, u32), (u32, u32));
/// Why the vector unit is not a member, or a run has no state.
#[derive(Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum DspError {
Unavailable(String),
Circuit(QuantumError),
TooWide { n_qubits: u8, max: u8 },
OutOfRange { gate: u32 },
Internal(&'static str),
Probe { a: i32, b: i32 },
}
impl std::fmt::Display for DspError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
DspError::Unavailable(why) => write!(f, "no DSP vector unit: {why}"),
DspError::Circuit(e) => write!(f, "{e}"),
DspError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"),
DspError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"),
DspError::Internal(why) => write!(f, "host defect: {why}"),
DspError::Probe { a, b } => write!(f, "the kernel's arithmetic differs from the CPU's at ({a}, {b})"),
}
}
}
impl std::error::Error for DspError {}
impl From<DspError> for MemberError {
fn from(e: DspError) -> MemberError {
match e {
DspError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max },
DspError::OutOfRange { gate } => MemberError::OutOfRange { gate },
DspError::Circuit(e) => MemberError::Circuit(e),
other => MemberError::Other(other.to_string()),
}
}
}
/// The kernels a member runs: the conforming one, or a mutant.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Variant {
Conforming,
/// The rounding constant is half what it should be.
M1,
/// The product is shifted by one bit too few.
M2,
/// Row 0's real part adds its last term instead of subtracting it.
M3,
}
impl Variant {
pub const MUTANTS: [Variant; 3] = [Variant::M1, Variant::M2, Variant::M3];
/// The kernel's constants: the rounding constant, the shift, and whether
/// row 0's last real term is added.
pub fn constants(self) -> (i32, i32, bool) {
match self {
Variant::Conforming => (1 << 29, 30, false),
Variant::M1 => (1 << 28, 30, false),
Variant::M2 => (1 << 29, 29, false),
Variant::M3 => (1 << 29, 30, true),
}
}
}
/// The text the kernels are compiled from.
pub const KERNEL_TEXT: &str = include_str!("quantum_fabric/dsp.rs");
/// The implementation name the id starts with.
pub const IMPLEMENTATION: &str = "dsp-lanes64";
/// Version of the host layout: the gather of a gate's pairs into four
/// columns padded with zeros to whole vectors, the rows' high and low words
/// joined into 64 bits, and the host's range check. Part of the
/// implementation id.
pub const HOST_LAYOUT: u32 = 1;
/// `"dsp-lanes64/<16 hex>"` over the kernel text, a variant's constants and
/// the layout.
pub fn implementation_id_of(v: Variant) -> String {
let (rnd, sh, last_add) = v.constants();
let mut o = b"wai:quantum-fabric-dsp\x01".to_vec();
o.extend_from_slice(&(KERNEL_TEXT.len() as u64).to_be_bytes());
o.extend_from_slice(KERNEL_TEXT.as_bytes());
o.extend_from_slice(&rnd.to_be_bytes());
o.extend_from_slice(&sh.to_be_bytes());
o.push(u8::from(last_add));
o.extend_from_slice(&HOST_LAYOUT.to_be_bytes());
let h: String = blake3::hash(&o).as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect();
format!("{IMPLEMENTATION}/{h}")
}
pub fn implementation_id() -> String {
implementation_id_of(Variant::Conforming)
}
/// Whether this build runs on the signal processor's vector unit.
pub fn has_vector_unit() -> Result<(), String> {
if BUILT_FOR_UNIT {
Ok(())
} else {
Err(format!("this build ({}) is not for the signal processor with its vector unit in 128-byte mode", std::env::consts::ARCH))
}
}
/// What [`discover`] found.
#[derive(Debug)]
pub struct Discovery {
pub found: Vec<DspMember>,
pub refused: Vec<(String, DspError)>,
pub unavailable: Option<String>,
}
/// The vector unit, probed in this process, or why there is none.
pub fn discover() -> Discovery {
let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None };
if let Err(why) = has_vector_unit() {
d.unavailable = Some(why);
return d;
}
match DspMember::open(Variant::Conforming) {
Ok(m) => d.found.push(m),
Err(e) => d.refused.push((KEY.into(), e)),
}
d
}
/// The key a runner declares.
pub const KEY: &str = "dsp-vector";
/// The most qubits a run holds.
pub const MAX_DSP_QUBITS: u8 = if 24 < MAX_QUBITS { 24 } else { MAX_QUBITS };
/// The DSP's vector unit as a fabric member.
#[derive(Clone, Debug)]
pub struct DspMember {
variant: Variant,
id: String,
implementation: String,
detail: String,
provenance: Provenance,
max_qubits: u8,
}
/// `v` padded with zeros to whole vectors.
fn padded(v: &[i32]) -> Vec<i32> {
let mut p = v.to_vec();
p.resize(v.len().next_multiple_of(kernel::LANES), 0);
p
}
fn join(hi: i32, lo: i32) -> i64 {
(i64::from(hi) << 32) | i64::from(lo as u32)
}
impl DspMember {
fn open(variant: Variant) -> Result<DspMember, DspError> {
let emulator = std::env::var("WAI_QUANTUM_DSP_EMULATOR").ok().filter(|s| !s.trim().is_empty());
let provenance = emulator.as_ref().map_or(Provenance::Silicon, |e| Provenance::Emulator(e.trim().to_string()));
let mut o = b"wai:quantum-fabric-dsp-member\x01".to_vec();
o.extend_from_slice(std::env::consts::ARCH.as_bytes());
o.extend_from_slice(std::env::consts::OS.as_bytes());
o.extend_from_slice(provenance.label().as_bytes());
let id = format!("local/dsp-vector-{}", blake3::hash(&o).as_bytes()[..6].iter().map(|b| format!("{b:02x}")).collect::<String>());
let member = DspMember {
variant,
id,
implementation: implementation_id_of(variant),
detail: format!("dsp; the signal processor's vector unit, 128-byte vectors, 64-bit values as word pairs; {}; up to {MAX_DSP_QUBITS} qubits; unmetered", provenance.label()),
provenance,
max_qubits: MAX_DSP_QUBITS,
};
if variant == Variant::Conforming {
let pairs = lanes::probe_pairs(4096, 0x5eed);
let out = member.probe(&pairs)?;
if let Some((a, b)) = lanes::first_mismatch(&pairs, &out) {
return Err(DspError::Probe { a, b });
}
}
Ok(member)
}
/// The same processor running a mutant: for the mutation tests.
#[doc(hidden)]
pub fn with_variant(&self, v: Variant) -> DspMember {
DspMember { variant: v, implementation: implementation_id_of(v), ..self.clone() }
}
#[doc(hidden)]
pub fn with_max_qubits(&self, max_qubits: u8) -> DspMember {
DspMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() }
}
/// The kernel this member runs: the conforming one, or a mutant.
pub fn variant(&self) -> Variant {
self.variant
}
pub fn id(&self) -> &str {
&self.id
}
pub fn key(&self) -> &str {
KEY
}
pub fn class(&self) -> FabricClass {
FabricClass::Dsp
}
pub fn detail(&self) -> &str {
&self.detail
}
pub fn implementation(&self) -> &str {
&self.implementation
}
pub fn provenance(&self) -> &Provenance {
&self.provenance
}
pub fn max_qubits(&self) -> u8 {
self.max_qubits
}
pub fn config(&self) -> Config {
Config::PER_GATE
}
/// The kernel's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each pair.
pub fn probe(&self, pairs: &[(i32, i32)]) -> Result<Vec<ProbeOutput>, DspError> {
{
let a = padded(&pairs.iter().map(|p| p.0).collect::<Vec<i32>>());
let b = padded(&pairs.iter().map(|p| p.1).collect::<Vec<i32>>());
let n = a.len();
let (mut ph, mut pl, mut fh, mut fl) = (vec![0i32; n], vec![0i32; n], vec![0i32; n], vec![0i32; n]);
// SAFETY: this build targets the vector unit; every slice holds n
// values, a whole number of vectors.
unsafe {
let probe = match self.variant {
Variant::M1 => kernel::probe::<{ 1 << 28 }, 30>,
Variant::M2 => kernel::probe::<{ 1 << 29 }, 29>,
Variant::Conforming | Variant::M3 => kernel::probe::<{ 1 << 29 }, 30>,
};
probe(&a, &b, [&mut ph[..], &mut pl[..]], [&mut fh[..], &mut fl[..]]);
}
let split = |v: i64| ((v as u64 >> 32) as u32, v as u64 as u32);
Ok((0..pairs.len()).map(|k| (split(join(ph[k], pl[k])), split(join(fh[k], fl[k])))).collect())
}
}
/// The four rows of each pair, from the kernel.
fn rows(&self, m: &[i32; 8], cols: &[Vec<i32>; 4]) -> [Vec<i64>; 4] {
let len = cols[0].len();
let cols = [padded(&cols[0]), padded(&cols[1]), padded(&cols[2]), padded(&cols[3])];
let n = cols[0].len();
let mut hi = [vec![0i32; n], vec![0i32; n], vec![0i32; n], vec![0i32; n]];
let mut lo = hi.clone();
{
let [h0, h1, h2, h3] = &mut hi;
let [l0, l1, l2, l3] = &mut lo;
let c = [&cols[0][..], &cols[1][..], &cols[2][..], &cols[3][..]];
// SAFETY: this build targets the vector unit; every slice holds n
// values, a whole number of vectors.
unsafe {
let rows = match self.variant {
Variant::Conforming => kernel::rows::<{ 1 << 29 }, 30, false>,
Variant::M1 => kernel::rows::<{ 1 << 28 }, 30, false>,
Variant::M2 => kernel::rows::<{ 1 << 29 }, 29, false>,
Variant::M3 => kernel::rows::<{ 1 << 29 }, 30, true>,
};
rows(m, c, [&mut h0[..], &mut h1[..], &mut h2[..], &mut h3[..]], [&mut l0[..], &mut l1[..], &mut l2[..], &mut l3[..]]);
}
}
let row = |r: usize| (0..len).map(|k| join(hi[r][k], lo[r][k])).collect::<Vec<i64>>();
[row(0), row(1), row(2), row(3)]
}
/// Gates in order from `state`, as [`lanes::run_words`] runs them.
pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result<LaneRun, DspError> {
let len = state.len();
if len < 2 || !len.is_power_of_two() {
return Err(DspError::Internal("a state that is not 2^n lanes, n at least 1"));
}
let n = len.trailing_zeros() as u8;
if n > self.max_qubits {
return Err(DspError::TooWide { n_qubits: n, max: self.max_qubits });
}
let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect();
self.execute(state.to_vec(), &planned)
}
fn execute(&self, mut amps: Vec<Lane>, gates: &[(&GateWords, u32)]) -> Result<LaneRun, DspError> {
let len = amps.len();
for (w, _) in gates {
let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 {
return Err(DspError::Internal("malformed gate words for this state"));
}
}
{
let mut report: Option<u32> = None;
for &(w, index) in gates {
let m: [i32; 8] = w.m.map(|x| x as i32);
let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
let mut at = Vec::new();
let mut cols: [Vec<i32>; 4] = Default::default();
for p in 0..len / 2 {
let low = p & (tbit - 1);
let i = ((p - low) << 1) | low;
if i & ctrl != ctrl {
continue;
}
at.push(i);
let (a0, a1) = (amps[i], amps[i | tbit]);
cols[0].push(a0[0]);
cols[1].push(a0[1]);
cols[2].push(a1[0]);
cols[3].push(a1[1]);
}
let [o0r, o0i, o1r, o1i] = self.rows(&m, &cols);
for (k, &i) in at.iter().enumerate() {
let rows = [o0r[k], o0i[k], o1r[k], o1i[k]];
if rows.iter().any(|&v| i32::try_from(v).is_err()) {
report = Some(report.map_or(index, |r| r.min(index)));
}
let low = |v: i64| v as u64 as u32 as i32;
amps[i] = [low(rows[0]), low(rows[1])];
amps[i | tbit] = [low(rows[2]), low(rows[3])];
}
}
Ok(LaneRun { amps, out_of_range: report })
}
}
pub fn run_program(&self, p: &Program) -> Result<LaneRun, DspError> {
let n = p.n_qubits;
if n == 0 {
return Ok(LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None });
}
if n > self.max_qubits {
return Err(DspError::TooWide { n_qubits: n, max: self.max_qubits });
}
if p.start >= 1usize << n {
return Err(DspError::Internal("the start is outside the state"));
}
let mut gates = Vec::with_capacity(p.steps.len());
for step in &p.steps {
let Step::Gate { gate } = *step else { return Err(DspError::Internal("a tile step; this member runs gate steps only")) };
let g = p.gates.get(gate as usize).ok_or(DspError::Internal("a step names a gate the program does not hold"))?;
if !g.is_well_formed() {
return Err(DspError::Internal("malformed gate words for this state"));
}
gates.push((&g.words, g.index));
}
let mut amps = vec![[0i32, 0]; 1usize << n];
amps[p.start] = [ONE as i32, 0];
self.execute(amps, &gates)
}
pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result<StateVector, DspError> {
let p = plan::compile_with(c, start, &self.config()).map_err(DspError::Circuit)?;
if p.n_qubits > self.max_qubits {
return Err(DspError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits });
}
let run = self.run_program(&p)?;
if let Some(gate) = run.out_of_range {
return Err(DspError::OutOfRange { gate });
}
Ok(StateVector { n_qubits: p.n_qubits, amps: run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }).collect() })
}
pub fn simulate(&self, c: &Circuit) -> Result<StateVector, DspError> {
self.simulate_from(c, 0)
}
/// The certificate of [`apply_kernel`]`(gs)` on this member, with its
/// provenance.
pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result<Certificate, DspError> {
let mut cert = Certification::new();
for c in corpus.circuits(gs) {
let reference = lanes::reference_output(&c).map_err(DspError::Circuit)?;
let run = self.run_program(&plan::compile_with(&c, 0, &self.config()).map_err(DspError::Circuit)?)?;
if let Some(gate) = run.out_of_range {
return Err(DspError::OutOfRange { gate });
}
cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps));
}
Ok(cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)).with_provenance(self.provenance.clone()))
}
pub fn member(&self, corpus: Corpus) -> Result<Member, DspError> {
let mut kernels = Vec::new();
for gs in Gateset::ALL {
kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate: self.certify(gs, corpus)?, energy: None });
}
Ok(Member { id: self.id.clone(), class: FabricClass::Dsp, presence: Presence::Detected, kernels, detail: self.detail.clone() })
}
}
impl Runner for DspMember {
fn member_id(&self) -> String {
self.id.clone()
}
fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, _meter: &'r mut dyn RunMeter) -> RunFuture<'r> {
Box::pin(async move {
if implementation != self.implementation {
return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation)));
}
Ok((self.simulate_from(c, start)?, 0, None))
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn every_variant_has_its_own_implementation_id() {
let ids: std::collections::BTreeSet<String> = [Variant::Conforming, Variant::M1, Variant::M2, Variant::M3].into_iter().map(implementation_id_of).collect();
assert_eq!(ids.len(), 4);
assert!(implementation_id().starts_with("dsp-lanes64/"));
assert!(KERNEL_TEXT.contains("Q6_W_vmpye_VwVuh") && KERNEL_TEXT.contains("Q6_W_vmpyoacc_WVwVh"));
}
#[test]
fn the_constants_are_the_reference_arithmetic() {
assert_eq!(Variant::Conforming.constants(), (1 << 29, 30, false));
for v in Variant::MUTANTS {
assert_ne!(v.constants(), Variant::Conforming.constants());
}
}
#[test]
fn the_member_id_sorts_after_the_cpu_reference() {
assert!("local/dsp-vector-000000000000" > "local/cpu-aarch64");
assert!("local/dsp-vector-000000000000" > "local/cpu-x86_64");
}
#[test]
fn elsewhere_the_member_is_absent_and_says_why() {
if !BUILT_FOR_UNIT {
let d = discover();
assert!(d.found.is_empty() && d.refused.is_empty());
assert!(d.unavailable.as_deref().is_some_and(|why| why.contains("is not for the signal processor")), "{:?}", d.unavailable);
}
}
}

764
wai/wai-quantum/src/quantum_fabric_npu.rs Normal file
View file

@ -0,0 +1,764 @@
//! The fabric's neural-processor member: the gate arithmetic of
//! [`quantum_fabric::npu`](crate::quantum_fabric::npu) on the platform's neural
//! processor, through its model runtime. See that module for why every product
//! is exact.
//!
//! - **Discovery** ([`discover`]): on macOS, the limb-product model is written,
//! compiled by the platform's model runtime and loaded for the CPU and the
//! neural processor. The runtime's compute plan must place **every**
//! operation of the model on the neural processor; a model the runtime would
//! run elsewhere, even in part, is refused with the placement it reported,
//! so no CPU result is credited to the processor. The products are then
//! probed against [`lanes`] in this process. Elsewhere, nothing is found, and
//! [`Discovery::unavailable`] says why.
//! - **Running** ([`NpuMember::run_program`]): the host holds the state; for
//! each gate, every product the gate needs (sixteen a pair) goes to the
//! processor in batches, and the host assembles each product and applies
//! [`lanes::apply_words`]'s fixed-point arithmetic. A result that leaves 32
//! bits is [`NpuError::OutOfRange`], never a state.
//! - **Certification, placement and receipts** are as for every member.
//!
//! # Honest boundaries
//!
//! - **The processor multiplies; the host assembles.** Every limb product and
//! every limb sum is formed on the processor; the shifts that join the 15
//! coefficients of a product, the rounding and the row sums are the host's.
//! - **Placement is the runtime's report.** The compute plan is the runtime's
//! statement of where each operation runs; it is checked before the member
//! exists, and named in its detail.
//! - **Unmetered**: the processor's energy counter is not readable unprivileged.
use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector};
use crate::quantum_fabric::lanes::{self, GateWords, Lane};
use crate::quantum_fabric::npu::{self, BATCH, LIMBS, SHIFTS};
use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step};
use crate::quantum_fabric::{Certificate, Certification, Corpus, KernelEntry, Member, MemberError, Presence, RunFuture, RunMeter, Runner, apply_kernel};
/// One probe case: [`lanes::mul_wide`] and [`lanes::fxmul_wide`], each as
/// `(high, low)` words.
pub type ProbeOutput = ((u32, u32), (u32, u32));
/// Why the processor is not a member, or a run has no state.
#[derive(Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum NpuError {
/// No model runtime with a neural processor on this platform, and why.
Unavailable(String),
/// The runtime refused or failed the model.
Runtime(String),
/// The runtime would run part of the model elsewhere: each operation and
/// the device it reported.
Placement(String),
Circuit(QuantumError),
TooWide { n_qubits: u8, max: u8 },
OutOfRange { gate: u32 },
Internal(&'static str),
Probe { a: i32, b: i32 },
}
impl std::fmt::Display for NpuError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
NpuError::Unavailable(why) => write!(f, "no neural processor: {why}"),
NpuError::Runtime(e) => write!(f, "model runtime: {e}"),
NpuError::Placement(p) => write!(f, "the runtime would not run the whole model on the neural processor: {p}"),
NpuError::Circuit(e) => write!(f, "{e}"),
NpuError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"),
NpuError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"),
NpuError::Internal(why) => write!(f, "host defect: {why}"),
NpuError::Probe { a, b } => write!(f, "the processor's products differ from the CPU's at ({a}, {b})"),
}
}
}
impl std::error::Error for NpuError {}
impl From<NpuError> for MemberError {
fn from(e: NpuError) -> MemberError {
match e {
NpuError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max },
NpuError::OutOfRange { gate } => MemberError::OutOfRange { gate },
NpuError::Runtime(why) => MemberError::Device(why),
NpuError::Circuit(e) => MemberError::Circuit(e),
other => MemberError::Other(other.to_string()),
}
}
}
/// What [`discover`] found: at most one member, the platform's processor.
#[derive(Debug)]
pub struct Discovery {
pub found: Vec<NpuMember>,
/// The processor, when it was not a member, with why.
pub refused: Vec<(String, NpuError)>,
pub unavailable: Option<String>,
}
/// The class name the runtime gives the neural processor in its compute plan.
pub const NEURAL_PROCESSOR: &str = "MLNeuralEngineComputeDevice"; // allow-name: the runtime's device class name
/// The most qubits a run holds: the host holds the state, and every gate's
/// products cross to the processor, so wide runs are slow rather than refused.
pub const MAX_NPU_QUBITS: u8 = if 20 < MAX_QUBITS { 20 } else { MAX_QUBITS };
/// The sixteen products a pair needs, in the order [`apply_pair`] reads them.
fn pair_products(m: &[i32; 8], a0: Lane, a1: Lane, out: &mut Vec<(i32, i32)>) {
let (a0r, a0i, a1r, a1i) = (a0[0], a0[1], a1[0], a1[1]);
out.extend_from_slice(&[
(m[0], a0r),
(m[1], a0i),
(m[2], a1r),
(m[3], a1i),
(m[0], a0i),
(m[1], a0r),
(m[2], a1i),
(m[3], a1r),
(m[4], a0r),
(m[5], a0i),
(m[6], a1r),
(m[7], a1i),
(m[4], a0i),
(m[5], a0r),
(m[6], a1i),
(m[7], a1r),
]);
}
/// `fx` of an exact product: `(p + 2^29) >> 30`.
fn fx(p: i64) -> i64 {
(p + (1 << 29)) >> 30
}
fn fits32(v: i64) -> bool {
i32::try_from(v).is_ok()
}
/// A pair's two new lanes from its sixteen products, and whether a sum left
/// 32 bits: row 0 is `(m0 + i·m1)·a0 + (m2 + i·m3)·a1`, row 1 likewise with
/// `m4..m7`, as [`lanes::apply_words`] computes them.
fn apply_pair(p: &[i64]) -> (Lane, Lane, bool) {
debug_assert!(p.len() >= 16);
let f: [i64; 16] = std::array::from_fn(|k| fx(p[k]));
let o0r = f[0] - f[1] + f[2] - f[3];
let o0i = f[4] + f[5] + f[6] + f[7];
let o1r = f[8] - f[9] + f[10] - f[11];
let o1i = f[12] + f[13] + f[14] + f[15];
let bad = !(fits32(o0r) && fits32(o0i) && fits32(o1r) && fits32(o1i));
let low = |v: i64| v as u64 as u32 as i32;
([low(o0r), low(o0i)], [low(o1r), low(o1i)], bad)
}
#[cfg(target_os = "macos")]
mod coreml {
use super::NpuError;
use crate::quantum_fabric::npu::{BATCH, LIMBS, SHIFTS};
use objc2::AnyThread;
use objc2::rc::Retained;
use objc2::runtime::{AnyObject, ProtocolObject};
use objc2_core_ml::{
MLComputePlan, MLComputeUnits, MLDictionaryFeatureProvider, MLFeatureProvider, MLFeatureValue, MLModel, MLModelConfiguration, MLMultiArray, MLMultiArrayDataType,
};
use objc2_foundation::{NSArray, NSDictionary, NSError, NSNumber, NSString, NSURL};
use std::path::Path;
use std::sync::{Arc, Condvar, Mutex};
fn ns_error(e: &NSError) -> String {
e.localizedDescription().to_string()
}
/// A runtime object handed from the thread the runtime calls back on to the
/// thread that waits for it, or held under a lock.
pub struct Handed<T>(pub T);
// SAFETY: a handed object is moved across threads only through a mutex, so
// one thread at a time uses it, and its retain count is atomic; the runtime's
// objects may be used from any thread that way.
unsafe impl<T> Send for Handed<T> {}
/// A compiled, loaded model, and where its runtime places each operation.
pub struct Loaded {
pub model: Retained<MLModel>,
/// `(operation, device class)` for every operation but constants.
pub placement: Vec<(String, String)>,
}
fn config() -> Retained<MLModelConfiguration> {
// SAFETY: a fresh configuration, set before any use.
unsafe {
let c = MLModelConfiguration::new();
c.setComputeUnits(MLComputeUnits::CPUAndNeuralEngine);
c
}
}
/// Writes `model` under `dir`, compiles it, loads it for the CPU and the
/// neural processor, and reads the runtime's compute plan for it.
pub fn load(model: &[u8], dir: &Path) -> Result<Loaded, NpuError> {
std::fs::create_dir_all(dir).map_err(|e| NpuError::Runtime(format!("{}: {e}", dir.display())))?;
let compiled = compiled_in(model, dir)?;
let cfg = config();
// SAFETY: the compiled model's URL and a configuration.
let model = unsafe { MLModel::modelWithContentsOfURL_configuration_error(&compiled, &cfg) }.map_err(|e| NpuError::Runtime(format!("loading the model: {}", ns_error(&e))))?;
let placement = plan(&compiled, &cfg)?;
Ok(Loaded { model, placement })
}
/// The compiled form of `model`, kept in `dir` (`limbs.mlmodelc`): compiled
/// once, and loaded by later processes without compiling again. Two
/// processes compiling at once each move their copy in; the first stays.
fn compiled_in(model: &[u8], dir: &Path) -> Result<Retained<NSURL>, NpuError> {
let kept = dir.join("limbs.mlmodelc");
let url_of = |p: &Path| NSURL::fileURLWithPath(&NSString::from_str(&p.to_string_lossy()));
if kept.is_dir() {
return Ok(url_of(&kept));
}
let io = |e: std::io::Error| NpuError::Runtime(format!("{}: {e}", dir.display()));
let src = dir.join(format!("limbs-{}.mlmodel", std::process::id()));
std::fs::write(&src, model).map_err(io)?;
// SAFETY: a file URL to a model description just written.
#[allow(deprecated)] // The synchronous form: discovery is synchronous.
let compiled = unsafe { MLModel::compileModelAtURL_error(&url_of(&src)) };
let _ = std::fs::remove_file(&src);
let compiled = compiled.map_err(|e| NpuError::Runtime(format!("compiling the model: {}", ns_error(&e))))?;
let out = compiled.path().ok_or_else(|| NpuError::Runtime("the compiled model has no path".into()))?.to_string();
let out = std::path::PathBuf::from(out);
if std::fs::rename(&out, &kept).is_err() {
if !kept.is_dir() {
// Another volume: copy it in under a private name, then move.
let part = dir.join(format!("limbs.mlmodelc.{}", std::process::id()));
copy_dir(&out, &part).map_err(io)?;
if std::fs::rename(&part, &kept).is_err() {
let _ = std::fs::remove_dir_all(&part);
}
}
let _ = std::fs::remove_dir_all(&out);
}
if kept.is_dir() { Ok(url_of(&kept)) } else { Err(NpuError::Runtime(format!("{}: the compiled model was not kept", kept.display()))) }
}
fn copy_dir(from: &Path, to: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(to)?;
for e in std::fs::read_dir(from)? {
let e = e?;
let t = to.join(e.file_name());
if e.file_type()?.is_dir() { copy_dir(&e.path(), &t)? } else { std::fs::copy(e.path(), t).map(|_| ())? }
}
Ok(())
}
/// The runtime's compute plan for the compiled model: each operation's
/// preferred device.
fn plan(compiled: &NSURL, cfg: &MLModelConfiguration) -> Result<Vec<(String, String)>, NpuError> {
type Slot = (Mutex<Option<Result<Handed<Retained<MLComputePlan>>, String>>>, Condvar);
let slot: Arc<Slot> = Arc::new((Mutex::new(None), Condvar::new()));
let s = slot.clone();
let handler = block2::RcBlock::new(move |plan: *mut MLComputePlan, err: *mut NSError| {
// SAFETY: the runtime hands one of the two, live for the call.
let r = unsafe {
match Retained::retain(plan) {
Some(p) => Ok(Handed(p)),
None => Err(err.as_ref().map_or_else(|| "no plan".to_string(), ns_error)),
}
};
let (lock, cv) = &*s;
*lock.lock().unwrap_or_else(|p| p.into_inner()) = Some(r);
cv.notify_all();
});
// SAFETY: the URL of a compiled model, a configuration, and a handler
// that only stores its arguments.
unsafe { MLComputePlan::loadContentsOfURL_configuration_completionHandler(compiled, cfg, &handler) };
let (lock, cv) = &*slot;
let guard = lock.lock().unwrap_or_else(|p| p.into_inner());
let (mut guard, timeout) = cv.wait_timeout_while(guard, std::time::Duration::from_secs(120), |r| r.is_none()).unwrap_or_else(|p| p.into_inner());
if timeout.timed_out() {
return Err(NpuError::Runtime("the compute plan did not arrive".into()));
}
let plan = guard.take().expect("set before the notify").map_err(|e| NpuError::Runtime(format!("the compute plan: {e}")))?.0;
let mut out = Vec::new();
// SAFETY: reads of the plan's structure, on this thread.
unsafe {
let structure = plan.modelStructure();
let program = structure.program().ok_or_else(|| NpuError::Runtime("the plan has no program".into()))?;
let functions = program.functions();
let main = functions.objectForKey(&NSString::from_str("main")).ok_or_else(|| NpuError::Runtime("the program has no main function".into()))?;
for op in main.block().operations().iter() {
let name = op.operatorName().to_string();
if name == "const" {
continue;
}
let device = match plan.computeDeviceUsageForMLProgramOperation(&op) {
Some(usage) => {
let d = usage.preferredComputeDevice();
let any: &AnyObject = d.as_ref();
any.class().name().to_string_lossy().into_owned()
}
None => "unknown".into(),
};
out.push((name, device));
}
}
Ok(out)
}
/// The fp16 array `[BATCH, SHIFTS, LIMBS]` holding `bits`, written
/// through the strides the runtime gives it (which may pad a dimension).
fn array(bits: &[u16]) -> Result<Retained<MLMultiArray>, NpuError> {
assert_eq!(bits.len(), BATCH * SHIFTS * LIMBS);
let shape: Retained<NSArray<NSNumber>> = NSArray::from_retained_slice(&[NSNumber::new_usize(BATCH), NSNumber::new_usize(SHIFTS), NSNumber::new_usize(LIMBS)]);
// SAFETY: an fp16 array of the model's input shape; every write lands
// at an index its own strides give, inside the array.
unsafe {
let a = MLMultiArray::initWithShape_dataType_error(MLMultiArray::alloc(), &shape, MLMultiArrayDataType::Float16).map_err(|e| NpuError::Runtime(ns_error(&e)))?;
let strides: Vec<usize> = a.strides().iter().map(|n| n.as_usize()).collect();
let [s0, s1, s2] = strides[..] else { return Err(NpuError::Runtime(format!("the input has strides {strides:?}"))) };
if s2 != 1 || s1 < LIMBS || s0 < SHIFTS * s1 {
return Err(NpuError::Runtime(format!("the input's strides {strides:?} do not hold [{BATCH}, {SHIFTS}, {LIMBS}]")));
}
#[allow(deprecated)]
let p = a.dataPointer().as_ptr().cast::<u16>();
if (s0, s1) == (SHIFTS * LIMBS, LIMBS) {
std::ptr::copy_nonoverlapping(bits.as_ptr(), p, BATCH * SHIFTS * LIMBS);
} else {
for i in 0..BATCH {
for s in 0..SHIFTS {
std::ptr::copy_nonoverlapping(bits.as_ptr().add((i * SHIFTS + s) * LIMBS), p.add(i * s0 + s * s1), LIMBS);
}
}
}
Ok(a)
}
}
/// One batch through the model: `z`, `[BATCH, SHIFTS]`, as fp16 bits.
pub fn predict(model: &MLModel, x: &[u16], y: &[u16]) -> Result<Vec<u16>, NpuError> {
let (ax, ay) = (array(x)?, array(y)?);
// SAFETY: a feature provider of the model's two inputs, a prediction,
// and a read of its fp16 output of the shape the model declares.
unsafe {
let fx = MLFeatureValue::featureValueWithMultiArray(&ax);
let fy = MLFeatureValue::featureValueWithMultiArray(&ay);
let keys = [NSString::from_str("x"), NSString::from_str("y")];
let values: [Retained<AnyObject>; 2] = [Retained::into_super(Retained::into_super(fx)), Retained::into_super(Retained::into_super(fy))];
let dict: Retained<NSDictionary<NSString, AnyObject>> = NSDictionary::from_retained_objects(&[&*keys[0], &*keys[1]], &values);
let provider = MLDictionaryFeatureProvider::initWithDictionary_error(MLDictionaryFeatureProvider::alloc(), &dict).map_err(|e| NpuError::Runtime(ns_error(&e)))?;
let out = model.predictionFromFeatures_error(ProtocolObject::from_ref(&*provider)).map_err(|e| NpuError::Runtime(format!("predicting: {}", ns_error(&e))))?;
let z = out.featureValueForName(&NSString::from_str("z")).and_then(|v| v.multiArrayValue()).ok_or_else(|| NpuError::Runtime("no output z".into()))?;
if z.dataType() != MLMultiArrayDataType::Float16 || z.count() as usize != BATCH * SHIFTS {
return Err(NpuError::Runtime("the output is not the declared fp16 [batch, shifts]".into()));
}
// The runtime may pad each row (strides [32, 1] for 15 values); read
// through the strides it reports, which must hold each row whole.
let strides: Vec<usize> = z.strides().iter().map(|n| n.as_usize()).collect();
let [row, one] = strides[..] else { return Err(NpuError::Runtime(format!("the output has strides {strides:?}"))) };
if one != 1 || row < SHIFTS {
return Err(NpuError::Runtime(format!("the output's strides {strides:?} do not hold a row of {SHIFTS}")));
}
#[allow(deprecated)]
let p = z.dataPointer().as_ptr().cast::<u16>();
let all = std::slice::from_raw_parts(p, (BATCH - 1) * row + SHIFTS);
let mut out = Vec::with_capacity(BATCH * SHIFTS);
for r in 0..BATCH {
out.extend_from_slice(&all[r * row..r * row + SHIFTS]);
}
Ok(out)
}
}
}
/// Every neural processor on this host: on macOS, the platform's, built,
/// placed and probed in this process.
pub fn discover() -> Discovery {
let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None };
#[cfg(target_os = "macos")]
{
match open(&npu::model()) {
Ok(m) => d.found.push(m),
Err(e) => d.refused.push(("npu-0".into(), e)),
}
}
#[cfg(not(target_os = "macos"))]
{
d.unavailable = Some("this platform has no neural-processor runtime this member reaches".into());
}
d
}
/// The member: the loaded model behind a lock (the runtime's model object is
/// used from one thread at a time), and what discovery established.
pub struct NpuMember {
#[cfg(target_os = "macos")]
model: std::sync::Arc<std::sync::Mutex<coreml::Handed<objc2::rc::Retained<objc2_core_ml::MLModel>>>>,
id: String,
key: String,
implementation: String,
detail: String,
placement: Vec<(String, String)>,
max_qubits: u8,
}
impl Clone for NpuMember {
fn clone(&self) -> Self {
NpuMember {
#[cfg(target_os = "macos")]
model: self.model.clone(),
id: self.id.clone(),
key: self.key.clone(),
implementation: self.implementation.clone(),
detail: self.detail.clone(),
placement: self.placement.clone(),
max_qubits: self.max_qubits,
}
}
}
impl std::fmt::Debug for NpuMember {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("NpuMember").field("id", &self.id).field("implementation", &self.implementation).field("placement", &self.placement).finish()
}
}
#[cfg(target_os = "macos")]
/// Where a model's compiled form is kept: by the model's text and the
/// system's build, whose compiler made it.
fn cache_dir(model: &[u8]) -> std::path::PathBuf {
let base = std::env::var_os("WAI_QUANTUM_NPU_CACHE").map(std::path::PathBuf::from).unwrap_or_else(|| std::env::temp_dir().join("wai-quantum-npu"));
let os = std::process::Command::new("sysctl").args(["-n", "kern.osversion"]).output().map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()).unwrap_or_default();
let mut h = blake3::Hasher::new();
h.update(model);
h.update(os.as_bytes());
let tag: String = h.finalize().as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect();
base.join(tag)
}
/// The placement as text, if the runtime put every operation on the neural
/// processor; otherwise why not.
#[cfg(target_os = "macos")]
fn placed_on_the_processor(placement: &[(String, String)]) -> Result<String, NpuError> {
let elsewhere: Vec<String> = placement.iter().filter(|(_, d)| d != NEURAL_PROCESSOR).map(|(op, d)| format!("{op} on {d}")).collect();
if placement.is_empty() || !elsewhere.is_empty() {
return Err(NpuError::Placement(if elsewhere.is_empty() { "no operation was reported".into() } else { elsewhere.join("; ") }));
}
Ok(placement.iter().map(|(op, d)| format!("{op}: {d}")).collect::<Vec<_>>().join(", "))
}
#[cfg(target_os = "macos")]
fn detail_of(placement_text: &str) -> String {
format!(
"npu; the platform's neural processor through its model runtime; placement by the runtime's compute plan: {placement_text}; fp16 limb products, the host assembling; up to {MAX_NPU_QUBITS} qubits; unmetered"
)
}
/// The neural processor running `model`: loaded, its placement checked, and
/// its products probed.
#[cfg(target_os = "macos")]
fn open(model: &[u8]) -> Result<NpuMember, NpuError> {
let loaded = coreml::load(model, &cache_dir(model))?;
let placement_text = placed_on_the_processor(&loaded.placement)?;
let mut o = b"wai:quantum-fabric-npu-member\x01".to_vec();
o.extend_from_slice(std::env::consts::ARCH.as_bytes());
o.extend_from_slice(placement_text.as_bytes());
let hw = std::process::Command::new("sysctl").args(["-n", "machdep.cpu.brand_string"]).output().map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()).unwrap_or_default();
o.extend_from_slice(hw.as_bytes());
let id = format!("local/npu-{}", blake3::hash(&o).as_bytes()[..6].iter().map(|b| format!("{b:02x}")).collect::<String>());
let member = NpuMember {
model: std::sync::Arc::new(std::sync::Mutex::new(coreml::Handed(loaded.model))),
id,
key: "npu-0".into(),
implementation: npu::implementation_id_of(model),
detail: detail_of(&placement_text),
placement: loaded.placement,
max_qubits: MAX_NPU_QUBITS,
};
let pairs = lanes::probe_pairs(4096, 0x5eed);
let out = member.probe(&pairs)?;
match lanes::first_mismatch(&pairs, &out) {
None => Ok(member),
Some((a, b)) => Err(NpuError::Probe { a, b }),
}
}
impl NpuMember {
pub fn id(&self) -> &str {
&self.id
}
/// The key a runner declares: `"npu-0"`.
pub fn key(&self) -> &str {
&self.key
}
pub fn class(&self) -> crate::quantum_fabric::FabricClass {
npu::CLASS
}
pub fn detail(&self) -> &str {
&self.detail
}
pub fn implementation(&self) -> &str {
&self.implementation
}
pub fn max_qubits(&self) -> u8 {
self.max_qubits
}
/// Each operation of the model and the device the runtime placed it on.
pub fn placement(&self) -> &[(String, String)] {
&self.placement
}
/// Exact products `a·b`, every limb product and limb sum formed on the
/// processor.
pub fn products(&self, pairs: &[(i32, i32)]) -> Result<Vec<i64>, NpuError> {
let mut out = Vec::with_capacity(pairs.len());
let (mut x, mut y) = (vec![0u16; BATCH * SHIFTS * LIMBS], vec![0u16; BATCH * SHIFTS * LIMBS]);
for chunk in pairs.chunks(BATCH) {
npu::expand(chunk, &mut x, &mut y);
let z = self.predict(&x, &y)?;
for p in 0..chunk.len() {
let mut c = [0i64; SHIFTS];
for (s, slot) in c.iter_mut().enumerate() {
*slot = npu::int_of_f16(z[p * SHIFTS + s]).ok_or_else(|| NpuError::Runtime("a coefficient that is no integer of fp16's exact range".into()))?;
}
out.push(npu::assemble(&c));
}
}
Ok(out)
}
#[cfg(target_os = "macos")]
fn predict(&self, x: &[u16], y: &[u16]) -> Result<Vec<u16>, NpuError> {
let m = self.model.lock().unwrap_or_else(|p| p.into_inner());
coreml::predict(&m.0, x, y)
}
#[cfg(not(target_os = "macos"))]
fn predict(&self, _x: &[u16], _y: &[u16]) -> Result<Vec<u16>, NpuError> {
Err(NpuError::Unavailable("no runtime on this platform".into()))
}
/// The processor's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each pair.
pub fn probe(&self, pairs: &[(i32, i32)]) -> Result<Vec<ProbeOutput>, NpuError> {
let products = self.products(pairs)?;
let split = |v: i64| ((v as u64 >> 32) as u32, v as u64 as u32);
Ok(products.iter().map(|&p| (split(p), split(fx(p)))).collect())
}
pub fn config(&self) -> Config {
Config::PER_GATE
}
/// Gates in order from `state`, as [`lanes::run_words`] runs them.
pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result<LaneRun, NpuError> {
let len = state.len();
if len < 2 || !len.is_power_of_two() {
return Err(NpuError::Internal("a state that is not 2^n lanes, n at least 1"));
}
let n = len.trailing_zeros() as u8;
if n > self.max_qubits {
return Err(NpuError::TooWide { n_qubits: n, max: self.max_qubits });
}
let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect();
self.execute(state.to_vec(), &planned)
}
fn execute(&self, mut amps: Vec<Lane>, gates: &[(&GateWords, u32)]) -> Result<LaneRun, NpuError> {
let len = amps.len();
for (w, _) in gates {
let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 {
return Err(NpuError::Internal("malformed gate words for this state"));
}
}
let mut report: Option<u32> = None;
let mut products = Vec::new();
let mut pairs = Vec::new();
for &(w, index) in gates {
let m: [i32; 8] = w.m.map(|x| x as i32);
let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
pairs.clear();
products.clear();
for p in 0..len / 2 {
let low = p & (tbit - 1);
let i = ((p - low) << 1) | low;
if i & ctrl != ctrl {
continue;
}
pairs.push(i);
pair_products(&m, amps[i], amps[i | tbit], &mut products);
}
let exact = self.products(&products)?;
for (k, &i) in pairs.iter().enumerate() {
let (o0, o1, bad) = apply_pair(&exact[16 * k..16 * k + 16]);
if bad {
report = Some(report.map_or(index, |r| r.min(index)));
}
amps[i] = o0;
amps[i | tbit] = o1;
}
}
Ok(LaneRun { amps, out_of_range: report })
}
pub fn run_program(&self, p: &Program) -> Result<LaneRun, NpuError> {
let n = p.n_qubits;
if n == 0 {
return Ok(LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None });
}
if n > self.max_qubits {
return Err(NpuError::TooWide { n_qubits: n, max: self.max_qubits });
}
if p.start >= 1usize << n {
return Err(NpuError::Internal("the start is outside the state"));
}
let mut gates = Vec::with_capacity(p.steps.len());
for step in &p.steps {
let Step::Gate { gate } = *step else { return Err(NpuError::Internal("a tile step; this member runs gate steps only")) };
let g = p.gates.get(gate as usize).ok_or(NpuError::Internal("a step names a gate the program does not hold"))?;
if !g.is_well_formed() {
return Err(NpuError::Internal("malformed gate words for this state"));
}
gates.push((&g.words, g.index));
}
let mut amps = vec![[0i32, 0]; 1usize << n];
amps[p.start] = [ONE as i32, 0];
self.execute(amps, &gates)
}
pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result<StateVector, NpuError> {
let p = plan::compile_with(c, start, &self.config()).map_err(NpuError::Circuit)?;
if p.n_qubits > self.max_qubits {
return Err(NpuError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits });
}
let run = self.run_program(&p)?;
if let Some(gate) = run.out_of_range {
return Err(NpuError::OutOfRange { gate });
}
Ok(StateVector { n_qubits: p.n_qubits, amps: run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }).collect() })
}
pub fn simulate(&self, c: &Circuit) -> Result<StateVector, NpuError> {
self.simulate_from(c, 0)
}
/// The certificate of [`apply_kernel`]`(gs)` on this member over `corpus`.
pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result<Certificate, NpuError> {
let mut cert = Certification::new();
for c in corpus.circuits(gs) {
let reference = lanes::reference_output(&c).map_err(NpuError::Circuit)?;
let run = self.run_program(&plan::compile_with(&c, 0, &self.config()).map_err(NpuError::Circuit)?)?;
if let Some(gate) = run.out_of_range {
return Err(NpuError::OutOfRange { gate });
}
cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps));
}
Ok(cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)))
}
/// A fabric [`Member`] of class [`FabricClass::Npu`](crate::quantum_fabric::FabricClass::Npu).
pub fn member(&self, corpus: Corpus) -> Result<Member, NpuError> {
let mut kernels = Vec::new();
for gs in Gateset::ALL {
kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate: self.certify(gs, corpus)?, energy: None });
}
Ok(Member { id: self.id.clone(), class: npu::CLASS, presence: Presence::Detected, kernels, detail: self.detail.clone() })
}
/// The same member holding at most `max_qubits` qubits.
#[doc(hidden)]
pub fn with_max_qubits(&self, max_qubits: u8) -> NpuMember {
NpuMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() }
}
/// The same processor running another model text: for the mutation tests.
/// A mutant the runtime places elsewhere is refused at load, as any model is.
#[doc(hidden)]
pub fn with_model(&self, model: &[u8]) -> Result<NpuMember, NpuError> {
#[cfg(target_os = "macos")]
{
let loaded = coreml::load(model, &cache_dir(model))?;
let placement_text = placed_on_the_processor(&loaded.placement)?;
Ok(NpuMember {
model: std::sync::Arc::new(std::sync::Mutex::new(coreml::Handed(loaded.model))),
implementation: npu::implementation_id_of(model),
detail: detail_of(&placement_text),
placement: loaded.placement,
..self.clone()
})
}
#[cfg(not(target_os = "macos"))]
{
let _ = model;
Err(NpuError::Unavailable("no runtime on this platform".into()))
}
}
}
impl Runner for NpuMember {
fn member_id(&self) -> String {
self.id.clone()
}
fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, _meter: &'r mut dyn RunMeter) -> RunFuture<'r> {
Box::pin(async move {
if implementation != self.implementation {
return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation)));
}
let s = self.simulate_from(c, start)?;
Ok((s, 0, None))
})
}
}
#[cfg(test)]
mod tests {
use super::*;
/// The host's half of a gate, with the processor's products replaced by
/// exact ones: it must be `lanes::run_words`.
#[test]
fn the_hosts_arithmetic_is_run_words() {
let mut s = 0x1234_5678_u64;
let mut next = || {
s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
(s >> 32) as u32
};
for case in 0..300 {
let n = 1 + (next() % 6) as usize;
let state: Vec<Lane> = (0..1usize << n).map(|_| [next() as i32 >> (case % 3 * 4), next() as i32 >> (case % 3 * 4)]).collect();
let t = next() as usize % n;
let ctrl = (0..n).filter(|&b| b != t && next() % 3 == 0).fold(0u32, |a, b| a | 1 << b);
let w = GateWords { m: [0; 8].map(|_: u32| next() >> (case % 2 * 2)), tbit: 1 << t, ctrl };
let want = lanes::run_words(&state, std::slice::from_ref(&w)).unwrap();
let mut amps = state.clone();
let mut report = None;
let m: [i32; 8] = w.m.map(|x| x as i32);
for p in 0..amps.len() / 2 {
let low = p & ((1 << t) - 1);
let i = ((p - low) << 1) | low;
if i & ctrl as usize != ctrl as usize {
continue;
}
let mut pr = Vec::new();
pair_products(&m, amps[i], amps[i | 1 << t], &mut pr);
let exact: Vec<i64> = pr.iter().map(|&(a, b)| i64::from(a) * i64::from(b)).collect();
let (o0, o1, bad) = apply_pair(&exact);
if bad {
report = Some(0);
}
amps[i] = o0;
amps[i | 1 << t] = o1;
}
assert_eq!(report, want.out_of_range, "case {case}");
assert_eq!(amps, want.amps, "case {case}");
}
}
#[test]
fn errors_map_onto_the_fabric() {
assert!(MemberError::from(NpuError::OutOfRange { gate: 1 }).falls_back());
assert!(!MemberError::from(NpuError::Placement("mul on the CPU".into())).falls_back());
}
}

632
wai/wai-quantum/src/quantum_fabric_sme.rs Normal file
View file

@ -0,0 +1,632 @@
//! The fabric's matrix-extension member: the gate's fixed-point arithmetic in
//! the CPU's streaming vector mode (SME), where the matrix unit executes it.
//! Every instruction the kernel uses is base SME's: none needs SME2.
//!
//! **Why it is bit-exact.** The kernel is the same transcription of
//! [`lanes::apply_words`] as the CUDA kernel, in 64-bit vector lanes: each
//! operand is loaded sign-extended to 64 bits, each `fx(a, b) = (a·b + 2^29) >> 30`
//! is a 64-bit multiply, an add and an arithmetic shift, exact, and each row is
//! the sum of four. The host gathers the pairs a gate selects, the kernel forms
//! the four rows of eight pairs at a time (with 512-bit vectors), and the host
//! checks each row's range and keeps its low 32 bits, as
//! [`lanes::apply_words`] does.
//!
//! **Where it runs.** The kernel enters streaming mode (`smstart sm`) and
//! leaves it (`smstop sm`); its vector instructions exist only in that mode on
//! these processors, so a kernel that ran at all ran there. Discovery reads the
//! processor's feature flags (macOS `hw.optional.arm.FEAT_SME`; on Linux the
//! kernel's `AT_HWCAP2` bit 23, which an emulator supplies for its guest). Under an emulator, the runner declares it
//! (`WAI_QUANTUM_SME_EMULATOR`) and every certificate says so.
//!
//! The kernel's text, between the markers below, is the implementation id's
//! input; three mutants, compiled in, must each be caught.
use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector};
use crate::quantum_fabric::lanes::{self, GateWords, Lane};
use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step};
use crate::quantum_fabric::{Certificate, Certification, Corpus, FabricClass, KernelEntry, Member, MemberError, Presence, Provenance, RunFuture, RunMeter, Runner, apply_kernel};
/// One probe case: [`lanes::mul_wide`] and [`lanes::fxmul_wide`] as
/// `(high, low)` words.
pub type ProbeOutput = ((u32, u32), (u32, u32));
/// Why the matrix extension is not a member, or a run has no state.
#[derive(Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum SmeError {
Unavailable(String),
Circuit(QuantumError),
TooWide { n_qubits: u8, max: u8 },
OutOfRange { gate: u32 },
Internal(&'static str),
Probe { a: i32, b: i32 },
}
impl std::fmt::Display for SmeError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
SmeError::Unavailable(why) => write!(f, "no matrix extension: {why}"),
SmeError::Circuit(e) => write!(f, "{e}"),
SmeError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"),
SmeError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"),
SmeError::Internal(why) => write!(f, "host defect: {why}"),
SmeError::Probe { a, b } => write!(f, "the kernel's arithmetic differs from the CPU's at ({a}, {b})"),
}
}
}
impl std::error::Error for SmeError {}
impl From<SmeError> for MemberError {
fn from(e: SmeError) -> MemberError {
match e {
SmeError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max },
SmeError::OutOfRange { gate } => MemberError::OutOfRange { gate },
SmeError::Circuit(e) => MemberError::Circuit(e),
other => MemberError::Other(other.to_string()),
}
}
}
/// The kernels a member runs: the conforming one, or a mutant.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Variant {
Conforming,
/// The rounding constant is half what it should be.
M1,
/// The product is shifted by one bit too few.
M2,
/// Row 0's real part adds its last term instead of subtracting it.
M3,
}
impl Variant {
pub const MUTANTS: [Variant; 3] = [Variant::M1, Variant::M2, Variant::M3];
}
// kernel-begin
/// The kernels' text: the probe (each `a·b` and `fx(a, b)`) and the rows of a
/// gate. `$rnd` is the rounding constant, `$sh` the shift, `$s` the sign of
/// row 0's last real term.
macro_rules! rows_text {
($rnd:literal, $sh:literal, $s:literal) => {
concat!(
".arch_extension sme\n",
"smstart sm\n",
"ptrue p0.d\n",
"ldr x9, [{m}, #0]\nmov z16.d, x9\n",
"ldr x9, [{m}, #8]\nmov z17.d, x9\n",
"ldr x9, [{m}, #16]\nmov z18.d, x9\n",
"ldr x9, [{m}, #24]\nmov z19.d, x9\n",
"ldr x9, [{m}, #32]\nmov z20.d, x9\n",
"ldr x9, [{m}, #40]\nmov z21.d, x9\n",
"ldr x9, [{m}, #48]\nmov z22.d, x9\n",
"ldr x9, [{m}, #56]\nmov z23.d, x9\n",
"mov x9, #", $rnd, "\nmov z31.d, x9\n",
"mov x10, #0\n",
"2:\n",
"whilelt p1.d, x10, {n}\n",
"b.none 3f\n",
"ld1sw {{z0.d}}, p1/z, [{a0r}, x10, lsl #2]\n",
"ld1sw {{z1.d}}, p1/z, [{a0i}, x10, lsl #2]\n",
"ld1sw {{z2.d}}, p1/z, [{a1r}, x10, lsl #2]\n",
"ld1sw {{z3.d}}, p1/z, [{a1i}, x10, lsl #2]\n",
// Row 0, real.
"movprfx z4, z0\nmul z4.d, p0/m, z4.d, z16.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nmov z8.d, z4.d\n",
"movprfx z4, z1\nmul z4.d, p0/m, z4.d, z17.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nsub z8.d, z8.d, z4.d\n",
"movprfx z4, z2\nmul z4.d, p0/m, z4.d, z18.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z8.d, z8.d, z4.d\n",
"movprfx z4, z3\nmul z4.d, p0/m, z4.d, z19.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\n", $s, " z8.d, z8.d, z4.d\n",
// Row 0, imaginary.
"movprfx z4, z1\nmul z4.d, p0/m, z4.d, z16.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nmov z9.d, z4.d\n",
"movprfx z4, z0\nmul z4.d, p0/m, z4.d, z17.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z9.d, z9.d, z4.d\n",
"movprfx z4, z3\nmul z4.d, p0/m, z4.d, z18.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z9.d, z9.d, z4.d\n",
"movprfx z4, z2\nmul z4.d, p0/m, z4.d, z19.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z9.d, z9.d, z4.d\n",
// Row 1, real.
"movprfx z4, z0\nmul z4.d, p0/m, z4.d, z20.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nmov z10.d, z4.d\n",
"movprfx z4, z1\nmul z4.d, p0/m, z4.d, z21.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nsub z10.d, z10.d, z4.d\n",
"movprfx z4, z2\nmul z4.d, p0/m, z4.d, z22.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z10.d, z10.d, z4.d\n",
"movprfx z4, z3\nmul z4.d, p0/m, z4.d, z23.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nsub z10.d, z10.d, z4.d\n",
// Row 1, imaginary.
"movprfx z4, z1\nmul z4.d, p0/m, z4.d, z20.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nmov z11.d, z4.d\n",
"movprfx z4, z0\nmul z4.d, p0/m, z4.d, z21.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z11.d, z11.d, z4.d\n",
"movprfx z4, z3\nmul z4.d, p0/m, z4.d, z22.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z11.d, z11.d, z4.d\n",
"movprfx z4, z2\nmul z4.d, p0/m, z4.d, z23.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z11.d, z11.d, z4.d\n",
"st1d {{z8.d}}, p1, [{o0r}, x10, lsl #3]\n",
"st1d {{z9.d}}, p1, [{o0i}, x10, lsl #3]\n",
"st1d {{z10.d}}, p1, [{o1r}, x10, lsl #3]\n",
"st1d {{z11.d}}, p1, [{o1i}, x10, lsl #3]\n",
"incd x10\n",
"b 2b\n",
"3:\n",
"smstop sm\n",
)
};
}
macro_rules! probe_text {
($rnd:literal, $sh:literal) => {
concat!(
".arch_extension sme\n",
"smstart sm\n",
"ptrue p0.d\n",
"mov x9, #", $rnd, "\nmov z31.d, x9\n",
"mov x10, #0\n",
"2:\n",
"whilelt p1.d, x10, {n}\n",
"b.none 3f\n",
"ld1sw {{z0.d}}, p1/z, [{a}, x10, lsl #2]\n",
"ld1sw {{z1.d}}, p1/z, [{b}, x10, lsl #2]\n",
"movprfx z4, z0\nmul z4.d, p0/m, z4.d, z1.d\n",
"add z5.d, z4.d, z31.d\nasr z5.d, z5.d, #", $sh, "\n",
"st1d {{z4.d}}, p1, [{prod}, x10, lsl #3]\n",
"st1d {{z5.d}}, p1, [{fx}, x10, lsl #3]\n",
"incd x10\n",
"b 2b\n",
"3:\n",
"smstop sm\n",
)
};
}
// kernel-end
/// One kernel, conforming or a mutant: the rows of `n` pairs.
macro_rules! rows_fn {
($name:ident, $rnd:literal, $sh:literal, $s:literal) => {
#[cfg(target_arch = "aarch64")]
#[allow(clippy::too_many_arguments)]
/// # Safety
///
/// The processor has SME; every slice holds `n` values.
unsafe fn $name(m: &[i64; 8], n: usize, a0r: &[i32], a0i: &[i32], a1r: &[i32], a1i: &[i32], o0r: &mut [i64], o0i: &mut [i64], o1r: &mut [i64], o1i: &mut [i64]) {
debug_assert!([a0r.len(), a0i.len(), a1r.len(), a1i.len(), o0r.len(), o0i.len(), o1r.len(), o1i.len()].iter().all(|&l| l >= n));
// SAFETY: the caller's contract; streaming mode is entered and left
// inside the block, and every vector and predicate register it
// writes is declared clobbered (leaving streaming mode resets them).
unsafe {
std::arch::asm!(
rows_text!($rnd, $sh, $s),
m = in(reg) m.as_ptr(),
n = in(reg) n,
a0r = in(reg) a0r.as_ptr(),
a0i = in(reg) a0i.as_ptr(),
a1r = in(reg) a1r.as_ptr(),
a1i = in(reg) a1i.as_ptr(),
o0r = in(reg) o0r.as_mut_ptr(),
o0i = in(reg) o0i.as_mut_ptr(),
o1r = in(reg) o1r.as_mut_ptr(),
o1i = in(reg) o1i.as_mut_ptr(),
out("x9") _, out("x10") _, out("p0") _, out("p1") _,
out("v0") _, out("v1") _, out("v2") _, out("v3") _, out("v4") _, out("v5") _, out("v6") _, out("v7") _,
out("v8") _, out("v9") _, out("v10") _, out("v11") _, out("v12") _, out("v13") _, out("v14") _, out("v15") _,
out("v16") _, out("v17") _, out("v18") _, out("v19") _, out("v20") _, out("v21") _, out("v22") _, out("v23") _,
out("v24") _, out("v25") _, out("v26") _, out("v27") _, out("v28") _, out("v29") _, out("v30") _, out("v31") _,
options(nostack),
);
}
}
};
}
macro_rules! probe_fn {
($name:ident, $rnd:literal, $sh:literal) => {
#[cfg(target_arch = "aarch64")]
/// # Safety
///
/// The processor has SME; every slice holds `n` values.
unsafe fn $name(n: usize, a: &[i32], b: &[i32], prod: &mut [i64], fx: &mut [i64]) {
debug_assert!(a.len() >= n && b.len() >= n && prod.len() >= n && fx.len() >= n);
// SAFETY: as for the rows.
unsafe {
std::arch::asm!(
probe_text!($rnd, $sh),
n = in(reg) n,
a = in(reg) a.as_ptr(),
b = in(reg) b.as_ptr(),
prod = in(reg) prod.as_mut_ptr(),
fx = in(reg) fx.as_mut_ptr(),
out("x9") _, out("x10") _, out("p0") _, out("p1") _,
out("v0") _, out("v1") _, out("v2") _, out("v3") _, out("v4") _, out("v5") _, out("v6") _, out("v7") _,
out("v8") _, out("v9") _, out("v10") _, out("v11") _, out("v12") _, out("v13") _, out("v14") _, out("v15") _,
out("v16") _, out("v17") _, out("v18") _, out("v19") _, out("v20") _, out("v21") _, out("v22") _, out("v23") _,
out("v24") _, out("v25") _, out("v26") _, out("v27") _, out("v28") _, out("v29") _, out("v30") _, out("v31") _,
options(nostack),
);
}
}
};
}
rows_fn!(rows_conforming, "0x20000000", "30", "sub");
rows_fn!(rows_m1, "0x10000000", "30", "sub");
rows_fn!(rows_m2, "0x20000000", "29", "sub");
rows_fn!(rows_m3, "0x20000000", "30", "add");
probe_fn!(probe_conforming, "0x20000000", "30");
probe_fn!(probe_m1, "0x10000000", "30");
probe_fn!(probe_m2, "0x20000000", "29");
/// The text the conforming kernels are assembled from.
pub const KERNEL_TEXT: &str = concat!(rows_text!("0x20000000", "30", "sub"), probe_text!("0x20000000", "30"));
/// The implementation name the id starts with.
pub const IMPLEMENTATION: &str = "sme-lanes64";
/// Version of the host layout: the gather of a gate's pairs into four
/// columns, the 64-bit row outputs, and the host's range check. Part of the
/// implementation id.
pub const HOST_LAYOUT: u32 = 1;
/// `"sme-lanes64/<16 hex>"` over a variant's kernel text and the layout.
pub fn implementation_id_of(v: Variant) -> String {
let text = match v {
Variant::Conforming => KERNEL_TEXT,
Variant::M1 => concat!(rows_text!("0x10000000", "30", "sub"), probe_text!("0x10000000", "30")),
Variant::M2 => concat!(rows_text!("0x20000000", "29", "sub"), probe_text!("0x20000000", "29")),
Variant::M3 => concat!(rows_text!("0x20000000", "30", "add"), probe_text!("0x20000000", "30")),
};
let mut o = b"wai:quantum-fabric-sme\x01".to_vec();
o.extend_from_slice(&(text.len() as u64).to_be_bytes());
o.extend_from_slice(text.as_bytes());
o.extend_from_slice(&HOST_LAYOUT.to_be_bytes());
let h: String = blake3::hash(&o).as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect();
format!("{IMPLEMENTATION}/{h}")
}
pub fn implementation_id() -> String {
implementation_id_of(Variant::Conforming)
}
/// Whether this processor has SME, by its feature flags.
pub fn has_sme() -> Result<(), String> {
#[cfg(not(target_arch = "aarch64"))]
{
Err("this processor is not aarch64".into())
}
#[cfg(all(target_arch = "aarch64", target_os = "macos"))]
{
let out = std::process::Command::new("sysctl").args(["-n", "hw.optional.arm.FEAT_SME"]).output().map_err(|e| format!("sysctl: {e}"))?;
match String::from_utf8_lossy(&out.stdout).trim() {
"1" => Ok(()),
other => Err(format!("hw.optional.arm.FEAT_SME is {other:?}")),
}
}
#[cfg(all(target_arch = "aarch64", target_os = "linux"))]
{
// The kernel's capability bits for this process: AT_HWCAP2 (26), whose
// bit 23 is SME. An emulator supplies its own, where /proc/cpuinfo
// would describe the host instead.
const AT_HWCAP2: u64 = 26;
const HWCAP2_SME: u64 = 1 << 23;
let auxv = std::fs::read("/proc/self/auxv").map_err(|e| format!("/proc/self/auxv: {e}"))?;
let hwcap2 = auxv.as_chunks::<16>().0.iter().find_map(|e| {
let (k, v) = e.split_at(8);
(u64::from_ne_bytes(k.try_into().expect("8 bytes")) == AT_HWCAP2).then(|| u64::from_ne_bytes(v.try_into().expect("8 bytes")))
});
match hwcap2 {
Some(h) if h & HWCAP2_SME != 0 => Ok(()),
Some(_) => Err("the kernel reports no SME (AT_HWCAP2 bit 23)".into()),
None => Err("the kernel reports no AT_HWCAP2".into()),
}
}
#[cfg(all(target_arch = "aarch64", not(any(target_os = "macos", target_os = "linux"))))]
{
Err("feature detection is written for macOS and Linux".into())
}
}
/// What [`discover`] found.
#[derive(Debug)]
pub struct Discovery {
pub found: Vec<SmeMember>,
pub refused: Vec<(String, SmeError)>,
pub unavailable: Option<String>,
}
/// The matrix extension, probed in this process, or why there is none.
pub fn discover() -> Discovery {
let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None };
if let Err(why) = has_sme() {
d.unavailable = Some(why);
return d;
}
match SmeMember::open(Variant::Conforming) {
Ok(m) => d.found.push(m),
Err(e) => d.refused.push((KEY.into(), e)),
}
d
}
/// The key a runner declares.
pub const KEY: &str = "cpu-matrix-sme";
/// The most qubits a run holds.
pub const MAX_SME_QUBITS: u8 = if 24 < MAX_QUBITS { 24 } else { MAX_QUBITS };
/// The matrix extension as a fabric member.
#[derive(Clone, Debug)]
pub struct SmeMember {
variant: Variant,
id: String,
implementation: String,
detail: String,
provenance: Provenance,
max_qubits: u8,
}
impl SmeMember {
fn open(variant: Variant) -> Result<SmeMember, SmeError> {
let emulator = std::env::var("WAI_QUANTUM_SME_EMULATOR").ok().filter(|s| !s.trim().is_empty());
let provenance = emulator.as_ref().map_or(Provenance::Silicon, |e| Provenance::Emulator(e.trim().to_string()));
let mut o = b"wai:quantum-fabric-sme-member\x01".to_vec();
o.extend_from_slice(std::env::consts::ARCH.as_bytes());
o.extend_from_slice(std::env::consts::OS.as_bytes());
o.extend_from_slice(provenance.label().as_bytes());
let id = format!("local/cpu-matrix-sme-{}", blake3::hash(&o).as_bytes()[..6].iter().map(|b| format!("{b:02x}")).collect::<String>());
let member = SmeMember {
variant,
id,
implementation: implementation_id_of(variant),
detail: format!(
"cpu-matrix; the processor's matrix extension in streaming mode (SME), 64-bit lanes; {}; up to {MAX_SME_QUBITS} qubits; unmetered",
provenance.label()
),
provenance,
max_qubits: MAX_SME_QUBITS,
};
if variant == Variant::Conforming {
let pairs = lanes::probe_pairs(4096, 0x5eed);
let out = member.probe(&pairs)?;
if let Some((a, b)) = lanes::first_mismatch(&pairs, &out) {
return Err(SmeError::Probe { a, b });
}
}
Ok(member)
}
/// The same processor running a mutant: for the mutation tests.
#[doc(hidden)]
pub fn with_variant(&self, v: Variant) -> SmeMember {
SmeMember { variant: v, implementation: implementation_id_of(v), ..self.clone() }
}
#[doc(hidden)]
pub fn with_max_qubits(&self, max_qubits: u8) -> SmeMember {
SmeMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() }
}
pub fn id(&self) -> &str {
&self.id
}
pub fn key(&self) -> &str {
KEY
}
pub fn class(&self) -> FabricClass {
FabricClass::CpuMatrix
}
pub fn detail(&self) -> &str {
&self.detail
}
pub fn implementation(&self) -> &str {
&self.implementation
}
pub fn provenance(&self) -> &Provenance {
&self.provenance
}
pub fn max_qubits(&self) -> u8 {
self.max_qubits
}
pub fn config(&self) -> Config {
Config::PER_GATE
}
/// The kernel's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each pair.
pub fn probe(&self, pairs: &[(i32, i32)]) -> Result<Vec<ProbeOutput>, SmeError> {
#[cfg(target_arch = "aarch64")]
{
let n = pairs.len();
let a: Vec<i32> = pairs.iter().map(|p| p.0).collect();
let b: Vec<i32> = pairs.iter().map(|p| p.1).collect();
let (mut prod, mut fx) = (vec![0i64; n], vec![0i64; n]);
// SAFETY: discovery established SME; every slice holds n values.
unsafe {
match self.variant {
Variant::M1 => probe_m1(n, &a, &b, &mut prod, &mut fx),
Variant::M2 => probe_m2(n, &a, &b, &mut prod, &mut fx),
Variant::Conforming | Variant::M3 => probe_conforming(n, &a, &b, &mut prod, &mut fx),
}
}
let split = |v: i64| ((v as u64 >> 32) as u32, v as u64 as u32);
Ok(prod.iter().zip(&fx).map(|(&p, &f)| (split(p), split(f))).collect())
}
#[cfg(not(target_arch = "aarch64"))]
{
let _ = pairs;
Err(SmeError::Unavailable("not aarch64".into()))
}
}
/// The four rows of each pair, from the kernel.
#[cfg(target_arch = "aarch64")]
fn rows(&self, m: &[i64; 8], cols: &[Vec<i32>; 4]) -> [Vec<i64>; 4] {
let n = cols[0].len();
let mut o = [vec![0i64; n], vec![0i64; n], vec![0i64; n], vec![0i64; n]];
let [o0r, o0i, o1r, o1i] = &mut o;
// SAFETY: discovery established SME; every slice holds n values.
unsafe {
let f = match self.variant {
Variant::Conforming => rows_conforming,
Variant::M1 => rows_m1,
Variant::M2 => rows_m2,
Variant::M3 => rows_m3,
};
f(m, n, &cols[0], &cols[1], &cols[2], &cols[3], o0r, o0i, o1r, o1i);
}
o
}
/// Gates in order from `state`, as [`lanes::run_words`] runs them.
pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result<LaneRun, SmeError> {
let len = state.len();
if len < 2 || !len.is_power_of_two() {
return Err(SmeError::Internal("a state that is not 2^n lanes, n at least 1"));
}
let n = len.trailing_zeros() as u8;
if n > self.max_qubits {
return Err(SmeError::TooWide { n_qubits: n, max: self.max_qubits });
}
let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect();
self.execute(state.to_vec(), &planned)
}
fn execute(&self, mut amps: Vec<Lane>, gates: &[(&GateWords, u32)]) -> Result<LaneRun, SmeError> {
let len = amps.len();
for (w, _) in gates {
let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 {
return Err(SmeError::Internal("malformed gate words for this state"));
}
}
#[cfg(not(target_arch = "aarch64"))]
{
let _ = (&mut amps, gates);
return Err(SmeError::Unavailable("not aarch64".into()));
}
#[cfg(target_arch = "aarch64")]
{
let mut report: Option<u32> = None;
for &(w, index) in gates {
let m: [i64; 8] = w.m.map(|x| i64::from(x as i32));
let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize);
let mut at = Vec::new();
let mut cols: [Vec<i32>; 4] = Default::default();
for p in 0..len / 2 {
let low = p & (tbit - 1);
let i = ((p - low) << 1) | low;
if i & ctrl != ctrl {
continue;
}
at.push(i);
let (a0, a1) = (amps[i], amps[i | tbit]);
cols[0].push(a0[0]);
cols[1].push(a0[1]);
cols[2].push(a1[0]);
cols[3].push(a1[1]);
}
let [o0r, o0i, o1r, o1i] = self.rows(&m, &cols);
for (k, &i) in at.iter().enumerate() {
let rows = [o0r[k], o0i[k], o1r[k], o1i[k]];
if rows.iter().any(|&v| i32::try_from(v).is_err()) {
report = Some(report.map_or(index, |r| r.min(index)));
}
let low = |v: i64| v as u64 as u32 as i32;
amps[i] = [low(rows[0]), low(rows[1])];
amps[i | tbit] = [low(rows[2]), low(rows[3])];
}
}
Ok(LaneRun { amps, out_of_range: report })
}
}
pub fn run_program(&self, p: &Program) -> Result<LaneRun, SmeError> {
let n = p.n_qubits;
if n == 0 {
return Ok(LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None });
}
if n > self.max_qubits {
return Err(SmeError::TooWide { n_qubits: n, max: self.max_qubits });
}
if p.start >= 1usize << n {
return Err(SmeError::Internal("the start is outside the state"));
}
let mut gates = Vec::with_capacity(p.steps.len());
for step in &p.steps {
let Step::Gate { gate } = *step else { return Err(SmeError::Internal("a tile step; this member runs gate steps only")) };
let g = p.gates.get(gate as usize).ok_or(SmeError::Internal("a step names a gate the program does not hold"))?;
if !g.is_well_formed() {
return Err(SmeError::Internal("malformed gate words for this state"));
}
gates.push((&g.words, g.index));
}
let mut amps = vec![[0i32, 0]; 1usize << n];
amps[p.start] = [ONE as i32, 0];
self.execute(amps, &gates)
}
pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result<StateVector, SmeError> {
let p = plan::compile_with(c, start, &self.config()).map_err(SmeError::Circuit)?;
if p.n_qubits > self.max_qubits {
return Err(SmeError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits });
}
let run = self.run_program(&p)?;
if let Some(gate) = run.out_of_range {
return Err(SmeError::OutOfRange { gate });
}
Ok(StateVector { n_qubits: p.n_qubits, amps: run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }).collect() })
}
pub fn simulate(&self, c: &Circuit) -> Result<StateVector, SmeError> {
self.simulate_from(c, 0)
}
/// The certificate of [`apply_kernel`]`(gs)` on this member, with its
/// provenance.
pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result<Certificate, SmeError> {
let mut cert = Certification::new();
for c in corpus.circuits(gs) {
let reference = lanes::reference_output(&c).map_err(SmeError::Circuit)?;
let run = self.run_program(&plan::compile_with(&c, 0, &self.config()).map_err(SmeError::Circuit)?)?;
if let Some(gate) = run.out_of_range {
return Err(SmeError::OutOfRange { gate });
}
cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps));
}
Ok(cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)).with_provenance(self.provenance.clone()))
}
pub fn member(&self, corpus: Corpus) -> Result<Member, SmeError> {
let mut kernels = Vec::new();
for gs in Gateset::ALL {
kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate: self.certify(gs, corpus)?, energy: None });
}
Ok(Member { id: self.id.clone(), class: FabricClass::CpuMatrix, presence: Presence::Detected, kernels, detail: self.detail.clone() })
}
}
impl Runner for SmeMember {
fn member_id(&self) -> String {
self.id.clone()
}
fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, _meter: &'r mut dyn RunMeter) -> RunFuture<'r> {
Box::pin(async move {
if implementation != self.implementation {
return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation)));
}
Ok((self.simulate_from(c, start)?, 0, None))
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn every_variant_has_its_own_implementation_id() {
let ids: std::collections::BTreeSet<String> = [Variant::Conforming, Variant::M1, Variant::M2, Variant::M3].into_iter().map(implementation_id_of).collect();
assert_eq!(ids.len(), 4);
assert!(implementation_id().starts_with("sme-lanes64/"));
assert!(KERNEL_TEXT.contains("smstart sm") && KERNEL_TEXT.contains("smstop sm"));
}
#[test]
fn the_member_id_sorts_after_the_cpu_reference() {
assert!("local/cpu-matrix-sme-000000000000" > "local/cpu-aarch64");
}
}

1089
wai/wai-quantum/src/quantum_fabric_vector.rs Normal file
View file

File diff suppressed because it is too large Load diff

410
wai/wai-quantum/tests/dataflow_member.rs Normal file
View file

@ -0,0 +1,410 @@
//! The fabric's dataflow member on the machine that runs this.
//!
//! The machine declares the devices it must have in `WAI_QUANTUM_DATAFLOW_EXPECT`:
//! their keys, comma-separated and sorted, or `none`. Every test that needs a
//! member fails unless the devices found are exactly the declared ones, so a
//! machine whose GPU or driver went missing, or a run that declared nothing,
//! fails instead of passing over no member. Each test prints what it
//! established and, when `WAI_QUANTUM_DATAFLOW_REPORT` names a file, appends the
//! same lines to it, which a CI step can count per device.
//!
//! ```text
//! WAI_QUANTUM_DATAFLOW_LIB=<the device library> WAI_QUANTUM_DATAFLOW_EXPECT=dataflow-1e52-b140 \
//! cargo test --release --features quantum_fabric_dataflow --test dataflow_member -- --test-threads=1
//! ```
#![cfg(not(target_arch = "wasm32"))]
use std::io::Write;
use std::sync::OnceLock;
use wai_quantum::ed25519_dalek::SigningKey;
use wai_quantum::quantum::{BaseGate, Circuit, Gate, Gateset, MAX_QUBITS, QuantumError};
use wai_quantum::quantum_fabric::lanes::{self, GateWords, Lane};
use wai_quantum::quantum_fabric::plan::{Program, Step};
use wai_quantum::quantum_fabric::{
Corpus, CpuReference, FABRIC_DEFAULT, Fabric, FabricClass, FabricReceipt, MemberError, Parity, Presence, REFERENCE, apply_kernel, detect_local, place, ready,
};
use wai_quantum::quantum_fabric_dataflow::{self as dataflow, DataflowError, DataflowMember, Discovery, Variant, discover};
/// The CPU's corpus digests: random-v1 (seed 7, 200 cases) and fabric-v1
/// (seed 7, 200 + 24 cases), per gateset. A bit-exact member reproduces them.
const PINS: [(Gateset, &str, &str); 2] = [
(Gateset::V1, "d18454c639acfd9c7b66450f4ded1aac4ab46cc10a7bedbcf4958a96e204fb36", "41c726a0ac80f7defef92e15ecf893bdb7a3076b5e7f670fa0b7142e5c033df7"),
(Gateset::V2, "3dbde276c1095bf827faec71d5980e050f8a55a5ba7d97cd3bbdaec3f43a13b2", "9a84b55af68f1a8b07390b234cf562dd8d85cef3185a697ab85690c8790a7809"),
];
const RANDOM_V1: Corpus = Corpus::Random { seed: 7, cases: 200 };
static DISCOVERY: OnceLock<Discovery> = OnceLock::new();
fn discovery() -> &'static Discovery {
DISCOVERY.get_or_init(discover)
}
/// Prints `line` and appends it to the file `WAI_QUANTUM_DATAFLOW_REPORT` names.
fn report(line: &str) {
println!("{line}");
if let Some(path) = std::env::var_os("WAI_QUANTUM_DATAFLOW_REPORT") {
let mut f = std::fs::OpenOptions::new().create(true).append(true).open(&path).expect("the report file opens");
writeln!(f, "{line}").expect("the report line is written");
}
}
/// The declared device set, as written, if it is exactly what was found;
/// otherwise why not, with what was found and refused and how to declare it.
fn declared(d: &Discovery) -> Result<String, String> {
let mut found: Vec<&str> = d.found.iter().map(|m| m.key()).collect();
found.sort_unstable();
let found_list = if found.is_empty() { "none".to_string() } else { found.join(",") };
let refused: Vec<String> = d.refused.iter().map(|(k, e)| format!("{k}: {e}")).collect();
let refused_list = if refused.is_empty() { "none".to_string() } else { refused.join("; ") };
let unavailable = d.unavailable.as_deref().unwrap_or("no");
let how = format!("found: {found_list}; refused: {refused_list}; unavailable: {unavailable}. Declare the devices this machine must have: WAI_QUANTUM_DATAFLOW_EXPECT={found_list}");
let expect = match std::env::var("WAI_QUANTUM_DATAFLOW_EXPECT") {
Ok(v) if !v.is_empty() => v,
_ => return Err(format!("WAI_QUANTUM_DATAFLOW_EXPECT is not set. {how}")),
};
let keys: Vec<&str> = if expect == "none" { Vec::new() } else { expect.split(',').collect() };
let well_formed = keys.windows(2).all(|w| w[0] <= w[1]) && keys.iter().all(|k| !k.is_empty() && *k != "none" && k.trim() == *k);
if !well_formed {
return Err(format!("WAI_QUANTUM_DATAFLOW_EXPECT={expect:?} is not a sorted, comma-separated list of keys, nor none. {how}"));
}
for k in &keys {
if !found.contains(k)
&& let Some((_, e)) = d.refused.iter().find(|(key, _)| key == k)
{
return Err(format!("the declared device {k} was refused: {e}. {how}"));
}
}
if found != keys {
return Err(format!("the devices found are not the declared ones ({expect}). {how}"));
}
Ok(expect)
}
/// The members, once the declared devices are exactly the ones found.
fn members() -> &'static [DataflowMember] {
let d = discovery();
if let Err(why) = declared(d) {
panic!("{why}");
}
&d.found
}
fn hex(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}
fn next(s: &mut u64) -> u64 {
*s = s.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = *s;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^ (z >> 31)
}
/// A value in `[-bound, bound]`.
fn within(s: &mut u64, bound: i64) -> i32 {
((next(s) % (2 * bound as u64 + 1)) as i64 - bound) as i32
}
/// A random circuit of `ops` gates on `n` qubits: every base gate, up to three
/// distinct controls, `P(1..=32)`.
fn random_circuit(n: u8, ops: usize, seed: u64, gs: Gateset) -> Circuit {
let bases = [BaseGate::I, BaseGate::X, BaseGate::Y, BaseGate::Z, BaseGate::H, BaseGate::S, BaseGate::Sdg, BaseGate::T, BaseGate::Tdg, BaseGate::P];
let mut s = seed;
let mut c = Circuit::with_gateset(n, gs);
for _ in 0..ops {
let base = bases[(next(&mut s) % bases.len() as u64) as usize];
let target = (next(&mut s) % u64::from(n)) as u8;
let mut controls = Vec::new();
for _ in 0..next(&mut s) % 4 {
let q = (next(&mut s) % u64::from(n)) as u8;
if q != target && !controls.contains(&q) {
controls.push(q);
}
}
let param = if base == BaseGate::P { 1 + (next(&mut s) % 32) as u16 } else { 0 };
c.ops.push(Gate { base, controls, target, param });
}
c
}
/// Five circuits the native simulator refuses, in each gateset.
fn refused_circuits() -> Vec<Circuit> {
let gate = |base, controls: Vec<u8>, target, param| Gate { base, controls, target, param };
let bad = [
(1, vec![gate(BaseGate::H, vec![], 0, 0), gate(BaseGate::P, vec![], 0, 0)]),
(1, vec![gate(BaseGate::P, vec![], 0, 33)]),
(1, vec![gate(BaseGate::X, vec![], 3, 0)]),
(2, vec![gate(BaseGate::X, vec![0], 0, 0)]),
(2, vec![gate(BaseGate::X, vec![5], 1, 0)]),
];
let mut out = Vec::new();
for gs in Gateset::ALL {
for (n, ops) in bad.clone() {
let mut c = Circuit::with_gateset(n, gs);
c.ops = ops;
out.push(c);
}
}
out
}
/// `Z` on qubit 0: on `[(0, 0), (i32::MIN, 0)]` its exact result is `2^31`,
/// which leaves 32 bits.
fn z_words() -> GateWords {
lanes::gate_words(&Gate { base: BaseGate::Z, controls: vec![], target: 0, param: 0 }, Gateset::V2)
}
const Z_STATE: [Lane; 2] = [[0, 0], [i32::MIN, 0]];
#[test]
fn c0_devices_are_declared() {
let d = discovery();
match declared(d) {
Ok(expect) => report(&format!("dataflow devices: as declared ({expect})")),
Err(why) => panic!("{why}"),
}
for m in &d.found {
assert_eq!(m.class(), FabricClass::Dataflow);
assert!(m.id().starts_with("local/dataflow-"), "{}", m.id());
assert_eq!(m.implementation(), dataflow::implementation_id());
assert!(m.max_qubits() >= 17, "{}: holds only {} qubits", m.key(), m.max_qubits());
report(&format!("dataflow member: key={} id={} {}", m.key(), m.id(), m.detail()));
}
}
#[test]
fn c1_probe_matches_lanes() {
for m in members() {
let pairs = lanes::probe_pairs(1 << 16, 0x0cda);
let out = m.probe(&pairs).unwrap_or_else(|e| panic!("{}: {e}", m.key()));
assert_eq!(lanes::first_mismatch(&pairs, &out), None, "{}", m.key());
report(&format!("dataflow probe: key={} {} pairs agree", m.key(), pairs.len()));
}
}
#[test]
fn c2_kernel_matches_run_words() {
for m in members() {
let mut s = 0x6b65_726e_656c_u64;
let (mut flagged, mut clean) = (0, 0);
for case in 0..2000 {
let n = 1 + (next(&mut s) % 12) as u32;
// Three regimes: the whole 32-bit range (nearly always out of
// range), small states and entries (never), and states and
// entries up to 2^30 (either).
let mode = case % 3;
let value = |s: &mut u64| -> i32 {
match mode {
0 => next(s) as i32,
1 => within(s, 1 << 22),
_ => within(s, 1 << 30),
}
};
let entry = |s: &mut u64| -> u32 {
match mode {
0 => next(s) as u32,
1 => within(s, 1 << 29) as u32,
_ => within(s, 1 << 30) as u32,
}
};
let state: Vec<Lane> = (0..1usize << n).map(|_| [value(&mut s), value(&mut s)]).collect();
let gates: Vec<GateWords> = (0..1 + next(&mut s) % 8)
.map(|_| {
let t = (next(&mut s) % u64::from(n)) as u32;
let ctrl = (0..n).filter(|&b| b != t && next(&mut s).is_multiple_of(4)).fold(0u32, |acc, b| acc | 1 << b);
let mut w = [0u32; 8];
w.iter_mut().for_each(|x| *x = entry(&mut s));
GateWords { m: w, tbit: 1 << t, ctrl }
})
.collect();
let want = lanes::run_words(&state, &gates).expect("valid shapes");
let got = m.run_lanes(&state, &gates).unwrap_or_else(|e| panic!("{}: case {case}: {e}", m.key()));
assert_eq!(got.out_of_range, want.out_of_range, "{}: case {case}, n = {n}", m.key());
assert!(got.amps == want.amps, "{}: case {case}, n = {n}: the lanes differ", m.key());
if want.out_of_range.is_some() {
flagged += 1;
} else {
clean += 1;
}
}
assert!(flagged >= 100 && clean >= 100, "{}: {flagged} flagged, {clean} clean", m.key());
report(&format!("dataflow kernel: key={} ok", m.key()));
}
}
#[test]
fn c3_certificates_reproduce_the_pins() {
let ms = members();
for (gs, random_pin, fabric_pin) in PINS {
let cpu = lanes::certify_corpus(gs, "local/test", RANDOM_V1);
assert_eq!(hex(&cpu.corpus), random_pin, "{gs:?}");
let other = Corpus::Random { seed: 8, cases: 300 };
let cpu_other = lanes::certify_corpus(gs, "local/test", other);
for m in ms {
for (corpus, cases, pin) in [(RANDOM_V1, 200, random_pin), (FABRIC_DEFAULT, 224, fabric_pin)] {
let c = m.certify(gs, corpus).unwrap_or_else(|e| panic!("{} {gs:?} {}: {e}", m.key(), corpus.name()));
assert_eq!(c.parity, Parity::BitExact, "{} {gs:?} {}", m.key(), corpus.name());
assert_eq!(c.cases, cases);
assert_eq!(hex(&c.corpus), pin, "{} {gs:?} {}", m.key(), corpus.name());
assert_eq!((c.kernel.as_str(), c.implementation.as_str(), c.member.as_str()), (apply_kernel(gs), m.implementation(), m.id()));
assert_eq!(c.generator, corpus.generator(gs));
report(&format!(
"dataflow certified: key={} member={} kernel={} corpus={} implementation={} parity=bit-exact cases={} digest={}",
m.key(),
m.id(),
c.kernel,
corpus.name(),
c.implementation,
c.cases,
hex(&c.corpus)
));
}
// A corpus with no pin: the member and the CPU agree on it.
let c = m.certify(gs, other).unwrap_or_else(|e| panic!("{} {gs:?} seed 8: {e}", m.key()));
assert_eq!((c.parity, c.cases), (Parity::BitExact, 300), "{} {gs:?}", m.key());
assert_eq!(c.corpus, cpu_other.corpus, "{} {gs:?}", m.key());
}
}
}
#[test]
fn c4_wide_circuits_match_native() {
for m in members() {
let widest = m.max_qubits().min(20);
let mut s = 0x0077_6964_6500_u64;
for (k, n) in [12u8, 16, 18, widest].into_iter().enumerate() {
for gs in Gateset::ALL {
let c = random_circuit(n, 40, 0x1000 + k as u64, gs);
let start = (next(&mut s) as usize) % (1usize << n);
let want = c.simulate_from(start).expect("the native run");
let got = m.simulate_from(&c, start).unwrap_or_else(|e| panic!("{}: n = {n} {gs:?}: {e}", m.key()));
assert!(got == want, "{}: n = {n} {gs:?}: the states differ", m.key());
}
}
// The transforms the conformance corpus carries.
for n in [3u8, 5, 8, 12] {
let qft = Circuit::qft(n);
assert!(m.simulate(&qft).unwrap() == qft.simulate().unwrap(), "{}: QFT-{n}", m.key());
}
report(&format!("dataflow wide: key={} up to {widest} qubits match native", m.key()));
}
}
#[test]
fn c5_refusals_are_typed() {
for m in members() {
for c in refused_circuits() {
let native = c.simulate_from(0).unwrap_err();
assert_eq!(m.simulate_from(&c, 0).unwrap_err(), DataflowError::Circuit(native), "{c:?}");
}
assert_eq!(m.simulate(&Circuit::new(MAX_QUBITS + 1)).unwrap_err(), DataflowError::Circuit(QuantumError::TooManyQubits(MAX_QUBITS + 1)));
let small = m.with_max_qubits(4);
assert_eq!(small.simulate(&Circuit::new(5)).unwrap_err(), DataflowError::TooWide { n_qubits: 5, max: 4 });
let wide = Program { n_qubits: 5, start: 0, gates: Vec::new(), steps: Vec::new() };
assert_eq!(small.run_program(&wide).unwrap_err(), DataflowError::TooWide { n_qubits: 5, max: 4 });
// A result that leaves 32 bits is reported at its op, and the lane
// holds its low 32 bits.
let r = m.run_lanes(&Z_STATE, &[z_words()]).unwrap();
assert_eq!(r.out_of_range, Some(0), "{}", m.key());
assert_eq!(r.amps[1][0] as u32, 0x8000_0000);
// Malformed words, a state the host cannot lay out and a tile step are
// host defects, never results.
let bad = GateWords { tbit: 3, ..z_words() };
assert!(matches!(m.run_lanes(&[[0, 0]; 4], &[bad]), Err(DataflowError::Internal(_))));
assert!(matches!(m.run_lanes(&[[0, 0]; 3], &[z_words()]), Err(DataflowError::Internal(_))));
let tiled = Program { n_qubits: 2, start: 0, gates: Vec::new(), steps: vec![Step::Tile { first: 0, count: 1, size: 2 }] };
assert!(matches!(m.run_program(&tiled), Err(DataflowError::Internal(_))));
// The refusals that may still fall back to another member, do.
assert!(MemberError::from(DataflowError::TooWide { n_qubits: 5, max: 4 }).falls_back());
report(&format!("dataflow refusals: key={} typed", m.key()));
}
}
/// The kernel mutants, compiled in: each must be caught by the probe or by a
/// certificate.
#[test]
fn c6_mutants_are_killed() {
let edges = lanes::probe_pairs(0, 0);
for m in members() {
let mut killed = 0;
for v in Variant::MUTANTS {
let mm = m.with_variant(v);
assert_ne!(mm.implementation(), m.implementation());
let out = mm.probe(&edges).expect("the mutant's probe runs");
let differing = edges.iter().zip(&out).filter(|&(&(a, b), o)| *o != (lanes::mul_wide(a, b), lanes::fxmul_wide(a, b))).count();
let caught = differing > 0
|| Gateset::ALL.into_iter().all(|gs| match mm.certify(gs, RANDOM_V1) {
Err(_) => true,
Ok(c) => c.parity != Parity::BitExact || c.corpus != lanes::certify_corpus(gs, "local/test", RANDOM_V1).corpus,
});
println!("dataflow mutant {v:?} on {}: {differing} of {} edge pairs differ", m.key(), edges.len());
assert!(caught, "{}: mutant {v:?} survived", m.key());
killed += 1;
}
report(&format!("dataflow mutants: key={} killed {killed} of 3", m.key()));
}
}
#[test]
fn c7_receipt_placement_and_the_fabric() {
for m in members() {
let member = m.member(FABRIC_DEFAULT).unwrap_or_else(|e| panic!("{}: {e}", m.key()));
assert_eq!((member.id.as_str(), member.class, member.presence), (m.id(), FabricClass::Dataflow, Presence::Detected));
assert_eq!(member.kernels.len(), 2);
for (entry, gs) in member.kernels.iter().zip(Gateset::ALL) {
assert_eq!((entry.kernel.as_str(), entry.implementation.as_str()), (apply_kernel(gs), m.implementation()));
assert!(entry.energy.is_none());
assert_eq!(entry.certificate.parity, Parity::BitExact, "{gs:?}");
}
// A receipt over a member's run.
let qft = Circuit::qft(5);
let state = m.simulate(&qft).unwrap();
assert_eq!(state, qft.simulate().unwrap());
let entry = member.kernels.iter().find(|k| k.kernel == apply_kernel(qft.gateset)).unwrap();
let signer = SigningKey::from_bytes(&[7; 32]);
let receipt = FabricReceipt::seal(&signer, "local/test", state.statevector_hash(), &member, entry, 0, None);
assert!(receipt.verify());
assert!(receipt.certificate_matches(&entry.certificate));
// Placement: beside the CPU reference with no energy figures, the
// reference; alone, the member.
let local = detect_local();
for gs in Gateset::ALL {
let both = [local.clone(), member.clone()];
let (placed, k) = place(&both, apply_kernel(gs), &Parity::BitExact).unwrap();
assert_eq!((placed.id.as_str(), k.implementation.as_str()), (local.id.as_str(), REFERENCE));
let (placed, k) = place(std::slice::from_ref(&member), apply_kernel(gs), &Parity::BitExact).unwrap();
assert_eq!((placed.id.as_str(), k.implementation.as_str()), (m.id(), m.implementation()));
}
// On the fabric: the member alone runs the circuit; a pairing with
// another member's entries is refused.
let mut fabric = Fabric::new();
fabric.add(member.clone(), m);
let run = ready(fabric.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs");
assert_eq!(run.state, state);
assert_eq!(run.receipt.member_id, m.id());
assert!(run.receipt.verify());
let mut mismatched = Fabric::new();
mismatched.add(local.clone(), m);
assert!(ready(mismatched.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").is_err());
let mut both = Fabric::new();
both.add(local.clone(), CpuReference).add(member.clone(), m);
let run = ready(both.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs");
assert_eq!(run.state, state);
report(&format!("dataflow receipt: key={} ok", m.key()));
}
}
#[test]
fn c8_repeated_runs_agree() {
for m in members() {
let c = random_circuit(16, 60, 0xabc, Gateset::V2);
let first = m.simulate(&c).unwrap();
for _ in 0..5 {
assert!(m.simulate(&c).unwrap() == first, "{}", m.key());
}
report(&format!("dataflow repeat: key={} ok", m.key()));
}
}

410
wai/wai-quantum/tests/dsp_member.rs Normal file
View file

@ -0,0 +1,410 @@
//! The fabric's DSP vector-unit member on the machine that runs this.
//!
//! The machine declares the devices it must have in `WAI_QUANTUM_DSP_EXPECT`:
//! their keys, comma-separated and sorted, or `none`. Every test that needs a
//! member fails unless the devices found are exactly the declared ones, so a
//! machine whose GPU or driver went missing, or a run that declared nothing,
//! fails instead of passing over no member. Each test prints what it
//! established and, when `WAI_QUANTUM_DSP_REPORT` names a file, appends the
//! same lines to it, which a CI step can count per device.
//!
//! ```text
//! WAI_QUANTUM_DSP_EXPECT=dsp-vector \
//! cargo test --release --features quantum_fabric_dsp --target <the signal processor's> --test dsp_member -- --test-threads=1
//! ```
#![cfg(not(target_arch = "wasm32"))]
use std::io::Write;
use std::sync::OnceLock;
use wai_quantum::ed25519_dalek::SigningKey;
use wai_quantum::quantum::{BaseGate, Circuit, Gate, Gateset, MAX_QUBITS, QuantumError};
use wai_quantum::quantum_fabric::lanes::{self, GateWords, Lane};
use wai_quantum::quantum_fabric::plan::{Program, Step};
use wai_quantum::quantum_fabric::{
Corpus, CpuReference, FABRIC_DEFAULT, Fabric, FabricClass, FabricReceipt, MemberError, Parity, Presence, REFERENCE, apply_kernel, detect_local, place, ready,
};
use wai_quantum::quantum_fabric_dsp::{self as dsp, Discovery, DspError, DspMember, Variant, discover};
/// The CPU's corpus digests: random-v1 (seed 7, 200 cases) and fabric-v1
/// (seed 7, 200 + 24 cases), per gateset. A bit-exact member reproduces them.
const PINS: [(Gateset, &str, &str); 2] = [
(Gateset::V1, "d18454c639acfd9c7b66450f4ded1aac4ab46cc10a7bedbcf4958a96e204fb36", "41c726a0ac80f7defef92e15ecf893bdb7a3076b5e7f670fa0b7142e5c033df7"),
(Gateset::V2, "3dbde276c1095bf827faec71d5980e050f8a55a5ba7d97cd3bbdaec3f43a13b2", "9a84b55af68f1a8b07390b234cf562dd8d85cef3185a697ab85690c8790a7809"),
];
const RANDOM_V1: Corpus = Corpus::Random { seed: 7, cases: 200 };
static DISCOVERY: OnceLock<Discovery> = OnceLock::new();
fn discovery() -> &'static Discovery {
DISCOVERY.get_or_init(discover)
}
/// Prints `line` and appends it to the file `WAI_QUANTUM_DSP_REPORT` names.
fn report(line: &str) {
println!("{line}");
if let Some(path) = std::env::var_os("WAI_QUANTUM_DSP_REPORT") {
let mut f = std::fs::OpenOptions::new().create(true).append(true).open(&path).expect("the report file opens");
writeln!(f, "{line}").expect("the report line is written");
}
}
/// The declared device set, as written, if it is exactly what was found;
/// otherwise why not, with what was found and refused and how to declare it.
fn declared(d: &Discovery) -> Result<String, String> {
let mut found: Vec<&str> = d.found.iter().map(|m| m.key()).collect();
found.sort_unstable();
let found_list = if found.is_empty() { "none".to_string() } else { found.join(",") };
let refused: Vec<String> = d.refused.iter().map(|(k, e)| format!("{k}: {e}")).collect();
let refused_list = if refused.is_empty() { "none".to_string() } else { refused.join("; ") };
let unavailable = d.unavailable.as_deref().unwrap_or("no");
let how = format!("found: {found_list}; refused: {refused_list}; unavailable: {unavailable}. Declare the devices this machine must have: WAI_QUANTUM_DSP_EXPECT={found_list}");
let expect = match std::env::var("WAI_QUANTUM_DSP_EXPECT") {
Ok(v) if !v.is_empty() => v,
_ => return Err(format!("WAI_QUANTUM_DSP_EXPECT is not set. {how}")),
};
let keys: Vec<&str> = if expect == "none" { Vec::new() } else { expect.split(',').collect() };
let well_formed = keys.windows(2).all(|w| w[0] <= w[1]) && keys.iter().all(|k| !k.is_empty() && *k != "none" && k.trim() == *k);
if !well_formed {
return Err(format!("WAI_QUANTUM_DSP_EXPECT={expect:?} is not a sorted, comma-separated list of keys, nor none. {how}"));
}
for k in &keys {
if !found.contains(k)
&& let Some((_, e)) = d.refused.iter().find(|(key, _)| key == k)
{
return Err(format!("the declared device {k} was refused: {e}. {how}"));
}
}
if found != keys {
return Err(format!("the devices found are not the declared ones ({expect}). {how}"));
}
Ok(expect)
}
/// The members, once the declared devices are exactly the ones found.
fn members() -> &'static [DspMember] {
let d = discovery();
if let Err(why) = declared(d) {
panic!("{why}");
}
&d.found
}
fn hex(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}
fn next(s: &mut u64) -> u64 {
*s = s.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = *s;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^ (z >> 31)
}
/// A value in `[-bound, bound]`.
fn within(s: &mut u64, bound: i64) -> i32 {
((next(s) % (2 * bound as u64 + 1)) as i64 - bound) as i32
}
/// A random circuit of `ops` gates on `n` qubits: every base gate, up to three
/// distinct controls, `P(1..=32)`.
fn random_circuit(n: u8, ops: usize, seed: u64, gs: Gateset) -> Circuit {
let bases = [BaseGate::I, BaseGate::X, BaseGate::Y, BaseGate::Z, BaseGate::H, BaseGate::S, BaseGate::Sdg, BaseGate::T, BaseGate::Tdg, BaseGate::P];
let mut s = seed;
let mut c = Circuit::with_gateset(n, gs);
for _ in 0..ops {
let base = bases[(next(&mut s) % bases.len() as u64) as usize];
let target = (next(&mut s) % u64::from(n)) as u8;
let mut controls = Vec::new();
for _ in 0..next(&mut s) % 4 {
let q = (next(&mut s) % u64::from(n)) as u8;
if q != target && !controls.contains(&q) {
controls.push(q);
}
}
let param = if base == BaseGate::P { 1 + (next(&mut s) % 32) as u16 } else { 0 };
c.ops.push(Gate { base, controls, target, param });
}
c
}
/// Five circuits the native simulator refuses, in each gateset.
fn refused_circuits() -> Vec<Circuit> {
let gate = |base, controls: Vec<u8>, target, param| Gate { base, controls, target, param };
let bad = [
(1, vec![gate(BaseGate::H, vec![], 0, 0), gate(BaseGate::P, vec![], 0, 0)]),
(1, vec![gate(BaseGate::P, vec![], 0, 33)]),
(1, vec![gate(BaseGate::X, vec![], 3, 0)]),
(2, vec![gate(BaseGate::X, vec![0], 0, 0)]),
(2, vec![gate(BaseGate::X, vec![5], 1, 0)]),
];
let mut out = Vec::new();
for gs in Gateset::ALL {
for (n, ops) in bad.clone() {
let mut c = Circuit::with_gateset(n, gs);
c.ops = ops;
out.push(c);
}
}
out
}
/// `Z` on qubit 0: on `[(0, 0), (i32::MIN, 0)]` its exact result is `2^31`,
/// which leaves 32 bits.
fn z_words() -> GateWords {
lanes::gate_words(&Gate { base: BaseGate::Z, controls: vec![], target: 0, param: 0 }, Gateset::V2)
}
const Z_STATE: [Lane; 2] = [[0, 0], [i32::MIN, 0]];
#[test]
fn c0_devices_are_declared() {
let d = discovery();
match declared(d) {
Ok(expect) => report(&format!("dsp devices: as declared ({expect})")),
Err(why) => panic!("{why}"),
}
for m in &d.found {
assert_eq!(m.class(), FabricClass::Dsp);
assert!(m.id().starts_with("local/dsp-vector-"), "{}", m.id());
assert_eq!(m.implementation(), dsp::implementation_id());
assert!(m.max_qubits() >= 20, "{}: holds only {} qubits", m.key(), m.max_qubits());
report(&format!("dsp member: key={} id={} {}", m.key(), m.id(), m.detail()));
}
}
#[test]
fn c1_probe_matches_lanes() {
for m in members() {
let pairs = lanes::probe_pairs(1 << 16, 0x0cda);
let out = m.probe(&pairs).unwrap_or_else(|e| panic!("{}: {e}", m.key()));
assert_eq!(lanes::first_mismatch(&pairs, &out), None, "{}", m.key());
report(&format!("dsp probe: key={} {} pairs agree", m.key(), pairs.len()));
}
}
#[test]
fn c2_kernel_matches_run_words() {
for m in members() {
let mut s = 0x6b65_726e_656c_u64;
let (mut flagged, mut clean) = (0, 0);
for case in 0..2000 {
let n = 1 + (next(&mut s) % 12) as u32;
// Three regimes: the whole 32-bit range (nearly always out of
// range), small states and entries (never), and states and
// entries up to 2^30 (either).
let mode = case % 3;
let value = |s: &mut u64| -> i32 {
match mode {
0 => next(s) as i32,
1 => within(s, 1 << 22),
_ => within(s, 1 << 30),
}
};
let entry = |s: &mut u64| -> u32 {
match mode {
0 => next(s) as u32,
1 => within(s, 1 << 29) as u32,
_ => within(s, 1 << 30) as u32,
}
};
let state: Vec<Lane> = (0..1usize << n).map(|_| [value(&mut s), value(&mut s)]).collect();
let gates: Vec<GateWords> = (0..1 + next(&mut s) % 8)
.map(|_| {
let t = (next(&mut s) % u64::from(n)) as u32;
let ctrl = (0..n).filter(|&b| b != t && next(&mut s).is_multiple_of(4)).fold(0u32, |acc, b| acc | 1 << b);
let mut w = [0u32; 8];
w.iter_mut().for_each(|x| *x = entry(&mut s));
GateWords { m: w, tbit: 1 << t, ctrl }
})
.collect();
let want = lanes::run_words(&state, &gates).expect("valid shapes");
let got = m.run_lanes(&state, &gates).unwrap_or_else(|e| panic!("{}: case {case}: {e}", m.key()));
assert_eq!(got.out_of_range, want.out_of_range, "{}: case {case}, n = {n}", m.key());
assert!(got.amps == want.amps, "{}: case {case}, n = {n}: the lanes differ", m.key());
if want.out_of_range.is_some() {
flagged += 1;
} else {
clean += 1;
}
}
assert!(flagged >= 100 && clean >= 100, "{}: {flagged} flagged, {clean} clean", m.key());
report(&format!("dsp kernel: key={} ok", m.key()));
}
}
#[test]
fn c3_certificates_reproduce_the_pins() {
let ms = members();
for (gs, random_pin, fabric_pin) in PINS {
let cpu = lanes::certify_corpus(gs, "local/test", RANDOM_V1);
assert_eq!(hex(&cpu.corpus), random_pin, "{gs:?}");
let other = Corpus::Random { seed: 8, cases: 300 };
let cpu_other = lanes::certify_corpus(gs, "local/test", other);
for m in ms {
for (corpus, cases, pin) in [(RANDOM_V1, 200, random_pin), (FABRIC_DEFAULT, 224, fabric_pin)] {
let c = m.certify(gs, corpus).unwrap_or_else(|e| panic!("{} {gs:?} {}: {e}", m.key(), corpus.name()));
assert_eq!(c.parity, Parity::BitExact, "{} {gs:?} {}", m.key(), corpus.name());
assert_eq!(c.cases, cases);
assert_eq!(hex(&c.corpus), pin, "{} {gs:?} {}", m.key(), corpus.name());
assert_eq!((c.kernel.as_str(), c.implementation.as_str(), c.member.as_str()), (apply_kernel(gs), m.implementation(), m.id()));
assert_eq!(c.generator, corpus.generator(gs));
report(&format!(
"dsp certified: key={} member={} kernel={} corpus={} implementation={} parity=bit-exact cases={} digest={}",
m.key(),
m.id(),
c.kernel,
corpus.name(),
c.implementation,
c.cases,
hex(&c.corpus)
));
}
// A corpus with no pin: the member and the CPU agree on it.
let c = m.certify(gs, other).unwrap_or_else(|e| panic!("{} {gs:?} seed 8: {e}", m.key()));
assert_eq!((c.parity, c.cases), (Parity::BitExact, 300), "{} {gs:?}", m.key());
assert_eq!(c.corpus, cpu_other.corpus, "{} {gs:?}", m.key());
}
}
}
#[test]
fn c4_wide_circuits_match_native() {
for m in members() {
let widest = m.max_qubits().min(20);
let mut s = 0x0077_6964_6500_u64;
for (k, n) in [12u8, 16, 18, widest].into_iter().enumerate() {
for gs in Gateset::ALL {
let c = random_circuit(n, 40, 0x1000 + k as u64, gs);
let start = (next(&mut s) as usize) % (1usize << n);
let want = c.simulate_from(start).expect("the native run");
let got = m.simulate_from(&c, start).unwrap_or_else(|e| panic!("{}: n = {n} {gs:?}: {e}", m.key()));
assert!(got == want, "{}: n = {n} {gs:?}: the states differ", m.key());
}
}
// The transforms the conformance corpus carries.
for n in [3u8, 5, 8, 12] {
let qft = Circuit::qft(n);
assert!(m.simulate(&qft).unwrap() == qft.simulate().unwrap(), "{}: QFT-{n}", m.key());
}
report(&format!("dsp wide: key={} up to {widest} qubits match native", m.key()));
}
}
#[test]
fn c5_refusals_are_typed() {
for m in members() {
for c in refused_circuits() {
let native = c.simulate_from(0).unwrap_err();
assert_eq!(m.simulate_from(&c, 0).unwrap_err(), DspError::Circuit(native), "{c:?}");
}
assert_eq!(m.simulate(&Circuit::new(MAX_QUBITS + 1)).unwrap_err(), DspError::Circuit(QuantumError::TooManyQubits(MAX_QUBITS + 1)));
let small = m.with_max_qubits(4);
assert_eq!(small.simulate(&Circuit::new(5)).unwrap_err(), DspError::TooWide { n_qubits: 5, max: 4 });
let wide = Program { n_qubits: 5, start: 0, gates: Vec::new(), steps: Vec::new() };
assert_eq!(small.run_program(&wide).unwrap_err(), DspError::TooWide { n_qubits: 5, max: 4 });
// A result that leaves 32 bits is reported at its op, and the lane
// holds its low 32 bits.
let r = m.run_lanes(&Z_STATE, &[z_words()]).unwrap();
assert_eq!(r.out_of_range, Some(0), "{}", m.key());
assert_eq!(r.amps[1][0] as u32, 0x8000_0000);
// Malformed words, a state the host cannot lay out and a tile step are
// host defects, never results.
let bad = GateWords { tbit: 3, ..z_words() };
assert!(matches!(m.run_lanes(&[[0, 0]; 4], &[bad]), Err(DspError::Internal(_))));
assert!(matches!(m.run_lanes(&[[0, 0]; 3], &[z_words()]), Err(DspError::Internal(_))));
let tiled = Program { n_qubits: 2, start: 0, gates: Vec::new(), steps: vec![Step::Tile { first: 0, count: 1, size: 2 }] };
assert!(matches!(m.run_program(&tiled), Err(DspError::Internal(_))));
// The refusals that may still fall back to another member, do.
assert!(MemberError::from(DspError::TooWide { n_qubits: 5, max: 4 }).falls_back());
report(&format!("dsp refusals: key={} typed", m.key()));
}
}
/// The kernel mutants, compiled in: each must be caught by the probe or by a
/// certificate.
#[test]
fn c6_mutants_are_killed() {
let edges = lanes::probe_pairs(0, 0);
for m in members() {
let mut killed = 0;
for v in Variant::MUTANTS {
let mm = m.with_variant(v);
assert_ne!(mm.implementation(), m.implementation());
let out = mm.probe(&edges).expect("the mutant's probe runs");
let differing = edges.iter().zip(&out).filter(|&(&(a, b), o)| *o != (lanes::mul_wide(a, b), lanes::fxmul_wide(a, b))).count();
let caught = differing > 0
|| Gateset::ALL.into_iter().all(|gs| match mm.certify(gs, RANDOM_V1) {
Err(_) => true,
Ok(c) => c.parity != Parity::BitExact || c.corpus != lanes::certify_corpus(gs, "local/test", RANDOM_V1).corpus,
});
println!("dsp mutant {v:?} on {}: {differing} of {} edge pairs differ", m.key(), edges.len());
assert!(caught, "{}: mutant {v:?} survived", m.key());
killed += 1;
}
report(&format!("dsp mutants: key={} killed {killed} of 3", m.key()));
}
}
#[test]
fn c7_receipt_placement_and_the_fabric() {
for m in members() {
let member = m.member(FABRIC_DEFAULT).unwrap_or_else(|e| panic!("{}: {e}", m.key()));
assert_eq!((member.id.as_str(), member.class, member.presence), (m.id(), FabricClass::Dsp, Presence::Detected));
assert_eq!(member.kernels.len(), 2);
for (entry, gs) in member.kernels.iter().zip(Gateset::ALL) {
assert_eq!((entry.kernel.as_str(), entry.implementation.as_str()), (apply_kernel(gs), m.implementation()));
assert!(entry.energy.is_none());
assert_eq!(entry.certificate.parity, Parity::BitExact, "{gs:?}");
}
// A receipt over a member's run.
let qft = Circuit::qft(5);
let state = m.simulate(&qft).unwrap();
assert_eq!(state, qft.simulate().unwrap());
let entry = member.kernels.iter().find(|k| k.kernel == apply_kernel(qft.gateset)).unwrap();
let signer = SigningKey::from_bytes(&[7; 32]);
let receipt = FabricReceipt::seal(&signer, "local/test", state.statevector_hash(), &member, entry, 0, None);
assert!(receipt.verify());
assert!(receipt.certificate_matches(&entry.certificate));
// Placement: beside the CPU reference with no energy figures, the
// reference; alone, the member.
let local = detect_local();
for gs in Gateset::ALL {
let both = [local.clone(), member.clone()];
let (placed, k) = place(&both, apply_kernel(gs), &Parity::BitExact).unwrap();
assert_eq!((placed.id.as_str(), k.implementation.as_str()), (local.id.as_str(), REFERENCE));
let (placed, k) = place(std::slice::from_ref(&member), apply_kernel(gs), &Parity::BitExact).unwrap();
assert_eq!((placed.id.as_str(), k.implementation.as_str()), (m.id(), m.implementation()));
}
// On the fabric: the member alone runs the circuit; a pairing with
// another member's entries is refused.
let mut fabric = Fabric::new();
fabric.add(member.clone(), m);
let run = ready(fabric.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs");
assert_eq!(run.state, state);
assert_eq!(run.receipt.member_id, m.id());
assert!(run.receipt.verify());
let mut mismatched = Fabric::new();
mismatched.add(local.clone(), m);
assert!(ready(mismatched.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").is_err());
let mut both = Fabric::new();
both.add(local.clone(), CpuReference).add(member.clone(), m);
let run = ready(both.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs");
assert_eq!(run.state, state);
report(&format!("dsp receipt: key={} ok", m.key()));
}
}
#[test]
fn c8_repeated_runs_agree() {
for m in members() {
let c = random_circuit(16, 60, 0xabc, Gateset::V2);
let first = m.simulate(&c).unwrap();
for _ in 0..5 {
assert!(m.simulate(&c).unwrap() == first, "{}", m.key());
}
report(&format!("dsp repeat: key={} ok", m.key()));
}
}

413
wai/wai-quantum/tests/npu_member.rs Normal file
View file

@ -0,0 +1,413 @@
//! The fabric's neural-processor member on the machine that runs this.
//!
//! The machine declares the devices it must have in `WAI_QUANTUM_NPU_EXPECT`:
//! their keys, comma-separated and sorted, or `none`. Every test that needs a
//! member fails unless the devices found are exactly the declared ones, so a
//! machine whose GPU or driver went missing, or a run that declared nothing,
//! fails instead of passing over no member. Each test prints what it
//! established and, when `WAI_QUANTUM_NPU_REPORT` names a file, appends the
//! same lines to it, which a CI step can count per device.
//!
//! ```text
//! WAI_QUANTUM_NPU_EXPECT=npu-0 \
//! cargo test --release --features quantum_fabric_npu --test npu_member -- --test-threads=1
//! ```
#![cfg(not(target_arch = "wasm32"))]
use std::io::Write;
use std::sync::OnceLock;
use wai_quantum::ed25519_dalek::SigningKey;
use wai_quantum::quantum::{BaseGate, Circuit, Gate, Gateset, MAX_QUBITS, QuantumError};
use wai_quantum::quantum_fabric::npu;
use wai_quantum::quantum_fabric::lanes::{self, GateWords, Lane};
use wai_quantum::quantum_fabric::plan::{Program, Step};
use wai_quantum::quantum_fabric::{
Corpus, CpuReference, FABRIC_DEFAULT, Fabric, FabricClass, FabricReceipt, MemberError, Parity, Presence, REFERENCE, apply_kernel, detect_local, place, ready,
};
use wai_quantum::quantum_fabric_npu::{Discovery, NpuError, NpuMember, discover};
/// The CPU's corpus digests: random-v1 (seed 7, 200 cases) and fabric-v1
/// (seed 7, 200 + 24 cases), per gateset. A bit-exact member reproduces them.
const PINS: [(Gateset, &str, &str); 2] = [
(Gateset::V1, "d18454c639acfd9c7b66450f4ded1aac4ab46cc10a7bedbcf4958a96e204fb36", "41c726a0ac80f7defef92e15ecf893bdb7a3076b5e7f670fa0b7142e5c033df7"),
(Gateset::V2, "3dbde276c1095bf827faec71d5980e050f8a55a5ba7d97cd3bbdaec3f43a13b2", "9a84b55af68f1a8b07390b234cf562dd8d85cef3185a697ab85690c8790a7809"),
];
const RANDOM_V1: Corpus = Corpus::Random { seed: 7, cases: 200 };
static DISCOVERY: OnceLock<Discovery> = OnceLock::new();
fn discovery() -> &'static Discovery {
DISCOVERY.get_or_init(discover)
}
/// Prints `line` and appends it to the file `WAI_QUANTUM_NPU_REPORT` names.
fn report(line: &str) {
println!("{line}");
if let Some(path) = std::env::var_os("WAI_QUANTUM_NPU_REPORT") {
let mut f = std::fs::OpenOptions::new().create(true).append(true).open(&path).expect("the report file opens");
writeln!(f, "{line}").expect("the report line is written");
}
}
/// The declared device set, as written, if it is exactly what was found;
/// otherwise why not, with what was found and refused and how to declare it.
fn declared(d: &Discovery) -> Result<String, String> {
let mut found: Vec<&str> = d.found.iter().map(|m| m.key()).collect();
found.sort_unstable();
let found_list = if found.is_empty() { "none".to_string() } else { found.join(",") };
let refused: Vec<String> = d.refused.iter().map(|(k, e)| format!("{k}: {e}")).collect();
let refused_list = if refused.is_empty() { "none".to_string() } else { refused.join("; ") };
let unavailable = d.unavailable.as_deref().unwrap_or("no");
let how = format!("found: {found_list}; refused: {refused_list}; unavailable: {unavailable}. Declare the devices this machine must have: WAI_QUANTUM_NPU_EXPECT={found_list}");
let expect = match std::env::var("WAI_QUANTUM_NPU_EXPECT") {
Ok(v) if !v.is_empty() => v,
_ => return Err(format!("WAI_QUANTUM_NPU_EXPECT is not set. {how}")),
};
let keys: Vec<&str> = if expect == "none" { Vec::new() } else { expect.split(',').collect() };
let well_formed = keys.windows(2).all(|w| w[0] <= w[1]) && keys.iter().all(|k| !k.is_empty() && *k != "none" && k.trim() == *k);
if !well_formed {
return Err(format!("WAI_QUANTUM_NPU_EXPECT={expect:?} is not a sorted, comma-separated list of keys, nor none. {how}"));
}
for k in &keys {
if !found.contains(k)
&& let Some((_, e)) = d.refused.iter().find(|(key, _)| key == k)
{
return Err(format!("the declared device {k} was refused: {e}. {how}"));
}
}
if found != keys {
return Err(format!("the devices found are not the declared ones ({expect}). {how}"));
}
Ok(expect)
}
/// The members, once the declared devices are exactly the ones found.
fn members() -> &'static [NpuMember] {
let d = discovery();
if let Err(why) = declared(d) {
panic!("{why}");
}
&d.found
}
fn hex(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}
fn next(s: &mut u64) -> u64 {
*s = s.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = *s;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^ (z >> 31)
}
/// A value in `[-bound, bound]`.
fn within(s: &mut u64, bound: i64) -> i32 {
((next(s) % (2 * bound as u64 + 1)) as i64 - bound) as i32
}
/// A random circuit of `ops` gates on `n` qubits: every base gate, up to three
/// distinct controls, `P(1..=32)`.
fn random_circuit(n: u8, ops: usize, seed: u64, gs: Gateset) -> Circuit {
let bases = [BaseGate::I, BaseGate::X, BaseGate::Y, BaseGate::Z, BaseGate::H, BaseGate::S, BaseGate::Sdg, BaseGate::T, BaseGate::Tdg, BaseGate::P];
let mut s = seed;
let mut c = Circuit::with_gateset(n, gs);
for _ in 0..ops {
let base = bases[(next(&mut s) % bases.len() as u64) as usize];
let target = (next(&mut s) % u64::from(n)) as u8;
let mut controls = Vec::new();
for _ in 0..next(&mut s) % 4 {
let q = (next(&mut s) % u64::from(n)) as u8;
if q != target && !controls.contains(&q) {
controls.push(q);
}
}
let param = if base == BaseGate::P { 1 + (next(&mut s) % 32) as u16 } else { 0 };
c.ops.push(Gate { base, controls, target, param });
}
c
}
/// Five circuits the native simulator refuses, in each gateset.
fn refused_circuits() -> Vec<Circuit> {
let gate = |base, controls: Vec<u8>, target, param| Gate { base, controls, target, param };
let bad = [
(1, vec![gate(BaseGate::H, vec![], 0, 0), gate(BaseGate::P, vec![], 0, 0)]),
(1, vec![gate(BaseGate::P, vec![], 0, 33)]),
(1, vec![gate(BaseGate::X, vec![], 3, 0)]),
(2, vec![gate(BaseGate::X, vec![0], 0, 0)]),
(2, vec![gate(BaseGate::X, vec![5], 1, 0)]),
];
let mut out = Vec::new();
for gs in Gateset::ALL {
for (n, ops) in bad.clone() {
let mut c = Circuit::with_gateset(n, gs);
c.ops = ops;
out.push(c);
}
}
out
}
/// `Z` on qubit 0: on `[(0, 0), (i32::MIN, 0)]` its exact result is `2^31`,
/// which leaves 32 bits.
fn z_words() -> GateWords {
lanes::gate_words(&Gate { base: BaseGate::Z, controls: vec![], target: 0, param: 0 }, Gateset::V2)
}
const Z_STATE: [Lane; 2] = [[0, 0], [i32::MIN, 0]];
#[test]
fn c0_devices_are_declared() {
let d = discovery();
match declared(d) {
Ok(expect) => report(&format!("npu devices: as declared ({expect})")),
Err(why) => panic!("{why}"),
}
for m in &d.found {
assert_eq!(m.class(), FabricClass::Npu);
assert!(m.id().starts_with("local/npu-"), "{}", m.id());
assert_eq!(m.implementation(), npu::implementation_id());
assert!(!m.placement().is_empty() && m.placement().iter().all(|(_, d)| d == wai_quantum::quantum_fabric_npu::NEURAL_PROCESSOR), "{:?}", m.placement());
report(&format!("npu placement: key={} {}", m.key(), m.placement().iter().map(|(op, d)| format!("{op}={d}")).collect::<Vec<_>>().join(" ")));
assert!(m.max_qubits() >= 20, "{}: holds only {} qubits", m.key(), m.max_qubits());
report(&format!("npu member: key={} id={} {}", m.key(), m.id(), m.detail()));
}
}
#[test]
fn c1_probe_matches_lanes() {
for m in members() {
let pairs = lanes::probe_pairs(1 << 16, 0x0cda);
let out = m.probe(&pairs).unwrap_or_else(|e| panic!("{}: {e}", m.key()));
assert_eq!(lanes::first_mismatch(&pairs, &out), None, "{}", m.key());
report(&format!("npu probe: key={} {} pairs agree", m.key(), pairs.len()));
}
}
#[test]
fn c2_kernel_matches_run_words() {
for m in members() {
let mut s = 0x6b65_726e_656c_u64;
let (mut flagged, mut clean) = (0, 0);
// Fewer cases than the GPU members': each gate's products cross in batches.
for case in 0..600 {
let n = 1 + (next(&mut s) % 12) as u32;
// Three regimes: the whole 32-bit range (nearly always out of
// range), small states and entries (never), and states and
// entries up to 2^30 (either).
let mode = case % 3;
let value = |s: &mut u64| -> i32 {
match mode {
0 => next(s) as i32,
1 => within(s, 1 << 22),
_ => within(s, 1 << 30),
}
};
let entry = |s: &mut u64| -> u32 {
match mode {
0 => next(s) as u32,
1 => within(s, 1 << 29) as u32,
_ => within(s, 1 << 30) as u32,
}
};
let state: Vec<Lane> = (0..1usize << n).map(|_| [value(&mut s), value(&mut s)]).collect();
let gates: Vec<GateWords> = (0..1 + next(&mut s) % 8)
.map(|_| {
let t = (next(&mut s) % u64::from(n)) as u32;
let ctrl = (0..n).filter(|&b| b != t && next(&mut s).is_multiple_of(4)).fold(0u32, |acc, b| acc | 1 << b);
let mut w = [0u32; 8];
w.iter_mut().for_each(|x| *x = entry(&mut s));
GateWords { m: w, tbit: 1 << t, ctrl }
})
.collect();
let want = lanes::run_words(&state, &gates).expect("valid shapes");
let got = m.run_lanes(&state, &gates).unwrap_or_else(|e| panic!("{}: case {case}: {e}", m.key()));
assert_eq!(got.out_of_range, want.out_of_range, "{}: case {case}, n = {n}", m.key());
assert!(got.amps == want.amps, "{}: case {case}, n = {n}: the lanes differ", m.key());
if want.out_of_range.is_some() {
flagged += 1;
} else {
clean += 1;
}
}
assert!(flagged >= 100 && clean >= 100, "{}: {flagged} flagged, {clean} clean", m.key());
report(&format!("npu kernel: key={} ok", m.key()));
}
}
#[test]
fn c3_certificates_reproduce_the_pins() {
let ms = members();
for (gs, random_pin, fabric_pin) in PINS {
let cpu = lanes::certify_corpus(gs, "local/test", RANDOM_V1);
assert_eq!(hex(&cpu.corpus), random_pin, "{gs:?}");
let other = Corpus::Random { seed: 8, cases: 300 };
let cpu_other = lanes::certify_corpus(gs, "local/test", other);
for m in ms {
for (corpus, cases, pin) in [(RANDOM_V1, 200, random_pin), (FABRIC_DEFAULT, 224, fabric_pin)] {
let c = m.certify(gs, corpus).unwrap_or_else(|e| panic!("{} {gs:?} {}: {e}", m.key(), corpus.name()));
assert_eq!(c.parity, Parity::BitExact, "{} {gs:?} {}", m.key(), corpus.name());
assert_eq!(c.cases, cases);
assert_eq!(hex(&c.corpus), pin, "{} {gs:?} {}", m.key(), corpus.name());
assert_eq!((c.kernel.as_str(), c.implementation.as_str(), c.member.as_str()), (apply_kernel(gs), m.implementation(), m.id()));
assert_eq!(c.generator, corpus.generator(gs));
report(&format!(
"npu certified: key={} member={} kernel={} corpus={} implementation={} parity=bit-exact cases={} digest={}",
m.key(),
m.id(),
c.kernel,
corpus.name(),
c.implementation,
c.cases,
hex(&c.corpus)
));
}
// A corpus with no pin: the member and the CPU agree on it.
let c = m.certify(gs, other).unwrap_or_else(|e| panic!("{} {gs:?} seed 8: {e}", m.key()));
assert_eq!((c.parity, c.cases), (Parity::BitExact, 300), "{} {gs:?}", m.key());
assert_eq!(c.corpus, cpu_other.corpus, "{} {gs:?}", m.key());
}
}
}
#[test]
fn c4_wide_circuits_match_native() {
for m in members() {
let widest = m.max_qubits().min(16);
let mut s = 0x0077_6964_6500_u64;
for (k, n) in [10u8, 12, 14, widest].into_iter().enumerate() {
for gs in Gateset::ALL {
let c = random_circuit(n, 40, 0x1000 + k as u64, gs);
let start = (next(&mut s) as usize) % (1usize << n);
let want = c.simulate_from(start).expect("the native run");
let got = m.simulate_from(&c, start).unwrap_or_else(|e| panic!("{}: n = {n} {gs:?}: {e}", m.key()));
assert!(got == want, "{}: n = {n} {gs:?}: the states differ", m.key());
}
}
// The transforms the conformance corpus carries.
for n in [3u8, 5, 8, 12] {
let qft = Circuit::qft(n);
assert!(m.simulate(&qft).unwrap() == qft.simulate().unwrap(), "{}: QFT-{n}", m.key());
}
report(&format!("npu wide: key={} up to {widest} qubits match native", m.key()));
}
}
#[test]
fn c5_refusals_are_typed() {
for m in members() {
for c in refused_circuits() {
let native = c.simulate_from(0).unwrap_err();
assert_eq!(m.simulate_from(&c, 0).unwrap_err(), NpuError::Circuit(native), "{c:?}");
}
assert_eq!(m.simulate(&Circuit::new(MAX_QUBITS + 1)).unwrap_err(), NpuError::Circuit(QuantumError::TooManyQubits(MAX_QUBITS + 1)));
let small = m.with_max_qubits(4);
assert_eq!(small.simulate(&Circuit::new(5)).unwrap_err(), NpuError::TooWide { n_qubits: 5, max: 4 });
let wide = Program { n_qubits: 5, start: 0, gates: Vec::new(), steps: Vec::new() };
assert_eq!(small.run_program(&wide).unwrap_err(), NpuError::TooWide { n_qubits: 5, max: 4 });
// A result that leaves 32 bits is reported at its op, and the lane
// holds its low 32 bits.
let r = m.run_lanes(&Z_STATE, &[z_words()]).unwrap();
assert_eq!(r.out_of_range, Some(0), "{}", m.key());
assert_eq!(r.amps[1][0] as u32, 0x8000_0000);
// Malformed words, a state the host cannot lay out and a tile step are
// host defects, never results.
let bad = GateWords { tbit: 3, ..z_words() };
assert!(matches!(m.run_lanes(&[[0, 0]; 4], &[bad]), Err(NpuError::Internal(_))));
assert!(matches!(m.run_lanes(&[[0, 0]; 3], &[z_words()]), Err(NpuError::Internal(_))));
let tiled = Program { n_qubits: 2, start: 0, gates: Vec::new(), steps: vec![Step::Tile { first: 0, count: 1, size: 2 }] };
assert!(matches!(m.run_program(&tiled), Err(NpuError::Internal(_))));
// The refusals that may still fall back to another member, do.
assert!(MemberError::from(NpuError::TooWide { n_qubits: 5, max: 4 }).falls_back());
report(&format!("npu refusals: key={} typed", m.key()));
}
}
/// The model mutants: one operation replaced. Each must be caught: refused at
/// load (the runtime places it elsewhere or rejects it) or by the probe.
#[test]
fn c6_mutants_are_killed() {
let edges = lanes::probe_pairs(0, 0);
for m in members() {
let mut killed = 0;
for (from, to) in [("mul", "add"), ("reduce_sum", "reduce_max")] {
match m.with_model(&npu::mutate_model(from, to)) {
Err(e) => println!("npu mutant {from}->{to} on {}: refused: {e}", m.key()),
Ok(mm) => {
assert_ne!(mm.implementation(), m.implementation());
let out = mm.probe(&edges).expect("the mutant's probe runs");
let differing = edges.iter().zip(&out).filter(|&(&(a, b), o)| *o != (lanes::mul_wide(a, b), lanes::fxmul_wide(a, b))).count();
println!("npu mutant {from}->{to} on {}: {differing} of {} edge pairs differ", m.key(), edges.len());
assert!(differing >= 1, "{}: mutant {from}->{to} survived", m.key());
}
}
killed += 1;
}
report(&format!("npu mutants: key={} killed {killed} of 2", m.key()));
}
}
#[test]
fn c7_receipt_placement_and_the_fabric() {
for m in members() {
let member = m.member(FABRIC_DEFAULT).unwrap_or_else(|e| panic!("{}: {e}", m.key()));
assert_eq!((member.id.as_str(), member.class, member.presence), (m.id(), FabricClass::Npu, Presence::Detected));
assert_eq!(member.kernels.len(), 2);
for (entry, gs) in member.kernels.iter().zip(Gateset::ALL) {
assert_eq!((entry.kernel.as_str(), entry.implementation.as_str()), (apply_kernel(gs), m.implementation()));
assert!(entry.energy.is_none());
assert_eq!(entry.certificate.parity, Parity::BitExact, "{gs:?}");
}
// A receipt over a member's run.
let qft = Circuit::qft(5);
let state = m.simulate(&qft).unwrap();
assert_eq!(state, qft.simulate().unwrap());
let entry = member.kernels.iter().find(|k| k.kernel == apply_kernel(qft.gateset)).unwrap();
let signer = SigningKey::from_bytes(&[7; 32]);
let receipt = FabricReceipt::seal(&signer, "local/test", state.statevector_hash(), &member, entry, 0, None);
assert!(receipt.verify());
assert!(receipt.certificate_matches(&entry.certificate));
// Placement: beside the CPU reference with no energy figures, the
// reference; alone, the member.
let local = detect_local();
for gs in Gateset::ALL {
let both = [local.clone(), member.clone()];
let (placed, k) = place(&both, apply_kernel(gs), &Parity::BitExact).unwrap();
assert_eq!((placed.id.as_str(), k.implementation.as_str()), (local.id.as_str(), REFERENCE));
let (placed, k) = place(std::slice::from_ref(&member), apply_kernel(gs), &Parity::BitExact).unwrap();
assert_eq!((placed.id.as_str(), k.implementation.as_str()), (m.id(), m.implementation()));
}
// On the fabric: the member alone runs the circuit; a pairing with
// another member's entries is refused.
let mut fabric = Fabric::new();
fabric.add(member.clone(), m);
let run = ready(fabric.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs");
assert_eq!(run.state, state);
assert_eq!(run.receipt.member_id, m.id());
assert!(run.receipt.verify());
let mut mismatched = Fabric::new();
mismatched.add(local.clone(), m);
assert!(ready(mismatched.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").is_err());
let mut both = Fabric::new();
both.add(local.clone(), CpuReference).add(member.clone(), m);
let run = ready(both.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs");
assert_eq!(run.state, state);
report(&format!("npu receipt: key={} ok", m.key()));
}
}
#[test]
fn c8_repeated_runs_agree() {
for m in members() {
let c = random_circuit(16, 60, 0xabc, Gateset::V2);
let first = m.simulate(&c).unwrap();
for _ in 0..5 {
assert!(m.simulate(&c).unwrap() == first, "{}", m.key());
}
report(&format!("npu repeat: key={} ok", m.key()));
}
}

410
wai/wai-quantum/tests/sme_member.rs Normal file
View file

@ -0,0 +1,410 @@
//! The fabric's matrix-extension member on the machine that runs this.
//!
//! The machine declares the devices it must have in `WAI_QUANTUM_SME_EXPECT`:
//! their keys, comma-separated and sorted, or `none`. Every test that needs a
//! member fails unless the devices found are exactly the declared ones, so a
//! machine whose GPU or driver went missing, or a run that declared nothing,
//! fails instead of passing over no member. Each test prints what it
//! established and, when `WAI_QUANTUM_SME_REPORT` names a file, appends the
//! same lines to it, which a CI step can count per device.
//!
//! ```text
//! WAI_QUANTUM_SME_EXPECT=cpu-matrix-sme \
//! cargo test --release --features quantum_fabric_sme --test sme_member -- --test-threads=1
//! ```
#![cfg(not(target_arch = "wasm32"))]
use std::io::Write;
use std::sync::OnceLock;
use wai_quantum::ed25519_dalek::SigningKey;
use wai_quantum::quantum::{BaseGate, Circuit, Gate, Gateset, MAX_QUBITS, QuantumError};
use wai_quantum::quantum_fabric::lanes::{self, GateWords, Lane};
use wai_quantum::quantum_fabric::plan::{Program, Step};
use wai_quantum::quantum_fabric::{
Corpus, CpuReference, FABRIC_DEFAULT, Fabric, FabricClass, FabricReceipt, MemberError, Parity, Presence, REFERENCE, apply_kernel, detect_local, place, ready,
};
use wai_quantum::quantum_fabric_sme::{self as sme, Discovery, SmeError, SmeMember, Variant, discover};
/// The CPU's corpus digests: random-v1 (seed 7, 200 cases) and fabric-v1
/// (seed 7, 200 + 24 cases), per gateset. A bit-exact member reproduces them.
const PINS: [(Gateset, &str, &str); 2] = [
(Gateset::V1, "d18454c639acfd9c7b66450f4ded1aac4ab46cc10a7bedbcf4958a96e204fb36", "41c726a0ac80f7defef92e15ecf893bdb7a3076b5e7f670fa0b7142e5c033df7"),
(Gateset::V2, "3dbde276c1095bf827faec71d5980e050f8a55a5ba7d97cd3bbdaec3f43a13b2", "9a84b55af68f1a8b07390b234cf562dd8d85cef3185a697ab85690c8790a7809"),
];
const RANDOM_V1: Corpus = Corpus::Random { seed: 7, cases: 200 };
static DISCOVERY: OnceLock<Discovery> = OnceLock::new();
fn discovery() -> &'static Discovery {
DISCOVERY.get_or_init(discover)
}
/// Prints `line` and appends it to the file `WAI_QUANTUM_SME_REPORT` names.
fn report(line: &str) {
println!("{line}");
if let Some(path) = std::env::var_os("WAI_QUANTUM_SME_REPORT") {
let mut f = std::fs::OpenOptions::new().create(true).append(true).open(&path).expect("the report file opens");
writeln!(f, "{line}").expect("the report line is written");
}
}
/// The declared device set, as written, if it is exactly what was found;
/// otherwise why not, with what was found and refused and how to declare it.
fn declared(d: &Discovery) -> Result<String, String> {
let mut found: Vec<&str> = d.found.iter().map(|m| m.key()).collect();
found.sort_unstable();
let found_list = if found.is_empty() { "none".to_string() } else { found.join(",") };
let refused: Vec<String> = d.refused.iter().map(|(k, e)| format!("{k}: {e}")).collect();
let refused_list = if refused.is_empty() { "none".to_string() } else { refused.join("; ") };
let unavailable = d.unavailable.as_deref().unwrap_or("no");
let how = format!("found: {found_list}; refused: {refused_list}; unavailable: {unavailable}. Declare the devices this machine must have: WAI_QUANTUM_SME_EXPECT={found_list}");
let expect = match std::env::var("WAI_QUANTUM_SME_EXPECT") {
Ok(v) if !v.is_empty() => v,
_ => return Err(format!("WAI_QUANTUM_SME_EXPECT is not set. {how}")),
};
let keys: Vec<&str> = if expect == "none" { Vec::new() } else { expect.split(',').collect() };
let well_formed = keys.windows(2).all(|w| w[0] <= w[1]) && keys.iter().all(|k| !k.is_empty() && *k != "none" && k.trim() == *k);
if !well_formed {
return Err(format!("WAI_QUANTUM_SME_EXPECT={expect:?} is not a sorted, comma-separated list of keys, nor none. {how}"));
}
for k in &keys {
if !found.contains(k)
&& let Some((_, e)) = d.refused.iter().find(|(key, _)| key == k)
{
return Err(format!("the declared device {k} was refused: {e}. {how}"));
}
}
if found != keys {
return Err(format!("the devices found are not the declared ones ({expect}). {how}"));
}
Ok(expect)
}
/// The members, once the declared devices are exactly the ones found.
fn members() -> &'static [SmeMember] {
let d = discovery();
if let Err(why) = declared(d) {
panic!("{why}");
}
&d.found
}
fn hex(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}
fn next(s: &mut u64) -> u64 {
*s = s.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = *s;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^ (z >> 31)
}
/// A value in `[-bound, bound]`.
fn within(s: &mut u64, bound: i64) -> i32 {
((next(s) % (2 * bound as u64 + 1)) as i64 - bound) as i32
}
/// A random circuit of `ops` gates on `n` qubits: every base gate, up to three
/// distinct controls, `P(1..=32)`.
fn random_circuit(n: u8, ops: usize, seed: u64, gs: Gateset) -> Circuit {
let bases = [BaseGate::I, BaseGate::X, BaseGate::Y, BaseGate::Z, BaseGate::H, BaseGate::S, BaseGate::Sdg, BaseGate::T, BaseGate::Tdg, BaseGate::P];
let mut s = seed;
let mut c = Circuit::with_gateset(n, gs);
for _ in 0..ops {
let base = bases[(next(&mut s) % bases.len() as u64) as usize];
let target = (next(&mut s) % u64::from(n)) as u8;
let mut controls = Vec::new();
for _ in 0..next(&mut s) % 4 {
let q = (next(&mut s) % u64::from(n)) as u8;
if q != target && !controls.contains(&q) {
controls.push(q);
}
}
let param = if base == BaseGate::P { 1 + (next(&mut s) % 32) as u16 } else { 0 };
c.ops.push(Gate { base, controls, target, param });
}
c
}
/// Five circuits the native simulator refuses, in each gateset.
fn refused_circuits() -> Vec<Circuit> {
let gate = |base, controls: Vec<u8>, target, param| Gate { base, controls, target, param };
let bad = [
(1, vec![gate(BaseGate::H, vec![], 0, 0), gate(BaseGate::P, vec![], 0, 0)]),
(1, vec![gate(BaseGate::P, vec![], 0, 33)]),
(1, vec![gate(BaseGate::X, vec![], 3, 0)]),
(2, vec![gate(BaseGate::X, vec![0], 0, 0)]),
(2, vec![gate(BaseGate::X, vec![5], 1, 0)]),
];
let mut out = Vec::new();
for gs in Gateset::ALL {
for (n, ops) in bad.clone() {
let mut c = Circuit::with_gateset(n, gs);
c.ops = ops;
out.push(c);
}
}
out
}
/// `Z` on qubit 0: on `[(0, 0), (i32::MIN, 0)]` its exact result is `2^31`,
/// which leaves 32 bits.
fn z_words() -> GateWords {
lanes::gate_words(&Gate { base: BaseGate::Z, controls: vec![], target: 0, param: 0 }, Gateset::V2)
}
const Z_STATE: [Lane; 2] = [[0, 0], [i32::MIN, 0]];
#[test]
fn c0_devices_are_declared() {
let d = discovery();
match declared(d) {
Ok(expect) => report(&format!("sme devices: as declared ({expect})")),
Err(why) => panic!("{why}"),
}
for m in &d.found {
assert_eq!(m.class(), FabricClass::CpuMatrix);
assert!(m.id().starts_with("local/cpu-matrix-sme-"), "{}", m.id());
assert_eq!(m.implementation(), sme::implementation_id());
assert!(m.max_qubits() >= 20, "{}: holds only {} qubits", m.key(), m.max_qubits());
report(&format!("sme member: key={} id={} {}", m.key(), m.id(), m.detail()));
}
}
#[test]
fn c1_probe_matches_lanes() {
for m in members() {
let pairs = lanes::probe_pairs(1 << 16, 0x0cda);
let out = m.probe(&pairs).unwrap_or_else(|e| panic!("{}: {e}", m.key()));
assert_eq!(lanes::first_mismatch(&pairs, &out), None, "{}", m.key());
report(&format!("sme probe: key={} {} pairs agree", m.key(), pairs.len()));
}
}
#[test]
fn c2_kernel_matches_run_words() {
for m in members() {
let mut s = 0x6b65_726e_656c_u64;
let (mut flagged, mut clean) = (0, 0);
for case in 0..2000 {
let n = 1 + (next(&mut s) % 12) as u32;
// Three regimes: the whole 32-bit range (nearly always out of
// range), small states and entries (never), and states and
// entries up to 2^30 (either).
let mode = case % 3;
let value = |s: &mut u64| -> i32 {
match mode {
0 => next(s) as i32,
1 => within(s, 1 << 22),
_ => within(s, 1 << 30),
}
};
let entry = |s: &mut u64| -> u32 {
match mode {
0 => next(s) as u32,
1 => within(s, 1 << 29) as u32,
_ => within(s, 1 << 30) as u32,
}
};
let state: Vec<Lane> = (0..1usize << n).map(|_| [value(&mut s), value(&mut s)]).collect();
let gates: Vec<GateWords> = (0..1 + next(&mut s) % 8)
.map(|_| {
let t = (next(&mut s) % u64::from(n)) as u32;
let ctrl = (0..n).filter(|&b| b != t && next(&mut s).is_multiple_of(4)).fold(0u32, |acc, b| acc | 1 << b);
let mut w = [0u32; 8];
w.iter_mut().for_each(|x| *x = entry(&mut s));
GateWords { m: w, tbit: 1 << t, ctrl }
})
.collect();
let want = lanes::run_words(&state, &gates).expect("valid shapes");
let got = m.run_lanes(&state, &gates).unwrap_or_else(|e| panic!("{}: case {case}: {e}", m.key()));
assert_eq!(got.out_of_range, want.out_of_range, "{}: case {case}, n = {n}", m.key());
assert!(got.amps == want.amps, "{}: case {case}, n = {n}: the lanes differ", m.key());
if want.out_of_range.is_some() {
flagged += 1;
} else {
clean += 1;
}
}
assert!(flagged >= 100 && clean >= 100, "{}: {flagged} flagged, {clean} clean", m.key());
report(&format!("sme kernel: key={} ok", m.key()));
}
}
#[test]
fn c3_certificates_reproduce_the_pins() {
let ms = members();
for (gs, random_pin, fabric_pin) in PINS {
let cpu = lanes::certify_corpus(gs, "local/test", RANDOM_V1);
assert_eq!(hex(&cpu.corpus), random_pin, "{gs:?}");
let other = Corpus::Random { seed: 8, cases: 300 };
let cpu_other = lanes::certify_corpus(gs, "local/test", other);
for m in ms {
for (corpus, cases, pin) in [(RANDOM_V1, 200, random_pin), (FABRIC_DEFAULT, 224, fabric_pin)] {
let c = m.certify(gs, corpus).unwrap_or_else(|e| panic!("{} {gs:?} {}: {e}", m.key(), corpus.name()));
assert_eq!(c.parity, Parity::BitExact, "{} {gs:?} {}", m.key(), corpus.name());
assert_eq!(c.cases, cases);
assert_eq!(hex(&c.corpus), pin, "{} {gs:?} {}", m.key(), corpus.name());
assert_eq!((c.kernel.as_str(), c.implementation.as_str(), c.member.as_str()), (apply_kernel(gs), m.implementation(), m.id()));
assert_eq!(c.generator, corpus.generator(gs));
report(&format!(
"sme certified: key={} member={} kernel={} corpus={} implementation={} parity=bit-exact cases={} digest={}",
m.key(),
m.id(),
c.kernel,
corpus.name(),
c.implementation,
c.cases,
hex(&c.corpus)
));
}
// A corpus with no pin: the member and the CPU agree on it.
let c = m.certify(gs, other).unwrap_or_else(|e| panic!("{} {gs:?} seed 8: {e}", m.key()));
assert_eq!((c.parity, c.cases), (Parity::BitExact, 300), "{} {gs:?}", m.key());
assert_eq!(c.corpus, cpu_other.corpus, "{} {gs:?}", m.key());
}
}
}
#[test]
fn c4_wide_circuits_match_native() {
for m in members() {
let widest = m.max_qubits().min(20);
let mut s = 0x0077_6964_6500_u64;
for (k, n) in [12u8, 16, 18, widest].into_iter().enumerate() {
for gs in Gateset::ALL {
let c = random_circuit(n, 40, 0x1000 + k as u64, gs);
let start = (next(&mut s) as usize) % (1usize << n);
let want = c.simulate_from(start).expect("the native run");
let got = m.simulate_from(&c, start).unwrap_or_else(|e| panic!("{}: n = {n} {gs:?}: {e}", m.key()));
assert!(got == want, "{}: n = {n} {gs:?}: the states differ", m.key());
}
}
// The transforms the conformance corpus carries.
for n in [3u8, 5, 8, 12] {
let qft = Circuit::qft(n);
assert!(m.simulate(&qft).unwrap() == qft.simulate().unwrap(), "{}: QFT-{n}", m.key());
}
report(&format!("sme wide: key={} up to {widest} qubits match native", m.key()));
}
}
#[test]
fn c5_refusals_are_typed() {
for m in members() {
for c in refused_circuits() {
let native = c.simulate_from(0).unwrap_err();
assert_eq!(m.simulate_from(&c, 0).unwrap_err(), SmeError::Circuit(native), "{c:?}");
}
assert_eq!(m.simulate(&Circuit::new(MAX_QUBITS + 1)).unwrap_err(), SmeError::Circuit(QuantumError::TooManyQubits(MAX_QUBITS + 1)));
let small = m.with_max_qubits(4);
assert_eq!(small.simulate(&Circuit::new(5)).unwrap_err(), SmeError::TooWide { n_qubits: 5, max: 4 });
let wide = Program { n_qubits: 5, start: 0, gates: Vec::new(), steps: Vec::new() };
assert_eq!(small.run_program(&wide).unwrap_err(), SmeError::TooWide { n_qubits: 5, max: 4 });
// A result that leaves 32 bits is reported at its op, and the lane
// holds its low 32 bits.
let r = m.run_lanes(&Z_STATE, &[z_words()]).unwrap();
assert_eq!(r.out_of_range, Some(0), "{}", m.key());
assert_eq!(r.amps[1][0] as u32, 0x8000_0000);
// Malformed words, a state the host cannot lay out and a tile step are
// host defects, never results.
let bad = GateWords { tbit: 3, ..z_words() };
assert!(matches!(m.run_lanes(&[[0, 0]; 4], &[bad]), Err(SmeError::Internal(_))));
assert!(matches!(m.run_lanes(&[[0, 0]; 3], &[z_words()]), Err(SmeError::Internal(_))));
let tiled = Program { n_qubits: 2, start: 0, gates: Vec::new(), steps: vec![Step::Tile { first: 0, count: 1, size: 2 }] };
assert!(matches!(m.run_program(&tiled), Err(SmeError::Internal(_))));
// The refusals that may still fall back to another member, do.
assert!(MemberError::from(SmeError::TooWide { n_qubits: 5, max: 4 }).falls_back());
report(&format!("sme refusals: key={} typed", m.key()));
}
}
/// The kernel mutants, compiled in: each must be caught by the probe or by a
/// certificate.
#[test]
fn c6_mutants_are_killed() {
let edges = lanes::probe_pairs(0, 0);
for m in members() {
let mut killed = 0;
for v in Variant::MUTANTS {
let mm = m.with_variant(v);
assert_ne!(mm.implementation(), m.implementation());
let out = mm.probe(&edges).expect("the mutant's probe runs");
let differing = edges.iter().zip(&out).filter(|&(&(a, b), o)| *o != (lanes::mul_wide(a, b), lanes::fxmul_wide(a, b))).count();
let caught = differing > 0
|| Gateset::ALL.into_iter().all(|gs| match mm.certify(gs, RANDOM_V1) {
Err(_) => true,
Ok(c) => c.parity != Parity::BitExact || c.corpus != lanes::certify_corpus(gs, "local/test", RANDOM_V1).corpus,
});
println!("sme mutant {v:?} on {}: {differing} of {} edge pairs differ", m.key(), edges.len());
assert!(caught, "{}: mutant {v:?} survived", m.key());
killed += 1;
}
report(&format!("sme mutants: key={} killed {killed} of 3", m.key()));
}
}
#[test]
fn c7_receipt_placement_and_the_fabric() {
for m in members() {
let member = m.member(FABRIC_DEFAULT).unwrap_or_else(|e| panic!("{}: {e}", m.key()));
assert_eq!((member.id.as_str(), member.class, member.presence), (m.id(), FabricClass::CpuMatrix, Presence::Detected));
assert_eq!(member.kernels.len(), 2);
for (entry, gs) in member.kernels.iter().zip(Gateset::ALL) {
assert_eq!((entry.kernel.as_str(), entry.implementation.as_str()), (apply_kernel(gs), m.implementation()));
assert!(entry.energy.is_none());
assert_eq!(entry.certificate.parity, Parity::BitExact, "{gs:?}");
}
// A receipt over a member's run.
let qft = Circuit::qft(5);
let state = m.simulate(&qft).unwrap();
assert_eq!(state, qft.simulate().unwrap());
let entry = member.kernels.iter().find(|k| k.kernel == apply_kernel(qft.gateset)).unwrap();
let signer = SigningKey::from_bytes(&[7; 32]);
let receipt = FabricReceipt::seal(&signer, "local/test", state.statevector_hash(), &member, entry, 0, None);
assert!(receipt.verify());
assert!(receipt.certificate_matches(&entry.certificate));
// Placement: beside the CPU reference with no energy figures, the
// reference; alone, the member.
let local = detect_local();
for gs in Gateset::ALL {
let both = [local.clone(), member.clone()];
let (placed, k) = place(&both, apply_kernel(gs), &Parity::BitExact).unwrap();
assert_eq!((placed.id.as_str(), k.implementation.as_str()), (local.id.as_str(), REFERENCE));
let (placed, k) = place(std::slice::from_ref(&member), apply_kernel(gs), &Parity::BitExact).unwrap();
assert_eq!((placed.id.as_str(), k.implementation.as_str()), (m.id(), m.implementation()));
}
// On the fabric: the member alone runs the circuit; a pairing with
// another member's entries is refused.
let mut fabric = Fabric::new();
fabric.add(member.clone(), m);
let run = ready(fabric.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs");
assert_eq!(run.state, state);
assert_eq!(run.receipt.member_id, m.id());
assert!(run.receipt.verify());
let mut mismatched = Fabric::new();
mismatched.add(local.clone(), m);
assert!(ready(mismatched.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").is_err());
let mut both = Fabric::new();
both.add(local.clone(), CpuReference).add(member.clone(), m);
let run = ready(both.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs");
assert_eq!(run.state, state);
report(&format!("sme receipt: key={} ok", m.key()));
}
}
#[test]
fn c8_repeated_runs_agree() {
for m in members() {
let c = random_circuit(16, 60, 0xabc, Gateset::V2);
let first = m.simulate(&c).unwrap();
for _ in 0..5 {
assert!(m.simulate(&c).unwrap() == first, "{}", m.key());
}
report(&format!("sme repeat: key={} ok", m.key()));
}
}

410
wai/wai-quantum/tests/vector_member.rs Normal file
View file

@ -0,0 +1,410 @@
//! The fabric's CPU vector-unit member on the machine that runs this.
//!
//! The machine declares the devices it must have in `WAI_QUANTUM_VECTOR_EXPECT`:
//! their keys, comma-separated and sorted, or `none`. Every test that needs a
//! member fails unless the devices found are exactly the declared ones, so a
//! machine whose GPU or driver went missing, or a run that declared nothing,
//! fails instead of passing over no member. Each test prints what it
//! established and, when `WAI_QUANTUM_VECTOR_REPORT` names a file, appends the
//! same lines to it, which a CI step can count per device.
//!
//! ```text
//! WAI_QUANTUM_VECTOR_EXPECT=cpu-vector-avx2 \
//! cargo test --release --features quantum_fabric_vector --test vector_member -- --test-threads=1
//! ```
#![cfg(not(target_arch = "wasm32"))]
use std::io::Write;
use std::sync::OnceLock;
use wai_quantum::ed25519_dalek::SigningKey;
use wai_quantum::quantum::{BaseGate, Circuit, Gate, Gateset, MAX_QUBITS, QuantumError};
use wai_quantum::quantum_fabric::lanes::{self, GateWords, Lane};
use wai_quantum::quantum_fabric::plan::{Program, Step};
use wai_quantum::quantum_fabric::{
Corpus, CpuReference, FABRIC_DEFAULT, Fabric, FabricClass, FabricReceipt, MemberError, Parity, Presence, REFERENCE, apply_kernel, detect_local, place, ready,
};
use wai_quantum::quantum_fabric_vector::{Discovery, VectorError, VectorMember, Variant, discover, implementation_id_of};
/// The CPU's corpus digests: random-v1 (seed 7, 200 cases) and fabric-v1
/// (seed 7, 200 + 24 cases), per gateset. A bit-exact member reproduces them.
const PINS: [(Gateset, &str, &str); 2] = [
(Gateset::V1, "d18454c639acfd9c7b66450f4ded1aac4ab46cc10a7bedbcf4958a96e204fb36", "41c726a0ac80f7defef92e15ecf893bdb7a3076b5e7f670fa0b7142e5c033df7"),
(Gateset::V2, "3dbde276c1095bf827faec71d5980e050f8a55a5ba7d97cd3bbdaec3f43a13b2", "9a84b55af68f1a8b07390b234cf562dd8d85cef3185a697ab85690c8790a7809"),
];
const RANDOM_V1: Corpus = Corpus::Random { seed: 7, cases: 200 };
static DISCOVERY: OnceLock<Discovery> = OnceLock::new();
fn discovery() -> &'static Discovery {
DISCOVERY.get_or_init(discover)
}
/// Prints `line` and appends it to the file `WAI_QUANTUM_VECTOR_REPORT` names.
fn report(line: &str) {
println!("{line}");
if let Some(path) = std::env::var_os("WAI_QUANTUM_VECTOR_REPORT") {
let mut f = std::fs::OpenOptions::new().create(true).append(true).open(&path).expect("the report file opens");
writeln!(f, "{line}").expect("the report line is written");
}
}
/// The declared device set, as written, if it is exactly what was found;
/// otherwise why not, with what was found and refused and how to declare it.
fn declared(d: &Discovery) -> Result<String, String> {
let mut found: Vec<&str> = d.found.iter().map(|m| m.key()).collect();
found.sort_unstable();
let found_list = if found.is_empty() { "none".to_string() } else { found.join(",") };
let refused: Vec<String> = d.refused.iter().map(|(k, e)| format!("{k}: {e}")).collect();
let refused_list = if refused.is_empty() { "none".to_string() } else { refused.join("; ") };
let unavailable = d.unavailable.as_deref().unwrap_or("no");
let how = format!("found: {found_list}; refused: {refused_list}; unavailable: {unavailable}. Declare the devices this machine must have: WAI_QUANTUM_VECTOR_EXPECT={found_list}");
let expect = match std::env::var("WAI_QUANTUM_VECTOR_EXPECT") {
Ok(v) if !v.is_empty() => v,
_ => return Err(format!("WAI_QUANTUM_VECTOR_EXPECT is not set. {how}")),
};
let keys: Vec<&str> = if expect == "none" { Vec::new() } else { expect.split(',').collect() };
let well_formed = keys.windows(2).all(|w| w[0] <= w[1]) && keys.iter().all(|k| !k.is_empty() && *k != "none" && k.trim() == *k);
if !well_formed {
return Err(format!("WAI_QUANTUM_VECTOR_EXPECT={expect:?} is not a sorted, comma-separated list of keys, nor none. {how}"));
}
for k in &keys {
if !found.contains(k)
&& let Some((_, e)) = d.refused.iter().find(|(key, _)| key == k)
{
return Err(format!("the declared device {k} was refused: {e}. {how}"));
}
}
if found != keys {
return Err(format!("the devices found are not the declared ones ({expect}). {how}"));
}
Ok(expect)
}
/// The members, once the declared devices are exactly the ones found.
fn members() -> &'static [VectorMember] {
let d = discovery();
if let Err(why) = declared(d) {
panic!("{why}");
}
&d.found
}
fn hex(b: &[u8]) -> String {
b.iter().map(|x| format!("{x:02x}")).collect()
}
fn next(s: &mut u64) -> u64 {
*s = s.wrapping_add(0x9E37_79B9_7F4A_7C15);
let mut z = *s;
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
z ^ (z >> 31)
}
/// A value in `[-bound, bound]`.
fn within(s: &mut u64, bound: i64) -> i32 {
((next(s) % (2 * bound as u64 + 1)) as i64 - bound) as i32
}
/// A random circuit of `ops` gates on `n` qubits: every base gate, up to three
/// distinct controls, `P(1..=32)`.
fn random_circuit(n: u8, ops: usize, seed: u64, gs: Gateset) -> Circuit {
let bases = [BaseGate::I, BaseGate::X, BaseGate::Y, BaseGate::Z, BaseGate::H, BaseGate::S, BaseGate::Sdg, BaseGate::T, BaseGate::Tdg, BaseGate::P];
let mut s = seed;
let mut c = Circuit::with_gateset(n, gs);
for _ in 0..ops {
let base = bases[(next(&mut s) % bases.len() as u64) as usize];
let target = (next(&mut s) % u64::from(n)) as u8;
let mut controls = Vec::new();
for _ in 0..next(&mut s) % 4 {
let q = (next(&mut s) % u64::from(n)) as u8;
if q != target && !controls.contains(&q) {
controls.push(q);
}
}
let param = if base == BaseGate::P { 1 + (next(&mut s) % 32) as u16 } else { 0 };
c.ops.push(Gate { base, controls, target, param });
}
c
}
/// Five circuits the native simulator refuses, in each gateset.
fn refused_circuits() -> Vec<Circuit> {
let gate = |base, controls: Vec<u8>, target, param| Gate { base, controls, target, param };
let bad = [
(1, vec![gate(BaseGate::H, vec![], 0, 0), gate(BaseGate::P, vec![], 0, 0)]),
(1, vec![gate(BaseGate::P, vec![], 0, 33)]),
(1, vec![gate(BaseGate::X, vec![], 3, 0)]),
(2, vec![gate(BaseGate::X, vec![0], 0, 0)]),
(2, vec![gate(BaseGate::X, vec![5], 1, 0)]),
];
let mut out = Vec::new();
for gs in Gateset::ALL {
for (n, ops) in bad.clone() {
let mut c = Circuit::with_gateset(n, gs);
c.ops = ops;
out.push(c);
}
}
out
}
/// `Z` on qubit 0: on `[(0, 0), (i32::MIN, 0)]` its exact result is `2^31`,
/// which leaves 32 bits.
fn z_words() -> GateWords {
lanes::gate_words(&Gate { base: BaseGate::Z, controls: vec![], target: 0, param: 0 }, Gateset::V2)
}
const Z_STATE: [Lane; 2] = [[0, 0], [i32::MIN, 0]];
#[test]
fn c0_devices_are_declared() {
let d = discovery();
match declared(d) {
Ok(expect) => report(&format!("vector devices: as declared ({expect})")),
Err(why) => panic!("{why}"),
}
for m in &d.found {
assert_eq!(m.class(), FabricClass::Cpu);
assert!(m.id().starts_with(&format!("local/cpu-{}-vector-", std::env::consts::ARCH)), "{}", m.id());
assert_eq!(m.implementation(), implementation_id_of(m.isa(), Variant::Conforming));
assert!(m.max_qubits() >= 20, "{}: holds only {} qubits", m.key(), m.max_qubits());
report(&format!("vector member: key={} id={} bits={} {}", m.key(), m.id(), m.vector_bits(), m.detail()));
}
}
#[test]
fn c1_probe_matches_lanes() {
for m in members() {
let pairs = lanes::probe_pairs(1 << 16, 0x0cda);
let out = m.probe(&pairs).unwrap_or_else(|e| panic!("{}: {e}", m.key()));
assert_eq!(lanes::first_mismatch(&pairs, &out), None, "{}", m.key());
report(&format!("vector probe: key={} {} pairs agree", m.key(), pairs.len()));
}
}
#[test]
fn c2_kernel_matches_run_words() {
for m in members() {
let mut s = 0x6b65_726e_656c_u64;
let (mut flagged, mut clean) = (0, 0);
for case in 0..2000 {
let n = 1 + (next(&mut s) % 12) as u32;
// Three regimes: the whole 32-bit range (nearly always out of
// range), small states and entries (never), and states and
// entries up to 2^30 (either).
let mode = case % 3;
let value = |s: &mut u64| -> i32 {
match mode {
0 => next(s) as i32,
1 => within(s, 1 << 22),
_ => within(s, 1 << 30),
}
};
let entry = |s: &mut u64| -> u32 {
match mode {
0 => next(s) as u32,
1 => within(s, 1 << 29) as u32,
_ => within(s, 1 << 30) as u32,
}
};
let state: Vec<Lane> = (0..1usize << n).map(|_| [value(&mut s), value(&mut s)]).collect();
let gates: Vec<GateWords> = (0..1 + next(&mut s) % 8)
.map(|_| {
let t = (next(&mut s) % u64::from(n)) as u32;
let ctrl = (0..n).filter(|&b| b != t && next(&mut s).is_multiple_of(4)).fold(0u32, |acc, b| acc | 1 << b);
let mut w = [0u32; 8];
w.iter_mut().for_each(|x| *x = entry(&mut s));
GateWords { m: w, tbit: 1 << t, ctrl }
})
.collect();
let want = lanes::run_words(&state, &gates).expect("valid shapes");
let got = m.run_lanes(&state, &gates).unwrap_or_else(|e| panic!("{}: case {case}: {e}", m.key()));
assert_eq!(got.out_of_range, want.out_of_range, "{}: case {case}, n = {n}", m.key());
assert!(got.amps == want.amps, "{}: case {case}, n = {n}: the lanes differ", m.key());
if want.out_of_range.is_some() {
flagged += 1;
} else {
clean += 1;
}
}
assert!(flagged >= 100 && clean >= 100, "{}: {flagged} flagged, {clean} clean", m.key());
report(&format!("vector kernel: key={} ok", m.key()));
}
}
#[test]
fn c3_certificates_reproduce_the_pins() {
let ms = members();
for (gs, random_pin, fabric_pin) in PINS {
let cpu = lanes::certify_corpus(gs, "local/test", RANDOM_V1);
assert_eq!(hex(&cpu.corpus), random_pin, "{gs:?}");
let other = Corpus::Random { seed: 8, cases: 300 };
let cpu_other = lanes::certify_corpus(gs, "local/test", other);
for m in ms {
for (corpus, cases, pin) in [(RANDOM_V1, 200, random_pin), (FABRIC_DEFAULT, 224, fabric_pin)] {
let c = m.certify(gs, corpus).unwrap_or_else(|e| panic!("{} {gs:?} {}: {e}", m.key(), corpus.name()));
assert_eq!(c.parity, Parity::BitExact, "{} {gs:?} {}", m.key(), corpus.name());
assert_eq!(c.cases, cases);
assert_eq!(hex(&c.corpus), pin, "{} {gs:?} {}", m.key(), corpus.name());
assert_eq!((c.kernel.as_str(), c.implementation.as_str(), c.member.as_str()), (apply_kernel(gs), m.implementation(), m.id()));
assert_eq!(c.generator, corpus.generator(gs));
report(&format!(
"vector certified: key={} member={} kernel={} corpus={} implementation={} parity=bit-exact cases={} digest={}",
m.key(),
m.id(),
c.kernel,
corpus.name(),
c.implementation,
c.cases,
hex(&c.corpus)
));
}
// A corpus with no pin: the member and the CPU agree on it.
let c = m.certify(gs, other).unwrap_or_else(|e| panic!("{} {gs:?} seed 8: {e}", m.key()));
assert_eq!((c.parity, c.cases), (Parity::BitExact, 300), "{} {gs:?}", m.key());
assert_eq!(c.corpus, cpu_other.corpus, "{} {gs:?}", m.key());
}
}
}
#[test]
fn c4_wide_circuits_match_native() {
for m in members() {
let widest = m.max_qubits().min(20);
let mut s = 0x0077_6964_6500_u64;
for (k, n) in [12u8, 16, 18, widest].into_iter().enumerate() {
for gs in Gateset::ALL {
let c = random_circuit(n, 40, 0x1000 + k as u64, gs);
let start = (next(&mut s) as usize) % (1usize << n);
let want = c.simulate_from(start).expect("the native run");
let got = m.simulate_from(&c, start).unwrap_or_else(|e| panic!("{}: n = {n} {gs:?}: {e}", m.key()));
assert!(got == want, "{}: n = {n} {gs:?}: the states differ", m.key());
}
}
// The transforms the conformance corpus carries.
for n in [3u8, 5, 8, 12] {
let qft = Circuit::qft(n);
assert!(m.simulate(&qft).unwrap() == qft.simulate().unwrap(), "{}: QFT-{n}", m.key());
}
report(&format!("vector wide: key={} up to {widest} qubits match native", m.key()));
}
}
#[test]
fn c5_refusals_are_typed() {
for m in members() {
for c in refused_circuits() {
let native = c.simulate_from(0).unwrap_err();
assert_eq!(m.simulate_from(&c, 0).unwrap_err(), VectorError::Circuit(native), "{c:?}");
}
assert_eq!(m.simulate(&Circuit::new(MAX_QUBITS + 1)).unwrap_err(), VectorError::Circuit(QuantumError::TooManyQubits(MAX_QUBITS + 1)));
let small = m.with_max_qubits(4);
assert_eq!(small.simulate(&Circuit::new(5)).unwrap_err(), VectorError::TooWide { n_qubits: 5, max: 4 });
let wide = Program { n_qubits: 5, start: 0, gates: Vec::new(), steps: Vec::new() };
assert_eq!(small.run_program(&wide).unwrap_err(), VectorError::TooWide { n_qubits: 5, max: 4 });
// A result that leaves 32 bits is reported at its op, and the lane
// holds its low 32 bits.
let r = m.run_lanes(&Z_STATE, &[z_words()]).unwrap();
assert_eq!(r.out_of_range, Some(0), "{}", m.key());
assert_eq!(r.amps[1][0] as u32, 0x8000_0000);
// Malformed words, a state the host cannot lay out and a tile step are
// host defects, never results.
let bad = GateWords { tbit: 3, ..z_words() };
assert!(matches!(m.run_lanes(&[[0, 0]; 4], &[bad]), Err(VectorError::Internal(_))));
assert!(matches!(m.run_lanes(&[[0, 0]; 3], &[z_words()]), Err(VectorError::Internal(_))));
let tiled = Program { n_qubits: 2, start: 0, gates: Vec::new(), steps: vec![Step::Tile { first: 0, count: 1, size: 2 }] };
assert!(matches!(m.run_program(&tiled), Err(VectorError::Internal(_))));
// The refusals that may still fall back to another member, do.
assert!(MemberError::from(VectorError::TooWide { n_qubits: 5, max: 4 }).falls_back());
report(&format!("vector refusals: key={} typed", m.key()));
}
}
/// The kernel mutants, compiled in: each must be caught by the probe or by a
/// certificate.
#[test]
fn c6_mutants_are_killed() {
let edges = lanes::probe_pairs(0, 0);
for m in members() {
let mut killed = 0;
for v in Variant::MUTANTS {
let mm = m.with_variant(v);
assert_ne!(mm.implementation(), m.implementation());
let out = mm.probe(&edges).expect("the mutant's probe runs");
let differing = edges.iter().zip(&out).filter(|&(&(a, b), o)| *o != (lanes::mul_wide(a, b), lanes::fxmul_wide(a, b))).count();
let caught = differing > 0
|| Gateset::ALL.into_iter().all(|gs| match mm.certify(gs, RANDOM_V1) {
Err(_) => true,
Ok(c) => c.parity != Parity::BitExact || c.corpus != lanes::certify_corpus(gs, "local/test", RANDOM_V1).corpus,
});
println!("vector mutant {v:?} on {}: {differing} of {} edge pairs differ", m.key(), edges.len());
assert!(caught, "{}: mutant {v:?} survived", m.key());
killed += 1;
}
report(&format!("vector mutants: key={} killed {killed} of 3", m.key()));
}
}
#[test]
fn c7_receipt_placement_and_the_fabric() {
for m in members() {
let member = m.member(FABRIC_DEFAULT).unwrap_or_else(|e| panic!("{}: {e}", m.key()));
assert_eq!((member.id.as_str(), member.class, member.presence), (m.id(), FabricClass::Cpu, Presence::Detected));
assert_eq!(member.kernels.len(), 2);
for (entry, gs) in member.kernels.iter().zip(Gateset::ALL) {
assert_eq!((entry.kernel.as_str(), entry.implementation.as_str()), (apply_kernel(gs), m.implementation()));
assert!(entry.energy.is_none());
assert_eq!(entry.certificate.parity, Parity::BitExact, "{gs:?}");
}
// A receipt over a member's run.
let qft = Circuit::qft(5);
let state = m.simulate(&qft).unwrap();
assert_eq!(state, qft.simulate().unwrap());
let entry = member.kernels.iter().find(|k| k.kernel == apply_kernel(qft.gateset)).unwrap();
let signer = SigningKey::from_bytes(&[7; 32]);
let receipt = FabricReceipt::seal(&signer, "local/test", state.statevector_hash(), &member, entry, 0, None);
assert!(receipt.verify());
assert!(receipt.certificate_matches(&entry.certificate));
// Placement: beside the CPU reference with no energy figures, the
// reference; alone, the member.
let local = detect_local();
for gs in Gateset::ALL {
let both = [local.clone(), member.clone()];
let (placed, k) = place(&both, apply_kernel(gs), &Parity::BitExact).unwrap();
assert_eq!((placed.id.as_str(), k.implementation.as_str()), (local.id.as_str(), REFERENCE));
let (placed, k) = place(std::slice::from_ref(&member), apply_kernel(gs), &Parity::BitExact).unwrap();
assert_eq!((placed.id.as_str(), k.implementation.as_str()), (m.id(), m.implementation()));
}
// On the fabric: the member alone runs the circuit; a pairing with
// another member's entries is refused.
let mut fabric = Fabric::new();
fabric.add(member.clone(), m);
let run = ready(fabric.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs");
assert_eq!(run.state, state);
assert_eq!(run.receipt.member_id, m.id());
assert!(run.receipt.verify());
let mut mismatched = Fabric::new();
mismatched.add(local.clone(), m);
assert!(ready(mismatched.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").is_err());
let mut both = Fabric::new();
both.add(local.clone(), CpuReference).add(member.clone(), m);
let run = ready(both.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs");
assert_eq!(run.state, state);
report(&format!("vector receipt: key={} ok", m.key()));
}
}
#[test]
fn c8_repeated_runs_agree() {
for m in members() {
let c = random_circuit(16, 60, 0xabc, Gateset::V2);
let first = m.simulate(&c).unwrap();
for _ in 0..5 {
assert!(m.simulate(&c).unwrap() == first, "{}", m.key());
}
report(&format!("vector repeat: key={} ok", m.key()));
}
}

2
wai/wai-rs/Cargo.lock generated
View file

@ -4300,7 +4300,7 @@ version = "0.1.0"
[[package]]
name = "wai-quantum"
version = "0.5.0"
version = "0.5.1"
dependencies = [
"blake3",
"ed25519-dalek",

2
wai/wai-web/Cargo.lock generated
View file

@ -1231,7 +1231,7 @@ version = "0.1.0"
[[package]]
name = "wai-quantum"
version = "0.5.0"
version = "0.5.1"
dependencies = [
"blake3",
"ed25519-dalek",