From 015e75fcbebbcad8c51a69ab071f0658f9c61b6c Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Wed, 14 Oct 2026 09:10:30 -0400 Subject: [PATCH 01/11] wai-quantum 0.5.1: the neural-processor member Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wai-quantum-npu.yml | 96 +++ wai/CHANGELOG.md | 20 + wai/tools/quantum-meter/Cargo.lock | 2 +- wai/tools/quantum-wgsl-check/Cargo.lock | 2 +- wai/wai-pack/Cargo.lock | 2 +- wai/wai-quantum-qpu/Cargo.lock | 2 +- wai/wai-quantum-serve/Cargo.lock | 2 +- wai/wai-quantum/Cargo.lock | 17 +- wai/wai-quantum/Cargo.toml | 21 +- wai/wai-quantum/README.md | 8 +- wai/wai-quantum/src/lib.rs | 9 +- wai/wai-quantum/src/quantum_fabric.rs | 1 + wai/wai-quantum/src/quantum_fabric/npu.rs | 408 +++++++++++++ wai/wai-quantum/src/quantum_fabric_npu.rs | 681 ++++++++++++++++++++++ wai/wai-quantum/tests/npu_member.rs | 413 +++++++++++++ wai/wai-rs/Cargo.lock | 2 +- wai/wai-web/Cargo.lock | 2 +- 17 files changed, 1673 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/wai-quantum-npu.yml create mode 100644 wai/wai-quantum/src/quantum_fabric/npu.rs create mode 100644 wai/wai-quantum/src/quantum_fabric_npu.rs create mode 100644 wai/wai-quantum/tests/npu_member.rs diff --git a/.github/workflows/wai-quantum-npu.yml b/.github/workflows/wai-quantum-npu.yml new file mode 100644 index 000000000..829e5158e --- /dev/null +++ b/.github/workflows/wai-quantum-npu.yml @@ -0,0 +1,96 @@ +name: wai-quantum neural-processor member + +# The fabric's neural-processor member: exact 32×32 products from 4-bit limbs, +# every limb product and limb sum formed in fp16 on the neural processor (each +# value an integer fp16 holds exactly), the host assembling; certified +# bit-exact against the CPU reference's pinned corpus digests. +# +# - npu-mac (the macOS runner): the platform's model runtime must place every +# operation of the model on the neural processor (its compute plan says so, +# or the member is refused); the device tests must find the declared +# processor, agree with the CPU on the probe and on random states, reproduce +# all four pinned certificates, and catch both model mutants. +# - npu-absent (a Linux container): the same build finds no member and says +# why; the device tests run over no member with the declaration `none`. + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: [main] + paths: + - 'wai/wai-quantum/**' + - 'rust-toolchain.toml' + - '.github/workflows/wai-quantum-npu.yml' + pull_request: + paths: + - 'wai/wai-quantum/**' + - 'rust-toolchain.toml' + - '.github/workflows/wai-quantum-npu.yml' + +jobs: + npu-mac: + name: member · neural processor (macos) + runs-on: macos + timeout-minutes: 120 + env: + WAI_QUANTUM_NPU_EXPECT: npu-0 + steps: + - uses: actions/checkout@v5 + # launchd's PATH omits cargo on the macOS host runner. + - name: Put cargo on PATH + shell: bash + run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: The member's unit tests + shell: bash + working-directory: wai/wai-quantum + run: | + set -euo pipefail + nice -n 10 cargo test --locked --features quantum_fabric_npu --lib -- quantum_fabric_npu:: quantum_fabric::npu:: 2>&1 | tee "$RUNNER_TEMP/unit.log" + grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } + - name: The member's device tests + shell: bash + working-directory: wai/wai-quantum + run: | + set -euo pipefail + report="$RUNNER_TEMP/npu-report.txt"; : > "$report" + export WAI_QUANTUM_NPU_CACHE="$RUNNER_TEMP/npu-cache" + WAI_QUANTUM_NPU_REPORT="$report" nice -n 10 \ + cargo test --locked --release --features quantum_fabric_npu --test npu_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log" + grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; } + cat "$report" + key="$WAI_QUANTUM_NPU_EXPECT" + grep -qx "npu devices: as declared ($key)" "$report" || { echo "::error::the processor found is not the declared one"; exit 1; } + grep -qE "^npu placement: key=$key( [a-z0-9_.]+=MLNeuralEngineComputeDevice)+$" "$report" || { echo "::error::an operation was not placed on the neural processor"; exit 1; } + n="$(grep -c "^npu certified: key=$key .* parity=bit-exact " "$report")" + [ "$n" = 4 ] || { echo "::error::$n of 4 certificates were earned"; exit 1; } + grep -qx "npu mutants: key=$key killed 2 of 2" "$report" || { echo "::error::a model mutant survived"; exit 1; } + + npu-absent: + name: member · neural processor absent (linux) + runs-on: linux + timeout-minutes: 60 + env: + WAI_QUANTUM_NPU_EXPECT: none + steps: + - uses: actions/checkout@v5 + - name: Install Rust 1.98.0 + shell: bash + run: | + export PATH="$HOME/.cargo/bin:$PATH" + command -v rustup >/dev/null 2>&1 || \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.98.0 + rustup toolchain install 1.98.0 --profile minimal + rustup default 1.98.0 + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: No neural processor here, and the member says so + shell: bash + working-directory: wai/wai-quantum + run: | + set -euo pipefail + report="$RUNNER_TEMP/npu-report.txt"; : > "$report" + WAI_QUANTUM_NPU_REPORT="$report" cargo test --locked --features quantum_fabric_npu --test npu_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log" + grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; } + grep -qx 'npu devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; } diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index 8700fcae4..b269e9c97 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -6,6 +6,26 @@ wire format may change and such changes are recorded here. ## v0.4.0 — current +### wai-quantum 0.5.1: the neural-processor member + +Additive. The default features, `full`, and every pinned digest are unchanged. + +- **The neural-processor member** (`quantum_fabric_npu`, opt-in, macOS only). A neural + processor computes in fp16 and does not document how it accumulates, so the member + never relies on it: each 32-bit operand is split into eight 4-bit limbs, and the + processor computes the fifteen coefficients `Σ_{i+j=s} a_i·b_j` of each product. Every + limb product is at most 225 in size and every coefficient at most 1800, so every value + it forms is an integer fp16 holds exactly, in any order. The host assembles each + product exactly and applies `lanes::apply_words`'s arithmetic. +- **Credited by placement.** The member writes its model (one multiply, one sum) as the + platform runtime's model description, compiles it at run time, and reads the + runtime's compute plan: every operation must be placed on the neural processor, or + the member is refused with the placement reported, so no CPU result is credited to the + processor. It is then probed and certified against the pinned corpus digests, and + the two model mutants (the multiply made an add, the sum a maximum) are caught. +- **CI.** `wai-quantum-npu.yml`: the device tests on the macOS runner's neural processor, + and the same build on Linux, where the member is absent and says so. + ### wai-quantum 0.5.0: provenance, sampling members, device programs, and three more members Breaking: `Certificate` gains a public field, `provenance`. A silicon certificate hashes diff --git a/wai/tools/quantum-meter/Cargo.lock b/wai/tools/quantum-meter/Cargo.lock index 26fd4127f..2e1ea15ea 100644 --- a/wai/tools/quantum-meter/Cargo.lock +++ b/wai/tools/quantum-meter/Cargo.lock @@ -1017,7 +1017,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wai-quantum" -version = "0.5.0" +version = "0.5.1" dependencies = [ "blake3", "ed25519-dalek", diff --git a/wai/tools/quantum-wgsl-check/Cargo.lock b/wai/tools/quantum-wgsl-check/Cargo.lock index c377a2234..f99c82d2c 100644 --- a/wai/tools/quantum-wgsl-check/Cargo.lock +++ b/wai/tools/quantum-wgsl-check/Cargo.lock @@ -599,7 +599,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wai-quantum" -version = "0.5.0" +version = "0.5.1" dependencies = [ "blake3", "ed25519-dalek", diff --git a/wai/wai-pack/Cargo.lock b/wai/wai-pack/Cargo.lock index de01c7a3c..ba68033a7 100644 --- a/wai/wai-pack/Cargo.lock +++ b/wai/wai-pack/Cargo.lock @@ -703,7 +703,7 @@ dependencies = [ [[package]] name = "wai-quantum" -version = "0.5.0" +version = "0.5.1" dependencies = [ "blake3", "ed25519-dalek", diff --git a/wai/wai-quantum-qpu/Cargo.lock b/wai/wai-quantum-qpu/Cargo.lock index 9f1f1e2ca..716a80636 100644 --- a/wai/wai-quantum-qpu/Cargo.lock +++ b/wai/wai-quantum-qpu/Cargo.lock @@ -422,7 +422,7 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wai-quantum" -version = "0.5.0" +version = "0.5.1" dependencies = [ "blake3", "ed25519-dalek", diff --git a/wai/wai-quantum-serve/Cargo.lock b/wai/wai-quantum-serve/Cargo.lock index 5b94eeacc..cb7ca6cb4 100644 --- a/wai/wai-quantum-serve/Cargo.lock +++ b/wai/wai-quantum-serve/Cargo.lock @@ -589,7 +589,7 @@ dependencies = [ [[package]] name = "wai-quantum" -version = "0.5.0" +version = "0.5.1" dependencies = [ "blake3", "ed25519-dalek", diff --git a/wai/wai-quantum/Cargo.lock b/wai/wai-quantum/Cargo.lock index 5de6b294e..78db5d69b 100644 --- a/wai/wai-quantum/Cargo.lock +++ b/wai/wai-quantum/Cargo.lock @@ -616,6 +616,17 @@ dependencies = [ "objc2-io-surface", ] +[[package]] +name = "objc2-core-ml" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "201b055e6acfa0f9f15568255d3f03ce2b54bc86d1814442dc69138e36813e18" +dependencies = [ + "block2", + "objc2", + "objc2-foundation", +] + [[package]] name = "objc2-encode" version = "4.1.0" @@ -1068,11 +1079,15 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wai-quantum" -version = "0.5.0" +version = "0.5.1" dependencies = [ "blake3", + "block2", "cudarc", "ed25519-dalek", + "objc2", + "objc2-core-ml", + "objc2-foundation", "opencl3", "serde_json", "wgpu", diff --git a/wai/wai-quantum/Cargo.toml b/wai/wai-quantum/Cargo.toml index ecc0c78bf..80feed38a 100644 --- a/wai/wai-quantum/Cargo.toml +++ b/wai/wai-quantum/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "wai-quantum" -version = "0.5.0" +version = "0.5.1" edition = "2024" rust-version = "1.97" license = "Apache-2.0" @@ -42,6 +42,19 @@ cudarc = { version = "0.19.10", optional = true, default-features = false, featu # feature needs no OpenCL installed and finds no member where there is none. opencl3 = { version = "0.12.3", optional = true, default-features = false, features = ["dynamic"] } +# The neural-processor member: the platform's model runtime on macOS, its +# framework loaded by the system; absent on every other target. +[target.'cfg(target_os = "macos")'.dependencies] +objc2 = { version = "0.6.5", optional = true } +block2 = { version = "0.6.2", optional = true } +objc2-foundation = { version = "0.3.2", optional = true, default-features = false, features = ["std", "NSArray", "NSDictionary", "NSError", "NSString", "NSURL", "NSValue"] } +objc2-core-ml = { version = "0.3.2", optional = true, default-features = false, features = [ + "std", "block2", "MLModel", "MLModel_MLModelCompilation", "MLModelConfiguration", "MLMultiArray", "MLDictionaryFeatureProvider", + "MLFeatureProvider", "MLFeatureValue", "MLComputePlan", "MLComputePlanDeviceUsage", "MLComputeDeviceProtocol", + "MLNeuralEngineComputeDevice", "MLModelStructure", "MLModelStructureProgram", "MLModelStructureProgramFunction", + "MLModelStructureProgramBlock", "MLModelStructureProgramOperation", +] } + [target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] wgpu = { version = "30.0.1", optional = true, default-features = false, features = ["std", "wgsl", "webgpu"] } @@ -152,6 +165,8 @@ quantum_fabric_cuda = ["quantum_fabric", "dep:cudarc"] quantum_fabric_opencl = ["quantum_fabric", "dep:opencl3"] # The simulated FPGA member: Verilator, invoked at run time, compiles the datapath. quantum_fabric_fpga = ["quantum_fabric"] +# The neural-processor member: exact fp16 limb products on the platform's neural processor. +quantum_fabric_npu = ["quantum_fabric", "dep:objc2", "dep:block2", "dep:objc2-foundation", "dep:objc2-core-ml"] # --- interference / ML layer ---------------------------------------------- quantum_vml = [] @@ -194,6 +209,10 @@ required-features = ["quantum_fabric_opencl", "quantum_sv"] name = "fpga_member" required-features = ["quantum_fabric_fpga"] +[[test]] +name = "npu_member" +required-features = ["quantum_fabric_npu"] + [[test]] name = "fabric_expected" required-features = ["quantum_fabric"] diff --git a/wai/wai-quantum/README.md b/wai/wai-quantum/README.md index 3f2113163..dc509e3d0 100644 --- a/wai/wai-quantum/README.md +++ b/wai/wai-quantum/README.md @@ -6,12 +6,13 @@ every stage. No QPU, no cloud service, no vendor SDK, and **no system libraries** — so the same source runs natively, in the browser via wasm, and as a WASI component at -the edge, producing **byte-identical results and receipts on all three**. Four +the edge, producing **byte-identical results and receipts on all three**. Five opt-in features reach outside the crate: `quantum_fabric_webgpu` (the platform's GPU interface), `quantum_fabric_cuda` (the CUDA driver and runtime compiler) and `quantum_fabric_opencl` (the OpenCL ICD loader), the last two -loaded when first used, and `quantum_fabric_fpga` (Verilator, run as a program -when first used); no default feature reaches a system library. +loaded when first used, `quantum_fabric_fpga` (Verilator, run as a program +when first used) and `quantum_fabric_npu` (the platform's model runtime, on +macOS); no default feature reaches a system library. ```toml [dependencies] @@ -102,6 +103,7 @@ never creates information. | `quantum_fabric_cuda` | **the fabric's CUDA member**, opt-in and not in `full`, native targets only: the gate kernel in CUDA C with exact 64-bit integers, compiled at run time for each device's own architecture, one launch per gate. The CUDA driver and runtime compiler are loaded when first used, so a build with the feature runs on a host without CUDA and finds no member there, saying why. Each device is probed against the CPU's arithmetic in the process that uses it and certified bit-exact by reproducing the CPU reference's pinned corpus digests; a result that would leave 32 bits is refused, never returned. With `quantum_sv` too, the **dense f64 state vector** runs on the device bit for bit as on the CPU: the CPU's own launches (diagonal, one-qubit and grouped formulas), each an IEEE binary64 operation with no fused multiply-add, certified against a pinned corpus digest | | `quantum_fabric_opencl` | **the fabric's OpenCL member**, opt-in and not in `full`, native targets only: the same exact 64-bit kernel in OpenCL C, built at run time by each device's own compiler, one launch per gate, on any OpenCL GPU or CPU (an OpenCL accelerator is refused: no fabric class names it). The ICD loader is opened when first used, so a build with the feature runs on a host without OpenCL and finds no member there, saying why. Each device is probed and certified as the CUDA member's are. With `quantum_sv` too, the dense f64 state vector runs bit for bit as on the CPU on every device with binary64 (contraction off); a device without it refuses, saying so | | `quantum_fabric_fpga` | **the fabric's FPGA member, simulated**, opt-in and not in `full`, native targets only: the gate datapath in Verilog (each 32×32 product from four 16-bit partial products, the size of an FPGA's multiplier blocks; a state RAM; one pair per two cycles), compiled at run time by Verilator into a cycle-accurate simulation driven over a pipe. Probed and certified as the other members are, and every certificate says it was earned in a simulator, so placement ranks it after silicon. A host without Verilator finds no member and says why | +| `quantum_fabric_npu` | **the fabric's neural-processor member**, opt-in and not in `full`, macOS only: each 32-bit operand split into eight 4-bit limbs, every limb product and limb sum formed in fp16 on the platform's neural processor (each value an integer fp16 holds exactly, whatever order it is added in), the host assembling each product and applying the fixed-point arithmetic. The member exists only where the platform model runtime's compute plan places every operation of the model on the neural processor; a model it would run elsewhere is refused with the placement it reported. Probed and certified as every member is | | `full` | everything | Default features are `quantum`, `quantum_toolchain`, `quantum_receipt`. diff --git a/wai/wai-quantum/src/lib.rs b/wai/wai-quantum/src/lib.rs index 88003215d..eee4dd06b 100644 --- a/wai/wai-quantum/src/lib.rs +++ b/wai/wai-quantum/src/lib.rs @@ -10,12 +10,13 @@ //! Rust, so the same source runs natively, in the browser via wasm, and as a //! WASI component at the edge, producing **byte-identical** results and receipts //! on all three. A fabric receipt is the exception by design: it names the -//! member that ran the work. Four opt-in features reach outside the crate: +//! member that ran the work. Five opt-in features reach outside the crate: //! `quantum_fabric_webgpu` (the platform's GPU interface), //! `quantum_fabric_cuda` (the CUDA driver and runtime compiler) and //! `quantum_fabric_opencl` (the OpenCL ICD loader), the last two loaded when -//! first used, and `quantum_fabric_fpga` (Verilator, run as a program when -//! first used); no default feature reaches a system library. +//! first used, `quantum_fabric_fpga` (Verilator, run as a program when first +//! used) and `quantum_fabric_npu` (the platform's model runtime, on macOS); no +//! default feature reaches a system library. //! //! Everything is behind a feature; enable only what you use. @@ -85,6 +86,8 @@ pub mod quantum_fabric_cuda; pub mod quantum_fabric_opencl; #[cfg(all(feature = "quantum_fabric_fpga", not(target_arch = "wasm32")))] pub mod quantum_fabric_fpga; +#[cfg(all(feature = "quantum_fabric_npu", not(target_arch = "wasm32")))] +pub mod quantum_fabric_npu; #[cfg(feature = "quantum_control")] pub mod quantum_control; diff --git a/wai/wai-quantum/src/quantum_fabric.rs b/wai/wai-quantum/src/quantum_fabric.rs index e5dc70077..7ef4c30d8 100644 --- a/wai/wai-quantum/src/quantum_fabric.rs +++ b/wai/wai-quantum/src/quantum_fabric.rs @@ -2035,6 +2035,7 @@ pub mod lanes { pub mod cuda; pub mod fpga; +pub mod npu; pub mod opencl; pub mod plan; pub mod sampling; diff --git a/wai/wai-quantum/src/quantum_fabric/npu.rs b/wai/wai-quantum/src/quantum_fabric/npu.rs new file mode 100644 index 000000000..c8ff579c4 --- /dev/null +++ b/wai/wai-quantum/src/quantum_fabric/npu.rs @@ -0,0 +1,408 @@ +//! The fabric's arithmetic on a neural processor, and what names that member: +//! the limb-product model, the layout of a batch, the exact assembly of each +//! product, and the implementation id. Nothing here touches a device, so all of +//! it is testable on any host; the member itself is `quantum_fabric_npu`. +//! +//! **Why the products are exact.** A neural processor computes in fp16, and does +//! not document how it accumulates. So each 32-bit operand is split into eight +//! 4-bit limbs (the top one signed, `−8..=7`, the rest `0..=15`), and the +//! processor computes, for each of the 15 shifts `s`, the limb sum +//! `Σ_{i+j=s} a_i·b_j`: the coefficients of the product of two polynomials in +//! 16. Every limb product is at most 225 in size and every sum of at most eight +//! of them at most 1800, below 2048, so every value the processor forms is an +//! integer fp16 holds exactly, whatever order it adds in. The host then +//! assembles `a·b = Σ_s z_s·16^s` exactly in 64 bits, and the gate's arithmetic +//! (`fx`, the row sums, the 32-bit lane and the range report) is +//! [`lanes::apply_words`](super::lanes::apply_words)'s. +//! +//! The model is one multiply and one sum over the last axis +//! ([`limb_product_model`]); the layout puts operand `a`'s limbs in `x[p, s, j]` +//! and `b`'s limb `s − j` in `y[p, s, j]` (zero outside), so +//! `z[p, s] = Σ_j x[p, s, j]·y[p, s, j]` is the coefficient `s`. +//! +//! **What the processor is credited with.** The member exists only where the +//! runtime's compute plan places every operation of the model on the neural +//! processor; a run the runtime would place on the CPU is refused, not counted. + +use super::FabricClass; + +/// Products per batch: the model's fixed first dimension. +pub const BATCH: usize = 16384; +/// Limbs per operand. +pub const LIMBS: usize = 8; +/// Coefficients of a product: `2·LIMBS − 1`. +pub const SHIFTS: usize = 2 * LIMBS - 1; + +/// Version of the host-side layout: the limb split, the `x`/`y` expansion, the +/// assembly, and the batch. Bumped on any change; it is part of the +/// implementation id. +pub const HOST_LAYOUT: u32 = 1; + +/// The implementation name the id starts with. +pub const IMPLEMENTATION: &str = "npu-limbs4"; + +const DOMAIN_IMPLEMENTATION: &[u8] = b"wai:quantum-fabric-npu\x01"; + +/// The class a neural-processor member has. +pub const CLASS: FabricClass = FabricClass::Npu; + +/// Protobuf wire types. +const VARINT: u32 = 0; +const LEN: u32 = 2; + +fn varint(o: &mut Vec, mut v: u64) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + o.push(b); + return; + } + o.push(b | 0x80); + } +} + +fn key(o: &mut Vec, field: u32, wire: u32) { + varint(o, u64::from(field << 3 | wire)); +} + +fn uint(o: &mut Vec, field: u32, v: u64) { + key(o, field, VARINT); + varint(o, v); +} + +/// A signed int64 field (two's complement, as protobuf's int64 is written). +fn int(o: &mut Vec, field: u32, v: i64) { + key(o, field, VARINT); + varint(o, v as u64); +} + +fn bytes(o: &mut Vec, field: u32, b: &[u8]) { + key(o, field, LEN); + varint(o, b.len() as u64); + o.extend_from_slice(b); +} + +fn msg(o: &mut Vec, field: u32, f: impl FnOnce(&mut Vec)) { + let mut inner = Vec::new(); + f(&mut inner); + bytes(o, field, &inner); +} + +// The model description's field numbers (its Model, FeatureTypes and MIL schemas). +const ARRAY_FLOAT16: u64 = 65552; +/// The operation set the program is written in. +const OPSET: &[u8] = b"CoreML8"; // allow-name: the runtime's operation-set identifier +const MIL_BOOL: u64 = 1; +const MIL_STRING: u64 = 2; +const MIL_FLOAT16: u64 = 10; +const MIL_INT32: u64 = 23; + +/// A MIL tensor type of `dtype` and `shape` (rank 0 for a scalar). +fn tensor_type(o: &mut Vec, dtype: u64, shape: &[u64]) { + // ValueType.tensorType = 1 + msg(o, 1, |t| { + uint(t, 1, dtype); // dataType + if !shape.is_empty() { + int(t, 2, shape.len() as i64); // rank + } + for &d in shape { + // dimensions = 3 { constant = 1 { size = 1 } } + msg(t, 3, |dim| msg(dim, 1, |c| uint(c, 1, d))); + } + }); +} + +/// A feature description: `name`, a multi-array of fp16 and `shape`. +fn feature(o: &mut Vec, field: u32, name: &str, shape: &[u64]) { + msg(o, field, |f| { + bytes(f, 1, name.as_bytes()); + // type = 3 { multiArrayType = 5 { shape = 1 (repeated), dataType = 2 } } + msg(f, 3, |t| { + msg(t, 5, |a| { + for &d in shape { + int(a, 1, d as i64); + } + uint(a, 2, ARRAY_FLOAT16); + }) + }); + }); +} + +/// A function input `name`: a MIL fp16 tensor of `shape`. +fn named_input(o: &mut Vec, name: &str, shape: &[u64]) { + // Function.inputs = 1 { name = 1, type = 2 } + msg(o, 1, |n| { + bytes(n, 1, name.as_bytes()); + msg(n, 2, |v| tensor_type(v, MIL_FLOAT16, shape)); + }); +} + +/// An operation's `name` attribute. +fn name_attribute(o: &mut Vec, name: &str) { + // attributes = 5 (map) { key = 1, value = 2 } + msg(o, 5, |e| { + bytes(e, 1, b"name"); + msg(e, 2, |v| { + msg(v, 2, |t| tensor_type(t, MIL_STRING, &[])); // Value.type + // immediateValue = 3 { tensor = 1 { strings = 4 { values = 1 } } } + msg(v, 3, |iv| msg(iv, 1, |tv| msg(tv, 4, |s| bytes(s, 1, name.as_bytes())))); + }); + }); +} + +/// An operation input binding: parameter `param` reads variable `var`. +fn input_binding(o: &mut Vec, param: &str, var: &str) { + // inputs = 2 (map) { key, value = Argument { arguments = 1 { name = 1 } } } + msg(o, 2, |e| { + bytes(e, 1, param.as_bytes()); + msg(e, 2, |arg| msg(arg, 1, |b| bytes(b, 1, var.as_bytes()))); + }); +} + +/// An operation output `name` of `dtype` and `shape`. +fn op_output(o: &mut Vec, name: &str, dtype: u64, shape: &[u64]) { + // outputs = 3 { name = 1, type = 2 } + msg(o, 3, |n| { + bytes(n, 1, name.as_bytes()); + msg(n, 2, |v| tensor_type(v, dtype, shape)); + }); +} + +/// A `const` operation `name` holding `value` (written by `write`, a +/// TensorValue) of `dtype` and `shape`. +fn const_op(o: &mut Vec, name: &str, dtype: u64, shape: &[u64], write: impl FnOnce(&mut Vec)) { + // Block.operations = 3 + msg(o, 3, |op| { + bytes(op, 1, b"const"); + op_output(op, name, dtype, shape); + msg(op, 5, |e| { + bytes(e, 1, b"val"); + msg(e, 2, |v| { + msg(v, 2, |t| tensor_type(t, dtype, shape)); + msg(v, 3, |iv| msg(iv, 1, write)); + }); + }); + name_attribute(op, name); + }); +} + +/// The model: `z[b, s] = Σ_j x[b, s, j] · y[b, s, j]`, all fp16, for +/// `batch` products of `shifts` limb sums of `limbs` terms. +pub fn limb_product_model(batch: u64, shifts: u64, limbs: u64) -> Vec { + let full = [batch, shifts, limbs]; + let out = [batch, shifts]; + let mut m = Vec::new(); + uint(&mut m, 1, 9); // specificationVersion + msg(&mut m, 2, |d| { + feature(d, 1, "x", &full); + feature(d, 1, "y", &full); + feature(d, 10, "z", &out); + }); + // mlProgram = 502 { version = 1, functions = 2 (map) } + msg(&mut m, 502, |p| { + int(p, 1, 1); + msg(p, 2, |e| { + bytes(e, 1, b"main"); + msg(e, 2, |f| { + named_input(f, "x", &full); + named_input(f, "y", &full); + bytes(f, 2, OPSET); // opset + // block_specializations = 3 (map) + msg(f, 3, |e| { + bytes(e, 1, OPSET); + msg(e, 2, |b| { + bytes(b, 2, b"z"); // Block.outputs + // mul + msg(b, 3, |op| { + bytes(op, 1, b"mul"); + input_binding(op, "x", "x"); + input_binding(op, "y", "y"); + op_output(op, "mul_0", MIL_FLOAT16, &full); + name_attribute(op, "mul_0"); + }); + const_op(b, "z_axes_0", MIL_INT32, &[1], |tv| msg(tv, 2, |ints| int(ints, 1, -1))); + const_op(b, "z_keep_dims_0", MIL_BOOL, &[], |tv| msg(tv, 3, |bools| uint(bools, 1, 0))); + // reduce_sum + msg(b, 3, |op| { + bytes(op, 1, b"reduce_sum"); + input_binding(op, "x", "mul_0"); + input_binding(op, "axes", "z_axes_0"); + input_binding(op, "keep_dims", "z_keep_dims_0"); + op_output(op, "z", MIL_FLOAT16, &out); + name_attribute(op, "z"); + }); + }); + }); + }); + }); + }); + m +} + + +/// `"npu-limbs4/<16 hex>"` over the model's bytes and the host layout. +pub fn implementation_id_of(model: &[u8]) -> String { + let mut o = DOMAIN_IMPLEMENTATION.to_vec(); + o.extend_from_slice(&(model.len() as u64).to_be_bytes()); + o.extend_from_slice(model); + o.extend_from_slice(&HOST_LAYOUT.to_be_bytes()); + let h: String = blake3::hash(&o).as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect(); + format!("{IMPLEMENTATION}/{h}") +} + +/// The model this member runs. +pub fn model() -> Vec { + limb_product_model(BATCH as u64, SHIFTS as u64, LIMBS as u64) +} + +pub fn implementation_id() -> String { + implementation_id_of(&model()) +} + +/// The eight 4-bit limbs of `v`, least significant first, the top one signed. +pub fn limbs(v: i32) -> [i32; LIMBS] { + let u = v as u32; + let mut out = [0i32; LIMBS]; + for (i, l) in out.iter_mut().enumerate() { + *l = ((u >> (4 * i)) & 15) as i32; + } + if out[LIMBS - 1] >= 8 { + out[LIMBS - 1] -= 16; + } + out +} + +/// The fp16 bits of an integer `v` with `|v| ≤ 2048`, exactly. +pub fn f16_of(v: i32) -> u16 { + debug_assert!(v.abs() <= 2048); + if v == 0 { + return 0; + } + let sign: u16 = if v < 0 { 0x8000 } else { 0 }; + let m = v.unsigned_abs(); + let e = 31 - m.leading_zeros(); // floor(log2 m), 0..=11 + // Significand bits below the leading one, left-aligned in 10 bits. + let frac = ((m - (1 << e)) << (10 - e)) as u16; + sign | (((e + 15) as u16) << 10) | frac +} + +/// The integer an fp16 value holds, or `None` if it is not an integer of at most +/// 2048 in size. +pub fn int_of_f16(bits: u16) -> Option { + let sign = if bits & 0x8000 != 0 { -1 } else { 1 }; + let e = i32::from((bits >> 10) & 0x1f); + let frac = i64::from(bits & 0x3ff); + if e == 0 { + return (frac == 0).then_some(0); + } + if e == 31 { + return None; + } + let p = e - 15; // the exponent + if !(0..=11).contains(&p) { + return None; + } + let m = (1024 + frac) << p; // value · 2^10 + (m & 1023 == 0).then_some(sign * (m >> 10)) +} + +/// Writes the inputs of one batch for `pairs` (at most [`BATCH`]; the rest is +/// zero): `x[p, s, j] = limb j of a`, `y[p, s, j] = limb s − j of b`, as fp16 +/// bits, row-major. +pub fn expand(pairs: &[(i32, i32)], x: &mut [u16], y: &mut [u16]) { + assert!(pairs.len() <= BATCH && x.len() == BATCH * SHIFTS * LIMBS && y.len() == x.len()); + x.fill(0); + y.fill(0); + for (p, &(a, b)) in pairs.iter().enumerate() { + let (la, lb) = (limbs(a), limbs(b)); + for s in 0..SHIFTS { + for j in 0..LIMBS { + if s >= j && s - j < LIMBS { + let at = (p * SHIFTS + s) * LIMBS + j; + x[at] = f16_of(la[j]); + y[at] = f16_of(lb[s - j]); + } + } + } + } +} + +/// The product `a·b` from its 15 coefficients: `Σ_s z_s·16^s`, exact in 64 bits. +pub fn assemble(z: &[i64]) -> i64 { + z.iter().rev().fold(0i64, |acc, &c| acc.wrapping_mul(16).wrapping_add(c)) +} + +/// The model's arithmetic in integers, on the host: what a processor that +/// computes it exactly returns. For the tests, and for nothing else. +pub fn replay(x: &[u16], y: &[u16]) -> Vec { + x.chunks(LIMBS).zip(y.chunks(LIMBS)).map(|(a, b)| a.iter().zip(b).map(|(&p, &q)| int_of_f16(p).unwrap_or(0) * int_of_f16(q).unwrap_or(0)).sum()).collect() +} + +/// The model with one operation's type replaced: for the mutation tests. +pub fn mutate_model(op_from: &str, op_to: &str) -> Vec { + let m = model(); + let (from, to) = (op_from.as_bytes(), op_to.as_bytes()); + assert_eq!(from.len(), to.len(), "a mutant keeps the operation name's length"); + let at = m.windows(from.len()).position(|w| w == from).expect("the operation is in the model"); + let mut out = m; + out[at..at + to.len()].copy_from_slice(to); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::quantum_fabric::lanes; + + #[test] + fn small_integers_round_trip_through_fp16_exactly() { + for v in -2048..=2048 { + assert_eq!(int_of_f16(f16_of(v)), Some(i64::from(v)), "{v}"); + } + assert_eq!(f16_of(1), 0x3c00); + assert_eq!(f16_of(-2), 0xc000); + assert_eq!(f16_of(1800), 0x6708); + assert_eq!(int_of_f16(0x3800), None, "one half is no integer"); + assert_eq!(int_of_f16(0x7c00), None, "infinity is no integer"); + } + + #[test] + fn limbs_reassemble_and_every_value_stays_exact_in_fp16() { + for v in [0, 1, -1, 7, -8, i32::MAX, i32::MIN, 0x7654_3210, -0x0f0f_0f0f] { + let l = limbs(v); + assert_eq!(l.iter().rev().fold(0i64, |a, &x| a * 16 + i64::from(x)), i64::from(v), "{v}"); + assert!(l[..LIMBS - 1].iter().all(|x| (0..=15).contains(x)) && (-8..=7).contains(&l[LIMBS - 1])); + } + // The largest coefficient a batch can form. + assert!(LIMBS as i64 * 15 * 15 <= 2048); + } + + #[test] + fn a_batch_replayed_assembles_every_product_exactly() { + let mut pairs = lanes::probe_pairs(4096, 0x11); + pairs.truncate(BATCH); + let (mut x, mut y) = (vec![0u16; BATCH * SHIFTS * LIMBS], vec![0u16; BATCH * SHIFTS * LIMBS]); + expand(&pairs, &mut x, &mut y); + let z = replay(&x, &y); + assert!(z.iter().all(|c| c.abs() <= 1800)); + for (p, &(a, b)) in pairs.iter().enumerate() { + let got = assemble(&z[p * SHIFTS..(p + 1) * SHIFTS]); + assert_eq!(got, i64::from(a) * i64::from(b), "({a}, {b})"); + } + } + + #[test] + fn the_model_is_the_program_it_names_and_its_mutants_differ() { + let m = model(); + assert_eq!(m[0], 0x08, "field 1, specificationVersion, comes first"); + assert_eq!(m[1], 9); + for op in [&b"mul"[..], b"reduce_sum", OPSET] { + assert!(m.windows(op.len()).any(|w| w == op)); + } + let id = implementation_id(); + assert!(id.starts_with("npu-limbs4/") && id.len() == "npu-limbs4/".len() + 16); + assert_ne!(implementation_id_of(&mutate_model("mul", "add")), id); + assert_ne!(implementation_id_of(&mutate_model("reduce_sum", "reduce_max")), id); + } +} diff --git a/wai/wai-quantum/src/quantum_fabric_npu.rs b/wai/wai-quantum/src/quantum_fabric_npu.rs new file mode 100644 index 000000000..d6c4c7e64 --- /dev/null +++ b/wai/wai-quantum/src/quantum_fabric_npu.rs @@ -0,0 +1,681 @@ +//! The fabric's neural-processor member: the gate arithmetic of +//! [`quantum_fabric::npu`](crate::quantum_fabric::npu) on the platform's neural +//! processor, through its model runtime. See that module for why every product +//! is exact. +//! +//! - **Discovery** ([`discover`]): on macOS, the limb-product model is written, +//! compiled by the platform's model runtime and loaded for the CPU and the +//! neural processor. The runtime's compute plan must place **every** +//! operation of the model on the neural processor; a model the runtime would +//! run elsewhere, even in part, is refused with the placement it reported, +//! so no CPU result is credited to the processor. The products are then +//! probed against [`lanes`] in this process. Elsewhere, nothing is found, and +//! [`Discovery::unavailable`] says why. +//! - **Running** ([`NpuMember::run_program`]): the host holds the state; for +//! each gate, every product the gate needs (sixteen a pair) goes to the +//! processor in batches, and the host assembles each product and applies +//! [`lanes::apply_words`]'s fixed-point arithmetic. A result that leaves 32 +//! bits is [`NpuError::OutOfRange`], never a state. +//! - **Certification, placement and receipts** are as for every member. +//! +//! # Honest boundaries +//! +//! - **The processor multiplies; the host assembles.** Every limb product and +//! every limb sum is formed on the processor; the shifts that join the 15 +//! coefficients of a product, the rounding and the row sums are the host's. +//! - **Placement is the runtime's report.** The compute plan is the runtime's +//! statement of where each operation runs; it is checked before the member +//! exists, and named in its detail. +//! - **Unmetered**: the processor's energy counter is not readable unprivileged. + +use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector}; +use crate::quantum_fabric::lanes::{self, GateWords, Lane}; +use crate::quantum_fabric::npu::{self, BATCH, LIMBS, SHIFTS}; +use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step}; +use crate::quantum_fabric::{Certificate, Certification, Corpus, KernelEntry, Member, MemberError, Presence, RunFuture, RunMeter, Runner, apply_kernel}; + +/// One probe case: [`lanes::mul_wide`] and [`lanes::fxmul_wide`], each as +/// `(high, low)` words. +pub type ProbeOutput = ((u32, u32), (u32, u32)); + +/// Why the processor is not a member, or a run has no state. +#[derive(Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum NpuError { + /// No model runtime with a neural processor on this platform, and why. + Unavailable(String), + /// The runtime refused or failed the model. + Runtime(String), + /// The runtime would run part of the model elsewhere: each operation and + /// the device it reported. + Placement(String), + Circuit(QuantumError), + TooWide { n_qubits: u8, max: u8 }, + OutOfRange { gate: u32 }, + Internal(&'static str), + Probe { a: i32, b: i32 }, +} + +impl std::fmt::Display for NpuError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + NpuError::Unavailable(why) => write!(f, "no neural processor: {why}"), + NpuError::Runtime(e) => write!(f, "model runtime: {e}"), + NpuError::Placement(p) => write!(f, "the runtime would not run the whole model on the neural processor: {p}"), + NpuError::Circuit(e) => write!(f, "{e}"), + NpuError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"), + NpuError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"), + NpuError::Internal(why) => write!(f, "host defect: {why}"), + NpuError::Probe { a, b } => write!(f, "the processor's products differ from the CPU's at ({a}, {b})"), + } + } +} + +impl std::error::Error for NpuError {} + +impl From for MemberError { + fn from(e: NpuError) -> MemberError { + match e { + NpuError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max }, + NpuError::OutOfRange { gate } => MemberError::OutOfRange { gate }, + NpuError::Runtime(why) => MemberError::Device(why), + NpuError::Circuit(e) => MemberError::Circuit(e), + other => MemberError::Other(other.to_string()), + } + } +} + +/// What [`discover`] found: at most one member, the platform's processor. +#[derive(Debug)] +pub struct Discovery { + pub found: Vec, + /// The processor, when it was not a member, with why. + pub refused: Vec<(String, NpuError)>, + pub unavailable: Option, +} + +/// The class name the runtime gives the neural processor in its compute plan. +pub const NEURAL_PROCESSOR: &str = "MLNeuralEngineComputeDevice"; // allow-name: the runtime's device class name + +/// The most qubits a run holds: the host holds the state, and every gate's +/// products cross to the processor, so wide runs are slow rather than refused. +pub const MAX_NPU_QUBITS: u8 = 20; + +/// The sixteen products a pair needs, in the order [`apply_pair`] reads them. +fn pair_products(m: &[i32; 8], a0: Lane, a1: Lane, out: &mut Vec<(i32, i32)>) { + let (a0r, a0i, a1r, a1i) = (a0[0], a0[1], a1[0], a1[1]); + out.extend_from_slice(&[ + (m[0], a0r), + (m[1], a0i), + (m[2], a1r), + (m[3], a1i), + (m[0], a0i), + (m[1], a0r), + (m[2], a1i), + (m[3], a1r), + (m[4], a0r), + (m[5], a0i), + (m[6], a1r), + (m[7], a1i), + (m[4], a0i), + (m[5], a0r), + (m[6], a1i), + (m[7], a1r), + ]); +} + +/// `fx` of an exact product: `(p + 2^29) >> 30`. +fn fx(p: i64) -> i64 { + (p + (1 << 29)) >> 30 +} + +fn fits32(v: i64) -> bool { + i32::try_from(v).is_ok() +} + +/// A pair's two new lanes from its sixteen products, and whether a sum left +/// 32 bits: row 0 is `(m0 + i·m1)·a0 + (m2 + i·m3)·a1`, row 1 likewise with +/// `m4..m7`, as [`lanes::apply_words`] computes them. +fn apply_pair(p: &[i64]) -> (Lane, Lane, bool) { + let f: Vec = p.iter().map(|&x| fx(x)).collect(); + let o0r = f[0] - f[1] + f[2] - f[3]; + let o0i = f[4] + f[5] + f[6] + f[7]; + let o1r = f[8] - f[9] + f[10] - f[11]; + let o1i = f[12] + f[13] + f[14] + f[15]; + let bad = !(fits32(o0r) && fits32(o0i) && fits32(o1r) && fits32(o1i)); + let low = |v: i64| v as u64 as u32 as i32; + ([low(o0r), low(o0i)], [low(o1r), low(o1i)], bad) +} + +#[cfg(target_os = "macos")] +mod coreml { + use super::NpuError; + use crate::quantum_fabric::npu::{BATCH, LIMBS, SHIFTS}; + use objc2::AnyThread; + use objc2::rc::Retained; + use objc2::runtime::{AnyObject, ProtocolObject}; + use objc2_core_ml::{ + MLComputePlan, MLComputeUnits, MLDictionaryFeatureProvider, MLFeatureProvider, MLFeatureValue, MLModel, MLModelConfiguration, MLMultiArray, MLMultiArrayDataType, + }; + use objc2_foundation::{NSArray, NSDictionary, NSError, NSNumber, NSString, NSURL}; + use std::path::Path; + use std::sync::{Arc, Condvar, Mutex}; + + fn ns_error(e: &NSError) -> String { + e.localizedDescription().to_string() + } + + /// A compiled, loaded model, and where its runtime places each operation. + pub struct Loaded { + pub model: Retained, + /// `(operation, device class)` for every operation but constants. + pub placement: Vec<(String, String)>, + } + + fn config() -> Retained { + // SAFETY: a fresh configuration, set before any use. + unsafe { + let c = MLModelConfiguration::new(); + c.setComputeUnits(MLComputeUnits::CPUAndNeuralEngine); + c + } + } + + /// Writes `model` under `dir`, compiles it, loads it for the CPU and the + /// neural processor, and reads the runtime's compute plan for it. + pub fn load(model: &[u8], dir: &Path) -> Result { + std::fs::create_dir_all(dir).map_err(|e| NpuError::Runtime(format!("{}: {e}", dir.display())))?; + let src = dir.join("limbs.mlmodel"); + std::fs::write(&src, model).map_err(|e| NpuError::Runtime(e.to_string()))?; + let url = NSURL::fileURLWithPath(&NSString::from_str(&src.to_string_lossy())); + // SAFETY: a file URL to a model description just written. + #[allow(deprecated)] // The synchronous form: discovery is synchronous. + let compiled = unsafe { MLModel::compileModelAtURL_error(&url) }.map_err(|e| NpuError::Runtime(format!("compiling the model: {}", ns_error(&e))))?; + let cfg = config(); + // SAFETY: the compiled model's URL and a configuration. + let model = unsafe { MLModel::modelWithContentsOfURL_configuration_error(&compiled, &cfg) }.map_err(|e| NpuError::Runtime(format!("loading the model: {}", ns_error(&e))))?; + let placement = plan(&compiled, &cfg)?; + Ok(Loaded { model, placement }) + } + + /// The runtime's compute plan for the compiled model: each operation's + /// preferred device. + fn plan(compiled: &NSURL, cfg: &MLModelConfiguration) -> Result, NpuError> { + type Slot = (Mutex, String>>>, Condvar); + let slot: Arc = Arc::new((Mutex::new(None), Condvar::new())); + let s = slot.clone(); + let handler = block2::RcBlock::new(move |plan: *mut MLComputePlan, err: *mut NSError| { + // SAFETY: the runtime hands one of the two, live for the call. + let r = unsafe { + match Retained::retain(plan) { + Some(p) => Ok(p), + None => Err(err.as_ref().map_or_else(|| "no plan".to_string(), ns_error)), + } + }; + let (lock, cv) = &*s; + *lock.lock().unwrap_or_else(|p| p.into_inner()) = Some(r); + cv.notify_all(); + }); + // SAFETY: the URL of a compiled model, a configuration, and a handler + // that only stores its arguments. + unsafe { MLComputePlan::loadContentsOfURL_configuration_completionHandler(compiled, cfg, &handler) }; + let (lock, cv) = &*slot; + let guard = lock.lock().unwrap_or_else(|p| p.into_inner()); + let (mut guard, timeout) = cv.wait_timeout_while(guard, std::time::Duration::from_secs(120), |r| r.is_none()).unwrap_or_else(|p| p.into_inner()); + if timeout.timed_out() { + return Err(NpuError::Runtime("the compute plan did not arrive".into())); + } + let plan = guard.take().expect("set before the notify").map_err(|e| NpuError::Runtime(format!("the compute plan: {e}")))?; + let mut out = Vec::new(); + // SAFETY: reads of the plan's structure, on this thread. + unsafe { + let structure = plan.modelStructure(); + let program = structure.program().ok_or_else(|| NpuError::Runtime("the plan has no program".into()))?; + let functions = program.functions(); + let main = functions.objectForKey(&NSString::from_str("main")).ok_or_else(|| NpuError::Runtime("the program has no main function".into()))?; + for op in main.block().operations().iter() { + let name = op.operatorName().to_string(); + if name == "const" { + continue; + } + let device = match plan.computeDeviceUsageForMLProgramOperation(&op) { + Some(usage) => { + let d = usage.preferredComputeDevice(); + let any: &AnyObject = d.as_ref(); + any.class().name().to_string_lossy().into_owned() + } + None => "unknown".into(), + }; + out.push((name, device)); + } + } + Ok(out) + } + + /// The fp16 array `[BATCH, SHIFTS, LIMBS]` holding `bits`. + fn array(bits: &[u16]) -> Result, NpuError> { + let shape: Retained> = NSArray::from_retained_slice(&[NSNumber::new_usize(BATCH), NSNumber::new_usize(SHIFTS), NSNumber::new_usize(LIMBS)]); + // SAFETY: a contiguous fp16 array of the model's input shape, then + // filled with exactly that many values. + unsafe { + let a = MLMultiArray::initWithShape_dataType_error(MLMultiArray::alloc(), &shape, MLMultiArrayDataType::Float16).map_err(|e| NpuError::Runtime(ns_error(&e)))?; + #[allow(deprecated)] + let p = a.dataPointer().as_ptr().cast::(); + std::ptr::copy_nonoverlapping(bits.as_ptr(), p, BATCH * SHIFTS * LIMBS); + Ok(a) + } + } + + /// One batch through the model: `z`, `[BATCH, SHIFTS]`, as fp16 bits. + pub fn predict(model: &MLModel, x: &[u16], y: &[u16]) -> Result, NpuError> { + let (ax, ay) = (array(x)?, array(y)?); + // SAFETY: a feature provider of the model's two inputs, a prediction, + // and a read of its fp16 output of the shape the model declares. + unsafe { + let fx = MLFeatureValue::featureValueWithMultiArray(&ax); + let fy = MLFeatureValue::featureValueWithMultiArray(&ay); + let keys = [NSString::from_str("x"), NSString::from_str("y")]; + let values: [Retained; 2] = [Retained::into_super(Retained::into_super(fx)), Retained::into_super(Retained::into_super(fy))]; + let dict: Retained> = NSDictionary::from_retained_objects(&[&*keys[0], &*keys[1]], &values); + let provider = MLDictionaryFeatureProvider::initWithDictionary_error(MLDictionaryFeatureProvider::alloc(), &dict).map_err(|e| NpuError::Runtime(ns_error(&e)))?; + let out = model.predictionFromFeatures_error(ProtocolObject::from_ref(&*provider)).map_err(|e| NpuError::Runtime(format!("predicting: {}", ns_error(&e))))?; + let z = out.featureValueForName(&NSString::from_str("z")).and_then(|v| v.multiArrayValue()).ok_or_else(|| NpuError::Runtime("no output z".into()))?; + if z.dataType() != MLMultiArrayDataType::Float16 || z.count() as usize != BATCH * SHIFTS { + return Err(NpuError::Runtime("the output is not the declared fp16 [batch, shifts]".into())); + } + // The runtime may pad each row (strides [32, 1] for 15 values); read + // through the strides it reports, which must hold each row whole. + let strides: Vec = z.strides().iter().map(|n| n.as_usize()).collect(); + let [row, one] = strides[..] else { return Err(NpuError::Runtime(format!("the output has strides {strides:?}"))) }; + if one != 1 || row < SHIFTS { + return Err(NpuError::Runtime(format!("the output's strides {strides:?} do not hold a row of {SHIFTS}"))); + } + #[allow(deprecated)] + let p = z.dataPointer().as_ptr().cast::(); + let all = std::slice::from_raw_parts(p, (BATCH - 1) * row + SHIFTS); + let mut out = Vec::with_capacity(BATCH * SHIFTS); + for r in 0..BATCH { + out.extend_from_slice(&all[r * row..r * row + SHIFTS]); + } + Ok(out) + } + } +} + +/// Every neural processor on this host: on macOS, the platform's, built, +/// placed and probed in this process. +pub fn discover() -> Discovery { + let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None }; + #[cfg(target_os = "macos")] + { + match open(&npu::model()) { + Ok(m) => d.found.push(m), + Err(e) => d.refused.push(("npu-0".into(), e)), + } + } + #[cfg(not(target_os = "macos"))] + { + d.unavailable = Some("this platform has no neural-processor runtime this member reaches".into()); + } + d +} + +/// The member: the loaded model behind a lock (the runtime's model object is +/// used from one thread at a time), and what discovery established. +pub struct NpuMember { + #[cfg(target_os = "macos")] + model: std::sync::Arc>>, + id: String, + key: String, + implementation: String, + detail: String, + placement: Vec<(String, String)>, + max_qubits: u8, +} + +// SAFETY: the runtime's model object is reached only through the mutex, one +// thread at a time; everything else is plain data. +unsafe impl Send for NpuMember {} +// SAFETY: as above. +unsafe impl Sync for NpuMember {} + +impl Clone for NpuMember { + fn clone(&self) -> Self { + NpuMember { + #[cfg(target_os = "macos")] + model: self.model.clone(), + id: self.id.clone(), + key: self.key.clone(), + implementation: self.implementation.clone(), + detail: self.detail.clone(), + placement: self.placement.clone(), + max_qubits: self.max_qubits, + } + } +} + +impl std::fmt::Debug for NpuMember { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NpuMember").field("id", &self.id).field("implementation", &self.implementation).field("placement", &self.placement).finish() + } +} + +#[cfg(target_os = "macos")] +fn cache_dir(model: &[u8]) -> std::path::PathBuf { + let base = std::env::var_os("WAI_QUANTUM_NPU_CACHE").map(std::path::PathBuf::from).unwrap_or_else(|| std::env::temp_dir().join("wai-quantum-npu")); + let tag: String = blake3::hash(model).as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect(); + base.join(format!("{tag}-{}", std::process::id())) +} + +/// The neural processor running `model`: loaded, its placement checked, and +/// its products probed. +#[cfg(target_os = "macos")] +fn open(model: &[u8]) -> Result { + let loaded = coreml::load(model, &cache_dir(model))?; + let elsewhere: Vec = loaded.placement.iter().filter(|(_, d)| d != NEURAL_PROCESSOR).map(|(op, d)| format!("{op} on {d}")).collect(); + let placement_text = loaded.placement.iter().map(|(op, d)| format!("{op}: {d}")).collect::>().join(", "); + if loaded.placement.is_empty() || !elsewhere.is_empty() { + return Err(NpuError::Placement(if elsewhere.is_empty() { "no operation was reported".into() } else { elsewhere.join("; ") })); + } + let mut o = b"wai:quantum-fabric-npu-member\x01".to_vec(); + o.extend_from_slice(std::env::consts::ARCH.as_bytes()); + o.extend_from_slice(placement_text.as_bytes()); + let hw = std::process::Command::new("sysctl").args(["-n", "machdep.cpu.brand_string"]).output().map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()).unwrap_or_default(); + o.extend_from_slice(hw.as_bytes()); + let id = format!("local/npu-{}", blake3::hash(&o).as_bytes()[..6].iter().map(|b| format!("{b:02x}")).collect::()); + let member = NpuMember { + model: std::sync::Arc::new(std::sync::Mutex::new(loaded.model)), + id, + key: "npu-0".into(), + implementation: npu::implementation_id_of(model), + detail: format!( + "npu; the platform's neural processor through its model runtime; placement by the runtime's compute plan: {placement_text}; fp16 limb products, the host assembling; up to {MAX_NPU_QUBITS} qubits; unmetered" + ), + placement: loaded.placement, + max_qubits: MAX_NPU_QUBITS.min(MAX_QUBITS), + }; + let pairs = lanes::probe_pairs(4096, 0x5eed); + let out = member.probe(&pairs)?; + match lanes::first_mismatch(&pairs, &out) { + None => Ok(member), + Some((a, b)) => Err(NpuError::Probe { a, b }), + } +} + +impl NpuMember { + pub fn id(&self) -> &str { + &self.id + } + + /// The key a runner declares: `"npu-0"`. + pub fn key(&self) -> &str { + &self.key + } + + pub fn class(&self) -> crate::quantum_fabric::FabricClass { + npu::CLASS + } + + pub fn detail(&self) -> &str { + &self.detail + } + + pub fn implementation(&self) -> &str { + &self.implementation + } + + pub fn max_qubits(&self) -> u8 { + self.max_qubits + } + + /// Each operation of the model and the device the runtime placed it on. + pub fn placement(&self) -> &[(String, String)] { + &self.placement + } + + /// Exact products `a·b`, every limb product and limb sum formed on the + /// processor. + pub fn products(&self, pairs: &[(i32, i32)]) -> Result, NpuError> { + let mut out = Vec::with_capacity(pairs.len()); + let (mut x, mut y) = (vec![0u16; BATCH * SHIFTS * LIMBS], vec![0u16; BATCH * SHIFTS * LIMBS]); + for chunk in pairs.chunks(BATCH) { + npu::expand(chunk, &mut x, &mut y); + let z = self.predict(&x, &y)?; + for p in 0..chunk.len() { + let mut c = [0i64; SHIFTS]; + for (s, slot) in c.iter_mut().enumerate() { + *slot = npu::int_of_f16(z[p * SHIFTS + s]).ok_or(NpuError::Runtime("a coefficient that is no integer of fp16's exact range".into()))?; + } + out.push(npu::assemble(&c)); + } + } + Ok(out) + } + + #[cfg(target_os = "macos")] + fn predict(&self, x: &[u16], y: &[u16]) -> Result, NpuError> { + let m = self.model.lock().unwrap_or_else(|p| p.into_inner()); + coreml::predict(&m, x, y) + } + + #[cfg(not(target_os = "macos"))] + fn predict(&self, _x: &[u16], _y: &[u16]) -> Result, NpuError> { + Err(NpuError::Unavailable("no runtime on this platform".into())) + } + + /// The processor's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each pair. + pub fn probe(&self, pairs: &[(i32, i32)]) -> Result, NpuError> { + let products = self.products(pairs)?; + let split = |v: i64| ((v as u64 >> 32) as u32, v as u64 as u32); + Ok(products.iter().map(|&p| (split(p), split(fx(p)))).collect()) + } + + pub fn config(&self) -> Config { + Config::PER_GATE + } + + /// Gates in order from `state`, as [`lanes::run_words`] runs them. + pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result { + let len = state.len(); + if len < 2 || !len.is_power_of_two() { + return Err(NpuError::Internal("a state that is not 2^n lanes, n at least 1")); + } + let n = len.trailing_zeros() as u8; + if n > self.max_qubits { + return Err(NpuError::TooWide { n_qubits: n, max: self.max_qubits }); + } + let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect(); + self.execute(state.to_vec(), &planned) + } + + fn execute(&self, mut amps: Vec, gates: &[(&GateWords, u32)]) -> Result { + let len = amps.len(); + for (w, _) in gates { + let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize); + if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 { + return Err(NpuError::Internal("malformed gate words for this state")); + } + } + let mut report: Option = None; + let mut products = Vec::new(); + let mut pairs = Vec::new(); + for &(w, index) in gates { + let m: [i32; 8] = w.m.map(|x| x as i32); + let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize); + pairs.clear(); + products.clear(); + for p in 0..len / 2 { + let low = p & (tbit - 1); + let i = ((p - low) << 1) | low; + if i & ctrl != ctrl { + continue; + } + pairs.push(i); + pair_products(&m, amps[i], amps[i | tbit], &mut products); + } + let exact = self.products(&products)?; + for (k, &i) in pairs.iter().enumerate() { + let (o0, o1, bad) = apply_pair(&exact[16 * k..16 * k + 16]); + if bad { + report = Some(report.map_or(index, |r| r.min(index))); + } + amps[i] = o0; + amps[i | tbit] = o1; + } + } + Ok(LaneRun { amps, out_of_range: report }) + } + + pub fn run_program(&self, p: &Program) -> Result { + let n = p.n_qubits; + if n == 0 { + return Ok(LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None }); + } + if n > self.max_qubits { + return Err(NpuError::TooWide { n_qubits: n, max: self.max_qubits }); + } + if p.start >= 1usize << n { + return Err(NpuError::Internal("the start is outside the state")); + } + let mut gates = Vec::with_capacity(p.steps.len()); + for step in &p.steps { + let Step::Gate { gate } = *step else { return Err(NpuError::Internal("a tile step; this member runs gate steps only")) }; + let g = p.gates.get(gate as usize).ok_or(NpuError::Internal("a step names a gate the program does not hold"))?; + if !g.is_well_formed() { + return Err(NpuError::Internal("malformed gate words for this state")); + } + gates.push((&g.words, g.index)); + } + let mut amps = vec![[0i32, 0]; 1usize << n]; + amps[p.start] = [ONE as i32, 0]; + self.execute(amps, &gates) + } + + pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result { + let p = plan::compile_with(c, start, &self.config()).map_err(NpuError::Circuit)?; + if p.n_qubits > self.max_qubits { + return Err(NpuError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits }); + } + let run = self.run_program(&p)?; + if let Some(gate) = run.out_of_range { + return Err(NpuError::OutOfRange { gate }); + } + Ok(StateVector { n_qubits: p.n_qubits, amps: run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }).collect() }) + } + + pub fn simulate(&self, c: &Circuit) -> Result { + self.simulate_from(c, 0) + } + + /// The certificate of [`apply_kernel`]`(gs)` on this member over `corpus`. + pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result { + let mut cert = Certification::new(); + for c in corpus.circuits(gs) { + let reference = lanes::reference_output(&c).map_err(NpuError::Circuit)?; + let run = self.run_program(&plan::compile_with(&c, 0, &self.config()).map_err(NpuError::Circuit)?)?; + if let Some(gate) = run.out_of_range { + return Err(NpuError::OutOfRange { gate }); + } + cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps)); + } + Ok(cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs))) + } + + /// A fabric [`Member`] of class [`FabricClass::Npu`](crate::quantum_fabric::FabricClass::Npu). + pub fn member(&self, corpus: Corpus) -> Result { + let mut kernels = Vec::new(); + for gs in Gateset::ALL { + kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate: self.certify(gs, corpus)?, energy: None }); + } + Ok(Member { id: self.id.clone(), class: npu::CLASS, presence: Presence::Detected, kernels, detail: self.detail.clone() }) + } + + /// The same member holding at most `max_qubits` qubits. + #[doc(hidden)] + pub fn with_max_qubits(&self, max_qubits: u8) -> NpuMember { + NpuMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() } + } + + /// The same processor running another model text: for the mutation tests. + /// A mutant the runtime places elsewhere is refused at load, as any model is. + #[doc(hidden)] + pub fn with_model(&self, model: &[u8]) -> Result { + #[cfg(target_os = "macos")] + { + let loaded = coreml::load(model, &cache_dir(model))?; + Ok(NpuMember { model: std::sync::Arc::new(std::sync::Mutex::new(loaded.model)), implementation: npu::implementation_id_of(model), placement: loaded.placement, ..self.clone() }) + } + #[cfg(not(target_os = "macos"))] + { + let _ = model; + Err(NpuError::Unavailable("no runtime on this platform".into())) + } + } +} + +impl Runner for NpuMember { + fn member_id(&self) -> String { + self.id.clone() + } + + fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, _meter: &'r mut dyn RunMeter) -> RunFuture<'r> { + Box::pin(async move { + if implementation != self.implementation { + return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation))); + } + let s = self.simulate_from(c, start)?; + Ok((s, 0, None)) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The host's half of a gate, with the processor's products replaced by + /// exact ones: it must be `lanes::run_words`. + #[test] + fn the_hosts_arithmetic_is_run_words() { + let mut s = 0x1234_5678_u64; + let mut next = || { + s = s.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (s >> 32) as u32 + }; + for case in 0..300 { + let n = 1 + (next() % 6) as usize; + let state: Vec = (0..1usize << n).map(|_| [next() as i32 >> (case % 3 * 4), next() as i32 >> (case % 3 * 4)]).collect(); + let t = next() as usize % n; + let ctrl = (0..n).filter(|&b| b != t && next() % 3 == 0).fold(0u32, |a, b| a | 1 << b); + let w = GateWords { m: [0; 8].map(|_: u32| next() >> (case % 2 * 2)), tbit: 1 << t, ctrl }; + let want = lanes::run_words(&state, std::slice::from_ref(&w)).unwrap(); + let mut amps = state.clone(); + let mut report = None; + let m: [i32; 8] = w.m.map(|x| x as i32); + for p in 0..amps.len() / 2 { + let low = p & ((1 << t) - 1); + let i = ((p - low) << 1) | low; + if i & ctrl as usize != ctrl as usize { + continue; + } + let mut pr = Vec::new(); + pair_products(&m, amps[i], amps[i | 1 << t], &mut pr); + let exact: Vec = pr.iter().map(|&(a, b)| i64::from(a) * i64::from(b)).collect(); + let (o0, o1, bad) = apply_pair(&exact); + if bad { + report = Some(0); + } + amps[i] = o0; + amps[i | 1 << t] = o1; + } + assert_eq!(report, want.out_of_range, "case {case}"); + assert_eq!(amps, want.amps, "case {case}"); + } + } + + #[test] + fn errors_map_onto_the_fabric() { + assert!(MemberError::from(NpuError::OutOfRange { gate: 1 }).falls_back()); + assert!(!MemberError::from(NpuError::Placement("mul on the CPU".into())).falls_back()); + } +} diff --git a/wai/wai-quantum/tests/npu_member.rs b/wai/wai-quantum/tests/npu_member.rs new file mode 100644 index 000000000..9a1c81ebc --- /dev/null +++ b/wai/wai-quantum/tests/npu_member.rs @@ -0,0 +1,413 @@ +//! The fabric's neural-processor member on the machine that runs this. +//! +//! The machine declares the devices it must have in `WAI_QUANTUM_NPU_EXPECT`: +//! their keys, comma-separated and sorted, or `none`. Every test that needs a +//! member fails unless the devices found are exactly the declared ones, so a +//! machine whose GPU or driver went missing, or a run that declared nothing, +//! fails instead of passing over no member. Each test prints what it +//! established and, when `WAI_QUANTUM_NPU_REPORT` names a file, appends the +//! same lines to it, which a CI step can count per device. +//! +//! ```text +//! WAI_QUANTUM_NPU_EXPECT=npu-0 \ +//! cargo test --release --features quantum_fabric_npu --test npu_member -- --test-threads=1 +//! ``` +#![cfg(not(target_arch = "wasm32"))] + +use std::io::Write; +use std::sync::OnceLock; +use wai_quantum::ed25519_dalek::SigningKey; +use wai_quantum::quantum::{BaseGate, Circuit, Gate, Gateset, MAX_QUBITS, QuantumError}; +use wai_quantum::quantum_fabric::npu; +use wai_quantum::quantum_fabric::lanes::{self, GateWords, Lane}; +use wai_quantum::quantum_fabric::plan::{Program, Step}; +use wai_quantum::quantum_fabric::{ + Corpus, CpuReference, FABRIC_DEFAULT, Fabric, FabricClass, FabricReceipt, MemberError, Parity, Presence, REFERENCE, apply_kernel, detect_local, place, ready, +}; +use wai_quantum::quantum_fabric_npu::{Discovery, NpuError, NpuMember, discover}; + +/// The CPU's corpus digests: random-v1 (seed 7, 200 cases) and fabric-v1 +/// (seed 7, 200 + 24 cases), per gateset. A bit-exact member reproduces them. +const PINS: [(Gateset, &str, &str); 2] = [ + (Gateset::V1, "d18454c639acfd9c7b66450f4ded1aac4ab46cc10a7bedbcf4958a96e204fb36", "41c726a0ac80f7defef92e15ecf893bdb7a3076b5e7f670fa0b7142e5c033df7"), + (Gateset::V2, "3dbde276c1095bf827faec71d5980e050f8a55a5ba7d97cd3bbdaec3f43a13b2", "9a84b55af68f1a8b07390b234cf562dd8d85cef3185a697ab85690c8790a7809"), +]; + +const RANDOM_V1: Corpus = Corpus::Random { seed: 7, cases: 200 }; + + +static DISCOVERY: OnceLock = OnceLock::new(); + +fn discovery() -> &'static Discovery { + DISCOVERY.get_or_init(discover) +} + +/// Prints `line` and appends it to the file `WAI_QUANTUM_NPU_REPORT` names. +fn report(line: &str) { + println!("{line}"); + if let Some(path) = std::env::var_os("WAI_QUANTUM_NPU_REPORT") { + let mut f = std::fs::OpenOptions::new().create(true).append(true).open(&path).expect("the report file opens"); + writeln!(f, "{line}").expect("the report line is written"); + } +} + +/// The declared device set, as written, if it is exactly what was found; +/// otherwise why not, with what was found and refused and how to declare it. +fn declared(d: &Discovery) -> Result { + let mut found: Vec<&str> = d.found.iter().map(|m| m.key()).collect(); + found.sort_unstable(); + let found_list = if found.is_empty() { "none".to_string() } else { found.join(",") }; + let refused: Vec = d.refused.iter().map(|(k, e)| format!("{k}: {e}")).collect(); + let refused_list = if refused.is_empty() { "none".to_string() } else { refused.join("; ") }; + let unavailable = d.unavailable.as_deref().unwrap_or("no"); + let how = format!("found: {found_list}; refused: {refused_list}; unavailable: {unavailable}. Declare the devices this machine must have: WAI_QUANTUM_NPU_EXPECT={found_list}"); + let expect = match std::env::var("WAI_QUANTUM_NPU_EXPECT") { + Ok(v) if !v.is_empty() => v, + _ => return Err(format!("WAI_QUANTUM_NPU_EXPECT is not set. {how}")), + }; + let keys: Vec<&str> = if expect == "none" { Vec::new() } else { expect.split(',').collect() }; + let well_formed = keys.windows(2).all(|w| w[0] <= w[1]) && keys.iter().all(|k| !k.is_empty() && *k != "none" && k.trim() == *k); + if !well_formed { + return Err(format!("WAI_QUANTUM_NPU_EXPECT={expect:?} is not a sorted, comma-separated list of keys, nor none. {how}")); + } + for k in &keys { + if !found.contains(k) + && let Some((_, e)) = d.refused.iter().find(|(key, _)| key == k) + { + return Err(format!("the declared device {k} was refused: {e}. {how}")); + } + } + if found != keys { + return Err(format!("the devices found are not the declared ones ({expect}). {how}")); + } + Ok(expect) +} + +/// The members, once the declared devices are exactly the ones found. +fn members() -> &'static [NpuMember] { + let d = discovery(); + if let Err(why) = declared(d) { + panic!("{why}"); + } + &d.found +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn next(s: &mut u64) -> u64 { + *s = s.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *s; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// A value in `[-bound, bound]`. +fn within(s: &mut u64, bound: i64) -> i32 { + ((next(s) % (2 * bound as u64 + 1)) as i64 - bound) as i32 +} + +/// A random circuit of `ops` gates on `n` qubits: every base gate, up to three +/// distinct controls, `P(1..=32)`. +fn random_circuit(n: u8, ops: usize, seed: u64, gs: Gateset) -> Circuit { + let bases = [BaseGate::I, BaseGate::X, BaseGate::Y, BaseGate::Z, BaseGate::H, BaseGate::S, BaseGate::Sdg, BaseGate::T, BaseGate::Tdg, BaseGate::P]; + let mut s = seed; + let mut c = Circuit::with_gateset(n, gs); + for _ in 0..ops { + let base = bases[(next(&mut s) % bases.len() as u64) as usize]; + let target = (next(&mut s) % u64::from(n)) as u8; + let mut controls = Vec::new(); + for _ in 0..next(&mut s) % 4 { + let q = (next(&mut s) % u64::from(n)) as u8; + if q != target && !controls.contains(&q) { + controls.push(q); + } + } + let param = if base == BaseGate::P { 1 + (next(&mut s) % 32) as u16 } else { 0 }; + c.ops.push(Gate { base, controls, target, param }); + } + c +} + +/// Five circuits the native simulator refuses, in each gateset. +fn refused_circuits() -> Vec { + let gate = |base, controls: Vec, target, param| Gate { base, controls, target, param }; + let bad = [ + (1, vec![gate(BaseGate::H, vec![], 0, 0), gate(BaseGate::P, vec![], 0, 0)]), + (1, vec![gate(BaseGate::P, vec![], 0, 33)]), + (1, vec![gate(BaseGate::X, vec![], 3, 0)]), + (2, vec![gate(BaseGate::X, vec![0], 0, 0)]), + (2, vec![gate(BaseGate::X, vec![5], 1, 0)]), + ]; + let mut out = Vec::new(); + for gs in Gateset::ALL { + for (n, ops) in bad.clone() { + let mut c = Circuit::with_gateset(n, gs); + c.ops = ops; + out.push(c); + } + } + out +} + +/// `Z` on qubit 0: on `[(0, 0), (i32::MIN, 0)]` its exact result is `2^31`, +/// which leaves 32 bits. +fn z_words() -> GateWords { + lanes::gate_words(&Gate { base: BaseGate::Z, controls: vec![], target: 0, param: 0 }, Gateset::V2) +} + +const Z_STATE: [Lane; 2] = [[0, 0], [i32::MIN, 0]]; + +#[test] +fn c0_devices_are_declared() { + let d = discovery(); + match declared(d) { + Ok(expect) => report(&format!("npu devices: as declared ({expect})")), + Err(why) => panic!("{why}"), + } + for m in &d.found { + assert_eq!(m.class(), FabricClass::Npu); + assert!(m.id().starts_with("local/npu-"), "{}", m.id()); + assert_eq!(m.implementation(), npu::implementation_id()); + assert!(!m.placement().is_empty() && m.placement().iter().all(|(_, d)| d == wai_quantum::quantum_fabric_npu::NEURAL_PROCESSOR), "{:?}", m.placement()); + report(&format!("npu placement: key={} {}", m.key(), m.placement().iter().map(|(op, d)| format!("{op}={d}")).collect::>().join(" "))); + assert!(m.max_qubits() >= 20, "{}: holds only {} qubits", m.key(), m.max_qubits()); + report(&format!("npu member: key={} id={} {}", m.key(), m.id(), m.detail())); + } +} + +#[test] +fn c1_probe_matches_lanes() { + for m in members() { + let pairs = lanes::probe_pairs(1 << 16, 0x0cda); + let out = m.probe(&pairs).unwrap_or_else(|e| panic!("{}: {e}", m.key())); + assert_eq!(lanes::first_mismatch(&pairs, &out), None, "{}", m.key()); + report(&format!("npu probe: key={} {} pairs agree", m.key(), pairs.len())); + } +} + +#[test] +fn c2_kernel_matches_run_words() { + for m in members() { + let mut s = 0x6b65_726e_656c_u64; + let (mut flagged, mut clean) = (0, 0); + // Fewer cases than the GPU members': each gate's products cross in batches. + for case in 0..600 { + let n = 1 + (next(&mut s) % 12) as u32; + // Three regimes: the whole 32-bit range (nearly always out of + // range), small states and entries (never), and states and + // entries up to 2^30 (either). + let mode = case % 3; + let value = |s: &mut u64| -> i32 { + match mode { + 0 => next(s) as i32, + 1 => within(s, 1 << 22), + _ => within(s, 1 << 30), + } + }; + let entry = |s: &mut u64| -> u32 { + match mode { + 0 => next(s) as u32, + 1 => within(s, 1 << 29) as u32, + _ => within(s, 1 << 30) as u32, + } + }; + let state: Vec = (0..1usize << n).map(|_| [value(&mut s), value(&mut s)]).collect(); + let gates: Vec = (0..1 + next(&mut s) % 8) + .map(|_| { + let t = (next(&mut s) % u64::from(n)) as u32; + let ctrl = (0..n).filter(|&b| b != t && next(&mut s).is_multiple_of(4)).fold(0u32, |acc, b| acc | 1 << b); + let mut w = [0u32; 8]; + w.iter_mut().for_each(|x| *x = entry(&mut s)); + GateWords { m: w, tbit: 1 << t, ctrl } + }) + .collect(); + let want = lanes::run_words(&state, &gates).expect("valid shapes"); + let got = m.run_lanes(&state, &gates).unwrap_or_else(|e| panic!("{}: case {case}: {e}", m.key())); + assert_eq!(got.out_of_range, want.out_of_range, "{}: case {case}, n = {n}", m.key()); + assert!(got.amps == want.amps, "{}: case {case}, n = {n}: the lanes differ", m.key()); + if want.out_of_range.is_some() { + flagged += 1; + } else { + clean += 1; + } + } + assert!(flagged >= 100 && clean >= 100, "{}: {flagged} flagged, {clean} clean", m.key()); + report(&format!("npu kernel: key={} ok", m.key())); + } +} + +#[test] +fn c3_certificates_reproduce_the_pins() { + let ms = members(); + for (gs, random_pin, fabric_pin) in PINS { + let cpu = lanes::certify_corpus(gs, "local/test", RANDOM_V1); + assert_eq!(hex(&cpu.corpus), random_pin, "{gs:?}"); + let other = Corpus::Random { seed: 8, cases: 300 }; + let cpu_other = lanes::certify_corpus(gs, "local/test", other); + for m in ms { + for (corpus, cases, pin) in [(RANDOM_V1, 200, random_pin), (FABRIC_DEFAULT, 224, fabric_pin)] { + let c = m.certify(gs, corpus).unwrap_or_else(|e| panic!("{} {gs:?} {}: {e}", m.key(), corpus.name())); + assert_eq!(c.parity, Parity::BitExact, "{} {gs:?} {}", m.key(), corpus.name()); + assert_eq!(c.cases, cases); + assert_eq!(hex(&c.corpus), pin, "{} {gs:?} {}", m.key(), corpus.name()); + assert_eq!((c.kernel.as_str(), c.implementation.as_str(), c.member.as_str()), (apply_kernel(gs), m.implementation(), m.id())); + assert_eq!(c.generator, corpus.generator(gs)); + report(&format!( + "npu certified: key={} member={} kernel={} corpus={} implementation={} parity=bit-exact cases={} digest={}", + m.key(), + m.id(), + c.kernel, + corpus.name(), + c.implementation, + c.cases, + hex(&c.corpus) + )); + } + // A corpus with no pin: the member and the CPU agree on it. + let c = m.certify(gs, other).unwrap_or_else(|e| panic!("{} {gs:?} seed 8: {e}", m.key())); + assert_eq!((c.parity, c.cases), (Parity::BitExact, 300), "{} {gs:?}", m.key()); + assert_eq!(c.corpus, cpu_other.corpus, "{} {gs:?}", m.key()); + } + } +} + +#[test] +fn c4_wide_circuits_match_native() { + for m in members() { + let widest = m.max_qubits().min(16); + let mut s = 0x0077_6964_6500_u64; + for (k, n) in [10u8, 12, 14, widest].into_iter().enumerate() { + for gs in Gateset::ALL { + let c = random_circuit(n, 40, 0x1000 + k as u64, gs); + let start = (next(&mut s) as usize) % (1usize << n); + let want = c.simulate_from(start).expect("the native run"); + let got = m.simulate_from(&c, start).unwrap_or_else(|e| panic!("{}: n = {n} {gs:?}: {e}", m.key())); + assert!(got == want, "{}: n = {n} {gs:?}: the states differ", m.key()); + } + } + // The transforms the conformance corpus carries. + for n in [3u8, 5, 8, 12] { + let qft = Circuit::qft(n); + assert!(m.simulate(&qft).unwrap() == qft.simulate().unwrap(), "{}: QFT-{n}", m.key()); + } + report(&format!("npu wide: key={} up to {widest} qubits match native", m.key())); + } +} + +#[test] +fn c5_refusals_are_typed() { + for m in members() { + for c in refused_circuits() { + let native = c.simulate_from(0).unwrap_err(); + assert_eq!(m.simulate_from(&c, 0).unwrap_err(), NpuError::Circuit(native), "{c:?}"); + } + assert_eq!(m.simulate(&Circuit::new(MAX_QUBITS + 1)).unwrap_err(), NpuError::Circuit(QuantumError::TooManyQubits(MAX_QUBITS + 1))); + let small = m.with_max_qubits(4); + assert_eq!(small.simulate(&Circuit::new(5)).unwrap_err(), NpuError::TooWide { n_qubits: 5, max: 4 }); + let wide = Program { n_qubits: 5, start: 0, gates: Vec::new(), steps: Vec::new() }; + assert_eq!(small.run_program(&wide).unwrap_err(), NpuError::TooWide { n_qubits: 5, max: 4 }); + // A result that leaves 32 bits is reported at its op, and the lane + // holds its low 32 bits. + let r = m.run_lanes(&Z_STATE, &[z_words()]).unwrap(); + assert_eq!(r.out_of_range, Some(0), "{}", m.key()); + assert_eq!(r.amps[1][0] as u32, 0x8000_0000); + // Malformed words, a state the host cannot lay out and a tile step are + // host defects, never results. + let bad = GateWords { tbit: 3, ..z_words() }; + assert!(matches!(m.run_lanes(&[[0, 0]; 4], &[bad]), Err(NpuError::Internal(_)))); + assert!(matches!(m.run_lanes(&[[0, 0]; 3], &[z_words()]), Err(NpuError::Internal(_)))); + let tiled = Program { n_qubits: 2, start: 0, gates: Vec::new(), steps: vec![Step::Tile { first: 0, count: 1, size: 2 }] }; + assert!(matches!(m.run_program(&tiled), Err(NpuError::Internal(_)))); + // The refusals that may still fall back to another member, do. + assert!(MemberError::from(NpuError::TooWide { n_qubits: 5, max: 4 }).falls_back()); + report(&format!("npu refusals: key={} typed", m.key())); + } +} + +/// The model mutants: one operation replaced. Each must be caught: refused at +/// load (the runtime places it elsewhere or rejects it) or by the probe. +#[test] +fn c6_mutants_are_killed() { + let edges = lanes::probe_pairs(0, 0); + for m in members() { + let mut killed = 0; + for (from, to) in [("mul", "add"), ("reduce_sum", "reduce_max")] { + match m.with_model(&npu::mutate_model(from, to)) { + Err(e) => println!("npu mutant {from}->{to} on {}: refused: {e}", m.key()), + Ok(mm) => { + assert_ne!(mm.implementation(), m.implementation()); + let out = mm.probe(&edges).expect("the mutant's probe runs"); + let differing = edges.iter().zip(&out).filter(|&(&(a, b), o)| *o != (lanes::mul_wide(a, b), lanes::fxmul_wide(a, b))).count(); + println!("npu mutant {from}->{to} on {}: {differing} of {} edge pairs differ", m.key(), edges.len()); + assert!(differing >= 1, "{}: mutant {from}->{to} survived", m.key()); + } + } + killed += 1; + } + report(&format!("npu mutants: key={} killed {killed} of 2", m.key())); + } +} + +#[test] +fn c7_receipt_placement_and_the_fabric() { + for m in members() { + let member = m.member(FABRIC_DEFAULT).unwrap_or_else(|e| panic!("{}: {e}", m.key())); + assert_eq!((member.id.as_str(), member.class, member.presence), (m.id(), FabricClass::Npu, Presence::Detected)); + assert_eq!(member.kernels.len(), 2); + for (entry, gs) in member.kernels.iter().zip(Gateset::ALL) { + assert_eq!((entry.kernel.as_str(), entry.implementation.as_str()), (apply_kernel(gs), m.implementation())); + assert!(entry.energy.is_none()); + assert_eq!(entry.certificate.parity, Parity::BitExact, "{gs:?}"); + } + // A receipt over a member's run. + let qft = Circuit::qft(5); + let state = m.simulate(&qft).unwrap(); + assert_eq!(state, qft.simulate().unwrap()); + let entry = member.kernels.iter().find(|k| k.kernel == apply_kernel(qft.gateset)).unwrap(); + let signer = SigningKey::from_bytes(&[7; 32]); + let receipt = FabricReceipt::seal(&signer, "local/test", state.statevector_hash(), &member, entry, 0, None); + assert!(receipt.verify()); + assert!(receipt.certificate_matches(&entry.certificate)); + // Placement: beside the CPU reference with no energy figures, the + // reference; alone, the member. + let local = detect_local(); + for gs in Gateset::ALL { + let both = [local.clone(), member.clone()]; + let (placed, k) = place(&both, apply_kernel(gs), &Parity::BitExact).unwrap(); + assert_eq!((placed.id.as_str(), k.implementation.as_str()), (local.id.as_str(), REFERENCE)); + let (placed, k) = place(std::slice::from_ref(&member), apply_kernel(gs), &Parity::BitExact).unwrap(); + assert_eq!((placed.id.as_str(), k.implementation.as_str()), (m.id(), m.implementation())); + } + // On the fabric: the member alone runs the circuit; a pairing with + // another member's entries is refused. + let mut fabric = Fabric::new(); + fabric.add(member.clone(), m); + let run = ready(fabric.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs"); + assert_eq!(run.state, state); + assert_eq!(run.receipt.member_id, m.id()); + assert!(run.receipt.verify()); + let mut mismatched = Fabric::new(); + mismatched.add(local.clone(), m); + assert!(ready(mismatched.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").is_err()); + let mut both = Fabric::new(); + both.add(local.clone(), CpuReference).add(member.clone(), m); + let run = ready(both.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs"); + assert_eq!(run.state, state); + report(&format!("npu receipt: key={} ok", m.key())); + } +} + +#[test] +fn c8_repeated_runs_agree() { + for m in members() { + let c = random_circuit(16, 60, 0xabc, Gateset::V2); + let first = m.simulate(&c).unwrap(); + for _ in 0..5 { + assert!(m.simulate(&c).unwrap() == first, "{}", m.key()); + } + report(&format!("npu repeat: key={} ok", m.key())); + } +} diff --git a/wai/wai-rs/Cargo.lock b/wai/wai-rs/Cargo.lock index 4c2f88bda..e66d598b1 100644 --- a/wai/wai-rs/Cargo.lock +++ b/wai/wai-rs/Cargo.lock @@ -4464,7 +4464,7 @@ version = "0.1.0" [[package]] name = "wai-quantum" -version = "0.5.0" +version = "0.5.1" dependencies = [ "blake3", "ed25519-dalek", diff --git a/wai/wai-web/Cargo.lock b/wai/wai-web/Cargo.lock index 214ae1908..33a654a8c 100644 --- a/wai/wai-web/Cargo.lock +++ b/wai/wai-web/Cargo.lock @@ -1231,7 +1231,7 @@ version = "0.1.0" [[package]] name = "wai-quantum" -version = "0.5.0" +version = "0.5.1" dependencies = [ "blake3", "ed25519-dalek", -- 2.54.0 (Apple Git-157) From 3d2e153c165a1b82bbab92a44e1d1a31299d7db9 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Wed, 14 Oct 2026 09:17:03 -0400 Subject: [PATCH 02/11] npu member: the compute plan and the model cross threads through explicit, lock-guarded handoffs; the qubit limit on every platform Co-Authored-By: Claude Opus 5.5 --- wai/wai-quantum/src/quantum_fabric_npu.rs | 32 +++++++++++++---------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/wai/wai-quantum/src/quantum_fabric_npu.rs b/wai/wai-quantum/src/quantum_fabric_npu.rs index d6c4c7e64..cabc92b92 100644 --- a/wai/wai-quantum/src/quantum_fabric_npu.rs +++ b/wai/wai-quantum/src/quantum_fabric_npu.rs @@ -99,7 +99,7 @@ pub const NEURAL_PROCESSOR: &str = "MLNeuralEngineComputeDevice"; // allow-name: /// The most qubits a run holds: the host holds the state, and every gate's /// products cross to the processor, so wide runs are slow rather than refused. -pub const MAX_NPU_QUBITS: u8 = 20; +pub const MAX_NPU_QUBITS: u8 = if 20 < MAX_QUBITS { 20 } else { MAX_QUBITS }; /// The sixteen products a pair needs, in the order [`apply_pair`] reads them. fn pair_products(m: &[i32; 8], a0: Lane, a1: Lane, out: &mut Vec<(i32, i32)>) { @@ -165,6 +165,15 @@ mod coreml { e.localizedDescription().to_string() } + /// A runtime object handed from the thread the runtime calls back on to the + /// thread that waits for it, or held under a lock. + pub struct Handed(pub T); + + // SAFETY: a handed object is moved across threads only through a mutex, so + // one thread at a time uses it, and its retain count is atomic; the runtime's + // objects may be used from any thread that way. + unsafe impl Send for Handed {} + /// A compiled, loaded model, and where its runtime places each operation. pub struct Loaded { pub model: Retained, @@ -201,14 +210,14 @@ mod coreml { /// The runtime's compute plan for the compiled model: each operation's /// preferred device. fn plan(compiled: &NSURL, cfg: &MLModelConfiguration) -> Result, NpuError> { - type Slot = (Mutex, String>>>, Condvar); + type Slot = (Mutex>, String>>>, Condvar); let slot: Arc = Arc::new((Mutex::new(None), Condvar::new())); let s = slot.clone(); let handler = block2::RcBlock::new(move |plan: *mut MLComputePlan, err: *mut NSError| { // SAFETY: the runtime hands one of the two, live for the call. let r = unsafe { match Retained::retain(plan) { - Some(p) => Ok(p), + Some(p) => Ok(Handed(p)), None => Err(err.as_ref().map_or_else(|| "no plan".to_string(), ns_error)), } }; @@ -225,7 +234,7 @@ mod coreml { if timeout.timed_out() { return Err(NpuError::Runtime("the compute plan did not arrive".into())); } - let plan = guard.take().expect("set before the notify").map_err(|e| NpuError::Runtime(format!("the compute plan: {e}")))?; + let plan = guard.take().expect("set before the notify").map_err(|e| NpuError::Runtime(format!("the compute plan: {e}")))?.0; let mut out = Vec::new(); // SAFETY: reads of the plan's structure, on this thread. unsafe { @@ -324,7 +333,7 @@ pub fn discover() -> Discovery { /// used from one thread at a time), and what discovery established. pub struct NpuMember { #[cfg(target_os = "macos")] - model: std::sync::Arc>>, + model: std::sync::Arc>>>, id: String, key: String, implementation: String, @@ -333,11 +342,6 @@ pub struct NpuMember { max_qubits: u8, } -// SAFETY: the runtime's model object is reached only through the mutex, one -// thread at a time; everything else is plain data. -unsafe impl Send for NpuMember {} -// SAFETY: as above. -unsafe impl Sync for NpuMember {} impl Clone for NpuMember { fn clone(&self) -> Self { @@ -384,7 +388,7 @@ fn open(model: &[u8]) -> Result { o.extend_from_slice(hw.as_bytes()); let id = format!("local/npu-{}", blake3::hash(&o).as_bytes()[..6].iter().map(|b| format!("{b:02x}")).collect::()); let member = NpuMember { - model: std::sync::Arc::new(std::sync::Mutex::new(loaded.model)), + model: std::sync::Arc::new(std::sync::Mutex::new(coreml::Handed(loaded.model))), id, key: "npu-0".into(), implementation: npu::implementation_id_of(model), @@ -392,7 +396,7 @@ fn open(model: &[u8]) -> Result { "npu; the platform's neural processor through its model runtime; placement by the runtime's compute plan: {placement_text}; fp16 limb products, the host assembling; up to {MAX_NPU_QUBITS} qubits; unmetered" ), placement: loaded.placement, - max_qubits: MAX_NPU_QUBITS.min(MAX_QUBITS), + max_qubits: MAX_NPU_QUBITS, }; let pairs = lanes::probe_pairs(4096, 0x5eed); let out = member.probe(&pairs)?; @@ -455,7 +459,7 @@ impl NpuMember { #[cfg(target_os = "macos")] fn predict(&self, x: &[u16], y: &[u16]) -> Result, NpuError> { let m = self.model.lock().unwrap_or_else(|p| p.into_inner()); - coreml::predict(&m, x, y) + coreml::predict(&m.0, x, y) } #[cfg(not(target_os = "macos"))] @@ -603,7 +607,7 @@ impl NpuMember { #[cfg(target_os = "macos")] { let loaded = coreml::load(model, &cache_dir(model))?; - Ok(NpuMember { model: std::sync::Arc::new(std::sync::Mutex::new(loaded.model)), implementation: npu::implementation_id_of(model), placement: loaded.placement, ..self.clone() }) + Ok(NpuMember { model: std::sync::Arc::new(std::sync::Mutex::new(coreml::Handed(loaded.model))), implementation: npu::implementation_id_of(model), placement: loaded.placement, ..self.clone() }) } #[cfg(not(target_os = "macos"))] { -- 2.54.0 (Apple Git-157) From 36b4ddd9bae4def6d58443c394251d9a8f610c85 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Wed, 14 Oct 2026 09:26:05 -0400 Subject: [PATCH 03/11] wai-quantum 0.5.1: the matrix-extension member (SME), silicon and emulated Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wai-quantum-sme.yml | 113 ++++ wai/CHANGELOG.md | 12 +- wai/wai-quantum/Cargo.toml | 6 + wai/wai-quantum/README.md | 6 +- wai/wai-quantum/src/lib.rs | 9 +- wai/wai-quantum/src/quantum_fabric_sme.rs | 632 ++++++++++++++++++++++ wai/wai-quantum/tests/sme_member.rs | 410 ++++++++++++++ 7 files changed, 1182 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/wai-quantum-sme.yml create mode 100644 wai/wai-quantum/src/quantum_fabric_sme.rs create mode 100644 wai/wai-quantum/tests/sme_member.rs diff --git a/.github/workflows/wai-quantum-sme.yml b/.github/workflows/wai-quantum-sme.yml new file mode 100644 index 000000000..c8d8625b8 --- /dev/null +++ b/.github/workflows/wai-quantum-sme.yml @@ -0,0 +1,113 @@ +name: wai-quantum matrix-extension member + +# The fabric's matrix-extension member: the gate's fixed-point arithmetic in the +# processor's streaming vector mode (SME), 64-bit lanes, certified bit-exact +# against the CPU reference's pinned corpus digests. +# +# - sme-emulated (the container runner): the member's device tests for aarch64 +# Linux, run through the host's user-mode emulator (its binfmt handler, which +# emulates SME); the runner declares the emulator, and every certificate says +# it was earned there, never on silicon. +# - sme-absent (the macOS runner, whose processor has no SME): the same build +# finds no member and says why. +# +# Silicon: a processor with SME runs the same tests with the declaration +# `cpu-matrix-sme` and no emulator. + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: [main] + paths: + - 'wai/wai-quantum/**' + - 'rust-toolchain.toml' + - '.github/workflows/wai-quantum-sme.yml' + pull_request: + paths: + - 'wai/wai-quantum/**' + - 'rust-toolchain.toml' + - '.github/workflows/wai-quantum-sme.yml' + +jobs: + sme-emulated: + name: member · matrix extension, emulated (aarch64 Linux) + runs-on: linux + timeout-minutes: 90 + container: + image: ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:7f8308a8734d9fcd2ebbe9a3e4bdea74af293f0799d80c3cc341e340cda49a4c + options: --memory=6g --memory-swap=6g + env: + WAI_QUANTUM_SME_EXPECT: cpu-matrix-sme + # The host's binfmt handler runs every aarch64 binary; its emulator is + # what the certificates name. + WAI_QUANTUM_SME_EMULATOR: qemu-user 8.2.2 (the host's binfmt handler) + # Run the test binary directly, so the host's handler (which emulates + # SME) runs it, not the image's older emulator. + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER: env + steps: + # The image has no Node for a checkout action: the forge's archive of the + # exact commit, its embedded commit id checked. + - name: Fetch the source + shell: bash + run: | + set -euo pipefail + want="${{ github.event.pull_request.head.sha || github.sha }}" + rm -rf src src.tar.gz src.tar && mkdir src + curl -sSf -o src.tar.gz "http://100.64.224.0:3400/Transaction-Science/open-standards/archive/$want.tar.gz" + gzip -dc src.tar.gz > src.tar + got="$(git get-tar-commit-id < src.tar)" + [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } + tar -xf src.tar -C src --strip-components=1 + rm src.tar.gz src.tar + - name: The member's device tests under the emulator + shell: bash + working-directory: src/wai/wai-quantum + run: | + set -euo pipefail + cache=/ci-cache/wai-quantum-sme + mkdir -p "$cache" + exec 9>"$cache/.lock" + flock 9 + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" + export PATH="$CARGO_HOME/bin:$PATH" + if ! command -v rustup >/dev/null; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain none + fi + rustup toolchain install + rustup target add aarch64-unknown-linux-gnu + report="$(mktemp)" + WAI_QUANTUM_SME_REPORT="$report" cargo test --locked --release -j 6 --target aarch64-unknown-linux-gnu --features quantum_fabric_sme --test sme_member -- --test-threads=1 2>&1 | tee "$cache/device.log" + grep -qE '^test result: ok\. 9 passed; 0 failed' "$cache/device.log" || { echo "::error::not every device test ran and passed"; exit 1; } + cat "$report" + key="$WAI_QUANTUM_SME_EXPECT" + grep -qx "sme devices: as declared ($key)" "$report" || { echo "::error::the member found is not the declared one"; exit 1; } + grep -q "^sme member: key=$key .*emulator:qemu-user" "$report" || { echo "::error::the member does not say it ran under the emulator"; exit 1; } + n="$(grep -c "^sme certified: key=$key .* parity=bit-exact " "$report")" + [ "$n" = 4 ] || { echo "::error::$n of 4 certificates were earned"; exit 1; } + grep -qx "sme mutants: key=$key killed 3 of 3" "$report" || { echo "::error::a kernel mutant survived"; exit 1; } + + sme-absent: + name: member · matrix extension absent (macos) + runs-on: macos + timeout-minutes: 60 + env: + WAI_QUANTUM_SME_EXPECT: none + steps: + - uses: actions/checkout@v5 + # launchd's PATH omits cargo on the macOS host runner. + - name: Put cargo on PATH + shell: bash + run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: No matrix extension here, and the member says so + shell: bash + working-directory: wai/wai-quantum + run: | + set -euo pipefail + [ "$(sysctl -n hw.optional.arm.FEAT_SME 2>/dev/null || echo 0)" = 0 ] || { echo "::error::this runner has SME; declare it and run the silicon tests"; exit 1; } + report="$RUNNER_TEMP/sme-report.txt"; : > "$report" + WAI_QUANTUM_SME_REPORT="$report" nice -n 10 cargo test --locked --features quantum_fabric_sme --test sme_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log" + grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; } + grep -qx 'sme devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; } diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index b269e9c97..7d4db6424 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -6,7 +6,7 @@ wire format may change and such changes are recorded here. ## v0.4.0 — current -### wai-quantum 0.5.1: the neural-processor member +### wai-quantum 0.5.1: the neural-processor and matrix-extension members Additive. The default features, `full`, and every pinned digest are unchanged. @@ -23,8 +23,18 @@ Additive. The default features, `full`, and every pinned digest are unchanged. the member is refused with the placement reported, so no CPU result is credited to the processor. It is then probed and certified against the pinned corpus digests, and the two model mutants (the multiply made an add, the sum a maximum) are caught. +- **The matrix-extension member** (`quantum_fabric_sme`, opt-in, aarch64 only): the + gate's fixed-point arithmetic in the processor's streaming vector mode (SME), 64-bit + lanes, by inline assembly; the host gathers each gate's pairs and checks each row's + range. Its instructions exist only in that mode, so a kernel that ran, ran there. + Detection reads the processor's feature flags (on Linux, `AT_HWCAP2`, which a + user-mode emulator supplies for its guest); a runner under an emulator declares it, + and every certificate carries that provenance. Three kernel mutants are compiled in. - **CI.** `wai-quantum-npu.yml`: the device tests on the macOS runner's neural processor, and the same build on Linux, where the member is absent and says so. + `wai-quantum-sme.yml`: the matrix-extension member's device tests for aarch64 Linux + under the host's user-mode emulator, certificates labelled emulator; and the macOS + runner, whose processor has no SME, finding no member. ### wai-quantum 0.5.0: provenance, sampling members, device programs, and three more members diff --git a/wai/wai-quantum/Cargo.toml b/wai/wai-quantum/Cargo.toml index 80feed38a..a27e7b600 100644 --- a/wai/wai-quantum/Cargo.toml +++ b/wai/wai-quantum/Cargo.toml @@ -167,6 +167,8 @@ quantum_fabric_opencl = ["quantum_fabric", "dep:opencl3"] quantum_fabric_fpga = ["quantum_fabric"] # The neural-processor member: exact fp16 limb products on the platform's neural processor. quantum_fabric_npu = ["quantum_fabric", "dep:objc2", "dep:block2", "dep:objc2-foundation", "dep:objc2-core-ml"] +# The matrix-extension member: the gate's arithmetic in the processor's streaming vector mode (SME). +quantum_fabric_sme = ["quantum_fabric"] # --- interference / ML layer ---------------------------------------------- quantum_vml = [] @@ -213,6 +215,10 @@ required-features = ["quantum_fabric_fpga"] name = "npu_member" required-features = ["quantum_fabric_npu"] +[[test]] +name = "sme_member" +required-features = ["quantum_fabric_sme"] + [[test]] name = "fabric_expected" required-features = ["quantum_fabric"] diff --git a/wai/wai-quantum/README.md b/wai/wai-quantum/README.md index dc509e3d0..c0237bace 100644 --- a/wai/wai-quantum/README.md +++ b/wai/wai-quantum/README.md @@ -6,13 +6,14 @@ every stage. No QPU, no cloud service, no vendor SDK, and **no system libraries** — so the same source runs natively, in the browser via wasm, and as a WASI component at -the edge, producing **byte-identical results and receipts on all three**. Five +the edge, producing **byte-identical results and receipts on all three**. Six opt-in features reach outside the crate: `quantum_fabric_webgpu` (the platform's GPU interface), `quantum_fabric_cuda` (the CUDA driver and runtime compiler) and `quantum_fabric_opencl` (the OpenCL ICD loader), the last two loaded when first used, `quantum_fabric_fpga` (Verilator, run as a program when first used) and `quantum_fabric_npu` (the platform's model runtime, on -macOS); no default feature reaches a system library. +macOS) and `quantum_fabric_sme` (the processor's streaming vector mode, by +inline assembly); no default feature reaches a system library. ```toml [dependencies] @@ -104,6 +105,7 @@ never creates information. | `quantum_fabric_opencl` | **the fabric's OpenCL member**, opt-in and not in `full`, native targets only: the same exact 64-bit kernel in OpenCL C, built at run time by each device's own compiler, one launch per gate, on any OpenCL GPU or CPU (an OpenCL accelerator is refused: no fabric class names it). The ICD loader is opened when first used, so a build with the feature runs on a host without OpenCL and finds no member there, saying why. Each device is probed and certified as the CUDA member's are. With `quantum_sv` too, the dense f64 state vector runs bit for bit as on the CPU on every device with binary64 (contraction off); a device without it refuses, saying so | | `quantum_fabric_fpga` | **the fabric's FPGA member, simulated**, opt-in and not in `full`, native targets only: the gate datapath in Verilog (each 32×32 product from four 16-bit partial products, the size of an FPGA's multiplier blocks; a state RAM; one pair per two cycles), compiled at run time by Verilator into a cycle-accurate simulation driven over a pipe. Probed and certified as the other members are, and every certificate says it was earned in a simulator, so placement ranks it after silicon. A host without Verilator finds no member and says why | | `quantum_fabric_npu` | **the fabric's neural-processor member**, opt-in and not in `full`, macOS only: each 32-bit operand split into eight 4-bit limbs, every limb product and limb sum formed in fp16 on the platform's neural processor (each value an integer fp16 holds exactly, whatever order it is added in), the host assembling each product and applying the fixed-point arithmetic. The member exists only where the platform model runtime's compute plan places every operation of the model on the neural processor; a model it would run elsewhere is refused with the placement it reported. Probed and certified as every member is | +| `quantum_fabric_sme` | **the fabric's matrix-extension member**, opt-in and not in `full`, aarch64 only: the gate's fixed-point arithmetic in the processor's streaming vector mode (SME), 64-bit lanes, the host gathering each gate's pairs and checking each row's range. Every instruction it uses exists only in that mode, so a kernel that ran, ran there. Detected by the processor's feature flags (on Linux, the kernel's `AT_HWCAP2`, which a user-mode emulator supplies); under an emulator the runner says so (`WAI_QUANTUM_SME_EMULATOR`) and every certificate carries it. Three kernel mutants are compiled in and must be caught | | `full` | everything | Default features are `quantum`, `quantum_toolchain`, `quantum_receipt`. diff --git a/wai/wai-quantum/src/lib.rs b/wai/wai-quantum/src/lib.rs index eee4dd06b..300e72955 100644 --- a/wai/wai-quantum/src/lib.rs +++ b/wai/wai-quantum/src/lib.rs @@ -10,13 +10,14 @@ //! Rust, so the same source runs natively, in the browser via wasm, and as a //! WASI component at the edge, producing **byte-identical** results and receipts //! on all three. A fabric receipt is the exception by design: it names the -//! member that ran the work. Five opt-in features reach outside the crate: +//! member that ran the work. Six opt-in features reach outside the crate: //! `quantum_fabric_webgpu` (the platform's GPU interface), //! `quantum_fabric_cuda` (the CUDA driver and runtime compiler) and //! `quantum_fabric_opencl` (the OpenCL ICD loader), the last two loaded when //! first used, `quantum_fabric_fpga` (Verilator, run as a program when first -//! used) and `quantum_fabric_npu` (the platform's model runtime, on macOS); no -//! default feature reaches a system library. +//! used), `quantum_fabric_npu` (the platform's model runtime, on macOS) and +//! `quantum_fabric_sme` (the processor's streaming vector mode); no default +//! feature reaches a system library. //! //! Everything is behind a feature; enable only what you use. @@ -88,6 +89,8 @@ pub mod quantum_fabric_opencl; pub mod quantum_fabric_fpga; #[cfg(all(feature = "quantum_fabric_npu", not(target_arch = "wasm32")))] pub mod quantum_fabric_npu; +#[cfg(all(feature = "quantum_fabric_sme", not(target_arch = "wasm32")))] +pub mod quantum_fabric_sme; #[cfg(feature = "quantum_control")] pub mod quantum_control; diff --git a/wai/wai-quantum/src/quantum_fabric_sme.rs b/wai/wai-quantum/src/quantum_fabric_sme.rs new file mode 100644 index 000000000..dafbbbdf6 --- /dev/null +++ b/wai/wai-quantum/src/quantum_fabric_sme.rs @@ -0,0 +1,632 @@ +//! The fabric's matrix-extension member: the gate's fixed-point arithmetic in +//! the CPU's streaming vector mode (SME), where the matrix unit executes it. +//! Every instruction the kernel uses is base SME's: none needs SME2. +//! +//! **Why it is bit-exact.** The kernel is the same transcription of +//! [`lanes::apply_words`] as the CUDA kernel, in 64-bit vector lanes: each +//! operand is loaded sign-extended to 64 bits, each `fx(a, b) = (a·b + 2^29) >> 30` +//! is a 64-bit multiply, an add and an arithmetic shift, exact, and each row is +//! the sum of four. The host gathers the pairs a gate selects, the kernel forms +//! the four rows of eight pairs at a time (with 512-bit vectors), and the host +//! checks each row's range and keeps its low 32 bits, as +//! [`lanes::apply_words`] does. +//! +//! **Where it runs.** The kernel enters streaming mode (`smstart sm`) and +//! leaves it (`smstop sm`); its vector instructions exist only in that mode on +//! these processors, so a kernel that ran at all ran there. Discovery reads the +//! processor's feature flags (macOS `hw.optional.arm.FEAT_SME`; on Linux the +//! kernel's `AT_HWCAP2` bit 23, which an emulator supplies for its guest). Under an emulator, the runner declares it +//! (`WAI_QUANTUM_SME_EMULATOR`) and every certificate says so. +//! +//! The kernel's text, between the markers below, is the implementation id's +//! input; three mutants, compiled in, must each be caught. + +use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector}; +use crate::quantum_fabric::lanes::{self, GateWords, Lane}; +use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step}; +use crate::quantum_fabric::{Certificate, Certification, Corpus, FabricClass, KernelEntry, Member, MemberError, Presence, Provenance, RunFuture, RunMeter, Runner, apply_kernel}; + +/// One probe case: [`lanes::mul_wide`] and [`lanes::fxmul_wide`] as +/// `(high, low)` words. +pub type ProbeOutput = ((u32, u32), (u32, u32)); + +/// Why the matrix extension is not a member, or a run has no state. +#[derive(Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum SmeError { + Unavailable(String), + Circuit(QuantumError), + TooWide { n_qubits: u8, max: u8 }, + OutOfRange { gate: u32 }, + Internal(&'static str), + Probe { a: i32, b: i32 }, +} + +impl std::fmt::Display for SmeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SmeError::Unavailable(why) => write!(f, "no matrix extension: {why}"), + SmeError::Circuit(e) => write!(f, "{e}"), + SmeError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"), + SmeError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"), + SmeError::Internal(why) => write!(f, "host defect: {why}"), + SmeError::Probe { a, b } => write!(f, "the kernel's arithmetic differs from the CPU's at ({a}, {b})"), + } + } +} + +impl std::error::Error for SmeError {} + +impl From for MemberError { + fn from(e: SmeError) -> MemberError { + match e { + SmeError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max }, + SmeError::OutOfRange { gate } => MemberError::OutOfRange { gate }, + SmeError::Circuit(e) => MemberError::Circuit(e), + other => MemberError::Other(other.to_string()), + } + } +} + +/// The kernels a member runs: the conforming one, or a mutant. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Variant { + Conforming, + /// The rounding constant is half what it should be. + M1, + /// The product is shifted by one bit too few. + M2, + /// Row 0's real part adds its last term instead of subtracting it. + M3, +} + +impl Variant { + pub const MUTANTS: [Variant; 3] = [Variant::M1, Variant::M2, Variant::M3]; +} + +// kernel-begin +/// The kernels' text: the probe (each `a·b` and `fx(a, b)`) and the rows of a +/// gate. `$rnd` is the rounding constant, `$sh` the shift, `$s` the sign of +/// row 0's last real term. +macro_rules! rows_text { + ($rnd:literal, $sh:literal, $s:literal) => { + concat!( + ".arch_extension sme\n", + "smstart sm\n", + "ptrue p0.d\n", + "ldr x9, [{m}, #0]\nmov z16.d, x9\n", + "ldr x9, [{m}, #8]\nmov z17.d, x9\n", + "ldr x9, [{m}, #16]\nmov z18.d, x9\n", + "ldr x9, [{m}, #24]\nmov z19.d, x9\n", + "ldr x9, [{m}, #32]\nmov z20.d, x9\n", + "ldr x9, [{m}, #40]\nmov z21.d, x9\n", + "ldr x9, [{m}, #48]\nmov z22.d, x9\n", + "ldr x9, [{m}, #56]\nmov z23.d, x9\n", + "mov x9, #", $rnd, "\nmov z31.d, x9\n", + "mov x10, #0\n", + "2:\n", + "whilelt p1.d, x10, {n}\n", + "b.none 3f\n", + "ld1sw {{z0.d}}, p1/z, [{a0r}, x10, lsl #2]\n", + "ld1sw {{z1.d}}, p1/z, [{a0i}, x10, lsl #2]\n", + "ld1sw {{z2.d}}, p1/z, [{a1r}, x10, lsl #2]\n", + "ld1sw {{z3.d}}, p1/z, [{a1i}, x10, lsl #2]\n", + // Row 0, real. + "movprfx z4, z0\nmul z4.d, p0/m, z4.d, z16.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nmov z8.d, z4.d\n", + "movprfx z4, z1\nmul z4.d, p0/m, z4.d, z17.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nsub z8.d, z8.d, z4.d\n", + "movprfx z4, z2\nmul z4.d, p0/m, z4.d, z18.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z8.d, z8.d, z4.d\n", + "movprfx z4, z3\nmul z4.d, p0/m, z4.d, z19.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\n", $s, " z8.d, z8.d, z4.d\n", + // Row 0, imaginary. + "movprfx z4, z1\nmul z4.d, p0/m, z4.d, z16.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nmov z9.d, z4.d\n", + "movprfx z4, z0\nmul z4.d, p0/m, z4.d, z17.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z9.d, z9.d, z4.d\n", + "movprfx z4, z3\nmul z4.d, p0/m, z4.d, z18.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z9.d, z9.d, z4.d\n", + "movprfx z4, z2\nmul z4.d, p0/m, z4.d, z19.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z9.d, z9.d, z4.d\n", + // Row 1, real. + "movprfx z4, z0\nmul z4.d, p0/m, z4.d, z20.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nmov z10.d, z4.d\n", + "movprfx z4, z1\nmul z4.d, p0/m, z4.d, z21.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nsub z10.d, z10.d, z4.d\n", + "movprfx z4, z2\nmul z4.d, p0/m, z4.d, z22.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z10.d, z10.d, z4.d\n", + "movprfx z4, z3\nmul z4.d, p0/m, z4.d, z23.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nsub z10.d, z10.d, z4.d\n", + // Row 1, imaginary. + "movprfx z4, z1\nmul z4.d, p0/m, z4.d, z20.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nmov z11.d, z4.d\n", + "movprfx z4, z0\nmul z4.d, p0/m, z4.d, z21.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z11.d, z11.d, z4.d\n", + "movprfx z4, z3\nmul z4.d, p0/m, z4.d, z22.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z11.d, z11.d, z4.d\n", + "movprfx z4, z2\nmul z4.d, p0/m, z4.d, z23.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z11.d, z11.d, z4.d\n", + "st1d {{z8.d}}, p1, [{o0r}, x10, lsl #3]\n", + "st1d {{z9.d}}, p1, [{o0i}, x10, lsl #3]\n", + "st1d {{z10.d}}, p1, [{o1r}, x10, lsl #3]\n", + "st1d {{z11.d}}, p1, [{o1i}, x10, lsl #3]\n", + "incd x10\n", + "b 2b\n", + "3:\n", + "smstop sm\n", + ) + }; +} + +macro_rules! probe_text { + ($rnd:literal, $sh:literal) => { + concat!( + ".arch_extension sme\n", + "smstart sm\n", + "ptrue p0.d\n", + "mov x9, #", $rnd, "\nmov z31.d, x9\n", + "mov x10, #0\n", + "2:\n", + "whilelt p1.d, x10, {n}\n", + "b.none 3f\n", + "ld1sw {{z0.d}}, p1/z, [{a}, x10, lsl #2]\n", + "ld1sw {{z1.d}}, p1/z, [{b}, x10, lsl #2]\n", + "movprfx z4, z0\nmul z4.d, p0/m, z4.d, z1.d\n", + "add z5.d, z4.d, z31.d\nasr z5.d, z5.d, #", $sh, "\n", + "st1d {{z4.d}}, p1, [{prod}, x10, lsl #3]\n", + "st1d {{z5.d}}, p1, [{fx}, x10, lsl #3]\n", + "incd x10\n", + "b 2b\n", + "3:\n", + "smstop sm\n", + ) + }; +} +// kernel-end + +/// One kernel, conforming or a mutant: the rows of `n` pairs. +macro_rules! rows_fn { + ($name:ident, $rnd:literal, $sh:literal, $s:literal) => { + #[cfg(target_arch = "aarch64")] + #[allow(clippy::too_many_arguments)] + /// # Safety + /// + /// The processor has SME; every slice holds `n` values. + unsafe fn $name(m: &[i64; 8], n: usize, a0r: &[i32], a0i: &[i32], a1r: &[i32], a1i: &[i32], o0r: &mut [i64], o0i: &mut [i64], o1r: &mut [i64], o1i: &mut [i64]) { + debug_assert!([a0r.len(), a0i.len(), a1r.len(), a1i.len(), o0r.len(), o0i.len(), o1r.len(), o1i.len()].iter().all(|&l| l >= n)); + // SAFETY: the caller's contract; streaming mode is entered and left + // inside the block, and every vector register it writes is declared + // clobbered (leaving streaming mode resets them). + unsafe { + std::arch::asm!( + rows_text!($rnd, $sh, $s), + m = in(reg) m.as_ptr(), + n = in(reg) n, + a0r = in(reg) a0r.as_ptr(), + a0i = in(reg) a0i.as_ptr(), + a1r = in(reg) a1r.as_ptr(), + a1i = in(reg) a1i.as_ptr(), + o0r = in(reg) o0r.as_mut_ptr(), + o0i = in(reg) o0i.as_mut_ptr(), + o1r = in(reg) o1r.as_mut_ptr(), + o1i = in(reg) o1i.as_mut_ptr(), + out("x9") _, out("x10") _, + out("v0") _, out("v1") _, out("v2") _, out("v3") _, out("v4") _, out("v5") _, out("v6") _, out("v7") _, + out("v8") _, out("v9") _, out("v10") _, out("v11") _, out("v12") _, out("v13") _, out("v14") _, out("v15") _, + out("v16") _, out("v17") _, out("v18") _, out("v19") _, out("v20") _, out("v21") _, out("v22") _, out("v23") _, + out("v24") _, out("v25") _, out("v26") _, out("v27") _, out("v28") _, out("v29") _, out("v30") _, out("v31") _, + options(nostack), + ); + } + } + }; +} + +macro_rules! probe_fn { + ($name:ident, $rnd:literal, $sh:literal) => { + #[cfg(target_arch = "aarch64")] + /// # Safety + /// + /// The processor has SME; every slice holds `n` values. + unsafe fn $name(n: usize, a: &[i32], b: &[i32], prod: &mut [i64], fx: &mut [i64]) { + debug_assert!(a.len() >= n && b.len() >= n && prod.len() >= n && fx.len() >= n); + // SAFETY: as for the rows. + unsafe { + std::arch::asm!( + probe_text!($rnd, $sh), + n = in(reg) n, + a = in(reg) a.as_ptr(), + b = in(reg) b.as_ptr(), + prod = in(reg) prod.as_mut_ptr(), + fx = in(reg) fx.as_mut_ptr(), + out("x9") _, out("x10") _, + out("v0") _, out("v1") _, out("v2") _, out("v3") _, out("v4") _, out("v5") _, out("v6") _, out("v7") _, + out("v8") _, out("v9") _, out("v10") _, out("v11") _, out("v12") _, out("v13") _, out("v14") _, out("v15") _, + out("v16") _, out("v17") _, out("v18") _, out("v19") _, out("v20") _, out("v21") _, out("v22") _, out("v23") _, + out("v24") _, out("v25") _, out("v26") _, out("v27") _, out("v28") _, out("v29") _, out("v30") _, out("v31") _, + options(nostack), + ); + } + } + }; +} + +rows_fn!(rows_conforming, "0x20000000", "30", "sub"); +rows_fn!(rows_m1, "0x10000000", "30", "sub"); +rows_fn!(rows_m2, "0x20000000", "29", "sub"); +rows_fn!(rows_m3, "0x20000000", "30", "add"); +probe_fn!(probe_conforming, "0x20000000", "30"); +probe_fn!(probe_m1, "0x10000000", "30"); +probe_fn!(probe_m2, "0x20000000", "29"); + +/// The text the conforming kernels are assembled from. +pub const KERNEL_TEXT: &str = concat!(rows_text!("0x20000000", "30", "sub"), probe_text!("0x20000000", "30")); + +/// The implementation name the id starts with. +pub const IMPLEMENTATION: &str = "sme-lanes64"; + +/// Version of the host layout: the gather of a gate's pairs into four +/// columns, the 64-bit row outputs, and the host's range check. Part of the +/// implementation id. +pub const HOST_LAYOUT: u32 = 1; + +/// `"sme-lanes64/<16 hex>"` over a variant's kernel text and the layout. +pub fn implementation_id_of(v: Variant) -> String { + let text = match v { + Variant::Conforming => KERNEL_TEXT, + Variant::M1 => concat!(rows_text!("0x10000000", "30", "sub"), probe_text!("0x10000000", "30")), + Variant::M2 => concat!(rows_text!("0x20000000", "29", "sub"), probe_text!("0x20000000", "29")), + Variant::M3 => concat!(rows_text!("0x20000000", "30", "add"), probe_text!("0x20000000", "30")), + }; + let mut o = b"wai:quantum-fabric-sme\x01".to_vec(); + o.extend_from_slice(&(text.len() as u64).to_be_bytes()); + o.extend_from_slice(text.as_bytes()); + o.extend_from_slice(&HOST_LAYOUT.to_be_bytes()); + let h: String = blake3::hash(&o).as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect(); + format!("{IMPLEMENTATION}/{h}") +} + +pub fn implementation_id() -> String { + implementation_id_of(Variant::Conforming) +} + +/// Whether this processor has SME, by its feature flags. +pub fn has_sme() -> Result<(), String> { + #[cfg(not(target_arch = "aarch64"))] + { + Err("this processor is not aarch64".into()) + } + #[cfg(all(target_arch = "aarch64", target_os = "macos"))] + { + let out = std::process::Command::new("sysctl").args(["-n", "hw.optional.arm.FEAT_SME"]).output().map_err(|e| format!("sysctl: {e}"))?; + match String::from_utf8_lossy(&out.stdout).trim() { + "1" => Ok(()), + other => Err(format!("hw.optional.arm.FEAT_SME is {other:?}")), + } + } + #[cfg(all(target_arch = "aarch64", target_os = "linux"))] + { + // The kernel's capability bits for this process: AT_HWCAP2 (26), whose + // bit 23 is SME. An emulator supplies its own, where /proc/cpuinfo + // would describe the host instead. + const AT_HWCAP2: u64 = 26; + const HWCAP2_SME: u64 = 1 << 23; + let auxv = std::fs::read("/proc/self/auxv").map_err(|e| format!("/proc/self/auxv: {e}"))?; + let hwcap2 = auxv.as_chunks::<16>().0.iter().find_map(|e| { + let (k, v) = e.split_at(8); + (u64::from_ne_bytes(k.try_into().expect("8 bytes")) == AT_HWCAP2).then(|| u64::from_ne_bytes(v.try_into().expect("8 bytes"))) + }); + match hwcap2 { + Some(h) if h & HWCAP2_SME != 0 => Ok(()), + Some(_) => Err("the kernel reports no SME (AT_HWCAP2 bit 23)".into()), + None => Err("the kernel reports no AT_HWCAP2".into()), + } + } + #[cfg(all(target_arch = "aarch64", not(any(target_os = "macos", target_os = "linux"))))] + { + Err("feature detection is written for macOS and Linux".into()) + } +} + +/// What [`discover`] found. +#[derive(Debug)] +pub struct Discovery { + pub found: Vec, + pub refused: Vec<(String, SmeError)>, + pub unavailable: Option, +} + +/// The matrix extension, probed in this process, or why there is none. +pub fn discover() -> Discovery { + let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None }; + if let Err(why) = has_sme() { + d.unavailable = Some(why); + return d; + } + match SmeMember::open(Variant::Conforming) { + Ok(m) => d.found.push(m), + Err(e) => d.refused.push((KEY.into(), e)), + } + d +} + +/// The key a runner declares. +pub const KEY: &str = "cpu-matrix-sme"; + +/// The most qubits a run holds. +pub const MAX_SME_QUBITS: u8 = if 24 < MAX_QUBITS { 24 } else { MAX_QUBITS }; + +/// The matrix extension as a fabric member. +#[derive(Clone, Debug)] +pub struct SmeMember { + variant: Variant, + id: String, + implementation: String, + detail: String, + provenance: Provenance, + max_qubits: u8, +} + +impl SmeMember { + fn open(variant: Variant) -> Result { + let emulator = std::env::var("WAI_QUANTUM_SME_EMULATOR").ok().filter(|s| !s.trim().is_empty()); + let provenance = emulator.as_ref().map_or(Provenance::Silicon, |e| Provenance::Emulator(e.trim().to_string())); + let mut o = b"wai:quantum-fabric-sme-member\x01".to_vec(); + o.extend_from_slice(std::env::consts::ARCH.as_bytes()); + o.extend_from_slice(std::env::consts::OS.as_bytes()); + o.extend_from_slice(provenance.label().as_bytes()); + let id = format!("local/cpu-matrix-sme-{}", blake3::hash(&o).as_bytes()[..6].iter().map(|b| format!("{b:02x}")).collect::()); + let member = SmeMember { + variant, + id, + implementation: implementation_id_of(variant), + detail: format!( + "cpu-matrix; the processor's matrix extension in streaming mode (SME), 64-bit lanes; {}; up to {MAX_SME_QUBITS} qubits; unmetered", + provenance.label() + ), + provenance, + max_qubits: MAX_SME_QUBITS, + }; + if variant == Variant::Conforming { + let pairs = lanes::probe_pairs(4096, 0x5eed); + let out = member.probe(&pairs)?; + if let Some((a, b)) = lanes::first_mismatch(&pairs, &out) { + return Err(SmeError::Probe { a, b }); + } + } + Ok(member) + } + + /// The same processor running a mutant: for the mutation tests. + #[doc(hidden)] + pub fn with_variant(&self, v: Variant) -> SmeMember { + SmeMember { variant: v, implementation: implementation_id_of(v), ..self.clone() } + } + + #[doc(hidden)] + pub fn with_max_qubits(&self, max_qubits: u8) -> SmeMember { + SmeMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() } + } + + pub fn id(&self) -> &str { + &self.id + } + + pub fn key(&self) -> &str { + KEY + } + + pub fn class(&self) -> FabricClass { + FabricClass::CpuMatrix + } + + pub fn detail(&self) -> &str { + &self.detail + } + + pub fn implementation(&self) -> &str { + &self.implementation + } + + pub fn provenance(&self) -> &Provenance { + &self.provenance + } + + pub fn max_qubits(&self) -> u8 { + self.max_qubits + } + + pub fn config(&self) -> Config { + Config::PER_GATE + } + + /// The kernel's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each pair. + pub fn probe(&self, pairs: &[(i32, i32)]) -> Result, SmeError> { + #[cfg(target_arch = "aarch64")] + { + let n = pairs.len(); + let a: Vec = pairs.iter().map(|p| p.0).collect(); + let b: Vec = pairs.iter().map(|p| p.1).collect(); + let (mut prod, mut fx) = (vec![0i64; n], vec![0i64; n]); + // SAFETY: discovery established SME; every slice holds n values. + unsafe { + match self.variant { + Variant::M1 => probe_m1(n, &a, &b, &mut prod, &mut fx), + Variant::M2 => probe_m2(n, &a, &b, &mut prod, &mut fx), + Variant::Conforming | Variant::M3 => probe_conforming(n, &a, &b, &mut prod, &mut fx), + } + } + let split = |v: i64| ((v as u64 >> 32) as u32, v as u64 as u32); + Ok(prod.iter().zip(&fx).map(|(&p, &f)| (split(p), split(f))).collect()) + } + #[cfg(not(target_arch = "aarch64"))] + { + let _ = pairs; + Err(SmeError::Unavailable("not aarch64".into())) + } + } + + /// The four rows of each pair, from the kernel. + #[cfg(target_arch = "aarch64")] + fn rows(&self, m: &[i64; 8], cols: &[Vec; 4]) -> [Vec; 4] { + let n = cols[0].len(); + let mut o = [vec![0i64; n], vec![0i64; n], vec![0i64; n], vec![0i64; n]]; + let [o0r, o0i, o1r, o1i] = &mut o; + // SAFETY: discovery established SME; every slice holds n values. + unsafe { + let f = match self.variant { + Variant::Conforming => rows_conforming, + Variant::M1 => rows_m1, + Variant::M2 => rows_m2, + Variant::M3 => rows_m3, + }; + f(m, n, &cols[0], &cols[1], &cols[2], &cols[3], o0r, o0i, o1r, o1i); + } + o + } + + /// Gates in order from `state`, as [`lanes::run_words`] runs them. + pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result { + let len = state.len(); + if len < 2 || !len.is_power_of_two() { + return Err(SmeError::Internal("a state that is not 2^n lanes, n at least 1")); + } + let n = len.trailing_zeros() as u8; + if n > self.max_qubits { + return Err(SmeError::TooWide { n_qubits: n, max: self.max_qubits }); + } + let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect(); + self.execute(state.to_vec(), &planned) + } + + fn execute(&self, mut amps: Vec, gates: &[(&GateWords, u32)]) -> Result { + let len = amps.len(); + for (w, _) in gates { + let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize); + if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 { + return Err(SmeError::Internal("malformed gate words for this state")); + } + } + #[cfg(not(target_arch = "aarch64"))] + { + let _ = (&mut amps, gates); + return Err(SmeError::Unavailable("not aarch64".into())); + } + #[cfg(target_arch = "aarch64")] + { + let mut report: Option = None; + for &(w, index) in gates { + let m: [i64; 8] = w.m.map(|x| i64::from(x as i32)); + let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize); + let mut at = Vec::new(); + let mut cols: [Vec; 4] = Default::default(); + for p in 0..len / 2 { + let low = p & (tbit - 1); + let i = ((p - low) << 1) | low; + if i & ctrl != ctrl { + continue; + } + at.push(i); + let (a0, a1) = (amps[i], amps[i | tbit]); + cols[0].push(a0[0]); + cols[1].push(a0[1]); + cols[2].push(a1[0]); + cols[3].push(a1[1]); + } + let [o0r, o0i, o1r, o1i] = self.rows(&m, &cols); + for (k, &i) in at.iter().enumerate() { + let rows = [o0r[k], o0i[k], o1r[k], o1i[k]]; + if rows.iter().any(|&v| i32::try_from(v).is_err()) { + report = Some(report.map_or(index, |r| r.min(index))); + } + let low = |v: i64| v as u64 as u32 as i32; + amps[i] = [low(rows[0]), low(rows[1])]; + amps[i | tbit] = [low(rows[2]), low(rows[3])]; + } + } + Ok(LaneRun { amps, out_of_range: report }) + } + } + + pub fn run_program(&self, p: &Program) -> Result { + let n = p.n_qubits; + if n == 0 { + return Ok(LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None }); + } + if n > self.max_qubits { + return Err(SmeError::TooWide { n_qubits: n, max: self.max_qubits }); + } + if p.start >= 1usize << n { + return Err(SmeError::Internal("the start is outside the state")); + } + let mut gates = Vec::with_capacity(p.steps.len()); + for step in &p.steps { + let Step::Gate { gate } = *step else { return Err(SmeError::Internal("a tile step; this member runs gate steps only")) }; + let g = p.gates.get(gate as usize).ok_or(SmeError::Internal("a step names a gate the program does not hold"))?; + if !g.is_well_formed() { + return Err(SmeError::Internal("malformed gate words for this state")); + } + gates.push((&g.words, g.index)); + } + let mut amps = vec![[0i32, 0]; 1usize << n]; + amps[p.start] = [ONE as i32, 0]; + self.execute(amps, &gates) + } + + pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result { + let p = plan::compile_with(c, start, &self.config()).map_err(SmeError::Circuit)?; + if p.n_qubits > self.max_qubits { + return Err(SmeError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits }); + } + let run = self.run_program(&p)?; + if let Some(gate) = run.out_of_range { + return Err(SmeError::OutOfRange { gate }); + } + Ok(StateVector { n_qubits: p.n_qubits, amps: run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }).collect() }) + } + + pub fn simulate(&self, c: &Circuit) -> Result { + self.simulate_from(c, 0) + } + + /// The certificate of [`apply_kernel`]`(gs)` on this member, with its + /// provenance. + pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result { + let mut cert = Certification::new(); + for c in corpus.circuits(gs) { + let reference = lanes::reference_output(&c).map_err(SmeError::Circuit)?; + let run = self.run_program(&plan::compile_with(&c, 0, &self.config()).map_err(SmeError::Circuit)?)?; + if let Some(gate) = run.out_of_range { + return Err(SmeError::OutOfRange { gate }); + } + cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps)); + } + Ok(cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)).with_provenance(self.provenance.clone())) + } + + pub fn member(&self, corpus: Corpus) -> Result { + let mut kernels = Vec::new(); + for gs in Gateset::ALL { + kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate: self.certify(gs, corpus)?, energy: None }); + } + Ok(Member { id: self.id.clone(), class: FabricClass::CpuMatrix, presence: Presence::Detected, kernels, detail: self.detail.clone() }) + } +} + +impl Runner for SmeMember { + fn member_id(&self) -> String { + self.id.clone() + } + + fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, _meter: &'r mut dyn RunMeter) -> RunFuture<'r> { + Box::pin(async move { + if implementation != self.implementation { + return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation))); + } + Ok((self.simulate_from(c, start)?, 0, None)) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_variant_has_its_own_implementation_id() { + let ids: std::collections::BTreeSet = [Variant::Conforming, Variant::M1, Variant::M2, Variant::M3].into_iter().map(implementation_id_of).collect(); + assert_eq!(ids.len(), 4); + assert!(implementation_id().starts_with("sme-lanes64/")); + assert!(KERNEL_TEXT.contains("smstart sm") && KERNEL_TEXT.contains("smstop sm")); + } + + #[test] + fn the_member_id_sorts_after_the_cpu_reference() { + assert!("local/cpu-matrix-sme-000000000000" > "local/cpu-aarch64"); + } +} diff --git a/wai/wai-quantum/tests/sme_member.rs b/wai/wai-quantum/tests/sme_member.rs new file mode 100644 index 000000000..945eced9f --- /dev/null +++ b/wai/wai-quantum/tests/sme_member.rs @@ -0,0 +1,410 @@ +//! The fabric's matrix-extension member on the machine that runs this. +//! +//! The machine declares the devices it must have in `WAI_QUANTUM_SME_EXPECT`: +//! their keys, comma-separated and sorted, or `none`. Every test that needs a +//! member fails unless the devices found are exactly the declared ones, so a +//! machine whose GPU or driver went missing, or a run that declared nothing, +//! fails instead of passing over no member. Each test prints what it +//! established and, when `WAI_QUANTUM_SME_REPORT` names a file, appends the +//! same lines to it, which a CI step can count per device. +//! +//! ```text +//! WAI_QUANTUM_SME_EXPECT=cpu-matrix-sme \ +//! cargo test --release --features quantum_fabric_sme --test sme_member -- --test-threads=1 +//! ``` +#![cfg(not(target_arch = "wasm32"))] + +use std::io::Write; +use std::sync::OnceLock; +use wai_quantum::ed25519_dalek::SigningKey; +use wai_quantum::quantum::{BaseGate, Circuit, Gate, Gateset, MAX_QUBITS, QuantumError}; +use wai_quantum::quantum_fabric::lanes::{self, GateWords, Lane}; +use wai_quantum::quantum_fabric::plan::{Program, Step}; +use wai_quantum::quantum_fabric::{ + Corpus, CpuReference, FABRIC_DEFAULT, Fabric, FabricClass, FabricReceipt, MemberError, Parity, Presence, REFERENCE, apply_kernel, detect_local, place, ready, +}; +use wai_quantum::quantum_fabric_sme::{self as sme, Discovery, SmeError, SmeMember, Variant, discover}; + +/// The CPU's corpus digests: random-v1 (seed 7, 200 cases) and fabric-v1 +/// (seed 7, 200 + 24 cases), per gateset. A bit-exact member reproduces them. +const PINS: [(Gateset, &str, &str); 2] = [ + (Gateset::V1, "d18454c639acfd9c7b66450f4ded1aac4ab46cc10a7bedbcf4958a96e204fb36", "41c726a0ac80f7defef92e15ecf893bdb7a3076b5e7f670fa0b7142e5c033df7"), + (Gateset::V2, "3dbde276c1095bf827faec71d5980e050f8a55a5ba7d97cd3bbdaec3f43a13b2", "9a84b55af68f1a8b07390b234cf562dd8d85cef3185a697ab85690c8790a7809"), +]; + +const RANDOM_V1: Corpus = Corpus::Random { seed: 7, cases: 200 }; + + +static DISCOVERY: OnceLock = OnceLock::new(); + +fn discovery() -> &'static Discovery { + DISCOVERY.get_or_init(discover) +} + +/// Prints `line` and appends it to the file `WAI_QUANTUM_SME_REPORT` names. +fn report(line: &str) { + println!("{line}"); + if let Some(path) = std::env::var_os("WAI_QUANTUM_SME_REPORT") { + let mut f = std::fs::OpenOptions::new().create(true).append(true).open(&path).expect("the report file opens"); + writeln!(f, "{line}").expect("the report line is written"); + } +} + +/// The declared device set, as written, if it is exactly what was found; +/// otherwise why not, with what was found and refused and how to declare it. +fn declared(d: &Discovery) -> Result { + let mut found: Vec<&str> = d.found.iter().map(|m| m.key()).collect(); + found.sort_unstable(); + let found_list = if found.is_empty() { "none".to_string() } else { found.join(",") }; + let refused: Vec = d.refused.iter().map(|(k, e)| format!("{k}: {e}")).collect(); + let refused_list = if refused.is_empty() { "none".to_string() } else { refused.join("; ") }; + let unavailable = d.unavailable.as_deref().unwrap_or("no"); + let how = format!("found: {found_list}; refused: {refused_list}; unavailable: {unavailable}. Declare the devices this machine must have: WAI_QUANTUM_SME_EXPECT={found_list}"); + let expect = match std::env::var("WAI_QUANTUM_SME_EXPECT") { + Ok(v) if !v.is_empty() => v, + _ => return Err(format!("WAI_QUANTUM_SME_EXPECT is not set. {how}")), + }; + let keys: Vec<&str> = if expect == "none" { Vec::new() } else { expect.split(',').collect() }; + let well_formed = keys.windows(2).all(|w| w[0] <= w[1]) && keys.iter().all(|k| !k.is_empty() && *k != "none" && k.trim() == *k); + if !well_formed { + return Err(format!("WAI_QUANTUM_SME_EXPECT={expect:?} is not a sorted, comma-separated list of keys, nor none. {how}")); + } + for k in &keys { + if !found.contains(k) + && let Some((_, e)) = d.refused.iter().find(|(key, _)| key == k) + { + return Err(format!("the declared device {k} was refused: {e}. {how}")); + } + } + if found != keys { + return Err(format!("the devices found are not the declared ones ({expect}). {how}")); + } + Ok(expect) +} + +/// The members, once the declared devices are exactly the ones found. +fn members() -> &'static [SmeMember] { + let d = discovery(); + if let Err(why) = declared(d) { + panic!("{why}"); + } + &d.found +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn next(s: &mut u64) -> u64 { + *s = s.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *s; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// A value in `[-bound, bound]`. +fn within(s: &mut u64, bound: i64) -> i32 { + ((next(s) % (2 * bound as u64 + 1)) as i64 - bound) as i32 +} + +/// A random circuit of `ops` gates on `n` qubits: every base gate, up to three +/// distinct controls, `P(1..=32)`. +fn random_circuit(n: u8, ops: usize, seed: u64, gs: Gateset) -> Circuit { + let bases = [BaseGate::I, BaseGate::X, BaseGate::Y, BaseGate::Z, BaseGate::H, BaseGate::S, BaseGate::Sdg, BaseGate::T, BaseGate::Tdg, BaseGate::P]; + let mut s = seed; + let mut c = Circuit::with_gateset(n, gs); + for _ in 0..ops { + let base = bases[(next(&mut s) % bases.len() as u64) as usize]; + let target = (next(&mut s) % u64::from(n)) as u8; + let mut controls = Vec::new(); + for _ in 0..next(&mut s) % 4 { + let q = (next(&mut s) % u64::from(n)) as u8; + if q != target && !controls.contains(&q) { + controls.push(q); + } + } + let param = if base == BaseGate::P { 1 + (next(&mut s) % 32) as u16 } else { 0 }; + c.ops.push(Gate { base, controls, target, param }); + } + c +} + +/// Five circuits the native simulator refuses, in each gateset. +fn refused_circuits() -> Vec { + let gate = |base, controls: Vec, target, param| Gate { base, controls, target, param }; + let bad = [ + (1, vec![gate(BaseGate::H, vec![], 0, 0), gate(BaseGate::P, vec![], 0, 0)]), + (1, vec![gate(BaseGate::P, vec![], 0, 33)]), + (1, vec![gate(BaseGate::X, vec![], 3, 0)]), + (2, vec![gate(BaseGate::X, vec![0], 0, 0)]), + (2, vec![gate(BaseGate::X, vec![5], 1, 0)]), + ]; + let mut out = Vec::new(); + for gs in Gateset::ALL { + for (n, ops) in bad.clone() { + let mut c = Circuit::with_gateset(n, gs); + c.ops = ops; + out.push(c); + } + } + out +} + +/// `Z` on qubit 0: on `[(0, 0), (i32::MIN, 0)]` its exact result is `2^31`, +/// which leaves 32 bits. +fn z_words() -> GateWords { + lanes::gate_words(&Gate { base: BaseGate::Z, controls: vec![], target: 0, param: 0 }, Gateset::V2) +} + +const Z_STATE: [Lane; 2] = [[0, 0], [i32::MIN, 0]]; + +#[test] +fn c0_devices_are_declared() { + let d = discovery(); + match declared(d) { + Ok(expect) => report(&format!("sme devices: as declared ({expect})")), + Err(why) => panic!("{why}"), + } + for m in &d.found { + assert_eq!(m.class(), FabricClass::CpuMatrix); + assert!(m.id().starts_with("local/cpu-matrix-sme-"), "{}", m.id()); + assert_eq!(m.implementation(), sme::implementation_id()); + assert!(m.max_qubits() >= 20, "{}: holds only {} qubits", m.key(), m.max_qubits()); + report(&format!("sme member: key={} id={} {}", m.key(), m.id(), m.detail())); + } +} + +#[test] +fn c1_probe_matches_lanes() { + for m in members() { + let pairs = lanes::probe_pairs(1 << 16, 0x0cda); + let out = m.probe(&pairs).unwrap_or_else(|e| panic!("{}: {e}", m.key())); + assert_eq!(lanes::first_mismatch(&pairs, &out), None, "{}", m.key()); + report(&format!("sme probe: key={} {} pairs agree", m.key(), pairs.len())); + } +} + +#[test] +fn c2_kernel_matches_run_words() { + for m in members() { + let mut s = 0x6b65_726e_656c_u64; + let (mut flagged, mut clean) = (0, 0); + for case in 0..2000 { + let n = 1 + (next(&mut s) % 12) as u32; + // Three regimes: the whole 32-bit range (nearly always out of + // range), small states and entries (never), and states and + // entries up to 2^30 (either). + let mode = case % 3; + let value = |s: &mut u64| -> i32 { + match mode { + 0 => next(s) as i32, + 1 => within(s, 1 << 22), + _ => within(s, 1 << 30), + } + }; + let entry = |s: &mut u64| -> u32 { + match mode { + 0 => next(s) as u32, + 1 => within(s, 1 << 29) as u32, + _ => within(s, 1 << 30) as u32, + } + }; + let state: Vec = (0..1usize << n).map(|_| [value(&mut s), value(&mut s)]).collect(); + let gates: Vec = (0..1 + next(&mut s) % 8) + .map(|_| { + let t = (next(&mut s) % u64::from(n)) as u32; + let ctrl = (0..n).filter(|&b| b != t && next(&mut s).is_multiple_of(4)).fold(0u32, |acc, b| acc | 1 << b); + let mut w = [0u32; 8]; + w.iter_mut().for_each(|x| *x = entry(&mut s)); + GateWords { m: w, tbit: 1 << t, ctrl } + }) + .collect(); + let want = lanes::run_words(&state, &gates).expect("valid shapes"); + let got = m.run_lanes(&state, &gates).unwrap_or_else(|e| panic!("{}: case {case}: {e}", m.key())); + assert_eq!(got.out_of_range, want.out_of_range, "{}: case {case}, n = {n}", m.key()); + assert!(got.amps == want.amps, "{}: case {case}, n = {n}: the lanes differ", m.key()); + if want.out_of_range.is_some() { + flagged += 1; + } else { + clean += 1; + } + } + assert!(flagged >= 100 && clean >= 100, "{}: {flagged} flagged, {clean} clean", m.key()); + report(&format!("sme kernel: key={} ok", m.key())); + } +} + +#[test] +fn c3_certificates_reproduce_the_pins() { + let ms = members(); + for (gs, random_pin, fabric_pin) in PINS { + let cpu = lanes::certify_corpus(gs, "local/test", RANDOM_V1); + assert_eq!(hex(&cpu.corpus), random_pin, "{gs:?}"); + let other = Corpus::Random { seed: 8, cases: 300 }; + let cpu_other = lanes::certify_corpus(gs, "local/test", other); + for m in ms { + for (corpus, cases, pin) in [(RANDOM_V1, 200, random_pin), (FABRIC_DEFAULT, 224, fabric_pin)] { + let c = m.certify(gs, corpus).unwrap_or_else(|e| panic!("{} {gs:?} {}: {e}", m.key(), corpus.name())); + assert_eq!(c.parity, Parity::BitExact, "{} {gs:?} {}", m.key(), corpus.name()); + assert_eq!(c.cases, cases); + assert_eq!(hex(&c.corpus), pin, "{} {gs:?} {}", m.key(), corpus.name()); + assert_eq!((c.kernel.as_str(), c.implementation.as_str(), c.member.as_str()), (apply_kernel(gs), m.implementation(), m.id())); + assert_eq!(c.generator, corpus.generator(gs)); + report(&format!( + "sme certified: key={} member={} kernel={} corpus={} implementation={} parity=bit-exact cases={} digest={}", + m.key(), + m.id(), + c.kernel, + corpus.name(), + c.implementation, + c.cases, + hex(&c.corpus) + )); + } + // A corpus with no pin: the member and the CPU agree on it. + let c = m.certify(gs, other).unwrap_or_else(|e| panic!("{} {gs:?} seed 8: {e}", m.key())); + assert_eq!((c.parity, c.cases), (Parity::BitExact, 300), "{} {gs:?}", m.key()); + assert_eq!(c.corpus, cpu_other.corpus, "{} {gs:?}", m.key()); + } + } +} + +#[test] +fn c4_wide_circuits_match_native() { + for m in members() { + let widest = m.max_qubits().min(20); + let mut s = 0x0077_6964_6500_u64; + for (k, n) in [12u8, 16, 18, widest].into_iter().enumerate() { + for gs in Gateset::ALL { + let c = random_circuit(n, 40, 0x1000 + k as u64, gs); + let start = (next(&mut s) as usize) % (1usize << n); + let want = c.simulate_from(start).expect("the native run"); + let got = m.simulate_from(&c, start).unwrap_or_else(|e| panic!("{}: n = {n} {gs:?}: {e}", m.key())); + assert!(got == want, "{}: n = {n} {gs:?}: the states differ", m.key()); + } + } + // The transforms the conformance corpus carries. + for n in [3u8, 5, 8, 12] { + let qft = Circuit::qft(n); + assert!(m.simulate(&qft).unwrap() == qft.simulate().unwrap(), "{}: QFT-{n}", m.key()); + } + report(&format!("sme wide: key={} up to {widest} qubits match native", m.key())); + } +} + +#[test] +fn c5_refusals_are_typed() { + for m in members() { + for c in refused_circuits() { + let native = c.simulate_from(0).unwrap_err(); + assert_eq!(m.simulate_from(&c, 0).unwrap_err(), SmeError::Circuit(native), "{c:?}"); + } + assert_eq!(m.simulate(&Circuit::new(MAX_QUBITS + 1)).unwrap_err(), SmeError::Circuit(QuantumError::TooManyQubits(MAX_QUBITS + 1))); + let small = m.with_max_qubits(4); + assert_eq!(small.simulate(&Circuit::new(5)).unwrap_err(), SmeError::TooWide { n_qubits: 5, max: 4 }); + let wide = Program { n_qubits: 5, start: 0, gates: Vec::new(), steps: Vec::new() }; + assert_eq!(small.run_program(&wide).unwrap_err(), SmeError::TooWide { n_qubits: 5, max: 4 }); + // A result that leaves 32 bits is reported at its op, and the lane + // holds its low 32 bits. + let r = m.run_lanes(&Z_STATE, &[z_words()]).unwrap(); + assert_eq!(r.out_of_range, Some(0), "{}", m.key()); + assert_eq!(r.amps[1][0] as u32, 0x8000_0000); + // Malformed words, a state the host cannot lay out and a tile step are + // host defects, never results. + let bad = GateWords { tbit: 3, ..z_words() }; + assert!(matches!(m.run_lanes(&[[0, 0]; 4], &[bad]), Err(SmeError::Internal(_)))); + assert!(matches!(m.run_lanes(&[[0, 0]; 3], &[z_words()]), Err(SmeError::Internal(_)))); + let tiled = Program { n_qubits: 2, start: 0, gates: Vec::new(), steps: vec![Step::Tile { first: 0, count: 1, size: 2 }] }; + assert!(matches!(m.run_program(&tiled), Err(SmeError::Internal(_)))); + // The refusals that may still fall back to another member, do. + assert!(MemberError::from(SmeError::TooWide { n_qubits: 5, max: 4 }).falls_back()); + report(&format!("sme refusals: key={} typed", m.key())); + } +} + +/// The kernel mutants, compiled in: each must be caught by the probe or by a +/// certificate. +#[test] +fn c6_mutants_are_killed() { + let edges = lanes::probe_pairs(0, 0); + for m in members() { + let mut killed = 0; + for v in Variant::MUTANTS { + let mm = m.with_variant(v); + assert_ne!(mm.implementation(), m.implementation()); + let out = mm.probe(&edges).expect("the mutant's probe runs"); + let differing = edges.iter().zip(&out).filter(|&(&(a, b), o)| *o != (lanes::mul_wide(a, b), lanes::fxmul_wide(a, b))).count(); + let caught = differing > 0 + || Gateset::ALL.into_iter().all(|gs| match mm.certify(gs, RANDOM_V1) { + Err(_) => true, + Ok(c) => c.parity != Parity::BitExact || c.corpus != lanes::certify_corpus(gs, "local/test", RANDOM_V1).corpus, + }); + println!("sme mutant {v:?} on {}: {differing} of {} edge pairs differ", m.key(), edges.len()); + assert!(caught, "{}: mutant {v:?} survived", m.key()); + killed += 1; + } + report(&format!("sme mutants: key={} killed {killed} of 3", m.key())); + } +} + +#[test] +fn c7_receipt_placement_and_the_fabric() { + for m in members() { + let member = m.member(FABRIC_DEFAULT).unwrap_or_else(|e| panic!("{}: {e}", m.key())); + assert_eq!((member.id.as_str(), member.class, member.presence), (m.id(), FabricClass::CpuMatrix, Presence::Detected)); + assert_eq!(member.kernels.len(), 2); + for (entry, gs) in member.kernels.iter().zip(Gateset::ALL) { + assert_eq!((entry.kernel.as_str(), entry.implementation.as_str()), (apply_kernel(gs), m.implementation())); + assert!(entry.energy.is_none()); + assert_eq!(entry.certificate.parity, Parity::BitExact, "{gs:?}"); + } + // A receipt over a member's run. + let qft = Circuit::qft(5); + let state = m.simulate(&qft).unwrap(); + assert_eq!(state, qft.simulate().unwrap()); + let entry = member.kernels.iter().find(|k| k.kernel == apply_kernel(qft.gateset)).unwrap(); + let signer = SigningKey::from_bytes(&[7; 32]); + let receipt = FabricReceipt::seal(&signer, "local/test", state.statevector_hash(), &member, entry, 0, None); + assert!(receipt.verify()); + assert!(receipt.certificate_matches(&entry.certificate)); + // Placement: beside the CPU reference with no energy figures, the + // reference; alone, the member. + let local = detect_local(); + for gs in Gateset::ALL { + let both = [local.clone(), member.clone()]; + let (placed, k) = place(&both, apply_kernel(gs), &Parity::BitExact).unwrap(); + assert_eq!((placed.id.as_str(), k.implementation.as_str()), (local.id.as_str(), REFERENCE)); + let (placed, k) = place(std::slice::from_ref(&member), apply_kernel(gs), &Parity::BitExact).unwrap(); + assert_eq!((placed.id.as_str(), k.implementation.as_str()), (m.id(), m.implementation())); + } + // On the fabric: the member alone runs the circuit; a pairing with + // another member's entries is refused. + let mut fabric = Fabric::new(); + fabric.add(member.clone(), m); + let run = ready(fabric.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs"); + assert_eq!(run.state, state); + assert_eq!(run.receipt.member_id, m.id()); + assert!(run.receipt.verify()); + let mut mismatched = Fabric::new(); + mismatched.add(local.clone(), m); + assert!(ready(mismatched.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").is_err()); + let mut both = Fabric::new(); + both.add(local.clone(), CpuReference).add(member.clone(), m); + let run = ready(both.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs"); + assert_eq!(run.state, state); + report(&format!("sme receipt: key={} ok", m.key())); + } +} + +#[test] +fn c8_repeated_runs_agree() { + for m in members() { + let c = random_circuit(16, 60, 0xabc, Gateset::V2); + let first = m.simulate(&c).unwrap(); + for _ in 0..5 { + assert!(m.simulate(&c).unwrap() == first, "{}", m.key()); + } + report(&format!("sme repeat: key={} ok", m.key())); + } +} -- 2.54.0 (Apple Git-157) From 437de5d6510841be9989077027f8c8c6d50e4fe3 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Wed, 14 Oct 2026 10:03:55 -0400 Subject: [PATCH 04/11] wai-quantum: the limb model in the portable format; f16_of at 2048 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit npu::onnx_limb_model writes the neural-processor member's program (one multiply, one sum over the last axis) as an ONNX opset-17 model with fp16 or int32 tensors, and npu::expand_i32 lays out the int32 inputs. The example npu_onnx writes both models, a batch of inputs and the exact products they must give, for a neural processor reached through a runtime that reads that format. Both models pass the full ONNX checker and give every product of a 16384-pair batch exactly on the reference runtime. Fix: f16_of(±2048) shifted by 10 - 11 and panicked in a debug build. Co-Authored-By: Claude Opus 5.5 --- wai/CHANGELOG.md | 6 + wai/wai-quantum/Cargo.toml | 4 + wai/wai-quantum/examples/npu_onnx.rs | 42 +++++++ wai/wai-quantum/src/quantum_fabric/npu.rs | 142 +++++++++++++++++++++- 4 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 wai/wai-quantum/examples/npu_onnx.rs diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index 7d4db6424..4ad506c16 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -23,6 +23,12 @@ Additive. The default features, `full`, and every pinned digest are unchanged. the member is refused with the placement reported, so no CPU result is credited to the processor. It is then probed and certified against the pinned corpus digests, and the two model mutants (the multiply made an add, the sum a maximum) are caught. +- **The same program, portable.** `npu::onnx_limb_model` writes it in the portable model + format (ONNX, opset 17: one `Mul`, one `ReduceSum`), with fp16 or int32 tensors, and + `npu::expand_i32` lays out the int32 inputs; `examples/npu_onnx.rs` writes both models, + a batch of inputs and the exact products they must give, so a neural processor + reached through a runtime that reads that format can be checked against them. +- **Fixed** before release: `npu::f16_of(±2048)` subtracted past zero in a debug build. - **The matrix-extension member** (`quantum_fabric_sme`, opt-in, aarch64 only): the gate's fixed-point arithmetic in the processor's streaming vector mode (SME), 64-bit lanes, by inline assembly; the host gathers each gate's pairs and checks each row's diff --git a/wai/wai-quantum/Cargo.toml b/wai/wai-quantum/Cargo.toml index a27e7b600..dc6db3fa8 100644 --- a/wai/wai-quantum/Cargo.toml +++ b/wai/wai-quantum/Cargo.toml @@ -223,6 +223,10 @@ required-features = ["quantum_fabric_sme"] name = "fabric_expected" required-features = ["quantum_fabric"] +[[example]] +name = "npu_onnx" +required-features = ["quantum_fabric"] + [[example]] name = "qio_gset" required-features = ["quantum_qio"] diff --git a/wai/wai-quantum/examples/npu_onnx.rs b/wai/wai-quantum/examples/npu_onnx.rs new file mode 100644 index 000000000..163740f6c --- /dev/null +++ b/wai/wai-quantum/examples/npu_onnx.rs @@ -0,0 +1,42 @@ +//! Writes the limb-product model in the portable format (fp16 and int32), and a +//! batch of inputs with the products they must give, for a neural-processor +//! service that runs such models: +//! +//! ```text +//! cargo run --example npu_onnx --features quantum_fabric -- +//! ``` +//! +//! In ``: `limbs_f16.onnx`, `limbs_i32.onnx`; `x_f16.bin`, `y_f16.bin` +//! (fp16 bits, little-endian), `x_i32.bin`, `y_i32.bin` (int32, little-endian), +//! each `[batch, 15, 8]`; `pairs.bin` (the operands, int32 pairs) and +//! `products.bin` (each exact `a·b`, int64). A service's output `z` +//! (`[batch, 15]`) is right when `Σ_s z[p, s]·16^s` is `products[p]` for every +//! `p`. +use std::io::Write; +use wai_quantum::quantum_fabric::lanes; +use wai_quantum::quantum_fabric::npu::{BATCH, Elem, LIMBS, SHIFTS, expand, expand_i32, onnx_limb_model}; + +fn write(path: &std::path::Path, bytes: &[u8]) { + std::fs::File::create(path).and_then(|mut f| f.write_all(bytes)).unwrap_or_else(|e| panic!("{}: {e}", path.display())); +} + +fn main() { + let dir = std::path::PathBuf::from(std::env::args().nth(1).expect("an output directory")); + std::fs::create_dir_all(&dir).expect("the directory"); + write(&dir.join("limbs_f16.onnx"), &onnx_limb_model(BATCH as u64, SHIFTS as u64, LIMBS as u64, Elem::F16)); + write(&dir.join("limbs_i32.onnx"), &onnx_limb_model(BATCH as u64, SHIFTS as u64, LIMBS as u64, Elem::I32)); + let mut pairs = lanes::probe_pairs(BATCH, 0x0a1b); + pairs.truncate(BATCH); + let n = BATCH * SHIFTS * LIMBS; + let (mut x, mut y) = (vec![0u16; n], vec![0u16; n]); + expand(&pairs, &mut x, &mut y); + write(&dir.join("x_f16.bin"), &x.iter().flat_map(|v| v.to_le_bytes()).collect::>()); + write(&dir.join("y_f16.bin"), &y.iter().flat_map(|v| v.to_le_bytes()).collect::>()); + let (mut xi, mut yi) = (vec![0i32; n], vec![0i32; n]); + expand_i32(&pairs, &mut xi, &mut yi); + write(&dir.join("x_i32.bin"), &xi.iter().flat_map(|v| v.to_le_bytes()).collect::>()); + write(&dir.join("y_i32.bin"), &yi.iter().flat_map(|v| v.to_le_bytes()).collect::>()); + write(&dir.join("pairs.bin"), &pairs.iter().flat_map(|&(a, b)| [a.to_le_bytes(), b.to_le_bytes()].concat()).collect::>()); + write(&dir.join("products.bin"), &pairs.iter().flat_map(|&(a, b)| (i64::from(a) * i64::from(b)).to_le_bytes()).collect::>()); + println!("wrote {} pairs of a batch of {BATCH} to {}", pairs.len(), dir.display()); +} diff --git a/wai/wai-quantum/src/quantum_fabric/npu.rs b/wai/wai-quantum/src/quantum_fabric/npu.rs index c8ff579c4..e3113a32a 100644 --- a/wai/wai-quantum/src/quantum_fabric/npu.rs +++ b/wai/wai-quantum/src/quantum_fabric/npu.rs @@ -241,6 +241,119 @@ pub fn limb_product_model(batch: u64, shifts: u64, limbs: u64) -> Vec { } +/// The element type of a portable model's tensors. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Elem { + /// fp16: every value the model forms is an integer of at most 1800 in size, + /// which fp16 holds exactly. + F16, + /// int32: exact on any processor that computes it in integers without + /// wrapping; a coefficient is at most 1800 in size. + I32, +} + +// ONNX's field numbers (onnx.proto) and element types. +const ONNX_FLOAT16: u64 = 10; +const ONNX_INT32: u64 = 6; +const ONNX_INT64: u64 = 7; +const ONNX_ATTR_INT: u64 = 2; + +/// An ONNX value description: `name`, a tensor of `elem` and `shape`. +fn onnx_value(o: &mut Vec, field: u32, name: &str, elem: u64, shape: &[u64]) { + msg(o, field, |v| { + bytes(v, 1, name.as_bytes()); + // type = 2 { tensor_type = 1 { elem_type = 1, shape = 2 { dim = 1 { dim_value = 1 } } } } + msg(v, 2, |t| { + msg(t, 1, |tt| { + uint(tt, 1, elem); + msg(tt, 2, |sh| { + for &d in shape { + msg(sh, 1, |dim| int(dim, 1, d as i64)); + } + }); + }) + }); + }); +} + +/// The same program in the portable model format (ONNX, opset 17): +/// `z = ReduceSum(Mul(x, y), axes = [-1], keepdims = 0)`, `x` and `y` +/// `[batch, shifts, limbs]`, all of `elem`. Neural-processor runtimes that +/// read this format take it as it is; the inputs are [`expand`]'s (as fp16 +/// bits) or [`expand_i32`]'s. +pub fn onnx_limb_model(batch: u64, shifts: u64, limbs: u64, elem: Elem) -> Vec { + let e = match elem { + Elem::F16 => ONNX_FLOAT16, + Elem::I32 => ONNX_INT32, + }; + let full = [batch, shifts, limbs]; + let out = [batch, shifts]; + let mut m = Vec::new(); + int(&mut m, 1, 8); // ir_version + bytes(&mut m, 2, b"wai-quantum"); // producer_name + // graph = 7 + msg(&mut m, 7, |g| { + // node = 1: Mul + msg(g, 1, |n| { + bytes(n, 1, b"x"); + bytes(n, 1, b"y"); + bytes(n, 2, b"p"); + bytes(n, 3, b"mul"); + bytes(n, 4, b"Mul"); + }); + // node = 1: ReduceSum over the last axis, the axes as an input + msg(g, 1, |n| { + bytes(n, 1, b"p"); + bytes(n, 1, b"axes"); + bytes(n, 2, b"z"); + bytes(n, 3, b"sum"); + bytes(n, 4, b"ReduceSum"); + msg(n, 5, |a| { + bytes(a, 1, b"keepdims"); + int(a, 3, 0); + uint(a, 20, ONNX_ATTR_INT); + }); + }); + bytes(g, 2, b"limb_products"); + // initializer = 5: axes = [-1], int64 + msg(g, 5, |t| { + int(t, 1, 1); // dims + uint(t, 2, ONNX_INT64); // data_type + int(t, 7, -1); // int64_data + bytes(t, 8, b"axes"); // name + }); + onnx_value(g, 11, "x", e, &full); + onnx_value(g, 11, "y", e, &full); + onnx_value(g, 12, "z", e, &out); + }); + // opset_import = 8 { domain = 1 "", version = 2 } + msg(&mut m, 8, |op| { + bytes(op, 1, b""); + int(op, 2, 17); + }); + m +} + +/// [`expand`] with int32 values: `x[p, s, j]` and `y[p, s, j]` as the limbs +/// themselves. +pub fn expand_i32(pairs: &[(i32, i32)], x: &mut [i32], y: &mut [i32]) { + assert!(pairs.len() <= BATCH && x.len() == BATCH * SHIFTS * LIMBS && y.len() == x.len()); + x.fill(0); + y.fill(0); + for (p, &(a, b)) in pairs.iter().enumerate() { + let (la, lb) = (limbs(a), limbs(b)); + for s in 0..SHIFTS { + for j in 0..LIMBS { + if s >= j && s - j < LIMBS { + let at = (p * SHIFTS + s) * LIMBS + j; + x[at] = la[j]; + y[at] = lb[s - j]; + } + } + } + } +} + /// `"npu-limbs4/<16 hex>"` over the model's bytes and the host layout. pub fn implementation_id_of(model: &[u8]) -> String { let mut o = DOMAIN_IMPLEMENTATION.to_vec(); @@ -282,8 +395,10 @@ pub fn f16_of(v: i32) -> u16 { let sign: u16 = if v < 0 { 0x8000 } else { 0 }; let m = v.unsigned_abs(); let e = 31 - m.leading_zeros(); // floor(log2 m), 0..=11 - // Significand bits below the leading one, left-aligned in 10 bits. - let frac = ((m - (1 << e)) << (10 - e)) as u16; + // Significand bits below the leading one, left-aligned in 10 bits (2048, + // the one value with e = 11, has none). + let below = m - (1 << e); + let frac = if e <= 10 { below << (10 - e) } else { below >> (e - 10) } as u16; sign | (((e + 15) as u16) << 10) | frac } @@ -392,6 +507,29 @@ mod tests { } } + #[test] + fn the_portable_model_names_its_program_and_its_element_type() { + for (elem, code) in [(Elem::F16, ONNX_FLOAT16), (Elem::I32, ONNX_INT32)] { + let m = onnx_limb_model(BATCH as u64, SHIFTS as u64, LIMBS as u64, elem); + assert_eq!(&m[..2], &[0x08, 8], "ir_version first"); + for op in [&b"Mul"[..], b"ReduceSum", b"keepdims", b"axes"] { + assert!(m.windows(op.len()).any(|w| w == op)); + } + // elem_type = code, inside each value description. + assert!(m.windows(2).any(|w| w == [0x08, code as u8])); + } + assert_ne!(onnx_limb_model(4, 15, 8, Elem::F16), onnx_limb_model(4, 15, 8, Elem::I32)); + // The int32 layout carries the limbs themselves. + let pairs = [(i32::MIN, -1), (0x7654_3210, 0x0123_4567)]; + let n = BATCH * SHIFTS * LIMBS; + let (mut x, mut y) = (vec![0i32; n], vec![0i32; n]); + expand_i32(&pairs, &mut x, &mut y); + for (p, &(a, b)) in pairs.iter().enumerate() { + let z: Vec = (0..SHIFTS).map(|s| (0..LIMBS).map(|j| i64::from(x[(p * SHIFTS + s) * LIMBS + j]) * i64::from(y[(p * SHIFTS + s) * LIMBS + j])).sum()).collect(); + assert_eq!(assemble(&z), i64::from(a) * i64::from(b)); + } + } + #[test] fn the_model_is_the_program_it_names_and_its_mutants_differ() { let m = model(); -- 2.54.0 (Apple Git-157) From 05fef6cce9aeedb7adbfac47bbbad1b2e95647c9 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Wed, 14 Oct 2026 10:47:59 -0400 Subject: [PATCH 05/11] wai-quantum: the signal processor, and its vector unit as a member The signal processor (DSP) as a processor: the bit-identity digests (23), the frame fixtures (2) and the byte-pinning library tests (137) pass for its Linux target under the user-mode emulator, the standard library built from source (a tier-3 target). Emulator releases before 11.0 test a whole predicate register, where the processor tests its low bit, in a branch on a predicate written in the same packet. Compiled code relies on the low bit (a predicate from a register's 1, xored with a comparison's 0xff, so 0xfe is "equal"), and one library test fails under such an emulator though the code is right. The probe ci/dsp/predicate_branch.c shows it; the workflow builds 11.1.1 from its release tarball (SHA-256 pinned, signature checked when pinned) and runs every suite on it. The member (feature quantum_fabric_dsp, class dsp): the gate's fixed-point arithmetic in the vector unit's 128-byte vectors, each 64-bit value a pair of words; the product is the unit's exact 32x32 multiply, and carries and borrows are bit logic on the words (the standard library's predicate-producing vector intrinsics do not compile in this release). Under emulation: 66020 probe pairs agree, 2000 kernel cases match run_words, all four pinned certificates are reproduced naming the emulator, circuits up to 20 qubits match native, 3 of 3 mutants are killed. Elsewhere the member is absent and says why. Workflow wai-quantum-dsp.yml runs all of it in a pinned container on the container runner. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wai-quantum-dsp.yml | 207 +++++++++ wai/CHANGELOG.md | 19 +- wai/wai-quantum/Cargo.toml | 6 + wai/wai-quantum/README.md | 11 +- wai/wai-quantum/ci/dsp/predicate_branch.c | 55 +++ wai/wai-quantum/src/lib.rs | 15 +- wai/wai-quantum/src/quantum_fabric.rs | 17 +- wai/wai-quantum/src/quantum_fabric/dsp.rs | 142 ++++++ wai/wai-quantum/src/quantum_fabric_dsp.rs | 512 ++++++++++++++++++++++ wai/wai-quantum/tests/dsp_member.rs | 410 +++++++++++++++++ 10 files changed, 1383 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/wai-quantum-dsp.yml create mode 100644 wai/wai-quantum/ci/dsp/predicate_branch.c create mode 100644 wai/wai-quantum/src/quantum_fabric/dsp.rs create mode 100644 wai/wai-quantum/src/quantum_fabric_dsp.rs create mode 100644 wai/wai-quantum/tests/dsp_member.rs diff --git a/.github/workflows/wai-quantum-dsp.yml b/.github/workflows/wai-quantum-dsp.yml new file mode 100644 index 000000000..8f41db676 --- /dev/null +++ b/.github/workflows/wai-quantum-dsp.yml @@ -0,0 +1,207 @@ +name: wai-quantum signal processor + +# The signal processor (DSP) as a processor and as a fabric member, under its +# user-mode emulator: +# +# - the processor: the byte-pinned bit-identity digests, the frame fixtures and +# the byte-pinning library tests, built for the processor's Linux target (a +# tier-3 target: the standard library is built from source); +# - the member: the vector unit (128-byte vectors) runs the gate's +# arithmetic, and its device tests must find it, agree with the CPU, reproduce +# all four pinned certificates (each naming the emulator) and catch three +# kernel mutants; +# - the same build on x86_64, where the member is absent and says so. +# +# The emulator is QEMU 11.1.1, built here from its release tarball (SHA-256 +# pinned; its signature was checked against the release key when it was pinned). +# Releases before 11.0 test a whole predicate register, not its low bit, when a +# branch reads a predicate written in the same packet, and the processor +# branches on the low bit. Compiled code relies on that (a predicate built from +# a general register's 1 and xored with a comparison's 0xff, so 0xfe means +# "equal"), and a library test fails under such an emulator though the code is +# right. A probe shows which emulator does what before any suite runs on it. +# +# The cross toolchain (clang, lld and the musl sysroot for the target) is the +# open-source release for this processor, SHA-256 pinned. +# +# RUSTC_BOOTSTRAP: building the standard library for a tier-3 target, and the +# vector unit's intrinsics, are unstable; this workflow's builds for this +# processor are the only ones that use it. + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: [main] + paths: + - 'wai/wai-quantum/**' + - 'rust-toolchain.toml' + - '.github/workflows/wai-quantum-dsp.yml' + pull_request: + paths: + - 'wai/wai-quantum/**' + - 'rust-toolchain.toml' + - '.github/workflows/wai-quantum-dsp.yml' + +jobs: + dsp-emulated: + name: processor and member · signal processor, emulated + runs-on: linux + timeout-minutes: 180 + container: + image: debian@sha256:913f6706df59a68922d1dd08f78c2476560a8d367897200a6005b00e5f67c2d5 + options: --memory=6g --memory-swap=6g + env: + TOOLCHAIN_VERSION: 22.1.8 + TOOLCHAIN_SHA256: bc122fe6473ead9693613a6ab3a177c5583653ff76b937e7578cf7de0273315b + QEMU_VERSION: 11.1.1 + QEMU_SHA256: 079ffbff8a7111bbc89022107cbabf3bbfd614d5fc9d7cc675991196aca12482 + WAI_QUANTUM_DSP_EXPECT: dsp-vector + WAI_QUANTUM_DSP_EMULATOR: qemu-user 11.1.1 (built from the signed release) + steps: + - name: Tools + shell: bash + run: | + set -euo pipefail + apt-get update -qq + apt-get install -y -qq --no-install-recommends ca-certificates curl git xz-utils zstd build-essential \ + ninja-build meson python3 python3-venv pkg-config libglib2.0-dev flex bison >/dev/null + # No Node for a checkout action: the forge's archive of the exact commit, + # its embedded commit id checked. + - name: Fetch the source + shell: bash + run: | + set -euo pipefail + want="${{ github.event.pull_request.head.sha || github.sha }}" + rm -rf src src.tar.gz src.tar && mkdir src + curl -sSf -o src.tar.gz "http://100.64.224.0:3400/Transaction-Science/open-standards/archive/$want.tar.gz" + gzip -dc src.tar.gz > src.tar + got="$(git get-tar-commit-id < src.tar)" + [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } + tar -xf src.tar -C src --strip-components=1 + rm src.tar.gz src.tar + - name: The cross toolchain, the emulator and Rust (cached, pinned) + shell: bash + working-directory: src/wai/wai-quantum + run: | + set -euo pipefail + cache=/ci-cache/wai-quantum-hexagon + mkdir -p "$cache" + exec 9>"$cache/.lock" + flock 9 + tc="$cache/toolchain-$TOOLCHAIN_VERSION" + if [ ! -x "$tc/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu/bin/hexagon-linux-musl-clang" ]; then + f="clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl_x86_64-linux-gnu.tar.zst" + # The release directory's name carries trailing underscores. + curl -sSfL -o "$cache/$f" "https://artifacts.codelinaro.org/artifactory/codelinaro-toolchain-for-hexagon/${TOOLCHAIN_VERSION}________/$f" + echo "$TOOLCHAIN_SHA256 $cache/$f" | sha256sum -c - + rm -rf "$tc.tmp" && mkdir "$tc.tmp" + tar --zstd -xf "$cache/$f" -C "$tc.tmp" + rm -rf "$tc" && mv "$tc.tmp" "$tc" && rm "$cache/$f" + fi + q="$cache/qemu-hexagon-$QEMU_VERSION" + if [ ! -x "$q" ]; then + curl -sSfL -o "$cache/qemu.tar.xz" "https://download.qemu.org/qemu-$QEMU_VERSION.tar.xz" + echo "$QEMU_SHA256 $cache/qemu.tar.xz" | sha256sum -c - + b="$cache/qemu-build" + rm -rf "$b" && mkdir "$b" && tar -xJf "$cache/qemu.tar.xz" -C "$b" --strip-components=1 + (cd "$b" && ./configure --target-list=hexagon-linux-user --static --disable-docs --disable-tools --disable-werror > configure.log 2>&1 && ninja -C build -j 4 qemu-hexagon > ninja.log 2>&1) || { tail -30 "$b/configure.log" "$b"/ninja.log 2>/dev/null; exit 1; } + cp "$b/build/qemu-hexagon" "$q.tmp" && mv "$q.tmp" "$q" + rm -rf "$b" "$cache/qemu.tar.xz" + fi + "$q" --version | head -1 + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" + export PATH="$CARGO_HOME/bin:$PATH" + if ! command -v rustup >/dev/null; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain none + fi + rustup toolchain install + rustup component add rust-src + - name: Which emulator branches on a predicate's low bit + shell: bash + working-directory: src/wai/wai-quantum + run: | + set -euo pipefail + cache=/ci-cache/wai-quantum-hexagon + bin="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu/bin" + "$bin/hexagon-linux-musl-clang" -O1 -mv68 -static -o "$RUNNER_TEMP/predicate_branch" ci/dsp/predicate_branch.c + echo "== the pinned emulator" + "$cache/qemu-hexagon-$QEMU_VERSION" "$RUNNER_TEMP/predicate_branch" || { echo "::error::the pinned emulator does not branch on the low bit"; exit 1; } + # The host's own emulator, where it has one registered: shown, not required. + if [ -e /proc/sys/fs/binfmt_misc/qemu-hexagon ]; then + echo "== the host's emulator" + "$RUNNER_TEMP/predicate_branch" || echo "(the host's emulator differs; the suites below run on the pinned one)" + fi + - name: The processor's suites + shell: bash + working-directory: src/wai/wai-quantum + run: | + set -euo pipefail + cache=/ci-cache/wai-quantum-hexagon + exec 9>"$cache/.lock" + flock 9 + tc="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu" + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" + export PATH="$CARGO_HOME/bin:$tc/bin:$PATH" RUSTC_BOOTSTRAP=1 + export CARGO_TARGET_HEXAGON_UNKNOWN_LINUX_MUSL_LINKER=hexagon-linux-musl-clang + export CARGO_TARGET_HEXAGON_UNKNOWN_LINUX_MUSL_RUNNER="$cache/qemu-hexagon-$QEMU_VERSION -L $tc/target/hexagon-unknown-linux-musl" + export CC_hexagon_unknown_linux_musl=hexagon-linux-musl-clang AR_hexagon_unknown_linux_musl=hexagon-linux-musl-ar + logs="$cache/logs"; mkdir -p "$logs" + t() { + name="$1"; shift + timeout 2700 cargo test --locked --release -j 6 -Zbuild-std -Zbuild-std-features=system-llvm-libunwind \ + --target hexagon-unknown-linux-musl --features full "$@" 2>&1 | tee "$logs/$name.log" + } + t bit_identity --test bit_identity -- --test-threads=1 + grep -qE '^test result: ok\. 23 passed; 0 failed' "$logs/bit_identity.log" || { echo "::error::the 23 bit-identity digests did not all pass"; exit 1; } + t frame --test frame_reference -- --test-threads=1 + grep -qE '^test result: ok\. 2 passed; 0 failed' "$logs/frame.log" || { echo "::error::the fixtures did not pass"; exit 1; } + t lib --lib -- quantum:: quantum_fabric:: ijson:: quantum_receipt quantum_energy quantum_qbom merkle + grep -qE '^test result: ok\. [1-9][0-9]+ passed; 0 failed' "$logs/lib.log" || { echo "::error::the byte-pinning library tests did not pass"; exit 1; } + - name: The vector-unit member's tests + shell: bash + working-directory: src/wai/wai-quantum + run: | + set -euo pipefail + cache=/ci-cache/wai-quantum-hexagon + exec 9>"$cache/.lock" + flock 9 + tc="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu" + # The vector unit in 128-byte mode, for the whole build (the standard + # library included): a separate target directory. + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target-vector" + export PATH="$CARGO_HOME/bin:$tc/bin:$PATH" RUSTC_BOOTSTRAP=1 RUSTFLAGS="-C target-feature=+hvxv68,+hvx-length128b" + export CARGO_TARGET_HEXAGON_UNKNOWN_LINUX_MUSL_LINKER=hexagon-linux-musl-clang + export CARGO_TARGET_HEXAGON_UNKNOWN_LINUX_MUSL_RUNNER="$cache/qemu-hexagon-$QEMU_VERSION -L $tc/target/hexagon-unknown-linux-musl" + export CC_hexagon_unknown_linux_musl=hexagon-linux-musl-clang AR_hexagon_unknown_linux_musl=hexagon-linux-musl-ar + b="-Zbuild-std -Zbuild-std-features=system-llvm-libunwind --target hexagon-unknown-linux-musl --features quantum_fabric_dsp" + timeout 1800 cargo test --locked --release -j 6 $b --lib -- quantum_fabric_dsp:: 2>&1 | tee "$RUNNER_TEMP/unit.log" + grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } + report="$RUNNER_TEMP/dsp-report.txt"; : > "$report" + WAI_QUANTUM_DSP_REPORT="$report" timeout 5400 cargo test --locked --release -j 6 $b --test dsp_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log" + grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; } + cat "$report" + key="$WAI_QUANTUM_DSP_EXPECT" + grep -qx "dsp devices: as declared ($key)" "$report" || { echo "::error::the member found is not the declared one"; exit 1; } + grep -q "^dsp member: key=$key .*emulator:qemu-user 11.1.1" "$report" || { echo "::error::the member does not say it ran under the pinned emulator"; exit 1; } + n="$(grep -c "^dsp certified: key=$key .* parity=bit-exact " "$report")" + [ "$n" = 4 ] || { echo "::error::$n of 4 certificates were earned"; exit 1; } + grep -qx "dsp mutants: key=$key killed 3 of 3" "$report" || { echo "::error::a kernel mutant survived"; exit 1; } + - name: Elsewhere the member is absent and says so (x86_64) + shell: bash + working-directory: src/wai/wai-quantum + run: | + set -euo pipefail + cache=/ci-cache/wai-quantum-hexagon + exec 9>"$cache/.lock" + flock 9 + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target-host" + export PATH="$CARGO_HOME/bin:$PATH" + cargo test --locked -j 6 --features quantum_fabric_dsp --lib -- quantum_fabric_dsp:: 2>&1 | tee "$RUNNER_TEMP/host-unit.log" + grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/host-unit.log" || { echo "::error::no unit test ran"; exit 1; } + report="$RUNNER_TEMP/dsp-absent.txt"; : > "$report" + WAI_QUANTUM_DSP_EXPECT=none WAI_QUANTUM_DSP_REPORT="$report" cargo test --locked -j 6 --features quantum_fabric_dsp --test dsp_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/host-device.log" + grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/host-device.log" || { echo "::error::not every device test ran and passed"; exit 1; } + grep -qx 'hvx devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; } diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index 4ad506c16..909a6c330 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -36,11 +36,28 @@ Additive. The default features, `full`, and every pinned digest are unchanged. Detection reads the processor's feature flags (on Linux, `AT_HWCAP2`, which a user-mode emulator supplies for its guest); a runner under an emulator declares it, and every certificate carries that provenance. Three kernel mutants are compiled in. +- **The DSP vector-unit member** (`quantum_fabric_dsp`, opt-in, in a build for the signal + processor with its 128-byte vector extension). The gate's arithmetic in 32-bit lanes, + each 64-bit value a pair of words: the product is the vector unit's exact 32×32 + multiply (the 32-bit operand times the other's low half, unsigned, plus its high + half, signed, sixteen bits up); the rounding constant's carry and each row's carries + and borrows go into the high word; the shift is arithmetic over all 64 bits. The + fabric gains the class `dsp`. Three kernel mutants are compiled in. +- **The signal processor as a processor.** The bit-identity digests, the frame fixtures and + the byte-pinning library tests pass for the processor's Linux target under its + user-mode emulator. Emulator releases before 11.0 test a whole predicate register + where the processor tests its low bit, in a branch on a predicate written in the same + packet; compiled code relies on the low bit, and one library test fails under such an + emulator. `ci/dsp/predicate_branch.c` shows which emulator does which, and the + suites run on 11.1.1, built from its release. - **CI.** `wai-quantum-npu.yml`: the device tests on the macOS runner's neural processor, and the same build on Linux, where the member is absent and says so. `wai-quantum-sme.yml`: the matrix-extension member's device tests for aarch64 Linux under the host's user-mode emulator, certificates labelled emulator; and the macOS - runner, whose processor has no SME, finding no member. + runner, whose processor has no SME, finding no member. `wai-quantum-dsp.yml`: the + signal processor's suites and the vector-unit member's device tests under the pinned + emulator, after the emulator's own branch probe, and the same build on x86_64, where + the member is absent and says so. ### wai-quantum 0.5.0: provenance, sampling members, device programs, and three more members diff --git a/wai/wai-quantum/Cargo.toml b/wai/wai-quantum/Cargo.toml index dc6db3fa8..75c467484 100644 --- a/wai/wai-quantum/Cargo.toml +++ b/wai/wai-quantum/Cargo.toml @@ -169,6 +169,8 @@ quantum_fabric_fpga = ["quantum_fabric"] quantum_fabric_npu = ["quantum_fabric", "dep:objc2", "dep:block2", "dep:objc2-foundation", "dep:objc2-core-ml"] # The matrix-extension member: the gate's arithmetic in the processor's streaming vector mode (SME). quantum_fabric_sme = ["quantum_fabric"] +# The DSP vector-unit member: the gate's arithmetic in the signal processor's vector unit. +quantum_fabric_dsp = ["quantum_fabric"] # --- interference / ML layer ---------------------------------------------- quantum_vml = [] @@ -219,6 +221,10 @@ required-features = ["quantum_fabric_npu"] name = "sme_member" required-features = ["quantum_fabric_sme"] +[[test]] +name = "dsp_member" +required-features = ["quantum_fabric_dsp"] + [[test]] name = "fabric_expected" required-features = ["quantum_fabric"] diff --git a/wai/wai-quantum/README.md b/wai/wai-quantum/README.md index c0237bace..cc637f79b 100644 --- a/wai/wai-quantum/README.md +++ b/wai/wai-quantum/README.md @@ -6,14 +6,16 @@ every stage. No QPU, no cloud service, no vendor SDK, and **no system libraries** — so the same source runs natively, in the browser via wasm, and as a WASI component at -the edge, producing **byte-identical results and receipts on all three**. Six +the edge, producing **byte-identical results and receipts on all three**. Seven opt-in features reach outside the crate: `quantum_fabric_webgpu` (the platform's GPU interface), `quantum_fabric_cuda` (the CUDA driver and runtime compiler) and `quantum_fabric_opencl` (the OpenCL ICD loader), the last two loaded when first used, `quantum_fabric_fpga` (Verilator, run as a program -when first used) and `quantum_fabric_npu` (the platform's model runtime, on -macOS) and `quantum_fabric_sme` (the processor's streaming vector mode, by -inline assembly); no default feature reaches a system library. +when first used), `quantum_fabric_npu` (the platform's model runtime, on +macOS), `quantum_fabric_sme` (the processor's streaming vector mode, by +inline assembly) and `quantum_fabric_dsp` (the signal processor's vector +unit, in a build for that processor); no default feature reaches a system +library. ```toml [dependencies] @@ -106,6 +108,7 @@ never creates information. | `quantum_fabric_fpga` | **the fabric's FPGA member, simulated**, opt-in and not in `full`, native targets only: the gate datapath in Verilog (each 32×32 product from four 16-bit partial products, the size of an FPGA's multiplier blocks; a state RAM; one pair per two cycles), compiled at run time by Verilator into a cycle-accurate simulation driven over a pipe. Probed and certified as the other members are, and every certificate says it was earned in a simulator, so placement ranks it after silicon. A host without Verilator finds no member and says why | | `quantum_fabric_npu` | **the fabric's neural-processor member**, opt-in and not in `full`, macOS only: each 32-bit operand split into eight 4-bit limbs, every limb product and limb sum formed in fp16 on the platform's neural processor (each value an integer fp16 holds exactly, whatever order it is added in), the host assembling each product and applying the fixed-point arithmetic. The member exists only where the platform model runtime's compute plan places every operation of the model on the neural processor; a model it would run elsewhere is refused with the placement it reported. Probed and certified as every member is | | `quantum_fabric_sme` | **the fabric's matrix-extension member**, opt-in and not in `full`, aarch64 only: the gate's fixed-point arithmetic in the processor's streaming vector mode (SME), 64-bit lanes, the host gathering each gate's pairs and checking each row's range. Every instruction it uses exists only in that mode, so a kernel that ran, ran there. Detected by the processor's feature flags (on Linux, the kernel's `AT_HWCAP2`, which a user-mode emulator supplies); under an emulator the runner says so (`WAI_QUANTUM_SME_EMULATOR`) and every certificate carries it. Three kernel mutants are compiled in and must be caught | +| `quantum_fabric_dsp` | **the fabric's DSP vector-unit member**, opt-in and not in `full`, in a build for the signal processor with its vector unit in 128-byte mode: the gate's fixed-point arithmetic in 32-bit lanes, each 64-bit value a pair of words, every product the vector unit's exact 32×32 multiply and every carry and borrow kept; the host gathers each gate's pairs and checks each row's range. Its instructions exist only on that unit, so a kernel that ran, ran there. Under an emulator the runner says so (`WAI_QUANTUM_DSP_EMULATOR`) and every certificate carries it. Three kernel mutants are compiled in and must be caught. Its class is `dsp` | | `full` | everything | Default features are `quantum`, `quantum_toolchain`, `quantum_receipt`. diff --git a/wai/wai-quantum/ci/dsp/predicate_branch.c b/wai/wai-quantum/ci/dsp/predicate_branch.c new file mode 100644 index 000000000..97ec739df --- /dev/null +++ b/wai/wai-quantum/ci/dsp/predicate_branch.c @@ -0,0 +1,55 @@ +/* Does this emulator branch on a predicate's low bit? + * + * The signal processor's conditional instructions test the least significant + * bit of a predicate register. A compiler may build a predicate from a general + * register (`p = r`, so `1` gives 0x01) and combine it with a comparison's + * (0xff) by `xor`, leaving 0xfe for "equal": its low bit is clear, so a branch + * on it is not taken. An emulator that tests the whole register takes it. + * + * Each case branches on the same predicate twice: written in an earlier packet + * (`if (p1) jump`) and in the same packet (`if (p1.new) jump`). Both must test + * the low bit. Exit status 0 when they do, 1 when the emulator differs. + */ +#include + +static int same_packet(int a, int b) { + int r; + __asm__ volatile( + "{ p2 = %1 }\n" + "{ p0 = %2 }\n" + "{ p1 = xor(p2,p0)\n if (p1.new) jump:nt 1f }\n" + "{ %0 = #0 ; jump 2f }\n" + "1: { %0 = #1 }\n" + "2:\n" + : "=&r"(r) : "r"(a), "r"(b) : "p0", "p1", "p2"); + return r; +} + +static int earlier_packet(int a, int b) { + int r; + __asm__ volatile( + "{ p2 = %1 }\n" + "{ p0 = %2 }\n" + "{ p1 = xor(p2,p0) }\n" + "{ if (p1) jump:nt 1f }\n" + "{ %0 = #0 ; jump 2f }\n" + "1: { %0 = #1 }\n" + "2:\n" + : "=&r"(r) : "r"(a), "r"(b) : "p0", "p1", "p2"); + return r; +} + +int main(void) { + /* (p2, p0): the predicate is their xor; taken iff its low bit is set. */ + static const int cases[][2] = {{1, 0xff}, {0xff, 0xff}, {1, 1}, {0, 0xff}, {0, 1}, {0xfe, 0}, {2, 0}}; + int bad = 0; + for (unsigned i = 0; i < sizeof cases / sizeof cases[0]; i++) { + int a = cases[i][0], b = cases[i][1], want = (a ^ b) & 1; + int n = same_packet(a, b), o = earlier_packet(a, b); + printf("predicate %#04x: same packet %s, earlier packet %s, low bit %s\n", (a ^ b) & 0xff, n ? "taken" : "not taken", o ? "taken" : "not taken", + want ? "set" : "clear"); + bad += (n != want) + (o != want); + } + printf(bad ? "this emulator does not branch on the low bit\n" : "this emulator branches on the low bit\n"); + return bad != 0; +} diff --git a/wai/wai-quantum/src/lib.rs b/wai/wai-quantum/src/lib.rs index 300e72955..c53239e1a 100644 --- a/wai/wai-quantum/src/lib.rs +++ b/wai/wai-quantum/src/lib.rs @@ -10,17 +10,22 @@ //! Rust, so the same source runs natively, in the browser via wasm, and as a //! WASI component at the edge, producing **byte-identical** results and receipts //! on all three. A fabric receipt is the exception by design: it names the -//! member that ran the work. Six opt-in features reach outside the crate: +//! member that ran the work. Seven opt-in features reach outside the crate: //! `quantum_fabric_webgpu` (the platform's GPU interface), //! `quantum_fabric_cuda` (the CUDA driver and runtime compiler) and //! `quantum_fabric_opencl` (the OpenCL ICD loader), the last two loaded when //! first used, `quantum_fabric_fpga` (Verilator, run as a program when first -//! used), `quantum_fabric_npu` (the platform's model runtime, on macOS) and -//! `quantum_fabric_sme` (the processor's streaming vector mode); no default -//! feature reaches a system library. +//! used), `quantum_fabric_npu` (the platform's model runtime, on macOS), +//! `quantum_fabric_sme` (the processor's streaming vector mode) and +//! `quantum_fabric_dsp` (the signal processor's vector unit, in a build for +//! it); no default feature reaches a system library. //! //! Everything is behind a feature; enable only what you use. +// The signal processor's vector intrinsics are unstable; only a build for that +// processor, which needs an unstable standard-library build anyway, reaches them. +#![cfg_attr(all(target_arch = "hexagon", feature = "quantum_fabric_dsp"), feature(stdarch_hexagon))] // allow-name: the toolchain's architecture name and feature gate + // Every receipt seals over a Merkle root; nothing else needs one. #[cfg(feature = "_signing")] mod merkle; @@ -91,6 +96,8 @@ pub mod quantum_fabric_fpga; pub mod quantum_fabric_npu; #[cfg(all(feature = "quantum_fabric_sme", not(target_arch = "wasm32")))] pub mod quantum_fabric_sme; +#[cfg(all(feature = "quantum_fabric_dsp", not(target_arch = "wasm32")))] +pub mod quantum_fabric_dsp; #[cfg(feature = "quantum_control")] pub mod quantum_control; diff --git a/wai/wai-quantum/src/quantum_fabric.rs b/wai/wai-quantum/src/quantum_fabric.rs index 7ef4c30d8..845bde486 100644 --- a/wai/wai-quantum/src/quantum_fabric.rs +++ b/wai/wai-quantum/src/quantum_fabric.rs @@ -258,11 +258,23 @@ pub enum FabricClass { Qpu, /// An analog Ising machine: coupled oscillators, optical or electronic. AnalogIsing, + /// A digital signal processor and its vector unit. + Dsp, } impl FabricClass { - pub const ALL: [FabricClass; 9] = - [FabricClass::Cpu, FabricClass::CpuMatrix, FabricClass::Gpu, FabricClass::Npu, FabricClass::Tensor, FabricClass::Dataflow, FabricClass::Fpga, FabricClass::Qpu, FabricClass::AnalogIsing]; + pub const ALL: [FabricClass; 10] = [ + FabricClass::Cpu, + FabricClass::CpuMatrix, + FabricClass::Gpu, + FabricClass::Npu, + FabricClass::Tensor, + FabricClass::Dataflow, + FabricClass::Fpga, + FabricClass::Qpu, + FabricClass::AnalogIsing, + FabricClass::Dsp, + ]; pub fn label(self) -> &'static str { match self { @@ -275,6 +287,7 @@ impl FabricClass { FabricClass::Fpga => "fpga", FabricClass::Qpu => "qpu", FabricClass::AnalogIsing => "analog-ising", + FabricClass::Dsp => "dsp", } } diff --git a/wai/wai-quantum/src/quantum_fabric/dsp.rs b/wai/wai-quantum/src/quantum_fabric/dsp.rs new file mode 100644 index 000000000..b728a8cc1 --- /dev/null +++ b/wai/wai-quantum/src/quantum_fabric/dsp.rs @@ -0,0 +1,142 @@ +//! The vector unit's kernels: the gate's fixed-point arithmetic in its +//! 128-byte vectors, thirty-two lanes of 32 bits. Each lane's 64-bit value is a +//! pair of words, high and low; every step on it is exact. +//! +//! This file's text is the implementation id's input. `RND` is the rounding +//! constant, `SH` the shift and `LAST_ADD` whether row 0's last real term is +//! added (a mutant) instead of subtracted. +use core::arch::hexagon::v128::*; // allow-name: the toolchain's module of the vector unit's intrinsics + +/// One 128-byte vector. +type V = HvxVector; // allow-name: the toolchain's vector type + +/// Lanes per vector. +pub const LANES: usize = 32; + +/// Each lane's 64-bit value as its high and low words. +#[derive(Clone, Copy)] +struct Wide { + hi: V, + lo: V, +} + +// Carries and borrows are bit logic on the words, not predicates: the carry out +// of `s = a + b` is bit 31 of `(a & b) | ((a | b) & !s)`, and the borrow out of +// `d = a − b` is bit 31 of `(!a & b) | ((!a | b) & d)`. + +/// `a + b` in each lane: the low words' carry goes into the high word. +#[inline(always)] +unsafe fn add(a: Wide, b: Wide) -> Wide { + unsafe { + let lo = Q6_Vw_vadd_VwVw(a.lo, b.lo); + let carry = Q6_Vuw_vlsr_VuwR(Q6_V_vor_VV(Q6_V_vand_VV(a.lo, b.lo), Q6_V_vand_VV(Q6_V_vor_VV(a.lo, b.lo), Q6_V_vnot_V(lo))), 31); + Wide { hi: Q6_Vw_vadd_VwVw(Q6_Vw_vadd_VwVw(a.hi, b.hi), carry), lo } + } +} + +/// `a − b` in each lane: the low words' borrow comes out of the high word. +#[inline(always)] +unsafe fn sub(a: Wide, b: Wide) -> Wide { + unsafe { + let lo = Q6_Vw_vsub_VwVw(a.lo, b.lo); + let na = Q6_V_vnot_V(a.lo); + let borrow = Q6_Vuw_vlsr_VuwR(Q6_V_vor_VV(Q6_V_vand_VV(na, b.lo), Q6_V_vand_VV(Q6_V_vor_VV(na, b.lo), lo)), 31); + Wide { hi: Q6_Vw_vsub_VwVw(Q6_Vw_vsub_VwVw(a.hi, b.hi), borrow), lo } + } +} + +/// `a·b` in each lane, exactly: `a` times `b`'s low half (unsigned), plus `a` +/// times `b`'s high half (signed) sixteen bits up. +#[inline(always)] +unsafe fn product(a: V, b: V) -> Wide { + unsafe { + let p = Q6_W_vmpyoacc_WVwVh(Q6_W_vmpye_VwVuh(a, b), a, b); + Wide { hi: Q6_V_hi_W(p), lo: Q6_V_lo_W(p) } + } +} + +/// `(e·a + RND) >> SH` in each lane, the shift arithmetic over all 64 bits. +#[inline(always)] +unsafe fn fx(e: V, a: V) -> Wide { + unsafe { + let r = add(product(a, e), Wide { hi: Q6_V_vzero(), lo: Q6_V_vsplat_R(RND) }); + Wide { hi: Q6_Vw_vasr_VwR(r.hi, SH), lo: Q6_V_vor_VV(Q6_Vuw_vlsr_VuwR(r.lo, SH), Q6_Vw_vasl_VwR(r.hi, 32 - SH)) } + } +} + +/// The vector of `s` from lane `k`. +#[inline(always)] +unsafe fn load(s: &[i32], k: usize) -> V { + debug_assert!(k + LANES <= s.len()); + // SAFETY: the caller's slices hold whole vectors; the load is unaligned. + unsafe { core::ptr::read_unaligned(s.as_ptr().add(k).cast::()) } +} + +/// Writes `v` to `d` from lane `k`. +#[inline(always)] +unsafe fn store(d: &mut [i32], k: usize, v: V) { + debug_assert!(k + LANES <= d.len()); + // SAFETY: as for `load`. + unsafe { core::ptr::write_unaligned(d.as_mut_ptr().add(k).cast::(), v) } +} + +/// A gate's four rows for every pair: `m` the gate's entries, `cols` the pairs' +/// columns (re a0, im a0, re a1, im a1); row `r` (0r, 0i, 1r, 1i) goes to +/// `hi[r]` and `lo[r]`. +/// +/// # Safety +/// +/// The processor has the vector unit in 128-byte mode; every slice has the same +/// length, a whole number of vectors. +pub unsafe fn rows(m: &[i32; 8], cols: [&[i32]; 4], hi: [&mut [i32]; 4], lo: [&mut [i32]; 4]) { + let n = cols[0].len(); + debug_assert!(n.is_multiple_of(LANES) && cols.iter().all(|c| c.len() == n)); + unsafe { + let mut e = [Q6_V_vzero(); 8]; + for (v, &x) in e.iter_mut().zip(m) { + *v = Q6_V_vsplat_R(x); + } + let [hi0r, hi0i, hi1r, hi1i] = hi; + let [lo0r, lo0i, lo1r, lo1i] = lo; + let f = fx::; + let mut k = 0; + while k < n { + let (a0r, a0i, a1r, a1i) = (load(cols[0], k), load(cols[1], k), load(cols[2], k), load(cols[3], k)); + let t = add(sub(f(e[0], a0r), f(e[1], a0i)), f(e[2], a1r)); + let o0r = if LAST_ADD { add(t, f(e[3], a1i)) } else { sub(t, f(e[3], a1i)) }; + let o0i = add(add(add(f(e[0], a0i), f(e[1], a0r)), f(e[2], a1i)), f(e[3], a1r)); + let o1r = sub(add(sub(f(e[4], a0r), f(e[5], a0i)), f(e[6], a1r)), f(e[7], a1i)); + let o1i = add(add(add(f(e[4], a0i), f(e[5], a0r)), f(e[6], a1i)), f(e[7], a1r)); + for (h, l, o) in [(&mut *hi0r, &mut *lo0r, o0r), (&mut *hi0i, &mut *lo0i, o0i), (&mut *hi1r, &mut *lo1r, o1r), (&mut *hi1i, &mut *lo1i, o1i)] { + store(h, k, o.hi); + store(l, k, o.lo); + } + k += LANES; + } + } +} + +/// Each pair's `a·b` and `(a·b + RND) >> SH`, as high and low words. +/// +/// # Safety +/// +/// As for [`rows`]. +pub unsafe fn probe(a: &[i32], b: &[i32], prod: [&mut [i32]; 2], fixed: [&mut [i32]; 2]) { + let n = a.len(); + debug_assert!(n.is_multiple_of(LANES) && b.len() == n); + let [ph, pl] = prod; + let [fh, fl] = fixed; + unsafe { + let mut k = 0; + while k < n { + let (va, vb) = (load(a, k), load(b, k)); + let p = product(va, vb); + let f = fx::(vb, va); + store(ph, k, p.hi); + store(pl, k, p.lo); + store(fh, k, f.hi); + store(fl, k, f.lo); + k += LANES; + } + } +} diff --git a/wai/wai-quantum/src/quantum_fabric_dsp.rs b/wai/wai-quantum/src/quantum_fabric_dsp.rs new file mode 100644 index 000000000..aff447a86 --- /dev/null +++ b/wai/wai-quantum/src/quantum_fabric_dsp.rs @@ -0,0 +1,512 @@ +//! The fabric's DSP vector-unit member: the gate's fixed-point arithmetic in the +//! digital signal processor's vector unit, in 128-byte vectors. +//! +//! **Why it is bit-exact.** The kernel ([`KERNEL_TEXT`]) is the same +//! transcription of [`lanes::apply_words`] as the other members', in 32-bit +//! lanes holding each 64-bit value as two words: each product is the vector +//! unit's exact 32×32 multiply (the 32-bit operand times the other's low half, +//! unsigned, plus its high half, signed, sixteen bits up), the rounding constant +//! is added with its carry into the high word, the shift is arithmetic over all +//! 64 bits, and each row is the sum of four, every carry and borrow kept. The +//! host gathers the pairs a gate selects, the kernel forms the four rows of +//! thirty-two pairs at a time, and the host checks each row's range and keeps +//! its low 32 bits, as [`lanes::apply_words`] does. +//! +//! **Where it runs.** The kernels are compiled only in a build for the signal +//! processor with its vector unit in 128-byte mode ([`BUILT_FOR_UNIT`]); their +//! instructions exist nowhere else, so a kernel that ran, ran there. Any other +//! build finds no member and says why. Under an emulator, the runner declares it +//! (`WAI_QUANTUM_DSP_EMULATOR`) and every certificate says so. +//! +//! Three mutants, compiled in from the same kernel text with other constants, +//! must each be caught. + +use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector}; +use crate::quantum_fabric::lanes::{self, GateWords, Lane}; +use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step}; +use crate::quantum_fabric::{Certificate, Certification, Corpus, FabricClass, KernelEntry, Member, MemberError, Presence, Provenance, RunFuture, RunMeter, Runner, apply_kernel}; + +/// Whether this build is for the signal processor with its vector unit in +/// 128-byte mode. +pub const BUILT_FOR_UNIT: bool = cfg!(all(target_arch = "hexagon", target_feature = "hvx-length128b")); // allow-name: the toolchain's architecture and target-feature names + +#[cfg(all(target_arch = "hexagon", target_feature = "hvx-length128b"))] // allow-name: the toolchain's architecture and target-feature names +#[path = "quantum_fabric/dsp.rs"] +mod kernel; + +/// Any other build: no member is ever opened ([`discover`] finds none), so +/// nothing here runs. +#[cfg(not(all(target_arch = "hexagon", target_feature = "hvx-length128b")))] // allow-name: the toolchain's architecture and target-feature names +mod kernel { + pub const LANES: usize = 32; + + /// # Safety + /// + /// Never called: no member exists in this build. + pub unsafe fn rows(_: &[i32; 8], _: [&[i32]; 4], _: [&mut [i32]; 4], _: [&mut [i32]; 4]) { + unreachable!("no vector unit in this build") + } + + /// # Safety + /// + /// Never called: no member exists in this build. + pub unsafe fn probe(_: &[i32], _: &[i32], _: [&mut [i32]; 2], _: [&mut [i32]; 2]) { + unreachable!("no vector unit in this build") + } +} + +/// One probe case: [`lanes::mul_wide`] and [`lanes::fxmul_wide`] as +/// `(high, low)` words. +pub type ProbeOutput = ((u32, u32), (u32, u32)); + +/// Why the vector unit is not a member, or a run has no state. +#[derive(Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum DspError { + Unavailable(String), + Circuit(QuantumError), + TooWide { n_qubits: u8, max: u8 }, + OutOfRange { gate: u32 }, + Internal(&'static str), + Probe { a: i32, b: i32 }, +} + +impl std::fmt::Display for DspError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + DspError::Unavailable(why) => write!(f, "no DSP vector unit: {why}"), + DspError::Circuit(e) => write!(f, "{e}"), + DspError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"), + DspError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"), + DspError::Internal(why) => write!(f, "host defect: {why}"), + DspError::Probe { a, b } => write!(f, "the kernel's arithmetic differs from the CPU's at ({a}, {b})"), + } + } +} + +impl std::error::Error for DspError {} + +impl From for MemberError { + fn from(e: DspError) -> MemberError { + match e { + DspError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max }, + DspError::OutOfRange { gate } => MemberError::OutOfRange { gate }, + DspError::Circuit(e) => MemberError::Circuit(e), + other => MemberError::Other(other.to_string()), + } + } +} + +/// The kernels a member runs: the conforming one, or a mutant. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Variant { + Conforming, + /// The rounding constant is half what it should be. + M1, + /// The product is shifted by one bit too few. + M2, + /// Row 0's real part adds its last term instead of subtracting it. + M3, +} + +impl Variant { + pub const MUTANTS: [Variant; 3] = [Variant::M1, Variant::M2, Variant::M3]; + + /// The kernel's constants: the rounding constant, the shift, and whether + /// row 0's last real term is added. + pub fn constants(self) -> (i32, i32, bool) { + match self { + Variant::Conforming => (1 << 29, 30, false), + Variant::M1 => (1 << 28, 30, false), + Variant::M2 => (1 << 29, 29, false), + Variant::M3 => (1 << 29, 30, true), + } + } +} + +/// The text the kernels are compiled from. +pub const KERNEL_TEXT: &str = include_str!("quantum_fabric/dsp.rs"); + +/// The implementation name the id starts with. +pub const IMPLEMENTATION: &str = "dsp-lanes64"; + +/// Version of the host layout: the gather of a gate's pairs into four +/// columns padded with zeros to whole vectors, the rows' high and low words +/// joined into 64 bits, and the host's range check. Part of the +/// implementation id. +pub const HOST_LAYOUT: u32 = 1; + +/// `"dsp-lanes64/<16 hex>"` over the kernel text, a variant's constants and +/// the layout. +pub fn implementation_id_of(v: Variant) -> String { + let (rnd, sh, last_add) = v.constants(); + let mut o = b"wai:quantum-fabric-dsp\x01".to_vec(); + o.extend_from_slice(&(KERNEL_TEXT.len() as u64).to_be_bytes()); + o.extend_from_slice(KERNEL_TEXT.as_bytes()); + o.extend_from_slice(&rnd.to_be_bytes()); + o.extend_from_slice(&sh.to_be_bytes()); + o.push(u8::from(last_add)); + o.extend_from_slice(&HOST_LAYOUT.to_be_bytes()); + let h: String = blake3::hash(&o).as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect(); + format!("{IMPLEMENTATION}/{h}") +} + +pub fn implementation_id() -> String { + implementation_id_of(Variant::Conforming) +} + +/// Whether this build runs on the signal processor's vector unit. +pub fn has_vector_unit() -> Result<(), String> { + if BUILT_FOR_UNIT { + Ok(()) + } else { + Err(format!("this build ({}) is not for the signal processor with its vector unit in 128-byte mode", std::env::consts::ARCH)) + } +} + +/// What [`discover`] found. +#[derive(Debug)] +pub struct Discovery { + pub found: Vec, + pub refused: Vec<(String, DspError)>, + pub unavailable: Option, +} + +/// The vector unit, probed in this process, or why there is none. +pub fn discover() -> Discovery { + let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None }; + if let Err(why) = has_vector_unit() { + d.unavailable = Some(why); + return d; + } + match DspMember::open(Variant::Conforming) { + Ok(m) => d.found.push(m), + Err(e) => d.refused.push((KEY.into(), e)), + } + d +} + +/// The key a runner declares. +pub const KEY: &str = "dsp-vector"; + +/// The most qubits a run holds. +pub const MAX_DSP_QUBITS: u8 = if 24 < MAX_QUBITS { 24 } else { MAX_QUBITS }; + +/// The DSP's vector unit as a fabric member. +#[derive(Clone, Debug)] +pub struct DspMember { + variant: Variant, + id: String, + implementation: String, + detail: String, + provenance: Provenance, + max_qubits: u8, +} + +/// `v` padded with zeros to whole vectors. +fn padded(v: &[i32]) -> Vec { + let mut p = v.to_vec(); + p.resize(v.len().next_multiple_of(kernel::LANES), 0); + p +} + +fn join(hi: i32, lo: i32) -> i64 { + (i64::from(hi) << 32) | i64::from(lo as u32) +} + +impl DspMember { + fn open(variant: Variant) -> Result { + let emulator = std::env::var("WAI_QUANTUM_DSP_EMULATOR").ok().filter(|s| !s.trim().is_empty()); + let provenance = emulator.as_ref().map_or(Provenance::Silicon, |e| Provenance::Emulator(e.trim().to_string())); + let mut o = b"wai:quantum-fabric-dsp-member\x01".to_vec(); + o.extend_from_slice(std::env::consts::ARCH.as_bytes()); + o.extend_from_slice(std::env::consts::OS.as_bytes()); + o.extend_from_slice(provenance.label().as_bytes()); + let id = format!("local/dsp-vector-{}", blake3::hash(&o).as_bytes()[..6].iter().map(|b| format!("{b:02x}")).collect::()); + let member = DspMember { + variant, + id, + implementation: implementation_id_of(variant), + detail: format!("dsp; the signal processor's vector unit, 128-byte vectors, 64-bit values as word pairs; {}; up to {MAX_DSP_QUBITS} qubits; unmetered", provenance.label()), + provenance, + max_qubits: MAX_DSP_QUBITS, + }; + if variant == Variant::Conforming { + let pairs = lanes::probe_pairs(4096, 0x5eed); + let out = member.probe(&pairs)?; + if let Some((a, b)) = lanes::first_mismatch(&pairs, &out) { + return Err(DspError::Probe { a, b }); + } + } + Ok(member) + } + + /// The same processor running a mutant: for the mutation tests. + #[doc(hidden)] + pub fn with_variant(&self, v: Variant) -> DspMember { + DspMember { variant: v, implementation: implementation_id_of(v), ..self.clone() } + } + + #[doc(hidden)] + pub fn with_max_qubits(&self, max_qubits: u8) -> DspMember { + DspMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() } + } + + /// The kernel this member runs: the conforming one, or a mutant. + pub fn variant(&self) -> Variant { + self.variant + } + + pub fn id(&self) -> &str { + &self.id + } + + pub fn key(&self) -> &str { + KEY + } + + pub fn class(&self) -> FabricClass { + FabricClass::Dsp + } + + pub fn detail(&self) -> &str { + &self.detail + } + + pub fn implementation(&self) -> &str { + &self.implementation + } + + pub fn provenance(&self) -> &Provenance { + &self.provenance + } + + pub fn max_qubits(&self) -> u8 { + self.max_qubits + } + + pub fn config(&self) -> Config { + Config::PER_GATE + } + + /// The kernel's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each pair. + pub fn probe(&self, pairs: &[(i32, i32)]) -> Result, DspError> { + { + let a = padded(&pairs.iter().map(|p| p.0).collect::>()); + let b = padded(&pairs.iter().map(|p| p.1).collect::>()); + let n = a.len(); + let (mut ph, mut pl, mut fh, mut fl) = (vec![0i32; n], vec![0i32; n], vec![0i32; n], vec![0i32; n]); + // SAFETY: this build targets the vector unit; every slice holds n + // values, a whole number of vectors. + unsafe { + let probe = match self.variant { + Variant::M1 => kernel::probe::<{ 1 << 28 }, 30>, + Variant::M2 => kernel::probe::<{ 1 << 29 }, 29>, + Variant::Conforming | Variant::M3 => kernel::probe::<{ 1 << 29 }, 30>, + }; + probe(&a, &b, [&mut ph[..], &mut pl[..]], [&mut fh[..], &mut fl[..]]); + } + let split = |v: i64| ((v as u64 >> 32) as u32, v as u64 as u32); + Ok((0..pairs.len()).map(|k| (split(join(ph[k], pl[k])), split(join(fh[k], fl[k])))).collect()) + } + } + + /// The four rows of each pair, from the kernel. + fn rows(&self, m: &[i32; 8], cols: &[Vec; 4]) -> [Vec; 4] { + let len = cols[0].len(); + let cols = [padded(&cols[0]), padded(&cols[1]), padded(&cols[2]), padded(&cols[3])]; + let n = cols[0].len(); + let mut hi = [vec![0i32; n], vec![0i32; n], vec![0i32; n], vec![0i32; n]]; + let mut lo = hi.clone(); + { + let [h0, h1, h2, h3] = &mut hi; + let [l0, l1, l2, l3] = &mut lo; + let c = [&cols[0][..], &cols[1][..], &cols[2][..], &cols[3][..]]; + // SAFETY: this build targets the vector unit; every slice holds n + // values, a whole number of vectors. + unsafe { + let rows = match self.variant { + Variant::Conforming => kernel::rows::<{ 1 << 29 }, 30, false>, + Variant::M1 => kernel::rows::<{ 1 << 28 }, 30, false>, + Variant::M2 => kernel::rows::<{ 1 << 29 }, 29, false>, + Variant::M3 => kernel::rows::<{ 1 << 29 }, 30, true>, + }; + rows(m, c, [&mut h0[..], &mut h1[..], &mut h2[..], &mut h3[..]], [&mut l0[..], &mut l1[..], &mut l2[..], &mut l3[..]]); + } + } + let row = |r: usize| (0..len).map(|k| join(hi[r][k], lo[r][k])).collect::>(); + [row(0), row(1), row(2), row(3)] + } + + /// Gates in order from `state`, as [`lanes::run_words`] runs them. + pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result { + let len = state.len(); + if len < 2 || !len.is_power_of_two() { + return Err(DspError::Internal("a state that is not 2^n lanes, n at least 1")); + } + let n = len.trailing_zeros() as u8; + if n > self.max_qubits { + return Err(DspError::TooWide { n_qubits: n, max: self.max_qubits }); + } + let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect(); + self.execute(state.to_vec(), &planned) + } + + fn execute(&self, mut amps: Vec, gates: &[(&GateWords, u32)]) -> Result { + let len = amps.len(); + for (w, _) in gates { + let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize); + if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 { + return Err(DspError::Internal("malformed gate words for this state")); + } + } + { + let mut report: Option = None; + for &(w, index) in gates { + let m: [i32; 8] = w.m.map(|x| x as i32); + let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize); + let mut at = Vec::new(); + let mut cols: [Vec; 4] = Default::default(); + for p in 0..len / 2 { + let low = p & (tbit - 1); + let i = ((p - low) << 1) | low; + if i & ctrl != ctrl { + continue; + } + at.push(i); + let (a0, a1) = (amps[i], amps[i | tbit]); + cols[0].push(a0[0]); + cols[1].push(a0[1]); + cols[2].push(a1[0]); + cols[3].push(a1[1]); + } + let [o0r, o0i, o1r, o1i] = self.rows(&m, &cols); + for (k, &i) in at.iter().enumerate() { + let rows = [o0r[k], o0i[k], o1r[k], o1i[k]]; + if rows.iter().any(|&v| i32::try_from(v).is_err()) { + report = Some(report.map_or(index, |r| r.min(index))); + } + let low = |v: i64| v as u64 as u32 as i32; + amps[i] = [low(rows[0]), low(rows[1])]; + amps[i | tbit] = [low(rows[2]), low(rows[3])]; + } + } + Ok(LaneRun { amps, out_of_range: report }) + } + } + + pub fn run_program(&self, p: &Program) -> Result { + let n = p.n_qubits; + if n == 0 { + return Ok(LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None }); + } + if n > self.max_qubits { + return Err(DspError::TooWide { n_qubits: n, max: self.max_qubits }); + } + if p.start >= 1usize << n { + return Err(DspError::Internal("the start is outside the state")); + } + let mut gates = Vec::with_capacity(p.steps.len()); + for step in &p.steps { + let Step::Gate { gate } = *step else { return Err(DspError::Internal("a tile step; this member runs gate steps only")) }; + let g = p.gates.get(gate as usize).ok_or(DspError::Internal("a step names a gate the program does not hold"))?; + if !g.is_well_formed() { + return Err(DspError::Internal("malformed gate words for this state")); + } + gates.push((&g.words, g.index)); + } + let mut amps = vec![[0i32, 0]; 1usize << n]; + amps[p.start] = [ONE as i32, 0]; + self.execute(amps, &gates) + } + + pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result { + let p = plan::compile_with(c, start, &self.config()).map_err(DspError::Circuit)?; + if p.n_qubits > self.max_qubits { + return Err(DspError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits }); + } + let run = self.run_program(&p)?; + if let Some(gate) = run.out_of_range { + return Err(DspError::OutOfRange { gate }); + } + Ok(StateVector { n_qubits: p.n_qubits, amps: run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }).collect() }) + } + + pub fn simulate(&self, c: &Circuit) -> Result { + self.simulate_from(c, 0) + } + + /// The certificate of [`apply_kernel`]`(gs)` on this member, with its + /// provenance. + pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result { + let mut cert = Certification::new(); + for c in corpus.circuits(gs) { + let reference = lanes::reference_output(&c).map_err(DspError::Circuit)?; + let run = self.run_program(&plan::compile_with(&c, 0, &self.config()).map_err(DspError::Circuit)?)?; + if let Some(gate) = run.out_of_range { + return Err(DspError::OutOfRange { gate }); + } + cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps)); + } + Ok(cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)).with_provenance(self.provenance.clone())) + } + + pub fn member(&self, corpus: Corpus) -> Result { + let mut kernels = Vec::new(); + for gs in Gateset::ALL { + kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate: self.certify(gs, corpus)?, energy: None }); + } + Ok(Member { id: self.id.clone(), class: FabricClass::Dsp, presence: Presence::Detected, kernels, detail: self.detail.clone() }) + } +} + +impl Runner for DspMember { + fn member_id(&self) -> String { + self.id.clone() + } + + fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, _meter: &'r mut dyn RunMeter) -> RunFuture<'r> { + Box::pin(async move { + if implementation != self.implementation { + return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation))); + } + Ok((self.simulate_from(c, start)?, 0, None)) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_variant_has_its_own_implementation_id() { + let ids: std::collections::BTreeSet = [Variant::Conforming, Variant::M1, Variant::M2, Variant::M3].into_iter().map(implementation_id_of).collect(); + assert_eq!(ids.len(), 4); + assert!(implementation_id().starts_with("dsp-lanes64/")); + assert!(KERNEL_TEXT.contains("Q6_W_vmpye_VwVuh") && KERNEL_TEXT.contains("Q6_W_vmpyoacc_WVwVh")); + } + + #[test] + fn the_constants_are_the_reference_arithmetic() { + assert_eq!(Variant::Conforming.constants(), (1 << 29, 30, false)); + for v in Variant::MUTANTS { + assert_ne!(v.constants(), Variant::Conforming.constants()); + } + } + + #[test] + fn the_member_id_sorts_after_the_cpu_reference() { + assert!("local/dsp-vector-000000000000" > "local/cpu-aarch64"); + assert!("local/dsp-vector-000000000000" > "local/cpu-x86_64"); + } + + #[test] + fn elsewhere_the_member_is_absent_and_says_why() { + if !BUILT_FOR_UNIT { + let d = discover(); + assert!(d.found.is_empty() && d.refused.is_empty()); + assert!(d.unavailable.as_deref().is_some_and(|why| why.contains("is not for the signal processor")), "{:?}", d.unavailable); + } + } +} diff --git a/wai/wai-quantum/tests/dsp_member.rs b/wai/wai-quantum/tests/dsp_member.rs new file mode 100644 index 000000000..79122bc47 --- /dev/null +++ b/wai/wai-quantum/tests/dsp_member.rs @@ -0,0 +1,410 @@ +//! The fabric's DSP vector-unit member on the machine that runs this. +//! +//! The machine declares the devices it must have in `WAI_QUANTUM_DSP_EXPECT`: +//! their keys, comma-separated and sorted, or `none`. Every test that needs a +//! member fails unless the devices found are exactly the declared ones, so a +//! machine whose GPU or driver went missing, or a run that declared nothing, +//! fails instead of passing over no member. Each test prints what it +//! established and, when `WAI_QUANTUM_DSP_REPORT` names a file, appends the +//! same lines to it, which a CI step can count per device. +//! +//! ```text +//! WAI_QUANTUM_DSP_EXPECT=dsp-vector \ +//! cargo test --release --features quantum_fabric_dsp --target --test dsp_member -- --test-threads=1 +//! ``` +#![cfg(not(target_arch = "wasm32"))] + +use std::io::Write; +use std::sync::OnceLock; +use wai_quantum::ed25519_dalek::SigningKey; +use wai_quantum::quantum::{BaseGate, Circuit, Gate, Gateset, MAX_QUBITS, QuantumError}; +use wai_quantum::quantum_fabric::lanes::{self, GateWords, Lane}; +use wai_quantum::quantum_fabric::plan::{Program, Step}; +use wai_quantum::quantum_fabric::{ + Corpus, CpuReference, FABRIC_DEFAULT, Fabric, FabricClass, FabricReceipt, MemberError, Parity, Presence, REFERENCE, apply_kernel, detect_local, place, ready, +}; +use wai_quantum::quantum_fabric_dsp::{self as dsp, Discovery, DspError, DspMember, Variant, discover}; + +/// The CPU's corpus digests: random-v1 (seed 7, 200 cases) and fabric-v1 +/// (seed 7, 200 + 24 cases), per gateset. A bit-exact member reproduces them. +const PINS: [(Gateset, &str, &str); 2] = [ + (Gateset::V1, "d18454c639acfd9c7b66450f4ded1aac4ab46cc10a7bedbcf4958a96e204fb36", "41c726a0ac80f7defef92e15ecf893bdb7a3076b5e7f670fa0b7142e5c033df7"), + (Gateset::V2, "3dbde276c1095bf827faec71d5980e050f8a55a5ba7d97cd3bbdaec3f43a13b2", "9a84b55af68f1a8b07390b234cf562dd8d85cef3185a697ab85690c8790a7809"), +]; + +const RANDOM_V1: Corpus = Corpus::Random { seed: 7, cases: 200 }; + + +static DISCOVERY: OnceLock = OnceLock::new(); + +fn discovery() -> &'static Discovery { + DISCOVERY.get_or_init(discover) +} + +/// Prints `line` and appends it to the file `WAI_QUANTUM_DSP_REPORT` names. +fn report(line: &str) { + println!("{line}"); + if let Some(path) = std::env::var_os("WAI_QUANTUM_DSP_REPORT") { + let mut f = std::fs::OpenOptions::new().create(true).append(true).open(&path).expect("the report file opens"); + writeln!(f, "{line}").expect("the report line is written"); + } +} + +/// The declared device set, as written, if it is exactly what was found; +/// otherwise why not, with what was found and refused and how to declare it. +fn declared(d: &Discovery) -> Result { + let mut found: Vec<&str> = d.found.iter().map(|m| m.key()).collect(); + found.sort_unstable(); + let found_list = if found.is_empty() { "none".to_string() } else { found.join(",") }; + let refused: Vec = d.refused.iter().map(|(k, e)| format!("{k}: {e}")).collect(); + let refused_list = if refused.is_empty() { "none".to_string() } else { refused.join("; ") }; + let unavailable = d.unavailable.as_deref().unwrap_or("no"); + let how = format!("found: {found_list}; refused: {refused_list}; unavailable: {unavailable}. Declare the devices this machine must have: WAI_QUANTUM_DSP_EXPECT={found_list}"); + let expect = match std::env::var("WAI_QUANTUM_DSP_EXPECT") { + Ok(v) if !v.is_empty() => v, + _ => return Err(format!("WAI_QUANTUM_DSP_EXPECT is not set. {how}")), + }; + let keys: Vec<&str> = if expect == "none" { Vec::new() } else { expect.split(',').collect() }; + let well_formed = keys.windows(2).all(|w| w[0] <= w[1]) && keys.iter().all(|k| !k.is_empty() && *k != "none" && k.trim() == *k); + if !well_formed { + return Err(format!("WAI_QUANTUM_DSP_EXPECT={expect:?} is not a sorted, comma-separated list of keys, nor none. {how}")); + } + for k in &keys { + if !found.contains(k) + && let Some((_, e)) = d.refused.iter().find(|(key, _)| key == k) + { + return Err(format!("the declared device {k} was refused: {e}. {how}")); + } + } + if found != keys { + return Err(format!("the devices found are not the declared ones ({expect}). {how}")); + } + Ok(expect) +} + +/// The members, once the declared devices are exactly the ones found. +fn members() -> &'static [DspMember] { + let d = discovery(); + if let Err(why) = declared(d) { + panic!("{why}"); + } + &d.found +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn next(s: &mut u64) -> u64 { + *s = s.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *s; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// A value in `[-bound, bound]`. +fn within(s: &mut u64, bound: i64) -> i32 { + ((next(s) % (2 * bound as u64 + 1)) as i64 - bound) as i32 +} + +/// A random circuit of `ops` gates on `n` qubits: every base gate, up to three +/// distinct controls, `P(1..=32)`. +fn random_circuit(n: u8, ops: usize, seed: u64, gs: Gateset) -> Circuit { + let bases = [BaseGate::I, BaseGate::X, BaseGate::Y, BaseGate::Z, BaseGate::H, BaseGate::S, BaseGate::Sdg, BaseGate::T, BaseGate::Tdg, BaseGate::P]; + let mut s = seed; + let mut c = Circuit::with_gateset(n, gs); + for _ in 0..ops { + let base = bases[(next(&mut s) % bases.len() as u64) as usize]; + let target = (next(&mut s) % u64::from(n)) as u8; + let mut controls = Vec::new(); + for _ in 0..next(&mut s) % 4 { + let q = (next(&mut s) % u64::from(n)) as u8; + if q != target && !controls.contains(&q) { + controls.push(q); + } + } + let param = if base == BaseGate::P { 1 + (next(&mut s) % 32) as u16 } else { 0 }; + c.ops.push(Gate { base, controls, target, param }); + } + c +} + +/// Five circuits the native simulator refuses, in each gateset. +fn refused_circuits() -> Vec { + let gate = |base, controls: Vec, target, param| Gate { base, controls, target, param }; + let bad = [ + (1, vec![gate(BaseGate::H, vec![], 0, 0), gate(BaseGate::P, vec![], 0, 0)]), + (1, vec![gate(BaseGate::P, vec![], 0, 33)]), + (1, vec![gate(BaseGate::X, vec![], 3, 0)]), + (2, vec![gate(BaseGate::X, vec![0], 0, 0)]), + (2, vec![gate(BaseGate::X, vec![5], 1, 0)]), + ]; + let mut out = Vec::new(); + for gs in Gateset::ALL { + for (n, ops) in bad.clone() { + let mut c = Circuit::with_gateset(n, gs); + c.ops = ops; + out.push(c); + } + } + out +} + +/// `Z` on qubit 0: on `[(0, 0), (i32::MIN, 0)]` its exact result is `2^31`, +/// which leaves 32 bits. +fn z_words() -> GateWords { + lanes::gate_words(&Gate { base: BaseGate::Z, controls: vec![], target: 0, param: 0 }, Gateset::V2) +} + +const Z_STATE: [Lane; 2] = [[0, 0], [i32::MIN, 0]]; + +#[test] +fn c0_devices_are_declared() { + let d = discovery(); + match declared(d) { + Ok(expect) => report(&format!("dsp devices: as declared ({expect})")), + Err(why) => panic!("{why}"), + } + for m in &d.found { + assert_eq!(m.class(), FabricClass::Dsp); + assert!(m.id().starts_with("local/dsp-vector-"), "{}", m.id()); + assert_eq!(m.implementation(), dsp::implementation_id()); + assert!(m.max_qubits() >= 20, "{}: holds only {} qubits", m.key(), m.max_qubits()); + report(&format!("dsp member: key={} id={} {}", m.key(), m.id(), m.detail())); + } +} + +#[test] +fn c1_probe_matches_lanes() { + for m in members() { + let pairs = lanes::probe_pairs(1 << 16, 0x0cda); + let out = m.probe(&pairs).unwrap_or_else(|e| panic!("{}: {e}", m.key())); + assert_eq!(lanes::first_mismatch(&pairs, &out), None, "{}", m.key()); + report(&format!("dsp probe: key={} {} pairs agree", m.key(), pairs.len())); + } +} + +#[test] +fn c2_kernel_matches_run_words() { + for m in members() { + let mut s = 0x6b65_726e_656c_u64; + let (mut flagged, mut clean) = (0, 0); + for case in 0..2000 { + let n = 1 + (next(&mut s) % 12) as u32; + // Three regimes: the whole 32-bit range (nearly always out of + // range), small states and entries (never), and states and + // entries up to 2^30 (either). + let mode = case % 3; + let value = |s: &mut u64| -> i32 { + match mode { + 0 => next(s) as i32, + 1 => within(s, 1 << 22), + _ => within(s, 1 << 30), + } + }; + let entry = |s: &mut u64| -> u32 { + match mode { + 0 => next(s) as u32, + 1 => within(s, 1 << 29) as u32, + _ => within(s, 1 << 30) as u32, + } + }; + let state: Vec = (0..1usize << n).map(|_| [value(&mut s), value(&mut s)]).collect(); + let gates: Vec = (0..1 + next(&mut s) % 8) + .map(|_| { + let t = (next(&mut s) % u64::from(n)) as u32; + let ctrl = (0..n).filter(|&b| b != t && next(&mut s).is_multiple_of(4)).fold(0u32, |acc, b| acc | 1 << b); + let mut w = [0u32; 8]; + w.iter_mut().for_each(|x| *x = entry(&mut s)); + GateWords { m: w, tbit: 1 << t, ctrl } + }) + .collect(); + let want = lanes::run_words(&state, &gates).expect("valid shapes"); + let got = m.run_lanes(&state, &gates).unwrap_or_else(|e| panic!("{}: case {case}: {e}", m.key())); + assert_eq!(got.out_of_range, want.out_of_range, "{}: case {case}, n = {n}", m.key()); + assert!(got.amps == want.amps, "{}: case {case}, n = {n}: the lanes differ", m.key()); + if want.out_of_range.is_some() { + flagged += 1; + } else { + clean += 1; + } + } + assert!(flagged >= 100 && clean >= 100, "{}: {flagged} flagged, {clean} clean", m.key()); + report(&format!("dsp kernel: key={} ok", m.key())); + } +} + +#[test] +fn c3_certificates_reproduce_the_pins() { + let ms = members(); + for (gs, random_pin, fabric_pin) in PINS { + let cpu = lanes::certify_corpus(gs, "local/test", RANDOM_V1); + assert_eq!(hex(&cpu.corpus), random_pin, "{gs:?}"); + let other = Corpus::Random { seed: 8, cases: 300 }; + let cpu_other = lanes::certify_corpus(gs, "local/test", other); + for m in ms { + for (corpus, cases, pin) in [(RANDOM_V1, 200, random_pin), (FABRIC_DEFAULT, 224, fabric_pin)] { + let c = m.certify(gs, corpus).unwrap_or_else(|e| panic!("{} {gs:?} {}: {e}", m.key(), corpus.name())); + assert_eq!(c.parity, Parity::BitExact, "{} {gs:?} {}", m.key(), corpus.name()); + assert_eq!(c.cases, cases); + assert_eq!(hex(&c.corpus), pin, "{} {gs:?} {}", m.key(), corpus.name()); + assert_eq!((c.kernel.as_str(), c.implementation.as_str(), c.member.as_str()), (apply_kernel(gs), m.implementation(), m.id())); + assert_eq!(c.generator, corpus.generator(gs)); + report(&format!( + "dsp certified: key={} member={} kernel={} corpus={} implementation={} parity=bit-exact cases={} digest={}", + m.key(), + m.id(), + c.kernel, + corpus.name(), + c.implementation, + c.cases, + hex(&c.corpus) + )); + } + // A corpus with no pin: the member and the CPU agree on it. + let c = m.certify(gs, other).unwrap_or_else(|e| panic!("{} {gs:?} seed 8: {e}", m.key())); + assert_eq!((c.parity, c.cases), (Parity::BitExact, 300), "{} {gs:?}", m.key()); + assert_eq!(c.corpus, cpu_other.corpus, "{} {gs:?}", m.key()); + } + } +} + +#[test] +fn c4_wide_circuits_match_native() { + for m in members() { + let widest = m.max_qubits().min(20); + let mut s = 0x0077_6964_6500_u64; + for (k, n) in [12u8, 16, 18, widest].into_iter().enumerate() { + for gs in Gateset::ALL { + let c = random_circuit(n, 40, 0x1000 + k as u64, gs); + let start = (next(&mut s) as usize) % (1usize << n); + let want = c.simulate_from(start).expect("the native run"); + let got = m.simulate_from(&c, start).unwrap_or_else(|e| panic!("{}: n = {n} {gs:?}: {e}", m.key())); + assert!(got == want, "{}: n = {n} {gs:?}: the states differ", m.key()); + } + } + // The transforms the conformance corpus carries. + for n in [3u8, 5, 8, 12] { + let qft = Circuit::qft(n); + assert!(m.simulate(&qft).unwrap() == qft.simulate().unwrap(), "{}: QFT-{n}", m.key()); + } + report(&format!("dsp wide: key={} up to {widest} qubits match native", m.key())); + } +} + +#[test] +fn c5_refusals_are_typed() { + for m in members() { + for c in refused_circuits() { + let native = c.simulate_from(0).unwrap_err(); + assert_eq!(m.simulate_from(&c, 0).unwrap_err(), DspError::Circuit(native), "{c:?}"); + } + assert_eq!(m.simulate(&Circuit::new(MAX_QUBITS + 1)).unwrap_err(), DspError::Circuit(QuantumError::TooManyQubits(MAX_QUBITS + 1))); + let small = m.with_max_qubits(4); + assert_eq!(small.simulate(&Circuit::new(5)).unwrap_err(), DspError::TooWide { n_qubits: 5, max: 4 }); + let wide = Program { n_qubits: 5, start: 0, gates: Vec::new(), steps: Vec::new() }; + assert_eq!(small.run_program(&wide).unwrap_err(), DspError::TooWide { n_qubits: 5, max: 4 }); + // A result that leaves 32 bits is reported at its op, and the lane + // holds its low 32 bits. + let r = m.run_lanes(&Z_STATE, &[z_words()]).unwrap(); + assert_eq!(r.out_of_range, Some(0), "{}", m.key()); + assert_eq!(r.amps[1][0] as u32, 0x8000_0000); + // Malformed words, a state the host cannot lay out and a tile step are + // host defects, never results. + let bad = GateWords { tbit: 3, ..z_words() }; + assert!(matches!(m.run_lanes(&[[0, 0]; 4], &[bad]), Err(DspError::Internal(_)))); + assert!(matches!(m.run_lanes(&[[0, 0]; 3], &[z_words()]), Err(DspError::Internal(_)))); + let tiled = Program { n_qubits: 2, start: 0, gates: Vec::new(), steps: vec![Step::Tile { first: 0, count: 1, size: 2 }] }; + assert!(matches!(m.run_program(&tiled), Err(DspError::Internal(_)))); + // The refusals that may still fall back to another member, do. + assert!(MemberError::from(DspError::TooWide { n_qubits: 5, max: 4 }).falls_back()); + report(&format!("dsp refusals: key={} typed", m.key())); + } +} + +/// The kernel mutants, compiled in: each must be caught by the probe or by a +/// certificate. +#[test] +fn c6_mutants_are_killed() { + let edges = lanes::probe_pairs(0, 0); + for m in members() { + let mut killed = 0; + for v in Variant::MUTANTS { + let mm = m.with_variant(v); + assert_ne!(mm.implementation(), m.implementation()); + let out = mm.probe(&edges).expect("the mutant's probe runs"); + let differing = edges.iter().zip(&out).filter(|&(&(a, b), o)| *o != (lanes::mul_wide(a, b), lanes::fxmul_wide(a, b))).count(); + let caught = differing > 0 + || Gateset::ALL.into_iter().all(|gs| match mm.certify(gs, RANDOM_V1) { + Err(_) => true, + Ok(c) => c.parity != Parity::BitExact || c.corpus != lanes::certify_corpus(gs, "local/test", RANDOM_V1).corpus, + }); + println!("dsp mutant {v:?} on {}: {differing} of {} edge pairs differ", m.key(), edges.len()); + assert!(caught, "{}: mutant {v:?} survived", m.key()); + killed += 1; + } + report(&format!("dsp mutants: key={} killed {killed} of 3", m.key())); + } +} + +#[test] +fn c7_receipt_placement_and_the_fabric() { + for m in members() { + let member = m.member(FABRIC_DEFAULT).unwrap_or_else(|e| panic!("{}: {e}", m.key())); + assert_eq!((member.id.as_str(), member.class, member.presence), (m.id(), FabricClass::Dsp, Presence::Detected)); + assert_eq!(member.kernels.len(), 2); + for (entry, gs) in member.kernels.iter().zip(Gateset::ALL) { + assert_eq!((entry.kernel.as_str(), entry.implementation.as_str()), (apply_kernel(gs), m.implementation())); + assert!(entry.energy.is_none()); + assert_eq!(entry.certificate.parity, Parity::BitExact, "{gs:?}"); + } + // A receipt over a member's run. + let qft = Circuit::qft(5); + let state = m.simulate(&qft).unwrap(); + assert_eq!(state, qft.simulate().unwrap()); + let entry = member.kernels.iter().find(|k| k.kernel == apply_kernel(qft.gateset)).unwrap(); + let signer = SigningKey::from_bytes(&[7; 32]); + let receipt = FabricReceipt::seal(&signer, "local/test", state.statevector_hash(), &member, entry, 0, None); + assert!(receipt.verify()); + assert!(receipt.certificate_matches(&entry.certificate)); + // Placement: beside the CPU reference with no energy figures, the + // reference; alone, the member. + let local = detect_local(); + for gs in Gateset::ALL { + let both = [local.clone(), member.clone()]; + let (placed, k) = place(&both, apply_kernel(gs), &Parity::BitExact).unwrap(); + assert_eq!((placed.id.as_str(), k.implementation.as_str()), (local.id.as_str(), REFERENCE)); + let (placed, k) = place(std::slice::from_ref(&member), apply_kernel(gs), &Parity::BitExact).unwrap(); + assert_eq!((placed.id.as_str(), k.implementation.as_str()), (m.id(), m.implementation())); + } + // On the fabric: the member alone runs the circuit; a pairing with + // another member's entries is refused. + let mut fabric = Fabric::new(); + fabric.add(member.clone(), m); + let run = ready(fabric.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs"); + assert_eq!(run.state, state); + assert_eq!(run.receipt.member_id, m.id()); + assert!(run.receipt.verify()); + let mut mismatched = Fabric::new(); + mismatched.add(local.clone(), m); + assert!(ready(mismatched.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").is_err()); + let mut both = Fabric::new(); + both.add(local.clone(), CpuReference).add(member.clone(), m); + let run = ready(both.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs"); + assert_eq!(run.state, state); + report(&format!("dsp receipt: key={} ok", m.key())); + } +} + +#[test] +fn c8_repeated_runs_agree() { + for m in members() { + let c = random_circuit(16, 60, 0xabc, Gateset::V2); + let first = m.simulate(&c).unwrap(); + for _ in 0..5 { + assert!(m.simulate(&c).unwrap() == first, "{}", m.key()); + } + report(&format!("dsp repeat: key={} ok", m.key())); + } +} -- 2.54.0 (Apple Git-157) From 47994f302b6322869c7d5d7439350e1060d69eea Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Wed, 14 Oct 2026 12:01:43 -0400 Subject: [PATCH 06/11] wai-quantum: the CPU vector-unit members (avx2, sve, rvv) The gate's fixed-point arithmetic in the processor's own vector extension, feature quantum_fabric_vector, class cpu: - avx2 (x86-64): 256-bit vectors of four 64-bit lanes; the exact signed 32x32 multiply of each lane's low word, and the 64-bit arithmetic shift the unit lacks built from a logical shift and the sign bit; - sve (aarch64) and rvv (riscv64): one assembly text each for every vector length (the length set per pass); the member reads the length from the unit and names it in its id and detail. Each member's id extends the CPU reference's, so with no energy figures placement keeps the reference. Three kernel mutants per instruction set are compiled in. Measured: avx2 on an x86-64 processor; sve at 128, 512 and 2048 bits and rvv at 128 and 1024 bits under the user-mode emulator. Every leg: 9 of 9 device tests, all four pinned certificates, circuits up to 20 qubits matching native, 3 of 3 mutants killed, and the length the member read is the one given. Apple silicon finds no member and says why. Workflow wai-quantum-vector.yml runs the six legs on the container runner and the absent case on the macOS runner. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wai-quantum-vector.yml | 171 +++ wai/CHANGELOG.md | 13 +- wai/wai-quantum/Cargo.toml | 6 + wai/wai-quantum/README.md | 6 +- wai/wai-quantum/src/lib.rs | 7 +- .../src/quantum_fabric/vector_avx2.rs | 102 ++ wai/wai-quantum/src/quantum_fabric_vector.rs | 1089 +++++++++++++++++ wai/wai-quantum/tests/vector_member.rs | 410 +++++++ 8 files changed, 1798 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/wai-quantum-vector.yml create mode 100644 wai/wai-quantum/src/quantum_fabric/vector_avx2.rs create mode 100644 wai/wai-quantum/src/quantum_fabric_vector.rs create mode 100644 wai/wai-quantum/tests/vector_member.rs diff --git a/.github/workflows/wai-quantum-vector.yml b/.github/workflows/wai-quantum-vector.yml new file mode 100644 index 000000000..6268585fe --- /dev/null +++ b/.github/workflows/wai-quantum-vector.yml @@ -0,0 +1,171 @@ +name: wai-quantum CPU vector units + +# The fabric's CPU vector-unit members: the gate's fixed-point arithmetic in +# the processor's own vector extension, certified bit-exact against the CPU +# reference's pinned corpus digests. +# +# - avx2 on silicon: the container runner's x86-64 processor; +# - sve and rvv under the host's user-mode emulator (its binfmt handler), at +# several vector lengths each: the scalable kernels set their length per +# pass, and every leg checks the length the member read from the unit is the +# one the emulator was given, so a leg cannot pass at the wrong length; +# - the macOS runner, whose processor has no vector extension these kernels +# use outside streaming mode, finding no member. +# +# Silicon for sve and rvv: a processor with them runs the same tests with no +# emulator declared. + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: [main] + paths: + - 'wai/wai-quantum/**' + - 'rust-toolchain.toml' + - '.github/workflows/wai-quantum-vector.yml' + pull_request: + paths: + - 'wai/wai-quantum/**' + - 'rust-toolchain.toml' + - '.github/workflows/wai-quantum-vector.yml' + +jobs: + vector: + name: member · vector unit, ${{ matrix.name }} + runs-on: linux + timeout-minutes: 120 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + include: + - name: avx2 (x86_64 silicon) + target: x86_64-unknown-linux-gnu + image: ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:7f8308a8734d9fcd2ebbe9a3e4bdea74af293f0799d80c3cc341e340cda49a4c + key: cpu-vector-avx2 + bits: 256 + emulator: '' + qemu_cpu: '' + - name: sve 128-bit (aarch64, emulated) + target: aarch64-unknown-linux-gnu + image: ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:7f8308a8734d9fcd2ebbe9a3e4bdea74af293f0799d80c3cc341e340cda49a4c + key: cpu-vector-sve + bits: 128 + emulator: qemu-user 8.2.2 (the host's binfmt handler) + qemu_cpu: max,sve-default-vector-length=16 + - name: sve 512-bit (aarch64, emulated) + target: aarch64-unknown-linux-gnu + image: ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:7f8308a8734d9fcd2ebbe9a3e4bdea74af293f0799d80c3cc341e340cda49a4c + key: cpu-vector-sve + bits: 512 + emulator: qemu-user 8.2.2 (the host's binfmt handler) + qemu_cpu: max,sve-default-vector-length=64 + - name: sve 2048-bit (aarch64, emulated) + target: aarch64-unknown-linux-gnu + image: ghcr.io/cross-rs/aarch64-unknown-linux-gnu@sha256:7f8308a8734d9fcd2ebbe9a3e4bdea74af293f0799d80c3cc341e340cda49a4c + key: cpu-vector-sve + bits: 2048 + emulator: qemu-user 8.2.2 (the host's binfmt handler) + qemu_cpu: max,sve-default-vector-length=256 + - name: rvv 128-bit (riscv64, emulated) + target: riscv64gc-unknown-linux-gnu + image: ghcr.io/cross-rs/riscv64gc-unknown-linux-gnu@sha256:67ee97dd4719390d299edfcf39f226360ccfb1539e08b078ec187d4848415794 + key: cpu-vector-rvv + bits: 128 + emulator: qemu-user 8.2.2 (the host's binfmt handler) + qemu_cpu: rv64,v=true,vlen=128,elen=64 + - name: rvv 1024-bit (riscv64, emulated) + target: riscv64gc-unknown-linux-gnu + image: ghcr.io/cross-rs/riscv64gc-unknown-linux-gnu@sha256:67ee97dd4719390d299edfcf39f226360ccfb1539e08b078ec187d4848415794 + key: cpu-vector-rvv + bits: 1024 + emulator: qemu-user 8.2.2 (the host's binfmt handler) + qemu_cpu: rv64,v=true,vlen=1024,elen=64 + container: + image: ${{ matrix.image }} + options: --memory=6g --memory-swap=6g + env: + WAI_QUANTUM_VECTOR_EXPECT: ${{ matrix.key }} + WAI_QUANTUM_VECTOR_EMULATOR: ${{ matrix.emulator }} + # Run test binaries directly, so the host's handler runs them with the + # processor given here, not the image's older emulator. + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER: env + CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_RUNNER: env + steps: + # The image has no Node for a checkout action: the forge's archive of the + # exact commit, its embedded commit id checked. + - name: Fetch the source + shell: bash + run: | + set -euo pipefail + want="${{ github.event.pull_request.head.sha || github.sha }}" + rm -rf src src.tar.gz src.tar && mkdir src + curl -sSf -o src.tar.gz "http://100.64.224.0:3400/Transaction-Science/open-standards/archive/$want.tar.gz" + gzip -dc src.tar.gz > src.tar + got="$(git get-tar-commit-id < src.tar)" + [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } + tar -xf src.tar -C src --strip-components=1 + rm src.tar.gz src.tar + - name: The member's tests + shell: bash + working-directory: src/wai/wai-quantum + run: | + set -euo pipefail + cache=/ci-cache/wai-quantum-vector + mkdir -p "$cache" + exec 9>"$cache/.lock" + flock 9 + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target-${{ matrix.target }}" + export PATH="$CARGO_HOME/bin:$PATH" + if ! command -v rustup >/dev/null; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain none + fi + rustup toolchain install + rustup target add ${{ matrix.target }} + # The emulated processor, for the test binaries only. + if [ -n "${{ matrix.qemu_cpu }}" ]; then export QEMU_CPU="${{ matrix.qemu_cpu }}"; fi + t="--target ${{ matrix.target }} --features quantum_fabric_vector" + cargo test --locked --release -j 6 $t --lib -- quantum_fabric_vector:: 2>&1 | tee "$cache/unit.log" + grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$cache/unit.log" || { echo "::error::no unit test ran"; exit 1; } + report="$(mktemp)" + WAI_QUANTUM_VECTOR_REPORT="$report" cargo test --locked --release -j 6 $t --test vector_member -- --test-threads=1 2>&1 | tee "$cache/device.log" + grep -qE '^test result: ok\. 9 passed; 0 failed' "$cache/device.log" || { echo "::error::not every device test ran and passed"; exit 1; } + cat "$report" + key="$WAI_QUANTUM_VECTOR_EXPECT" + grep -qx "vector devices: as declared ($key)" "$report" || { echo "::error::the unit found is not the declared one"; exit 1; } + grep -q "^vector member: key=$key .* bits=${{ matrix.bits }} " "$report" || { echo "::error::the unit's vector length is not ${{ matrix.bits }} bits"; exit 1; } + if [ -n "$WAI_QUANTUM_VECTOR_EMULATOR" ]; then + grep -q "^vector member: key=$key .*emulator:qemu-user" "$report" || { echo "::error::the member does not say it ran under the emulator"; exit 1; } + else + grep -q "^vector member: key=$key .*; silicon;" "$report" || { echo "::error::the member does not say it ran on silicon"; exit 1; } + fi + n="$(grep -c "^vector certified: key=$key .* parity=bit-exact " "$report")" + [ "$n" = 4 ] || { echo "::error::$n of 4 certificates were earned"; exit 1; } + grep -qx "vector mutants: key=$key killed 3 of 3" "$report" || { echo "::error::a kernel mutant survived"; exit 1; } + + vector-absent: + name: member · vector unit absent (macos) + runs-on: macos + timeout-minutes: 60 + env: + WAI_QUANTUM_VECTOR_EXPECT: none + steps: + - uses: actions/checkout@v5 + # launchd's PATH omits cargo on the macOS host runner. + - name: Put cargo on PATH + shell: bash + run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: No vector unit these kernels use, and the member says so + shell: bash + working-directory: wai/wai-quantum + run: | + set -euo pipefail + report="$RUNNER_TEMP/vector-report.txt"; : > "$report" + nice -n 10 cargo test --locked --features quantum_fabric_vector --lib -- quantum_fabric_vector:: 2>&1 | tee "$RUNNER_TEMP/unit.log" + grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } + WAI_QUANTUM_VECTOR_REPORT="$report" nice -n 10 cargo test --locked --features quantum_fabric_vector --test vector_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log" + grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; } + grep -qx 'vector devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; } diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index 909a6c330..40a32402c 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -6,7 +6,7 @@ wire format may change and such changes are recorded here. ## v0.4.0 — current -### wai-quantum 0.5.1: the neural-processor and matrix-extension members +### wai-quantum 0.5.1: neural-processor, matrix-extension, vector-unit and signal-processor members Additive. The default features, `full`, and every pinned digest are unchanged. @@ -43,6 +43,13 @@ Additive. The default features, `full`, and every pinned digest are unchanged. half, signed, sixteen bits up); the rounding constant's carry and each row's carries and borrows go into the high word; the shift is arithmetic over all 64 bits. The fabric gains the class `dsp`. Three kernel mutants are compiled in. +- **The CPU vector-unit members** (`quantum_fabric_vector`, opt-in): the gate's arithmetic + in the processor's own vector extension — `avx2` on x86-64 (the 64-bit arithmetic + shift, which the unit lacks, built from a logical shift and the sign bit), `sve` on + aarch64 and `rvv` on riscv64, both written once for every vector length (the length + set per pass and read back from the unit into the member's id). Silicon for `avx2`; + `sve` at 128, 512 and 2048 bits and `rvv` at 128 and 1024 bits under the user-mode + emulator. Three kernel mutants per instruction set. - **The signal processor as a processor.** The bit-identity digests, the frame fixtures and the byte-pinning library tests pass for the processor's Linux target under its user-mode emulator. Emulator releases before 11.0 test a whole predicate register @@ -57,7 +64,9 @@ Additive. The default features, `full`, and every pinned digest are unchanged. runner, whose processor has no SME, finding no member. `wai-quantum-dsp.yml`: the signal processor's suites and the vector-unit member's device tests under the pinned emulator, after the emulator's own branch probe, and the same build on x86_64, where - the member is absent and says so. + the member is absent and says so. `wai-quantum-vector.yml`: `avx2` on the container + runner's processor, `sve` and `rvv` under the host's emulator at the lengths above (each + leg checks the length the member read), and the macOS runner finding no member. ### wai-quantum 0.5.0: provenance, sampling members, device programs, and three more members diff --git a/wai/wai-quantum/Cargo.toml b/wai/wai-quantum/Cargo.toml index 75c467484..62c08fded 100644 --- a/wai/wai-quantum/Cargo.toml +++ b/wai/wai-quantum/Cargo.toml @@ -171,6 +171,8 @@ quantum_fabric_npu = ["quantum_fabric", "dep:objc2", "dep:block2", "dep:objc2-fo quantum_fabric_sme = ["quantum_fabric"] # The DSP vector-unit member: the gate's arithmetic in the signal processor's vector unit. quantum_fabric_dsp = ["quantum_fabric"] +# The CPU vector-unit members: the gate's arithmetic in the processor's vector extension (avx2, sve, rvv). +quantum_fabric_vector = ["quantum_fabric"] # --- interference / ML layer ---------------------------------------------- quantum_vml = [] @@ -225,6 +227,10 @@ required-features = ["quantum_fabric_sme"] name = "dsp_member" required-features = ["quantum_fabric_dsp"] +[[test]] +name = "vector_member" +required-features = ["quantum_fabric_vector"] + [[test]] name = "fabric_expected" required-features = ["quantum_fabric"] diff --git a/wai/wai-quantum/README.md b/wai/wai-quantum/README.md index cc637f79b..cd8f42fe0 100644 --- a/wai/wai-quantum/README.md +++ b/wai/wai-quantum/README.md @@ -6,14 +6,15 @@ every stage. No QPU, no cloud service, no vendor SDK, and **no system libraries** — so the same source runs natively, in the browser via wasm, and as a WASI component at -the edge, producing **byte-identical results and receipts on all three**. Seven +the edge, producing **byte-identical results and receipts on all three**. Eight opt-in features reach outside the crate: `quantum_fabric_webgpu` (the platform's GPU interface), `quantum_fabric_cuda` (the CUDA driver and runtime compiler) and `quantum_fabric_opencl` (the OpenCL ICD loader), the last two loaded when first used, `quantum_fabric_fpga` (Verilator, run as a program when first used), `quantum_fabric_npu` (the platform's model runtime, on macOS), `quantum_fabric_sme` (the processor's streaming vector mode, by -inline assembly) and `quantum_fabric_dsp` (the signal processor's vector +inline assembly), `quantum_fabric_vector` (the processor's vector extension) +and `quantum_fabric_dsp` (the signal processor's vector unit, in a build for that processor); no default feature reaches a system library. @@ -109,6 +110,7 @@ never creates information. | `quantum_fabric_npu` | **the fabric's neural-processor member**, opt-in and not in `full`, macOS only: each 32-bit operand split into eight 4-bit limbs, every limb product and limb sum formed in fp16 on the platform's neural processor (each value an integer fp16 holds exactly, whatever order it is added in), the host assembling each product and applying the fixed-point arithmetic. The member exists only where the platform model runtime's compute plan places every operation of the model on the neural processor; a model it would run elsewhere is refused with the placement it reported. Probed and certified as every member is | | `quantum_fabric_sme` | **the fabric's matrix-extension member**, opt-in and not in `full`, aarch64 only: the gate's fixed-point arithmetic in the processor's streaming vector mode (SME), 64-bit lanes, the host gathering each gate's pairs and checking each row's range. Every instruction it uses exists only in that mode, so a kernel that ran, ran there. Detected by the processor's feature flags (on Linux, the kernel's `AT_HWCAP2`, which a user-mode emulator supplies); under an emulator the runner says so (`WAI_QUANTUM_SME_EMULATOR`) and every certificate carries it. Three kernel mutants are compiled in and must be caught | | `quantum_fabric_dsp` | **the fabric's DSP vector-unit member**, opt-in and not in `full`, in a build for the signal processor with its vector unit in 128-byte mode: the gate's fixed-point arithmetic in 32-bit lanes, each 64-bit value a pair of words, every product the vector unit's exact 32×32 multiply and every carry and borrow kept; the host gathers each gate's pairs and checks each row's range. Its instructions exist only on that unit, so a kernel that ran, ran there. Under an emulator the runner says so (`WAI_QUANTUM_DSP_EMULATOR`) and every certificate carries it. Three kernel mutants are compiled in and must be caught. Its class is `dsp` | +| `quantum_fabric_vector` | **the fabric's CPU vector-unit members**, opt-in and not in `full`: the gate's fixed-point arithmetic in the processor's own vector extension, one kernel per instruction set — `avx2` (x86-64, 256-bit), `sve` (aarch64, the scalable vector extension, any length) and `rvv` (riscv64, the vector extension 1.0, any length) — each operand sign-extended to 64-bit lanes, the host gathering each gate's pairs and checking each row's range. The scalable kernels set their length per pass; the member reads the length from the unit and names it in its id and detail. Detected by the processor's feature flags (on Linux, `AT_HWCAP` for the vector extension, which a user-mode emulator supplies); under an emulator the runner says so (`WAI_QUANTUM_VECTOR_EMULATOR`). Its id extends the CPU reference's, so placement with no energy figures keeps the reference. Three kernel mutants per instruction set are compiled in and must be caught | | `full` | everything | Default features are `quantum`, `quantum_toolchain`, `quantum_receipt`. diff --git a/wai/wai-quantum/src/lib.rs b/wai/wai-quantum/src/lib.rs index c53239e1a..ac3e22ba0 100644 --- a/wai/wai-quantum/src/lib.rs +++ b/wai/wai-quantum/src/lib.rs @@ -10,13 +10,14 @@ //! Rust, so the same source runs natively, in the browser via wasm, and as a //! WASI component at the edge, producing **byte-identical** results and receipts //! on all three. A fabric receipt is the exception by design: it names the -//! member that ran the work. Seven opt-in features reach outside the crate: +//! member that ran the work. Eight opt-in features reach outside the crate: //! `quantum_fabric_webgpu` (the platform's GPU interface), //! `quantum_fabric_cuda` (the CUDA driver and runtime compiler) and //! `quantum_fabric_opencl` (the OpenCL ICD loader), the last two loaded when //! first used, `quantum_fabric_fpga` (Verilator, run as a program when first //! used), `quantum_fabric_npu` (the platform's model runtime, on macOS), -//! `quantum_fabric_sme` (the processor's streaming vector mode) and +//! `quantum_fabric_sme` (the processor's streaming vector mode), +//! `quantum_fabric_vector` (the processor's vector extension) and //! `quantum_fabric_dsp` (the signal processor's vector unit, in a build for //! it); no default feature reaches a system library. //! @@ -98,6 +99,8 @@ pub mod quantum_fabric_npu; pub mod quantum_fabric_sme; #[cfg(all(feature = "quantum_fabric_dsp", not(target_arch = "wasm32")))] pub mod quantum_fabric_dsp; +#[cfg(all(feature = "quantum_fabric_vector", not(target_arch = "wasm32")))] +pub mod quantum_fabric_vector; #[cfg(feature = "quantum_control")] pub mod quantum_control; diff --git a/wai/wai-quantum/src/quantum_fabric/vector_avx2.rs b/wai/wai-quantum/src/quantum_fabric/vector_avx2.rs new file mode 100644 index 000000000..5dd8e7724 --- /dev/null +++ b/wai/wai-quantum/src/quantum_fabric/vector_avx2.rs @@ -0,0 +1,102 @@ +//! The x86-64 vector unit's kernels (AVX2): the gate's fixed-point arithmetic +//! in 256-bit vectors, four 64-bit lanes, every step exact. +//! +//! This file's text is the implementation id's input. `RND` is the rounding +//! constant, `SH` the shift and `LAST_ADD` whether row 0's last real term is +//! added (a mutant) instead of subtracted. +use std::arch::x86_64::*; + +/// Lanes per vector. +pub const LANES: usize = 4; + +/// Four operands of `s` from `k`, sign-extended to 64 bits. +#[inline] +#[target_feature(enable = "avx2")] +unsafe fn load(s: &[i32], k: usize) -> __m256i { + debug_assert!(k + LANES <= s.len()); + // SAFETY: four values from `k` are in bounds; the load is unaligned. + _mm256_cvtepi32_epi64(unsafe { _mm_loadu_si128(s.as_ptr().add(k).cast()) }) +} + +/// Writes the four lanes of `v` to `d` from `k`. +#[inline] +#[target_feature(enable = "avx2")] +unsafe fn store(d: &mut [i64], k: usize, v: __m256i) { + debug_assert!(k + LANES <= d.len()); + // SAFETY: as for `load`. + unsafe { _mm256_storeu_si256(d.as_mut_ptr().add(k).cast(), v) } +} + +/// `(e·a + RND) >> SH` in each lane. The multiply takes each lane's low 32 +/// bits as signed and forms the exact 64-bit product; the unit has no 64-bit +/// arithmetic shift, so the shift is a logical one with the sign bit's +/// position flipped and subtracted back. +#[inline] +#[target_feature(enable = "avx2")] +fn fx(e: __m256i, a: __m256i) -> __m256i { + let r = _mm256_add_epi64(_mm256_mul_epi32(a, e), _mm256_set1_epi64x(RND)); + let sign = _mm256_set1_epi64x(1i64 << (63 - SH)); + _mm256_sub_epi64(_mm256_xor_si256(_mm256_srli_epi64::(r), sign), sign) +} + +/// A gate's four rows for every pair: `m` the gate's entries, `cols` the pairs' +/// columns (re a0, im a0, re a1, im a1), `out` the rows (0r, 0i, 1r, 1i). +/// +/// # Safety +/// +/// The processor has AVX2; every slice has the same length, a multiple of +/// [`LANES`]. +#[target_feature(enable = "avx2")] +pub unsafe fn rows(m: &[i32; 8], cols: [&[i32]; 4], out: [&mut [i64]; 4]) { + let n = cols[0].len(); + debug_assert!(n.is_multiple_of(LANES) && cols.iter().all(|c| c.len() == n) && out.iter().all(|o| o.len() == n)); + let mut e = [_mm256_setzero_si256(); 8]; + for (v, &x) in e.iter_mut().zip(m) { + *v = _mm256_set1_epi64x(i64::from(x)); + } + let [o0r, o0i, o1r, o1i] = out; + macro_rules! f { + ($e:expr, $a:expr) => { + fx::($e, $a) + }; + } + let mut k = 0; + while k < n { + // SAFETY: the caller's slices hold whole vectors. + let (a0r, a0i, a1r, a1i) = unsafe { (load(cols[0], k), load(cols[1], k), load(cols[2], k), load(cols[3], k)) }; + let t = _mm256_add_epi64(_mm256_sub_epi64(f!(e[0], a0r), f!(e[1], a0i)), f!(e[2], a1r)); + let r0r = if LAST_ADD { _mm256_add_epi64(t, f!(e[3], a1i)) } else { _mm256_sub_epi64(t, f!(e[3], a1i)) }; + let r0i = _mm256_add_epi64(_mm256_add_epi64(_mm256_add_epi64(f!(e[0], a0i), f!(e[1], a0r)), f!(e[2], a1i)), f!(e[3], a1r)); + let r1r = _mm256_sub_epi64(_mm256_add_epi64(_mm256_sub_epi64(f!(e[4], a0r), f!(e[5], a0i)), f!(e[6], a1r)), f!(e[7], a1i)); + let r1i = _mm256_add_epi64(_mm256_add_epi64(_mm256_add_epi64(f!(e[4], a0i), f!(e[5], a0r)), f!(e[6], a1i)), f!(e[7], a1r)); + // SAFETY: as for the loads. + unsafe { + store(o0r, k, r0r); + store(o0i, k, r0i); + store(o1r, k, r1r); + store(o1i, k, r1i); + } + k += LANES; + } +} + +/// Each pair's `a·b` and `(a·b + RND) >> SH`. +/// +/// # Safety +/// +/// As for [`rows`]. +#[target_feature(enable = "avx2")] +pub unsafe fn probe(a: &[i32], b: &[i32], prod: &mut [i64], fixed: &mut [i64]) { + let n = a.len(); + debug_assert!(n.is_multiple_of(LANES) && b.len() == n && prod.len() == n && fixed.len() == n); + let mut k = 0; + while k < n { + // SAFETY: the caller's slices hold whole vectors. + unsafe { + let (va, vb) = (load(a, k), load(b, k)); + store(prod, k, _mm256_mul_epi32(va, vb)); + store(fixed, k, fx::(vb, va)); + } + k += LANES; + } +} diff --git a/wai/wai-quantum/src/quantum_fabric_vector.rs b/wai/wai-quantum/src/quantum_fabric_vector.rs new file mode 100644 index 000000000..ca255b6dc --- /dev/null +++ b/wai/wai-quantum/src/quantum_fabric_vector.rs @@ -0,0 +1,1089 @@ +//! The fabric's CPU vector-unit members: the gate's fixed-point arithmetic in +//! the processor's own vector extension, one kernel per instruction set: +//! +//! - `avx2` (x86-64): 256-bit vectors, four 64-bit lanes ([`AVX2_TEXT`]); +//! - `sve` (aarch64, the scalable vector extension): 128 to 2048-bit vectors, +//! the length the processor's ([`SVE_TEXT`]); +//! - `rvv` (riscv64, the vector extension 1.0): any vector length +//! ([`RVV_TEXT`]). +//! +//! **Why it is bit-exact.** Each kernel is the same transcription of +//! [`lanes::apply_words`] as the other members': each operand sign-extended to +//! 64 bits, each `fx(a, b) = (a·b + 2^29) >> 30` an exact 64-bit multiply, an +//! add and an arithmetic shift, each row the sum of four. The host gathers the +//! pairs a gate selects, the kernel forms their four rows a vector at a time, +//! and the host checks each row's range and keeps its low 32 bits, as +//! [`lanes::apply_words`] does. The scalable kernels set their vector length +//! per pass, so one text runs at every length; CI runs them at several. +//! +//! **Where it runs.** Discovery reads the processor's feature flags (on Linux, +//! the kernel's `AT_HWCAP`, which a user-mode emulator supplies for its guest). +//! Under an emulator the runner says so (`WAI_QUANTUM_VECTOR_EMULATOR`) and every +//! certificate carries it. Three mutants per instruction set are compiled in and +//! must each be caught. + +use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector}; +use crate::quantum_fabric::lanes::{self, GateWords, Lane}; +use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step}; +use crate::quantum_fabric::{Certificate, Certification, Corpus, FabricClass, KernelEntry, Member, MemberError, Presence, Provenance, RunFuture, RunMeter, Runner, apply_kernel}; + +#[cfg(target_arch = "x86_64")] +#[path = "quantum_fabric/vector_avx2.rs"] +mod avx2; + +/// One probe case: [`lanes::mul_wide`] and [`lanes::fxmul_wide`] as +/// `(high, low)` words. +pub type ProbeOutput = ((u32, u32), (u32, u32)); + +/// Why a vector unit is not a member, or a run has no state. +#[derive(Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum VectorError { + Unavailable(String), + Circuit(QuantumError), + TooWide { n_qubits: u8, max: u8 }, + OutOfRange { gate: u32 }, + Internal(&'static str), + Probe { a: i32, b: i32 }, +} + +impl std::fmt::Display for VectorError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + VectorError::Unavailable(why) => write!(f, "no vector unit: {why}"), + VectorError::Circuit(e) => write!(f, "{e}"), + VectorError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"), + VectorError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"), + VectorError::Internal(why) => write!(f, "host defect: {why}"), + VectorError::Probe { a, b } => write!(f, "the kernel's arithmetic differs from the CPU's at ({a}, {b})"), + } + } +} + +impl std::error::Error for VectorError {} + +impl From for MemberError { + fn from(e: VectorError) -> MemberError { + match e { + VectorError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max }, + VectorError::OutOfRange { gate } => MemberError::OutOfRange { gate }, + VectorError::Circuit(e) => MemberError::Circuit(e), + other => MemberError::Other(other.to_string()), + } + } +} + +/// The instruction sets a member runs. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Isa { + Avx2, + Sve, + Rvv, +} + +impl Isa { + pub const ALL: [Isa; 3] = [Isa::Avx2, Isa::Sve, Isa::Rvv]; + + /// The toolchain's name for the extension. + pub fn label(self) -> &'static str { + match self { + Isa::Avx2 => "avx2", + Isa::Sve => "sve", + Isa::Rvv => "rvv", + } + } + + /// The key a runner declares. + pub fn key(self) -> &'static str { + match self { + Isa::Avx2 => "cpu-vector-avx2", + Isa::Sve => "cpu-vector-sve", + Isa::Rvv => "cpu-vector-rvv", + } + } + + /// The processor architecture the extension belongs to. + pub fn arch(self) -> &'static str { + match self { + Isa::Avx2 => "x86_64", + Isa::Sve => "aarch64", + Isa::Rvv => "riscv64", + } + } +} + +/// The kernels a member runs: the conforming one, or a mutant. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Variant { + Conforming, + /// The rounding constant is half what it should be. + M1, + /// The product is shifted by one bit too few. + M2, + /// Row 0's real part adds its last term instead of subtracting it. + M3, +} + +impl Variant { + pub const MUTANTS: [Variant; 3] = [Variant::M1, Variant::M2, Variant::M3]; + + /// The kernel's constants: the rounding constant, the shift, and whether + /// row 0's last real term is added. + pub fn constants(self) -> (i64, i32, bool) { + match self { + Variant::Conforming => (1 << 29, 30, false), + Variant::M1 => (1 << 28, 30, false), + Variant::M2 => (1 << 29, 29, false), + Variant::M3 => (1 << 29, 30, true), + } + } +} + +// kernel-begin +/// The scalable vector extension's kernels: the probe (each `a·b` and +/// `fx(a, b)`) and the rows of a gate. `$rnd` is the rounding constant, `$sh` +/// the shift, `$s` the sign of row 0's last real term. +macro_rules! sve_rows_text { + ($rnd:literal, $sh:literal, $s:literal) => { + concat!( + ".arch_extension sve\n", + "ptrue p0.d\n", + "ldr x9, [{m}, #0]\nmov z16.d, x9\n", + "ldr x9, [{m}, #8]\nmov z17.d, x9\n", + "ldr x9, [{m}, #16]\nmov z18.d, x9\n", + "ldr x9, [{m}, #24]\nmov z19.d, x9\n", + "ldr x9, [{m}, #32]\nmov z20.d, x9\n", + "ldr x9, [{m}, #40]\nmov z21.d, x9\n", + "ldr x9, [{m}, #48]\nmov z22.d, x9\n", + "ldr x9, [{m}, #56]\nmov z23.d, x9\n", + "mov x9, #", $rnd, "\nmov z31.d, x9\n", + "mov x10, #0\n", + "2:\n", + "whilelt p1.d, x10, {n}\n", + "b.none 3f\n", + "ld1sw {{z0.d}}, p1/z, [{a0r}, x10, lsl #2]\n", + "ld1sw {{z1.d}}, p1/z, [{a0i}, x10, lsl #2]\n", + "ld1sw {{z2.d}}, p1/z, [{a1r}, x10, lsl #2]\n", + "ld1sw {{z3.d}}, p1/z, [{a1i}, x10, lsl #2]\n", + // Row 0, real. + "movprfx z4, z0\nmul z4.d, p0/m, z4.d, z16.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nmov z8.d, z4.d\n", + "movprfx z4, z1\nmul z4.d, p0/m, z4.d, z17.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nsub z8.d, z8.d, z4.d\n", + "movprfx z4, z2\nmul z4.d, p0/m, z4.d, z18.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z8.d, z8.d, z4.d\n", + "movprfx z4, z3\nmul z4.d, p0/m, z4.d, z19.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\n", $s, " z8.d, z8.d, z4.d\n", + // Row 0, imaginary. + "movprfx z4, z1\nmul z4.d, p0/m, z4.d, z16.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nmov z9.d, z4.d\n", + "movprfx z4, z0\nmul z4.d, p0/m, z4.d, z17.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z9.d, z9.d, z4.d\n", + "movprfx z4, z3\nmul z4.d, p0/m, z4.d, z18.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z9.d, z9.d, z4.d\n", + "movprfx z4, z2\nmul z4.d, p0/m, z4.d, z19.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z9.d, z9.d, z4.d\n", + // Row 1, real. + "movprfx z4, z0\nmul z4.d, p0/m, z4.d, z20.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nmov z10.d, z4.d\n", + "movprfx z4, z1\nmul z4.d, p0/m, z4.d, z21.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nsub z10.d, z10.d, z4.d\n", + "movprfx z4, z2\nmul z4.d, p0/m, z4.d, z22.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z10.d, z10.d, z4.d\n", + "movprfx z4, z3\nmul z4.d, p0/m, z4.d, z23.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nsub z10.d, z10.d, z4.d\n", + // Row 1, imaginary. + "movprfx z4, z1\nmul z4.d, p0/m, z4.d, z20.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nmov z11.d, z4.d\n", + "movprfx z4, z0\nmul z4.d, p0/m, z4.d, z21.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z11.d, z11.d, z4.d\n", + "movprfx z4, z3\nmul z4.d, p0/m, z4.d, z22.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z11.d, z11.d, z4.d\n", + "movprfx z4, z2\nmul z4.d, p0/m, z4.d, z23.d\nadd z4.d, z4.d, z31.d\nasr z4.d, z4.d, #", $sh, "\nadd z11.d, z11.d, z4.d\n", + "st1d {{z8.d}}, p1, [{o0r}, x10, lsl #3]\n", + "st1d {{z9.d}}, p1, [{o0i}, x10, lsl #3]\n", + "st1d {{z10.d}}, p1, [{o1r}, x10, lsl #3]\n", + "st1d {{z11.d}}, p1, [{o1i}, x10, lsl #3]\n", + "incd x10\n", + "b 2b\n", + "3:\n", + ) + }; +} + +macro_rules! sve_probe_text { + ($rnd:literal, $sh:literal) => { + concat!( + ".arch_extension sve\n", + "ptrue p0.d\n", + "mov x9, #", $rnd, "\nmov z31.d, x9\n", + "mov x10, #0\n", + "2:\n", + "whilelt p1.d, x10, {n}\n", + "b.none 3f\n", + "ld1sw {{z0.d}}, p1/z, [{a}, x10, lsl #2]\n", + "ld1sw {{z1.d}}, p1/z, [{b}, x10, lsl #2]\n", + "movprfx z4, z0\nmul z4.d, p0/m, z4.d, z1.d\n", + "add z5.d, z4.d, z31.d\nasr z5.d, z5.d, #", $sh, "\n", + "st1d {{z4.d}}, p1, [{prod}, x10, lsl #3]\n", + "st1d {{z5.d}}, p1, [{fx}, x10, lsl #3]\n", + "incd x10\n", + "b 2b\n", + "3:\n", + ) + }; +} + +/// One term of a riscv64 row: `fx(m[$k], $a)` into v12, then `$op` into `$acc` +/// (`vmv.v.v` starts the row). +macro_rules! rvv_term { + ($k:literal, $a:literal, $sh:literal, $op:literal, $acc:literal) => { + concat!( + "ld t1, ", $k, "({m})\n", + "vmul.vx v12, ", $a, ", t1\n", + "vadd.vx v12, v12, t2\n", + "vsra.vi v12, v12, ", $sh, "\n", + $op, " ", $acc, ", ", $acc, ", v12\n", + ) + }; +} + +/// The riscv64 vector extension's kernels. Each pass sets the vector length +/// for what remains (`vsetvli`), loads 32-bit operands and sign-extends them to +/// 64 bits. +macro_rules! rvv_rows_text { + ($rnd:literal, $sh:literal, $s:literal) => { + concat!( + ".option push\n", + ".option arch, +v\n", + "li t2, ", $rnd, "\n", + "2:\n", + "vsetvli t0, {n}, e32, mf2, ta, ma\n", + "beqz t0, 3f\n", + "vle32.v v1, ({a0r})\n", + "vle32.v v2, ({a0i})\n", + "vle32.v v3, ({a1r})\n", + "vle32.v v4, ({a1i})\n", + "vsetvli zero, t0, e64, m1, ta, ma\n", + "vsext.vf2 v8, v1\n", + "vsext.vf2 v9, v2\n", + "vsext.vf2 v10, v3\n", + "vsext.vf2 v11, v4\n", + "vmv.v.i v16, 0\nvmv.v.i v17, 0\nvmv.v.i v18, 0\nvmv.v.i v19, 0\n", + // Row 0, real. + rvv_term!("0", "v8", $sh, "vadd.vv", "v16"), + rvv_term!("8", "v9", $sh, "vsub.vv", "v16"), + rvv_term!("16", "v10", $sh, "vadd.vv", "v16"), + rvv_term!("24", "v11", $sh, $s, "v16"), + // Row 0, imaginary. + rvv_term!("0", "v9", $sh, "vadd.vv", "v17"), + rvv_term!("8", "v8", $sh, "vadd.vv", "v17"), + rvv_term!("16", "v11", $sh, "vadd.vv", "v17"), + rvv_term!("24", "v10", $sh, "vadd.vv", "v17"), + // Row 1, real. + rvv_term!("32", "v8", $sh, "vadd.vv", "v18"), + rvv_term!("40", "v9", $sh, "vsub.vv", "v18"), + rvv_term!("48", "v10", $sh, "vadd.vv", "v18"), + rvv_term!("56", "v11", $sh, "vsub.vv", "v18"), + // Row 1, imaginary. + rvv_term!("32", "v9", $sh, "vadd.vv", "v19"), + rvv_term!("40", "v8", $sh, "vadd.vv", "v19"), + rvv_term!("48", "v11", $sh, "vadd.vv", "v19"), + rvv_term!("56", "v10", $sh, "vadd.vv", "v19"), + "vse64.v v16, ({o0r})\n", + "vse64.v v17, ({o0i})\n", + "vse64.v v18, ({o1r})\n", + "vse64.v v19, ({o1i})\n", + "slli t3, t0, 2\n", + "add {a0r}, {a0r}, t3\nadd {a0i}, {a0i}, t3\nadd {a1r}, {a1r}, t3\nadd {a1i}, {a1i}, t3\n", + "slli t3, t0, 3\n", + "add {o0r}, {o0r}, t3\nadd {o0i}, {o0i}, t3\nadd {o1r}, {o1r}, t3\nadd {o1i}, {o1i}, t3\n", + "sub {n}, {n}, t0\n", + "j 2b\n", + "3:\n", + ".option pop\n", + ) + }; +} + +macro_rules! rvv_probe_text { + ($rnd:literal, $sh:literal) => { + concat!( + ".option push\n", + ".option arch, +v\n", + "li t2, ", $rnd, "\n", + "2:\n", + "vsetvli t0, {n}, e32, mf2, ta, ma\n", + "beqz t0, 3f\n", + "vle32.v v1, ({a})\n", + "vle32.v v2, ({b})\n", + "vsetvli zero, t0, e64, m1, ta, ma\n", + "vsext.vf2 v8, v1\n", + "vsext.vf2 v9, v2\n", + "vmul.vv v12, v8, v9\n", + "vse64.v v12, ({prod})\n", + "vadd.vx v13, v12, t2\n", + "vsra.vi v13, v13, ", $sh, "\n", + "vse64.v v13, ({fx})\n", + "slli t3, t0, 2\n", + "add {a}, {a}, t3\nadd {b}, {b}, t3\n", + "slli t3, t0, 3\n", + "add {prod}, {prod}, t3\nadd {fx}, {fx}, t3\n", + "sub {n}, {n}, t0\n", + "j 2b\n", + "3:\n", + ".option pop\n", + ) + }; +} +// kernel-end + +/// One scalable kernel: the rows of `n` pairs. +macro_rules! sve_rows_fn { + ($name:ident, $rnd:literal, $sh:literal, $s:literal) => { + #[cfg(target_arch = "aarch64")] + #[allow(clippy::too_many_arguments)] + /// # Safety + /// + /// The processor has the scalable vector extension; every slice holds + /// `n` values. + unsafe fn $name(m: &[i64; 8], n: usize, a0r: &[i32], a0i: &[i32], a1r: &[i32], a1i: &[i32], o0r: &mut [i64], o0i: &mut [i64], o1r: &mut [i64], o1i: &mut [i64]) { + debug_assert!([a0r.len(), a0i.len(), a1r.len(), a1i.len(), o0r.len(), o0i.len(), o1r.len(), o1i.len()].iter().all(|&l| l >= n)); + // SAFETY: the caller's contract; every vector and predicate + // register the kernel writes is declared clobbered. + unsafe { + std::arch::asm!( + sve_rows_text!($rnd, $sh, $s), + m = in(reg) m.as_ptr(), + n = in(reg) n, + a0r = in(reg) a0r.as_ptr(), + a0i = in(reg) a0i.as_ptr(), + a1r = in(reg) a1r.as_ptr(), + a1i = in(reg) a1i.as_ptr(), + o0r = in(reg) o0r.as_mut_ptr(), + o0i = in(reg) o0i.as_mut_ptr(), + o1r = in(reg) o1r.as_mut_ptr(), + o1i = in(reg) o1i.as_mut_ptr(), + out("x9") _, out("x10") _, out("p0") _, out("p1") _, + out("v0") _, out("v1") _, out("v2") _, out("v3") _, out("v4") _, out("v5") _, out("v6") _, out("v7") _, + out("v8") _, out("v9") _, out("v10") _, out("v11") _, out("v12") _, out("v13") _, out("v14") _, out("v15") _, + out("v16") _, out("v17") _, out("v18") _, out("v19") _, out("v20") _, out("v21") _, out("v22") _, out("v23") _, + out("v24") _, out("v25") _, out("v26") _, out("v27") _, out("v28") _, out("v29") _, out("v30") _, out("v31") _, + options(nostack), + ); + } + } + }; +} + +macro_rules! sve_probe_fn { + ($name:ident, $rnd:literal, $sh:literal) => { + #[cfg(target_arch = "aarch64")] + /// # Safety + /// + /// The processor has the scalable vector extension; every slice holds + /// `n` values. + unsafe fn $name(n: usize, a: &[i32], b: &[i32], prod: &mut [i64], fx: &mut [i64]) { + debug_assert!(a.len() >= n && b.len() >= n && prod.len() >= n && fx.len() >= n); + // SAFETY: as for the rows. + unsafe { + std::arch::asm!( + sve_probe_text!($rnd, $sh), + n = in(reg) n, + a = in(reg) a.as_ptr(), + b = in(reg) b.as_ptr(), + prod = in(reg) prod.as_mut_ptr(), + fx = in(reg) fx.as_mut_ptr(), + out("x9") _, out("x10") _, out("p0") _, out("p1") _, + out("v0") _, out("v1") _, out("v2") _, out("v3") _, out("v4") _, out("v5") _, out("v6") _, out("v7") _, + out("v8") _, out("v9") _, out("v10") _, out("v11") _, out("v12") _, out("v13") _, out("v14") _, out("v15") _, + out("v16") _, out("v17") _, out("v18") _, out("v19") _, out("v20") _, out("v21") _, out("v22") _, out("v23") _, + out("v24") _, out("v25") _, out("v26") _, out("v27") _, out("v28") _, out("v29") _, out("v30") _, out("v31") _, + options(nostack), + ); + } + } + }; +} + +macro_rules! rvv_rows_fn { + ($name:ident, $rnd:literal, $sh:literal, $s:literal) => { + #[cfg(target_arch = "riscv64")] + #[allow(clippy::too_many_arguments)] + /// # Safety + /// + /// The processor has the vector extension; every slice holds `n` + /// values. + unsafe fn $name(m: &[i64; 8], n: usize, a0r: &[i32], a0i: &[i32], a1r: &[i32], a1i: &[i32], o0r: &mut [i64], o0i: &mut [i64], o1r: &mut [i64], o1i: &mut [i64]) { + debug_assert!([a0r.len(), a0i.len(), a1r.len(), a1i.len(), o0r.len(), o0i.len(), o1r.len(), o1i.len()].iter().all(|&l| l >= n)); + // SAFETY: the caller's contract; the pointers and count are + // advanced in registers the kernel owns, and every vector register + // and temporary it writes is declared clobbered. + unsafe { + std::arch::asm!( + rvv_rows_text!($rnd, $sh, $s), + m = in(reg) m.as_ptr(), + n = inout(reg) n => _, + a0r = inout(reg) a0r.as_ptr() => _, + a0i = inout(reg) a0i.as_ptr() => _, + a1r = inout(reg) a1r.as_ptr() => _, + a1i = inout(reg) a1i.as_ptr() => _, + o0r = inout(reg) o0r.as_mut_ptr() => _, + o0i = inout(reg) o0i.as_mut_ptr() => _, + o1r = inout(reg) o1r.as_mut_ptr() => _, + o1i = inout(reg) o1i.as_mut_ptr() => _, + out("t0") _, out("t1") _, out("t2") _, out("t3") _, + out("v0") _, out("v1") _, out("v2") _, out("v3") _, out("v4") _, out("v5") _, out("v6") _, out("v7") _, + out("v8") _, out("v9") _, out("v10") _, out("v11") _, out("v12") _, out("v13") _, out("v14") _, out("v15") _, + out("v16") _, out("v17") _, out("v18") _, out("v19") _, out("v20") _, out("v21") _, out("v22") _, out("v23") _, + out("v24") _, out("v25") _, out("v26") _, out("v27") _, out("v28") _, out("v29") _, out("v30") _, out("v31") _, + options(nostack), + ); + } + } + }; +} + +macro_rules! rvv_probe_fn { + ($name:ident, $rnd:literal, $sh:literal) => { + #[cfg(target_arch = "riscv64")] + /// # Safety + /// + /// The processor has the vector extension; every slice holds `n` + /// values. + unsafe fn $name(n: usize, a: &[i32], b: &[i32], prod: &mut [i64], fx: &mut [i64]) { + debug_assert!(a.len() >= n && b.len() >= n && prod.len() >= n && fx.len() >= n); + // SAFETY: as for the rows. + unsafe { + std::arch::asm!( + rvv_probe_text!($rnd, $sh), + n = inout(reg) n => _, + a = inout(reg) a.as_ptr() => _, + b = inout(reg) b.as_ptr() => _, + prod = inout(reg) prod.as_mut_ptr() => _, + fx = inout(reg) fx.as_mut_ptr() => _, + out("t0") _, out("t1") _, out("t2") _, out("t3") _, + out("v0") _, out("v1") _, out("v2") _, out("v3") _, out("v4") _, out("v5") _, out("v6") _, out("v7") _, + out("v8") _, out("v9") _, out("v10") _, out("v11") _, out("v12") _, out("v13") _, out("v14") _, out("v15") _, + out("v16") _, out("v17") _, out("v18") _, out("v19") _, out("v20") _, out("v21") _, out("v22") _, out("v23") _, + out("v24") _, out("v25") _, out("v26") _, out("v27") _, out("v28") _, out("v29") _, out("v30") _, out("v31") _, + options(nostack), + ); + } + } + }; +} + +sve_rows_fn!(sve_rows_conforming, "0x20000000", "30", "sub"); +sve_rows_fn!(sve_rows_m1, "0x10000000", "30", "sub"); +sve_rows_fn!(sve_rows_m2, "0x20000000", "29", "sub"); +sve_rows_fn!(sve_rows_m3, "0x20000000", "30", "add"); +sve_probe_fn!(sve_probe_conforming, "0x20000000", "30"); +sve_probe_fn!(sve_probe_m1, "0x10000000", "30"); +sve_probe_fn!(sve_probe_m2, "0x20000000", "29"); +rvv_rows_fn!(rvv_rows_conforming, "0x20000000", "30", "vsub.vv"); +rvv_rows_fn!(rvv_rows_m1, "0x10000000", "30", "vsub.vv"); +rvv_rows_fn!(rvv_rows_m2, "0x20000000", "29", "vsub.vv"); +rvv_rows_fn!(rvv_rows_m3, "0x20000000", "30", "vadd.vv"); +rvv_probe_fn!(rvv_probe_conforming, "0x20000000", "30"); +rvv_probe_fn!(rvv_probe_m1, "0x10000000", "30"); +rvv_probe_fn!(rvv_probe_m2, "0x20000000", "29"); + +/// The text the conforming x86-64 kernels are compiled from. +pub const AVX2_TEXT: &str = include_str!("quantum_fabric/vector_avx2.rs"); + +/// The text the conforming scalable kernels are assembled from. +pub const SVE_TEXT: &str = concat!(sve_rows_text!("0x20000000", "30", "sub"), sve_probe_text!("0x20000000", "30")); + +/// The text the conforming riscv64 kernels are assembled from. +pub const RVV_TEXT: &str = concat!(rvv_rows_text!("0x20000000", "30", "vsub.vv"), rvv_probe_text!("0x20000000", "30")); + +/// A variant's kernel text for `isa`. +fn text_of(isa: Isa, v: Variant) -> &'static str { + match (isa, v) { + (Isa::Avx2, _) => AVX2_TEXT, + (Isa::Sve, Variant::Conforming) => SVE_TEXT, + (Isa::Sve, Variant::M1) => concat!(sve_rows_text!("0x10000000", "30", "sub"), sve_probe_text!("0x10000000", "30")), + (Isa::Sve, Variant::M2) => concat!(sve_rows_text!("0x20000000", "29", "sub"), sve_probe_text!("0x20000000", "29")), + (Isa::Sve, Variant::M3) => concat!(sve_rows_text!("0x20000000", "30", "add"), sve_probe_text!("0x20000000", "30")), + (Isa::Rvv, Variant::Conforming) => RVV_TEXT, + (Isa::Rvv, Variant::M1) => concat!(rvv_rows_text!("0x10000000", "30", "vsub.vv"), rvv_probe_text!("0x10000000", "30")), + (Isa::Rvv, Variant::M2) => concat!(rvv_rows_text!("0x20000000", "29", "vsub.vv"), rvv_probe_text!("0x20000000", "29")), + (Isa::Rvv, Variant::M3) => concat!(rvv_rows_text!("0x20000000", "30", "vadd.vv"), rvv_probe_text!("0x20000000", "30")), + } +} + +/// Version of the host layout: the gather of a gate's pairs into four +/// columns (padded with zeros to whole vectors where the kernel needs it), the +/// 64-bit row outputs, and the host's range check. Part of the implementation +/// id. +pub const HOST_LAYOUT: u32 = 1; + +/// `"-lanes64/<16 hex>"` over the instruction set, a variant's kernel +/// text and constants, and the layout. +pub fn implementation_id_of(isa: Isa, v: Variant) -> String { + let text = text_of(isa, v); + let (rnd, sh, last_add) = v.constants(); + let mut o = b"wai:quantum-fabric-vector\x01".to_vec(); + o.extend_from_slice(isa.label().as_bytes()); + o.push(0); + o.extend_from_slice(&(text.len() as u64).to_be_bytes()); + o.extend_from_slice(text.as_bytes()); + o.extend_from_slice(&rnd.to_be_bytes()); + o.extend_from_slice(&sh.to_be_bytes()); + o.push(u8::from(last_add)); + o.extend_from_slice(&HOST_LAYOUT.to_be_bytes()); + let h: String = blake3::hash(&o).as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect(); + format!("{}-lanes64/{h}", isa.label()) +} + +/// The instruction set this processor's architecture would use. +pub fn native_isa() -> Option { + Isa::ALL.into_iter().find(|i| i.arch() == std::env::consts::ARCH) +} + +/// The kernel's `AT_HWCAP` on Linux: the word whose bits name the base +/// extensions (an emulator supplies its guest's). +#[cfg(all(target_os = "linux", target_arch = "riscv64"))] +fn hwcap() -> Result { + const AT_HWCAP: u64 = 16; + let auxv = std::fs::read("/proc/self/auxv").map_err(|e| format!("/proc/self/auxv: {e}"))?; + auxv.as_chunks::<16>() + .0 + .iter() + .find_map(|e| { + let (k, v) = e.split_at(8); + (u64::from_ne_bytes(k.try_into().expect("8 bytes")) == AT_HWCAP).then(|| u64::from_ne_bytes(v.try_into().expect("8 bytes"))) + }) + .ok_or_else(|| "the kernel reports no AT_HWCAP".to_string()) +} + +/// Whether this processor has `isa`'s vector unit, by its feature flags. +pub fn has(isa: Isa) -> Result<(), String> { + if isa.arch() != std::env::consts::ARCH { + return Err(format!("this processor is {}, not {}", std::env::consts::ARCH, isa.arch())); + } + match isa { + Isa::Avx2 => { + #[cfg(target_arch = "x86_64")] + { + if std::arch::is_x86_feature_detected!("avx2") { Ok(()) } else { Err("the processor reports no avx2".into()) } + } + #[cfg(not(target_arch = "x86_64"))] + { + unreachable!("the architecture was checked") + } + } + Isa::Sve => { + #[cfg(target_arch = "aarch64")] + { + if std::arch::is_aarch64_feature_detected!("sve") { Ok(()) } else { Err("the processor reports no sve".into()) } + } + #[cfg(not(target_arch = "aarch64"))] + { + unreachable!("the architecture was checked") + } + } + Isa::Rvv => { + #[cfg(all(target_os = "linux", target_arch = "riscv64"))] + { + // Bit `'V' − 'A'` of AT_HWCAP is the vector extension. + const HWCAP_V: u64 = 1 << (b'V' - b'A'); + if hwcap()? & HWCAP_V != 0 { Ok(()) } else { Err("the kernel reports no vector extension (AT_HWCAP bit 21)".into()) } + } + #[cfg(all(target_arch = "riscv64", not(target_os = "linux")))] + { + Err("feature detection is written for Linux".into()) + } + #[cfg(not(target_arch = "riscv64"))] + { + unreachable!("the architecture was checked") + } + } + } +} + +/// The vector length in bits, read from the unit itself. +fn vector_bits(isa: Isa) -> u32 { + match isa { + Isa::Avx2 => 256, + Isa::Sve => { + #[cfg(target_arch = "aarch64")] + { + let bytes: u64; + // SAFETY: called only once the processor reports the extension; + // `cntb` reads the vector length and writes one register. + unsafe { std::arch::asm!(".arch_extension sve", "cntb {b}", b = out(reg) bytes, options(nomem, nostack, pure)) }; + (bytes * 8) as u32 + } + #[cfg(not(target_arch = "aarch64"))] + { + unreachable!("the architecture was checked") + } + } + Isa::Rvv => { + #[cfg(target_arch = "riscv64")] + { + let bytes: u64; + // SAFETY: called only once the kernel reports the extension; + // `vlenb` is a read-only register. + unsafe { std::arch::asm!(".option push", ".option arch, +v", "csrr {b}, vlenb", ".option pop", b = out(reg) bytes, options(nomem, nostack)) }; + (bytes * 8) as u32 + } + #[cfg(not(target_arch = "riscv64"))] + { + unreachable!("the architecture was checked") + } + } + } +} + +/// What [`discover`] found. +#[derive(Debug)] +pub struct Discovery { + pub found: Vec, + pub refused: Vec<(String, VectorError)>, + pub unavailable: Option, +} + +/// This processor's vector unit, probed in this process, or why there is none. +pub fn discover() -> Discovery { + let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None }; + let Some(isa) = native_isa() else { + d.unavailable = Some(format!("no vector kernel is written for {}", std::env::consts::ARCH)); + return d; + }; + if let Err(why) = has(isa) { + d.unavailable = Some(why); + return d; + } + match VectorMember::open(isa, Variant::Conforming) { + Ok(m) => d.found.push(m), + Err(e) => d.refused.push((isa.key().into(), e)), + } + d +} + +/// The most qubits a run holds. +pub const MAX_VECTOR_QUBITS: u8 = if 24 < MAX_QUBITS { 24 } else { MAX_QUBITS }; + +/// A processor's vector unit as a fabric member. +#[derive(Clone, Debug)] +pub struct VectorMember { + isa: Isa, + variant: Variant, + bits: u32, + id: String, + implementation: String, + detail: String, + provenance: Provenance, + max_qubits: u8, +} + +/// `v` padded with zeros to a whole number of `lanes`. +#[cfg(target_arch = "x86_64")] +fn padded(v: &[i32], lanes: usize) -> Vec { + let mut p = v.to_vec(); + p.resize(v.len().next_multiple_of(lanes), 0); + p +} + +impl VectorMember { + fn open(isa: Isa, variant: Variant) -> Result { + let emulator = std::env::var("WAI_QUANTUM_VECTOR_EMULATOR").ok().filter(|s| !s.trim().is_empty()); + let provenance = emulator.as_ref().map_or(Provenance::Silicon, |e| Provenance::Emulator(e.trim().to_string())); + let bits = vector_bits(isa); + let mut o = b"wai:quantum-fabric-vector-member\x01".to_vec(); + o.extend_from_slice(isa.label().as_bytes()); + o.extend_from_slice(&bits.to_be_bytes()); + o.extend_from_slice(std::env::consts::OS.as_bytes()); + o.extend_from_slice(provenance.label().as_bytes()); + let hex: String = blake3::hash(&o).as_bytes()[..6].iter().map(|b| format!("{b:02x}")).collect(); + // The CPU reference's id with the unit after it: it sorts after the + // reference, so with no energy figures placement keeps the reference. + let id = format!("local/cpu-{}-vector-{}-{hex}", isa.arch(), isa.label()); + let member = VectorMember { + isa, + variant, + bits, + id, + implementation: implementation_id_of(isa, variant), + detail: format!("cpu; the processor's vector extension ({}), {bits}-bit vectors, 64-bit lanes; {}; up to {MAX_VECTOR_QUBITS} qubits; unmetered", isa.label(), provenance.label()), + provenance, + max_qubits: MAX_VECTOR_QUBITS, + }; + if variant == Variant::Conforming { + let pairs = lanes::probe_pairs(4096, 0x5eed); + let out = member.probe(&pairs)?; + if let Some((a, b)) = lanes::first_mismatch(&pairs, &out) { + return Err(VectorError::Probe { a, b }); + } + } + Ok(member) + } + + /// The same processor running a mutant: for the mutation tests. + #[doc(hidden)] + pub fn with_variant(&self, v: Variant) -> VectorMember { + VectorMember { variant: v, implementation: implementation_id_of(self.isa, v), ..self.clone() } + } + + #[doc(hidden)] + pub fn with_max_qubits(&self, max_qubits: u8) -> VectorMember { + VectorMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() } + } + + pub fn isa(&self) -> Isa { + self.isa + } + + /// The kernel this member runs: the conforming one, or a mutant. + pub fn variant(&self) -> Variant { + self.variant + } + + /// The vector length, in bits. + pub fn vector_bits(&self) -> u32 { + self.bits + } + + pub fn id(&self) -> &str { + &self.id + } + + pub fn key(&self) -> &str { + self.isa.key() + } + + pub fn class(&self) -> FabricClass { + FabricClass::Cpu + } + + pub fn detail(&self) -> &str { + &self.detail + } + + pub fn implementation(&self) -> &str { + &self.implementation + } + + pub fn provenance(&self) -> &Provenance { + &self.provenance + } + + pub fn max_qubits(&self) -> u8 { + self.max_qubits + } + + pub fn config(&self) -> Config { + Config::PER_GATE + } + + /// The kernel's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each pair. + pub fn probe(&self, pairs: &[(i32, i32)]) -> Result, VectorError> { + let a: Vec = pairs.iter().map(|p| p.0).collect(); + let b: Vec = pairs.iter().map(|p| p.1).collect(); + let (prod, fx) = self.probe_kernel(&a, &b)?; + let split = |v: i64| ((v as u64 >> 32) as u32, v as u64 as u32); + Ok((0..pairs.len()).map(|k| (split(prod[k]), split(fx[k]))).collect()) + } + + #[allow(unreachable_code, unused_variables)] + fn probe_kernel(&self, a: &[i32], b: &[i32]) -> Result<(Vec, Vec), VectorError> { + let n = a.len(); + match self.isa { + Isa::Avx2 => { + #[cfg(target_arch = "x86_64")] + { + let (a, b) = (padded(a, avx2::LANES), padded(b, avx2::LANES)); + let (mut prod, mut fx) = (vec![0i64; a.len()], vec![0i64; a.len()]); + // SAFETY: the member exists only where the processor + // reports avx2; every slice holds whole vectors. + unsafe { + match self.variant { + Variant::M1 => avx2::probe::<{ 1 << 28 }, 30>(&a, &b, &mut prod, &mut fx), + Variant::M2 => avx2::probe::<{ 1 << 29 }, 29>(&a, &b, &mut prod, &mut fx), + Variant::Conforming | Variant::M3 => avx2::probe::<{ 1 << 29 }, 30>(&a, &b, &mut prod, &mut fx), + } + } + prod.truncate(n); + fx.truncate(n); + return Ok((prod, fx)); + } + } + Isa::Sve => { + #[cfg(target_arch = "aarch64")] + { + let (mut prod, mut fx) = (vec![0i64; n], vec![0i64; n]); + // SAFETY: the member exists only where the processor + // reports the extension; every slice holds n values. + unsafe { + match self.variant { + Variant::M1 => sve_probe_m1(n, a, b, &mut prod, &mut fx), + Variant::M2 => sve_probe_m2(n, a, b, &mut prod, &mut fx), + Variant::Conforming | Variant::M3 => sve_probe_conforming(n, a, b, &mut prod, &mut fx), + } + } + return Ok((prod, fx)); + } + } + Isa::Rvv => { + #[cfg(target_arch = "riscv64")] + { + let (mut prod, mut fx) = (vec![0i64; n], vec![0i64; n]); + // SAFETY: as above. + unsafe { + match self.variant { + Variant::M1 => rvv_probe_m1(n, a, b, &mut prod, &mut fx), + Variant::M2 => rvv_probe_m2(n, a, b, &mut prod, &mut fx), + Variant::Conforming | Variant::M3 => rvv_probe_conforming(n, a, b, &mut prod, &mut fx), + } + } + return Ok((prod, fx)); + } + } + } + Err(VectorError::Unavailable(format!("no {} kernel in a build for {}", self.isa.label(), std::env::consts::ARCH))) + } + + /// The four rows of each pair, from the kernel. + #[allow(unreachable_code, unused_variables)] + fn rows(&self, m: &[i32; 8], cols: &[Vec; 4]) -> Result<[Vec; 4], VectorError> { + let n = cols[0].len(); + match self.isa { + Isa::Avx2 => { + #[cfg(target_arch = "x86_64")] + { + let c = [padded(&cols[0], avx2::LANES), padded(&cols[1], avx2::LANES), padded(&cols[2], avx2::LANES), padded(&cols[3], avx2::LANES)]; + let len = c[0].len(); + let mut o = [vec![0i64; len], vec![0i64; len], vec![0i64; len], vec![0i64; len]]; + { + let [o0, o1, o2, o3] = &mut o; + let (c, out) = ([&c[0][..], &c[1][..], &c[2][..], &c[3][..]], [&mut o0[..], &mut o1[..], &mut o2[..], &mut o3[..]]); + // SAFETY: the member exists only where the processor + // reports avx2; every slice holds whole vectors. + unsafe { + match self.variant { + Variant::Conforming => avx2::rows::<{ 1 << 29 }, 30, false>(m, c, out), + Variant::M1 => avx2::rows::<{ 1 << 28 }, 30, false>(m, c, out), + Variant::M2 => avx2::rows::<{ 1 << 29 }, 29, false>(m, c, out), + Variant::M3 => avx2::rows::<{ 1 << 29 }, 30, true>(m, c, out), + } + } + } + for v in &mut o { + v.truncate(n); + } + return Ok(o); + } + } + Isa::Sve => { + #[cfg(target_arch = "aarch64")] + { + let m64 = m.map(i64::from); + let mut o = [vec![0i64; n], vec![0i64; n], vec![0i64; n], vec![0i64; n]]; + let [o0r, o0i, o1r, o1i] = &mut o; + // SAFETY: the member exists only where the processor + // reports the extension; every slice holds n values. + unsafe { + let f = match self.variant { + Variant::Conforming => sve_rows_conforming, + Variant::M1 => sve_rows_m1, + Variant::M2 => sve_rows_m2, + Variant::M3 => sve_rows_m3, + }; + f(&m64, n, &cols[0], &cols[1], &cols[2], &cols[3], o0r, o0i, o1r, o1i); + } + return Ok(o); + } + } + Isa::Rvv => { + #[cfg(target_arch = "riscv64")] + { + let m64 = m.map(i64::from); + let mut o = [vec![0i64; n], vec![0i64; n], vec![0i64; n], vec![0i64; n]]; + let [o0r, o0i, o1r, o1i] = &mut o; + // SAFETY: as above. + unsafe { + let f = match self.variant { + Variant::Conforming => rvv_rows_conforming, + Variant::M1 => rvv_rows_m1, + Variant::M2 => rvv_rows_m2, + Variant::M3 => rvv_rows_m3, + }; + f(&m64, n, &cols[0], &cols[1], &cols[2], &cols[3], o0r, o0i, o1r, o1i); + } + return Ok(o); + } + } + } + Err(VectorError::Unavailable(format!("no {} kernel in a build for {}", self.isa.label(), std::env::consts::ARCH))) + } + + /// Gates in order from `state`, as [`lanes::run_words`] runs them. + pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result { + let len = state.len(); + if len < 2 || !len.is_power_of_two() { + return Err(VectorError::Internal("a state that is not 2^n lanes, n at least 1")); + } + let n = len.trailing_zeros() as u8; + if n > self.max_qubits { + return Err(VectorError::TooWide { n_qubits: n, max: self.max_qubits }); + } + let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect(); + self.execute(state.to_vec(), &planned) + } + + fn execute(&self, mut amps: Vec, gates: &[(&GateWords, u32)]) -> Result { + let len = amps.len(); + for (w, _) in gates { + let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize); + if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 { + return Err(VectorError::Internal("malformed gate words for this state")); + } + } + let mut report: Option = None; + for &(w, index) in gates { + let m: [i32; 8] = w.m.map(|x| x as i32); + let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize); + let mut at = Vec::new(); + let mut cols: [Vec; 4] = Default::default(); + for p in 0..len / 2 { + let low = p & (tbit - 1); + let i = ((p - low) << 1) | low; + if i & ctrl != ctrl { + continue; + } + at.push(i); + let (a0, a1) = (amps[i], amps[i | tbit]); + cols[0].push(a0[0]); + cols[1].push(a0[1]); + cols[2].push(a1[0]); + cols[3].push(a1[1]); + } + let [o0r, o0i, o1r, o1i] = self.rows(&m, &cols)?; + for (k, &i) in at.iter().enumerate() { + let rows = [o0r[k], o0i[k], o1r[k], o1i[k]]; + if rows.iter().any(|&v| i32::try_from(v).is_err()) { + report = Some(report.map_or(index, |r| r.min(index))); + } + let low = |v: i64| v as u64 as u32 as i32; + amps[i] = [low(rows[0]), low(rows[1])]; + amps[i | tbit] = [low(rows[2]), low(rows[3])]; + } + } + Ok(LaneRun { amps, out_of_range: report }) + } + + pub fn run_program(&self, p: &Program) -> Result { + let n = p.n_qubits; + if n == 0 { + return Ok(LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None }); + } + if n > self.max_qubits { + return Err(VectorError::TooWide { n_qubits: n, max: self.max_qubits }); + } + if p.start >= 1usize << n { + return Err(VectorError::Internal("the start is outside the state")); + } + let mut gates = Vec::with_capacity(p.steps.len()); + for step in &p.steps { + let Step::Gate { gate } = *step else { return Err(VectorError::Internal("a tile step; this member runs gate steps only")) }; + let g = p.gates.get(gate as usize).ok_or(VectorError::Internal("a step names a gate the program does not hold"))?; + if !g.is_well_formed() { + return Err(VectorError::Internal("malformed gate words for this state")); + } + gates.push((&g.words, g.index)); + } + let mut amps = vec![[0i32, 0]; 1usize << n]; + amps[p.start] = [ONE as i32, 0]; + self.execute(amps, &gates) + } + + pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result { + let p = plan::compile_with(c, start, &self.config()).map_err(VectorError::Circuit)?; + if p.n_qubits > self.max_qubits { + return Err(VectorError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits }); + } + let run = self.run_program(&p)?; + if let Some(gate) = run.out_of_range { + return Err(VectorError::OutOfRange { gate }); + } + Ok(StateVector { n_qubits: p.n_qubits, amps: run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }).collect() }) + } + + pub fn simulate(&self, c: &Circuit) -> Result { + self.simulate_from(c, 0) + } + + /// The certificate of [`apply_kernel`]`(gs)` on this member, with its + /// provenance. + pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result { + let mut cert = Certification::new(); + for c in corpus.circuits(gs) { + let reference = lanes::reference_output(&c).map_err(VectorError::Circuit)?; + let run = self.run_program(&plan::compile_with(&c, 0, &self.config()).map_err(VectorError::Circuit)?)?; + if let Some(gate) = run.out_of_range { + return Err(VectorError::OutOfRange { gate }); + } + cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps)); + } + Ok(cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)).with_provenance(self.provenance.clone())) + } + + pub fn member(&self, corpus: Corpus) -> Result { + let mut kernels = Vec::new(); + for gs in Gateset::ALL { + kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate: self.certify(gs, corpus)?, energy: None }); + } + Ok(Member { id: self.id.clone(), class: FabricClass::Cpu, presence: Presence::Detected, kernels, detail: self.detail.clone() }) + } +} + +impl Runner for VectorMember { + fn member_id(&self) -> String { + self.id.clone() + } + + fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, _meter: &'r mut dyn RunMeter) -> RunFuture<'r> { + Box::pin(async move { + if implementation != self.implementation { + return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation))); + } + Ok((self.simulate_from(c, start)?, 0, None)) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_variant_of_every_isa_has_its_own_implementation_id() { + let ids: std::collections::BTreeSet = Isa::ALL + .into_iter() + .flat_map(|isa| [Variant::Conforming, Variant::M1, Variant::M2, Variant::M3].into_iter().map(move |v| implementation_id_of(isa, v))) + .collect(); + assert_eq!(ids.len(), 12); + for isa in Isa::ALL { + assert!(implementation_id_of(isa, Variant::Conforming).starts_with(&format!("{}-lanes64/", isa.label()))); + } + assert!(AVX2_TEXT.contains("_mm256_mul_epi32")); + assert!(SVE_TEXT.contains("whilelt") && !SVE_TEXT.contains("smstart")); + assert!(RVV_TEXT.contains("vsetvli") && RVV_TEXT.contains("vsext.vf2")); + } + + #[test] + fn the_mutants_differ_in_their_constants() { + assert_eq!(Variant::Conforming.constants(), (1 << 29, 30, false)); + for v in Variant::MUTANTS { + assert_ne!(v.constants(), Variant::Conforming.constants()); + } + } + + #[test] + fn the_member_id_sorts_after_the_cpu_reference() { + for isa in Isa::ALL { + let reference = format!("local/cpu-{}", isa.arch()); + let member = format!("local/cpu-{}-vector-{}-000000000000", isa.arch(), isa.label()); + assert!(member > reference, "{member} vs {reference}"); + } + } + + #[test] + fn a_processor_without_a_kernel_or_the_unit_says_why() { + let d = discover(); + if d.found.is_empty() { + assert!(d.refused.is_empty() && d.unavailable.is_some(), "{d:?}"); + } + for isa in Isa::ALL.into_iter().filter(|i| i.arch() != std::env::consts::ARCH) { + assert!(has(isa).unwrap_err().contains("this processor is")); + } + } +} diff --git a/wai/wai-quantum/tests/vector_member.rs b/wai/wai-quantum/tests/vector_member.rs new file mode 100644 index 000000000..1c91797a2 --- /dev/null +++ b/wai/wai-quantum/tests/vector_member.rs @@ -0,0 +1,410 @@ +//! The fabric's CPU vector-unit member on the machine that runs this. +//! +//! The machine declares the devices it must have in `WAI_QUANTUM_VECTOR_EXPECT`: +//! their keys, comma-separated and sorted, or `none`. Every test that needs a +//! member fails unless the devices found are exactly the declared ones, so a +//! machine whose GPU or driver went missing, or a run that declared nothing, +//! fails instead of passing over no member. Each test prints what it +//! established and, when `WAI_QUANTUM_VECTOR_REPORT` names a file, appends the +//! same lines to it, which a CI step can count per device. +//! +//! ```text +//! WAI_QUANTUM_VECTOR_EXPECT=cpu-vector-avx2 \ +//! cargo test --release --features quantum_fabric_vector --test vector_member -- --test-threads=1 +//! ``` +#![cfg(not(target_arch = "wasm32"))] + +use std::io::Write; +use std::sync::OnceLock; +use wai_quantum::ed25519_dalek::SigningKey; +use wai_quantum::quantum::{BaseGate, Circuit, Gate, Gateset, MAX_QUBITS, QuantumError}; +use wai_quantum::quantum_fabric::lanes::{self, GateWords, Lane}; +use wai_quantum::quantum_fabric::plan::{Program, Step}; +use wai_quantum::quantum_fabric::{ + Corpus, CpuReference, FABRIC_DEFAULT, Fabric, FabricClass, FabricReceipt, MemberError, Parity, Presence, REFERENCE, apply_kernel, detect_local, place, ready, +}; +use wai_quantum::quantum_fabric_vector::{Discovery, VectorError, VectorMember, Variant, discover, implementation_id_of}; + +/// The CPU's corpus digests: random-v1 (seed 7, 200 cases) and fabric-v1 +/// (seed 7, 200 + 24 cases), per gateset. A bit-exact member reproduces them. +const PINS: [(Gateset, &str, &str); 2] = [ + (Gateset::V1, "d18454c639acfd9c7b66450f4ded1aac4ab46cc10a7bedbcf4958a96e204fb36", "41c726a0ac80f7defef92e15ecf893bdb7a3076b5e7f670fa0b7142e5c033df7"), + (Gateset::V2, "3dbde276c1095bf827faec71d5980e050f8a55a5ba7d97cd3bbdaec3f43a13b2", "9a84b55af68f1a8b07390b234cf562dd8d85cef3185a697ab85690c8790a7809"), +]; + +const RANDOM_V1: Corpus = Corpus::Random { seed: 7, cases: 200 }; + + +static DISCOVERY: OnceLock = OnceLock::new(); + +fn discovery() -> &'static Discovery { + DISCOVERY.get_or_init(discover) +} + +/// Prints `line` and appends it to the file `WAI_QUANTUM_VECTOR_REPORT` names. +fn report(line: &str) { + println!("{line}"); + if let Some(path) = std::env::var_os("WAI_QUANTUM_VECTOR_REPORT") { + let mut f = std::fs::OpenOptions::new().create(true).append(true).open(&path).expect("the report file opens"); + writeln!(f, "{line}").expect("the report line is written"); + } +} + +/// The declared device set, as written, if it is exactly what was found; +/// otherwise why not, with what was found and refused and how to declare it. +fn declared(d: &Discovery) -> Result { + let mut found: Vec<&str> = d.found.iter().map(|m| m.key()).collect(); + found.sort_unstable(); + let found_list = if found.is_empty() { "none".to_string() } else { found.join(",") }; + let refused: Vec = d.refused.iter().map(|(k, e)| format!("{k}: {e}")).collect(); + let refused_list = if refused.is_empty() { "none".to_string() } else { refused.join("; ") }; + let unavailable = d.unavailable.as_deref().unwrap_or("no"); + let how = format!("found: {found_list}; refused: {refused_list}; unavailable: {unavailable}. Declare the devices this machine must have: WAI_QUANTUM_VECTOR_EXPECT={found_list}"); + let expect = match std::env::var("WAI_QUANTUM_VECTOR_EXPECT") { + Ok(v) if !v.is_empty() => v, + _ => return Err(format!("WAI_QUANTUM_VECTOR_EXPECT is not set. {how}")), + }; + let keys: Vec<&str> = if expect == "none" { Vec::new() } else { expect.split(',').collect() }; + let well_formed = keys.windows(2).all(|w| w[0] <= w[1]) && keys.iter().all(|k| !k.is_empty() && *k != "none" && k.trim() == *k); + if !well_formed { + return Err(format!("WAI_QUANTUM_VECTOR_EXPECT={expect:?} is not a sorted, comma-separated list of keys, nor none. {how}")); + } + for k in &keys { + if !found.contains(k) + && let Some((_, e)) = d.refused.iter().find(|(key, _)| key == k) + { + return Err(format!("the declared device {k} was refused: {e}. {how}")); + } + } + if found != keys { + return Err(format!("the devices found are not the declared ones ({expect}). {how}")); + } + Ok(expect) +} + +/// The members, once the declared devices are exactly the ones found. +fn members() -> &'static [VectorMember] { + let d = discovery(); + if let Err(why) = declared(d) { + panic!("{why}"); + } + &d.found +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn next(s: &mut u64) -> u64 { + *s = s.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *s; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// A value in `[-bound, bound]`. +fn within(s: &mut u64, bound: i64) -> i32 { + ((next(s) % (2 * bound as u64 + 1)) as i64 - bound) as i32 +} + +/// A random circuit of `ops` gates on `n` qubits: every base gate, up to three +/// distinct controls, `P(1..=32)`. +fn random_circuit(n: u8, ops: usize, seed: u64, gs: Gateset) -> Circuit { + let bases = [BaseGate::I, BaseGate::X, BaseGate::Y, BaseGate::Z, BaseGate::H, BaseGate::S, BaseGate::Sdg, BaseGate::T, BaseGate::Tdg, BaseGate::P]; + let mut s = seed; + let mut c = Circuit::with_gateset(n, gs); + for _ in 0..ops { + let base = bases[(next(&mut s) % bases.len() as u64) as usize]; + let target = (next(&mut s) % u64::from(n)) as u8; + let mut controls = Vec::new(); + for _ in 0..next(&mut s) % 4 { + let q = (next(&mut s) % u64::from(n)) as u8; + if q != target && !controls.contains(&q) { + controls.push(q); + } + } + let param = if base == BaseGate::P { 1 + (next(&mut s) % 32) as u16 } else { 0 }; + c.ops.push(Gate { base, controls, target, param }); + } + c +} + +/// Five circuits the native simulator refuses, in each gateset. +fn refused_circuits() -> Vec { + let gate = |base, controls: Vec, target, param| Gate { base, controls, target, param }; + let bad = [ + (1, vec![gate(BaseGate::H, vec![], 0, 0), gate(BaseGate::P, vec![], 0, 0)]), + (1, vec![gate(BaseGate::P, vec![], 0, 33)]), + (1, vec![gate(BaseGate::X, vec![], 3, 0)]), + (2, vec![gate(BaseGate::X, vec![0], 0, 0)]), + (2, vec![gate(BaseGate::X, vec![5], 1, 0)]), + ]; + let mut out = Vec::new(); + for gs in Gateset::ALL { + for (n, ops) in bad.clone() { + let mut c = Circuit::with_gateset(n, gs); + c.ops = ops; + out.push(c); + } + } + out +} + +/// `Z` on qubit 0: on `[(0, 0), (i32::MIN, 0)]` its exact result is `2^31`, +/// which leaves 32 bits. +fn z_words() -> GateWords { + lanes::gate_words(&Gate { base: BaseGate::Z, controls: vec![], target: 0, param: 0 }, Gateset::V2) +} + +const Z_STATE: [Lane; 2] = [[0, 0], [i32::MIN, 0]]; + +#[test] +fn c0_devices_are_declared() { + let d = discovery(); + match declared(d) { + Ok(expect) => report(&format!("vector devices: as declared ({expect})")), + Err(why) => panic!("{why}"), + } + for m in &d.found { + assert_eq!(m.class(), FabricClass::Cpu); + assert!(m.id().starts_with(&format!("local/cpu-{}-vector-", std::env::consts::ARCH)), "{}", m.id()); + assert_eq!(m.implementation(), implementation_id_of(m.isa(), Variant::Conforming)); + assert!(m.max_qubits() >= 20, "{}: holds only {} qubits", m.key(), m.max_qubits()); + report(&format!("vector member: key={} id={} bits={} {}", m.key(), m.id(), m.vector_bits(), m.detail())); + } +} + +#[test] +fn c1_probe_matches_lanes() { + for m in members() { + let pairs = lanes::probe_pairs(1 << 16, 0x0cda); + let out = m.probe(&pairs).unwrap_or_else(|e| panic!("{}: {e}", m.key())); + assert_eq!(lanes::first_mismatch(&pairs, &out), None, "{}", m.key()); + report(&format!("vector probe: key={} {} pairs agree", m.key(), pairs.len())); + } +} + +#[test] +fn c2_kernel_matches_run_words() { + for m in members() { + let mut s = 0x6b65_726e_656c_u64; + let (mut flagged, mut clean) = (0, 0); + for case in 0..2000 { + let n = 1 + (next(&mut s) % 12) as u32; + // Three regimes: the whole 32-bit range (nearly always out of + // range), small states and entries (never), and states and + // entries up to 2^30 (either). + let mode = case % 3; + let value = |s: &mut u64| -> i32 { + match mode { + 0 => next(s) as i32, + 1 => within(s, 1 << 22), + _ => within(s, 1 << 30), + } + }; + let entry = |s: &mut u64| -> u32 { + match mode { + 0 => next(s) as u32, + 1 => within(s, 1 << 29) as u32, + _ => within(s, 1 << 30) as u32, + } + }; + let state: Vec = (0..1usize << n).map(|_| [value(&mut s), value(&mut s)]).collect(); + let gates: Vec = (0..1 + next(&mut s) % 8) + .map(|_| { + let t = (next(&mut s) % u64::from(n)) as u32; + let ctrl = (0..n).filter(|&b| b != t && next(&mut s).is_multiple_of(4)).fold(0u32, |acc, b| acc | 1 << b); + let mut w = [0u32; 8]; + w.iter_mut().for_each(|x| *x = entry(&mut s)); + GateWords { m: w, tbit: 1 << t, ctrl } + }) + .collect(); + let want = lanes::run_words(&state, &gates).expect("valid shapes"); + let got = m.run_lanes(&state, &gates).unwrap_or_else(|e| panic!("{}: case {case}: {e}", m.key())); + assert_eq!(got.out_of_range, want.out_of_range, "{}: case {case}, n = {n}", m.key()); + assert!(got.amps == want.amps, "{}: case {case}, n = {n}: the lanes differ", m.key()); + if want.out_of_range.is_some() { + flagged += 1; + } else { + clean += 1; + } + } + assert!(flagged >= 100 && clean >= 100, "{}: {flagged} flagged, {clean} clean", m.key()); + report(&format!("vector kernel: key={} ok", m.key())); + } +} + +#[test] +fn c3_certificates_reproduce_the_pins() { + let ms = members(); + for (gs, random_pin, fabric_pin) in PINS { + let cpu = lanes::certify_corpus(gs, "local/test", RANDOM_V1); + assert_eq!(hex(&cpu.corpus), random_pin, "{gs:?}"); + let other = Corpus::Random { seed: 8, cases: 300 }; + let cpu_other = lanes::certify_corpus(gs, "local/test", other); + for m in ms { + for (corpus, cases, pin) in [(RANDOM_V1, 200, random_pin), (FABRIC_DEFAULT, 224, fabric_pin)] { + let c = m.certify(gs, corpus).unwrap_or_else(|e| panic!("{} {gs:?} {}: {e}", m.key(), corpus.name())); + assert_eq!(c.parity, Parity::BitExact, "{} {gs:?} {}", m.key(), corpus.name()); + assert_eq!(c.cases, cases); + assert_eq!(hex(&c.corpus), pin, "{} {gs:?} {}", m.key(), corpus.name()); + assert_eq!((c.kernel.as_str(), c.implementation.as_str(), c.member.as_str()), (apply_kernel(gs), m.implementation(), m.id())); + assert_eq!(c.generator, corpus.generator(gs)); + report(&format!( + "vector certified: key={} member={} kernel={} corpus={} implementation={} parity=bit-exact cases={} digest={}", + m.key(), + m.id(), + c.kernel, + corpus.name(), + c.implementation, + c.cases, + hex(&c.corpus) + )); + } + // A corpus with no pin: the member and the CPU agree on it. + let c = m.certify(gs, other).unwrap_or_else(|e| panic!("{} {gs:?} seed 8: {e}", m.key())); + assert_eq!((c.parity, c.cases), (Parity::BitExact, 300), "{} {gs:?}", m.key()); + assert_eq!(c.corpus, cpu_other.corpus, "{} {gs:?}", m.key()); + } + } +} + +#[test] +fn c4_wide_circuits_match_native() { + for m in members() { + let widest = m.max_qubits().min(20); + let mut s = 0x0077_6964_6500_u64; + for (k, n) in [12u8, 16, 18, widest].into_iter().enumerate() { + for gs in Gateset::ALL { + let c = random_circuit(n, 40, 0x1000 + k as u64, gs); + let start = (next(&mut s) as usize) % (1usize << n); + let want = c.simulate_from(start).expect("the native run"); + let got = m.simulate_from(&c, start).unwrap_or_else(|e| panic!("{}: n = {n} {gs:?}: {e}", m.key())); + assert!(got == want, "{}: n = {n} {gs:?}: the states differ", m.key()); + } + } + // The transforms the conformance corpus carries. + for n in [3u8, 5, 8, 12] { + let qft = Circuit::qft(n); + assert!(m.simulate(&qft).unwrap() == qft.simulate().unwrap(), "{}: QFT-{n}", m.key()); + } + report(&format!("vector wide: key={} up to {widest} qubits match native", m.key())); + } +} + +#[test] +fn c5_refusals_are_typed() { + for m in members() { + for c in refused_circuits() { + let native = c.simulate_from(0).unwrap_err(); + assert_eq!(m.simulate_from(&c, 0).unwrap_err(), VectorError::Circuit(native), "{c:?}"); + } + assert_eq!(m.simulate(&Circuit::new(MAX_QUBITS + 1)).unwrap_err(), VectorError::Circuit(QuantumError::TooManyQubits(MAX_QUBITS + 1))); + let small = m.with_max_qubits(4); + assert_eq!(small.simulate(&Circuit::new(5)).unwrap_err(), VectorError::TooWide { n_qubits: 5, max: 4 }); + let wide = Program { n_qubits: 5, start: 0, gates: Vec::new(), steps: Vec::new() }; + assert_eq!(small.run_program(&wide).unwrap_err(), VectorError::TooWide { n_qubits: 5, max: 4 }); + // A result that leaves 32 bits is reported at its op, and the lane + // holds its low 32 bits. + let r = m.run_lanes(&Z_STATE, &[z_words()]).unwrap(); + assert_eq!(r.out_of_range, Some(0), "{}", m.key()); + assert_eq!(r.amps[1][0] as u32, 0x8000_0000); + // Malformed words, a state the host cannot lay out and a tile step are + // host defects, never results. + let bad = GateWords { tbit: 3, ..z_words() }; + assert!(matches!(m.run_lanes(&[[0, 0]; 4], &[bad]), Err(VectorError::Internal(_)))); + assert!(matches!(m.run_lanes(&[[0, 0]; 3], &[z_words()]), Err(VectorError::Internal(_)))); + let tiled = Program { n_qubits: 2, start: 0, gates: Vec::new(), steps: vec![Step::Tile { first: 0, count: 1, size: 2 }] }; + assert!(matches!(m.run_program(&tiled), Err(VectorError::Internal(_)))); + // The refusals that may still fall back to another member, do. + assert!(MemberError::from(VectorError::TooWide { n_qubits: 5, max: 4 }).falls_back()); + report(&format!("vector refusals: key={} typed", m.key())); + } +} + +/// The kernel mutants, compiled in: each must be caught by the probe or by a +/// certificate. +#[test] +fn c6_mutants_are_killed() { + let edges = lanes::probe_pairs(0, 0); + for m in members() { + let mut killed = 0; + for v in Variant::MUTANTS { + let mm = m.with_variant(v); + assert_ne!(mm.implementation(), m.implementation()); + let out = mm.probe(&edges).expect("the mutant's probe runs"); + let differing = edges.iter().zip(&out).filter(|&(&(a, b), o)| *o != (lanes::mul_wide(a, b), lanes::fxmul_wide(a, b))).count(); + let caught = differing > 0 + || Gateset::ALL.into_iter().all(|gs| match mm.certify(gs, RANDOM_V1) { + Err(_) => true, + Ok(c) => c.parity != Parity::BitExact || c.corpus != lanes::certify_corpus(gs, "local/test", RANDOM_V1).corpus, + }); + println!("vector mutant {v:?} on {}: {differing} of {} edge pairs differ", m.key(), edges.len()); + assert!(caught, "{}: mutant {v:?} survived", m.key()); + killed += 1; + } + report(&format!("vector mutants: key={} killed {killed} of 3", m.key())); + } +} + +#[test] +fn c7_receipt_placement_and_the_fabric() { + for m in members() { + let member = m.member(FABRIC_DEFAULT).unwrap_or_else(|e| panic!("{}: {e}", m.key())); + assert_eq!((member.id.as_str(), member.class, member.presence), (m.id(), FabricClass::Cpu, Presence::Detected)); + assert_eq!(member.kernels.len(), 2); + for (entry, gs) in member.kernels.iter().zip(Gateset::ALL) { + assert_eq!((entry.kernel.as_str(), entry.implementation.as_str()), (apply_kernel(gs), m.implementation())); + assert!(entry.energy.is_none()); + assert_eq!(entry.certificate.parity, Parity::BitExact, "{gs:?}"); + } + // A receipt over a member's run. + let qft = Circuit::qft(5); + let state = m.simulate(&qft).unwrap(); + assert_eq!(state, qft.simulate().unwrap()); + let entry = member.kernels.iter().find(|k| k.kernel == apply_kernel(qft.gateset)).unwrap(); + let signer = SigningKey::from_bytes(&[7; 32]); + let receipt = FabricReceipt::seal(&signer, "local/test", state.statevector_hash(), &member, entry, 0, None); + assert!(receipt.verify()); + assert!(receipt.certificate_matches(&entry.certificate)); + // Placement: beside the CPU reference with no energy figures, the + // reference; alone, the member. + let local = detect_local(); + for gs in Gateset::ALL { + let both = [local.clone(), member.clone()]; + let (placed, k) = place(&both, apply_kernel(gs), &Parity::BitExact).unwrap(); + assert_eq!((placed.id.as_str(), k.implementation.as_str()), (local.id.as_str(), REFERENCE)); + let (placed, k) = place(std::slice::from_ref(&member), apply_kernel(gs), &Parity::BitExact).unwrap(); + assert_eq!((placed.id.as_str(), k.implementation.as_str()), (m.id(), m.implementation())); + } + // On the fabric: the member alone runs the circuit; a pairing with + // another member's entries is refused. + let mut fabric = Fabric::new(); + fabric.add(member.clone(), m); + let run = ready(fabric.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs"); + assert_eq!(run.state, state); + assert_eq!(run.receipt.member_id, m.id()); + assert!(run.receipt.verify()); + let mut mismatched = Fabric::new(); + mismatched.add(local.clone(), m); + assert!(ready(mismatched.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").is_err()); + let mut both = Fabric::new(); + both.add(local.clone(), CpuReference).add(member.clone(), m); + let run = ready(both.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs"); + assert_eq!(run.state, state); + report(&format!("vector receipt: key={} ok", m.key())); + } +} + +#[test] +fn c8_repeated_runs_agree() { + for m in members() { + let c = random_circuit(16, 60, 0xabc, Gateset::V2); + let first = m.simulate(&c).unwrap(); + for _ in 0..5 { + assert!(m.simulate(&c).unwrap() == first, "{}", m.key()); + } + report(&format!("vector repeat: key={} ok", m.key())); + } +} -- 2.54.0 (Apple Git-157) From 29fa40af8017c9bc9cda11416603bc6237c12352 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Wed, 14 Oct 2026 12:43:48 -0400 Subject: [PATCH 07/11] wai-quantum: the dataflow member, on 32 RISC-V cores of a simulated accelerator The gate's fixed-point arithmetic on the first core of each of 32 compute tiles of a dataflow accelerator, two device generations (PCI 1e52:b140 and 1e52:401e), feature quantum_fabric_dataflow, class dataflow. - quantum_fabric::rv32: an RV32IM encoder and the gate's programs, built as machine words and as the listing they encode (the implementation id's input). Every 64-bit value is a pair of words: mul/mulh for products, sltu for carries and borrows, the arithmetic shift from word shifts. An independent assembler turns every listing, conforming and mutant, into the same words; examples/rv32_listing writes them for that check. - The host loads the device's library at run time (WAI_QUANTUM_DATAFLOW_LIB) and drives it as a PCI endpoint: reads the ids and the first memory window's base from configuration space, opens two windows per tile (its memory and its registers), holds the tiles' cores in reset, writes the program at the reset address and each tile's parameters and slice of the gate's pairs beside it, releases the first cores, clocks the device until every tile reports done, and reads the rows back in order. - Spreading a batch over 32 tiles: the library's cost per clock is mostly fixed, so 32 active cores run about 23 times as many instructions a second as one. Measured on the device's open-source simulator, built from source: both generations 9 of 9 device tests (66020 probe pairs, 2000 kernel cases, all four pinned certificates, circuits up to 20 qubits matching native, 3 of 3 mutants killed), 75 s and 108 s. Every certificate is labelled simulator. Workflow wai-quantum-dataflow.yml builds the simulator at a pinned commit, checks the encoder against the assembler, and runs both generations; the macOS runner finds no member and says why. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wai-quantum-dataflow.yml | 171 ++++ wai/CHANGELOG.md | 10 +- wai/wai-quantum/Cargo.toml | 10 + wai/wai-quantum/README.md | 10 +- wai/wai-quantum/examples/rv32_listing.rs | 29 + wai/wai-quantum/src/lib.rs | 9 +- wai/wai-quantum/src/quantum_fabric.rs | 2 + wai/wai-quantum/src/quantum_fabric/rv32.rs | 380 +++++++++ .../src/quantum_fabric_dataflow.rs | 773 ++++++++++++++++++ wai/wai-quantum/tests/dataflow_member.rs | 410 ++++++++++ 10 files changed, 1796 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/wai-quantum-dataflow.yml create mode 100644 wai/wai-quantum/examples/rv32_listing.rs create mode 100644 wai/wai-quantum/src/quantum_fabric/rv32.rs create mode 100644 wai/wai-quantum/src/quantum_fabric_dataflow.rs create mode 100644 wai/wai-quantum/tests/dataflow_member.rs diff --git a/.github/workflows/wai-quantum-dataflow.yml b/.github/workflows/wai-quantum-dataflow.yml new file mode 100644 index 000000000..e202e71f6 --- /dev/null +++ b/.github/workflows/wai-quantum-dataflow.yml @@ -0,0 +1,171 @@ +name: wai-quantum dataflow member + +# The fabric's dataflow member: the gate's fixed-point arithmetic on the first +# RISC-V core of a dataflow accelerator's compute tile, driven through the +# device's library at the level of configuration space, memory windows and +# clocks. The library here is the device's open-source simulator (Apache-2.0), +# built from source at a pinned commit, so every certificate is labelled +# simulator. Two device generations, each its own leg. +# +# Before the device runs anything, the RV32IM encoder that builds its programs +# is checked against an independent assembler: every listing, conforming and +# mutant, must assemble to the encoder's words exactly. + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: [main] + paths: + - 'wai/wai-quantum/**' + - 'rust-toolchain.toml' + - '.github/workflows/wai-quantum-dataflow.yml' + pull_request: + paths: + - 'wai/wai-quantum/**' + - 'rust-toolchain.toml' + - '.github/workflows/wai-quantum-dataflow.yml' + +jobs: + dataflow: + name: member · dataflow device, simulated (pci ${{ matrix.pci }}) + runs-on: linux + timeout-minutes: 60 + strategy: + fail-fast: false + max-parallel: 1 + matrix: + include: + - pci: '1e52:b140' + build: bh + key: dataflow-1e52-b140 + - pci: '1e52:401e' + build: wh + key: dataflow-1e52-401e + container: + image: debian@sha256:913f6706df59a68922d1dd08f78c2476560a8d367897200a6005b00e5f67c2d5 + options: --memory=6g --memory-swap=6g + env: + SIM_REPO: https://github.com/tenstorrent/ttsim.git + SIM_COMMIT: 283f621e7384d86f01e4ac5a9c83b60704f77d92 + WAI_QUANTUM_DATAFLOW_EXPECT: ${{ matrix.key }} + WAI_QUANTUM_DATAFLOW_SIMULATOR: device simulator 283f621e (built from source, ${{ matrix.build }}) + steps: + - name: Tools + shell: bash + run: | + set -euo pipefail + apt-get update -qq + apt-get install -y -qq --no-install-recommends ca-certificates curl git build-essential python3 binutils-riscv64-linux-gnu >/dev/null + # No Node for a checkout action: the forge's archive of the exact commit, + # its embedded commit id checked. + - name: Fetch the source + shell: bash + run: | + set -euo pipefail + want="${{ github.event.pull_request.head.sha || github.sha }}" + rm -rf src src.tar.gz src.tar && mkdir src + curl -sSf -o src.tar.gz "http://100.64.224.0:3400/Transaction-Science/open-standards/archive/$want.tar.gz" + gzip -dc src.tar.gz > src.tar + got="$(git get-tar-commit-id < src.tar)" + [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } + tar -xf src.tar -C src --strip-components=1 + rm src.tar.gz src.tar + - name: The simulator, built from source at the pinned commit (cached), and Rust + shell: bash + working-directory: src/wai/wai-quantum + run: | + set -euo pipefail + cache=/ci-cache/wai-quantum-dataflow + mkdir -p "$cache" + exec 9>"$cache/.lock" + flock 9 + lib="$cache/sim-$SIM_COMMIT/${{ matrix.build }}/libttsim.so" + if [ ! -f "$lib" ]; then + b="$cache/sim-build" + rm -rf "$b" && git init -q "$b" + git -C "$b" fetch -q --depth 1 "$SIM_REPO" "$SIM_COMMIT" + git -C "$b" checkout -q FETCH_HEAD + [ "$(git -C "$b" rev-parse HEAD)" = "$SIM_COMMIT" ] || { echo "::error::the simulator is not at the pinned commit"; exit 1; } + (cd "$b" && ./make.py "src/_out/release_${{ matrix.build }}/libttsim.so" > build.log 2>&1) || { tail -30 "$b/build.log"; exit 1; } + mkdir -p "$(dirname "$lib")" && cp "$b/src/_out/release_${{ matrix.build }}/libttsim.so" "$lib.tmp" && mv "$lib.tmp" "$lib" + rm -rf "$b" + fi + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" + export PATH="$CARGO_HOME/bin:$PATH" + if ! command -v rustup >/dev/null; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal --default-toolchain none + fi + rustup toolchain install + - name: The encoder against an independent assembler + shell: bash + working-directory: src/wai/wai-quantum + run: | + set -euo pipefail + cache=/ci-cache/wai-quantum-dataflow + exec 9>"$cache/.lock" + flock 9 + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" + export PATH="$CARGO_HOME/bin:$PATH" + out="$RUNNER_TEMP/rv32" + cargo run --locked --release -q --example rv32_listing --features quantum_fabric_dataflow -- "$out" + riscv64-linux-gnu-as --version | head -1 + n=0 + for s in "$out"/*.s; do + p="${s%.s}" + riscv64-linux-gnu-as -march=rv32im -mabi=ilp32 "$s" -o "$p.o" + riscv64-linux-gnu-objcopy -O binary -j .text "$p.o" "$p.as.bin" + cmp "$p.bin" "$p.as.bin" || { echo "::error::$(basename "$p"): the encoder's words differ from the assembler's"; exit 1; } + n=$((n + 1)) + done + [ "$n" = 5 ] || { echo "::error::$n of 5 programs were compared"; exit 1; } + echo "all $n programs: the encoder's words are the assembler's" + - name: The member's tests + shell: bash + working-directory: src/wai/wai-quantum + run: | + set -euo pipefail + cache=/ci-cache/wai-quantum-dataflow + exec 9>"$cache/.lock" + flock 9 + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" + export PATH="$CARGO_HOME/bin:$PATH" + export WAI_QUANTUM_DATAFLOW_LIB="$cache/sim-$SIM_COMMIT/${{ matrix.build }}/libttsim.so" + cargo test --locked --release -j 6 --features quantum_fabric_dataflow --lib -- quantum_fabric_dataflow:: quantum_fabric::rv32:: 2>&1 | tee "$RUNNER_TEMP/unit.log" + grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } + report="$RUNNER_TEMP/dataflow-report.txt"; : > "$report" + WAI_QUANTUM_DATAFLOW_REPORT="$report" cargo test --locked --release -j 6 --features quantum_fabric_dataflow --test dataflow_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log" + grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; } + cat "$report" + key="$WAI_QUANTUM_DATAFLOW_EXPECT" + grep -qx "dataflow devices: as declared ($key)" "$report" || { echo "::error::the device found is not the declared one"; exit 1; } + grep -q "^dataflow member: key=$key .*simulator:device simulator 283f621e" "$report" || { echo "::error::the member does not say it ran on the pinned simulator"; exit 1; } + n="$(grep -c "^dataflow certified: key=$key .* parity=bit-exact " "$report")" + [ "$n" = 4 ] || { echo "::error::$n of 4 certificates were earned"; exit 1; } + grep -qx "dataflow mutants: key=$key killed 3 of 3" "$report" || { echo "::error::a kernel mutant survived"; exit 1; } + + dataflow-absent: + name: member · dataflow device absent (macos) + runs-on: macos + timeout-minutes: 60 + env: + WAI_QUANTUM_DATAFLOW_EXPECT: none + steps: + - uses: actions/checkout@v5 + # launchd's PATH omits cargo on the macOS host runner. + - name: Put cargo on PATH + shell: bash + run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: No device library, and the member says so + shell: bash + working-directory: wai/wai-quantum + run: | + set -euo pipefail + report="$RUNNER_TEMP/dataflow-report.txt"; : > "$report" + nice -n 10 cargo test --locked --features quantum_fabric_dataflow --lib -- quantum_fabric_dataflow:: quantum_fabric::rv32:: 2>&1 | tee "$RUNNER_TEMP/unit.log" + grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } + WAI_QUANTUM_DATAFLOW_REPORT="$report" nice -n 10 cargo test --locked --features quantum_fabric_dataflow --test dataflow_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log" + grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/device.log" || { echo "::error::not every device test ran and passed"; exit 1; } + grep -qx 'dataflow devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; } diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index 40a32402c..d70b8b22b 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -6,7 +6,7 @@ wire format may change and such changes are recorded here. ## v0.4.0 — current -### wai-quantum 0.5.1: neural-processor, matrix-extension, vector-unit and signal-processor members +### wai-quantum 0.5.1: neural-processor, matrix-extension, vector-unit, signal-processor and dataflow members Additive. The default features, `full`, and every pinned digest are unchanged. @@ -50,6 +50,14 @@ Additive. The default features, `full`, and every pinned digest are unchanged. set per pass and read back from the unit into the member's id). Silicon for `avx2`; `sve` at 128, 512 and 2048 bits and `rvv` at 128 and 1024 bits under the user-mode emulator. Three kernel mutants per instruction set. +- **The dataflow member** (`quantum_fabric_dataflow`, opt-in, Linux): the gate's arithmetic on + the first RISC-V core of each of 32 compute tiles of a dataflow accelerator, each over a + contiguous slice of a gate's pairs, two device generations (PCI 1e52:b140 and 1e52:401e), + driven through the device's own library at the level of configuration space, memory + windows, reset registers and clocks. The program comes from a new RV32IM + encoder (`quantum_fabric::rv32`), whose listings an independent assembler turns into + the same words; every 64-bit value is a pair of words. The library is the device's + open-source simulator, built from source, and every certificate is labelled simulator. - **The signal processor as a processor.** The bit-identity digests, the frame fixtures and the byte-pinning library tests pass for the processor's Linux target under its user-mode emulator. Emulator releases before 11.0 test a whole predicate register diff --git a/wai/wai-quantum/Cargo.toml b/wai/wai-quantum/Cargo.toml index 62c08fded..5771600a7 100644 --- a/wai/wai-quantum/Cargo.toml +++ b/wai/wai-quantum/Cargo.toml @@ -173,6 +173,8 @@ quantum_fabric_sme = ["quantum_fabric"] quantum_fabric_dsp = ["quantum_fabric"] # The CPU vector-unit members: the gate's arithmetic in the processor's vector extension (avx2, sve, rvv). quantum_fabric_vector = ["quantum_fabric"] +# The dataflow member: the gate's arithmetic on a dataflow accelerator's RISC-V cores, through a device library loaded at run time. +quantum_fabric_dataflow = ["quantum_fabric"] # --- interference / ML layer ---------------------------------------------- quantum_vml = [] @@ -231,10 +233,18 @@ required-features = ["quantum_fabric_dsp"] name = "vector_member" required-features = ["quantum_fabric_vector"] +[[test]] +name = "dataflow_member" +required-features = ["quantum_fabric_dataflow"] + [[test]] name = "fabric_expected" required-features = ["quantum_fabric"] +[[example]] +name = "rv32_listing" +required-features = ["quantum_fabric_dataflow"] + [[example]] name = "npu_onnx" required-features = ["quantum_fabric"] diff --git a/wai/wai-quantum/README.md b/wai/wai-quantum/README.md index cd8f42fe0..e9d6a06b8 100644 --- a/wai/wai-quantum/README.md +++ b/wai/wai-quantum/README.md @@ -6,16 +6,17 @@ every stage. No QPU, no cloud service, no vendor SDK, and **no system libraries** — so the same source runs natively, in the browser via wasm, and as a WASI component at -the edge, producing **byte-identical results and receipts on all three**. Eight +the edge, producing **byte-identical results and receipts on all three**. Nine opt-in features reach outside the crate: `quantum_fabric_webgpu` (the platform's GPU interface), `quantum_fabric_cuda` (the CUDA driver and runtime compiler) and `quantum_fabric_opencl` (the OpenCL ICD loader), the last two loaded when first used, `quantum_fabric_fpga` (Verilator, run as a program when first used), `quantum_fabric_npu` (the platform's model runtime, on macOS), `quantum_fabric_sme` (the processor's streaming vector mode, by -inline assembly), `quantum_fabric_vector` (the processor's vector extension) -and `quantum_fabric_dsp` (the signal processor's vector -unit, in a build for that processor); no default feature reaches a system +inline assembly), `quantum_fabric_vector` (the processor's vector extension), +`quantum_fabric_dsp` (the signal processor's vector +unit, in a build for that processor) and `quantum_fabric_dataflow` (a dataflow +device's library, loaded when first used); no default feature reaches a system library. ```toml @@ -111,6 +112,7 @@ never creates information. | `quantum_fabric_sme` | **the fabric's matrix-extension member**, opt-in and not in `full`, aarch64 only: the gate's fixed-point arithmetic in the processor's streaming vector mode (SME), 64-bit lanes, the host gathering each gate's pairs and checking each row's range. Every instruction it uses exists only in that mode, so a kernel that ran, ran there. Detected by the processor's feature flags (on Linux, the kernel's `AT_HWCAP2`, which a user-mode emulator supplies); under an emulator the runner says so (`WAI_QUANTUM_SME_EMULATOR`) and every certificate carries it. Three kernel mutants are compiled in and must be caught | | `quantum_fabric_dsp` | **the fabric's DSP vector-unit member**, opt-in and not in `full`, in a build for the signal processor with its vector unit in 128-byte mode: the gate's fixed-point arithmetic in 32-bit lanes, each 64-bit value a pair of words, every product the vector unit's exact 32×32 multiply and every carry and borrow kept; the host gathers each gate's pairs and checks each row's range. Its instructions exist only on that unit, so a kernel that ran, ran there. Under an emulator the runner says so (`WAI_QUANTUM_DSP_EMULATOR`) and every certificate carries it. Three kernel mutants are compiled in and must be caught. Its class is `dsp` | | `quantum_fabric_vector` | **the fabric's CPU vector-unit members**, opt-in and not in `full`: the gate's fixed-point arithmetic in the processor's own vector extension, one kernel per instruction set — `avx2` (x86-64, 256-bit), `sve` (aarch64, the scalable vector extension, any length) and `rvv` (riscv64, the vector extension 1.0, any length) — each operand sign-extended to 64-bit lanes, the host gathering each gate's pairs and checking each row's range. The scalable kernels set their length per pass; the member reads the length from the unit and names it in its id and detail. Detected by the processor's feature flags (on Linux, `AT_HWCAP` for the vector extension, which a user-mode emulator supplies); under an emulator the runner says so (`WAI_QUANTUM_VECTOR_EMULATOR`). Its id extends the CPU reference's, so placement with no energy figures keeps the reference. Three kernel mutants per instruction set are compiled in and must be caught | +| `quantum_fabric_dataflow` | **the fabric's dataflow member**, opt-in and not in `full`, Linux: the gate's fixed-point arithmetic on the first RISC-V core of each of 32 compute tiles of a dataflow accelerator, each over a contiguous slice of a gate's pairs. The program is built in this crate (`quantum_fabric::rv32`, an RV32IM encoder whose listings an independent assembler reproduces word for word): every 64-bit value a pair of words, `mul`/`mulh` for products, `sltu` for carries and borrows. The device's library (`WAI_QUANTUM_DATAFLOW_LIB`, loaded at run time) presents the device as a PCI endpoint; the host opens two memory windows on each tile, holds its cores in reset, writes the program at the reset address and that tile's parameters and slice beside it, releases the first cores and clocks the device until every tile reports done. The device is named by its PCI ids (`dataflow-1e52-`); the library is a simulator, and every certificate says so (`WAI_QUANTUM_DATAFLOW_SIMULATOR`). Three kernel mutants are compiled in | | `full` | everything | Default features are `quantum`, `quantum_toolchain`, `quantum_receipt`. diff --git a/wai/wai-quantum/examples/rv32_listing.rs b/wai/wai-quantum/examples/rv32_listing.rs new file mode 100644 index 000000000..b49b22607 --- /dev/null +++ b/wai/wai-quantum/examples/rv32_listing.rs @@ -0,0 +1,29 @@ +//! Writes the dataflow member's RV32IM programs (conforming and mutants) as +//! assembly listings and as the machine words this crate encodes, so an +//! independent assembler can be checked against the encoder: +//! +//! ```text +//! cargo run --example rv32_listing --features quantum_fabric_dataflow -- +//! ``` +//! +//! In ``, for each program ``: `.s` (the listing, with a +//! `.text` header) and `.bin` (the words, little-endian). +use wai_quantum::quantum_fabric::rv32::{Layout, probe_program, rows_program}; + +fn main() { + let dir = std::path::PathBuf::from(std::env::args().nth(1).expect("an output directory")); + std::fs::create_dir_all(&dir).expect("the directory"); + let l = Layout { base: 0x0001_0000, capacity: 1024 }; + let programs = [ + ("rows", rows_program(l, 1 << 29, 30, false)), + ("rows_m1", rows_program(l, 1 << 28, 30, false)), + ("rows_m2", rows_program(l, 1 << 29, 29, false)), + ("rows_m3", rows_program(l, 1 << 29, 30, true)), + ("probe", probe_program(l, 1 << 29, 30)), + ]; + for (name, (words, text)) in programs { + std::fs::write(dir.join(format!("{name}.s")), format!(".text\n.option norvc\n{text}")).expect("the listing"); + std::fs::write(dir.join(format!("{name}.bin")), words.iter().flat_map(|w| w.to_le_bytes()).collect::>()).expect("the words"); + println!("{name}: {} words", words.len()); + } +} diff --git a/wai/wai-quantum/src/lib.rs b/wai/wai-quantum/src/lib.rs index ac3e22ba0..8330a3059 100644 --- a/wai/wai-quantum/src/lib.rs +++ b/wai/wai-quantum/src/lib.rs @@ -10,16 +10,17 @@ //! Rust, so the same source runs natively, in the browser via wasm, and as a //! WASI component at the edge, producing **byte-identical** results and receipts //! on all three. A fabric receipt is the exception by design: it names the -//! member that ran the work. Eight opt-in features reach outside the crate: +//! member that ran the work. Nine opt-in features reach outside the crate: //! `quantum_fabric_webgpu` (the platform's GPU interface), //! `quantum_fabric_cuda` (the CUDA driver and runtime compiler) and //! `quantum_fabric_opencl` (the OpenCL ICD loader), the last two loaded when //! first used, `quantum_fabric_fpga` (Verilator, run as a program when first //! used), `quantum_fabric_npu` (the platform's model runtime, on macOS), //! `quantum_fabric_sme` (the processor's streaming vector mode), -//! `quantum_fabric_vector` (the processor's vector extension) and +//! `quantum_fabric_vector` (the processor's vector extension), //! `quantum_fabric_dsp` (the signal processor's vector unit, in a build for -//! it); no default feature reaches a system library. +//! it) and `quantum_fabric_dataflow` (a dataflow device's library, loaded when +//! first used); no default feature reaches a system library. //! //! Everything is behind a feature; enable only what you use. @@ -101,6 +102,8 @@ pub mod quantum_fabric_sme; pub mod quantum_fabric_dsp; #[cfg(all(feature = "quantum_fabric_vector", not(target_arch = "wasm32")))] pub mod quantum_fabric_vector; +#[cfg(all(feature = "quantum_fabric_dataflow", not(target_arch = "wasm32")))] +pub mod quantum_fabric_dataflow; #[cfg(feature = "quantum_control")] pub mod quantum_control; diff --git a/wai/wai-quantum/src/quantum_fabric.rs b/wai/wai-quantum/src/quantum_fabric.rs index 845bde486..ed9f5d5fb 100644 --- a/wai/wai-quantum/src/quantum_fabric.rs +++ b/wai/wai-quantum/src/quantum_fabric.rs @@ -2051,6 +2051,8 @@ pub mod fpga; pub mod npu; pub mod opencl; pub mod plan; +#[cfg(feature = "quantum_fabric_dataflow")] +pub mod rv32; pub mod sampling; #[cfg(feature = "quantum_sv")] pub mod sv64; diff --git a/wai/wai-quantum/src/quantum_fabric/rv32.rs b/wai/wai-quantum/src/quantum_fabric/rv32.rs new file mode 100644 index 000000000..ff5eda95d --- /dev/null +++ b/wai/wai-quantum/src/quantum_fabric/rv32.rs @@ -0,0 +1,380 @@ +//! A small RV32IM encoder and the gate's kernels for a 32-bit RISC-V core that +//! runs with no operating system: each program is built here, as machine words +//! and as the assembly listing they encode (the implementation id's input). +//! +//! The kernels read a parameter block at a base address in the core's memory and +//! write their results beside it ([`Layout`]); a done word tells the host they +//! finished. Every 64-bit value is a pair of words: the product is `mul` (low) +//! and `mulh` (high, signed); the rounding constant's carry is `sltu`; the +//! arithmetic shift is built from word shifts; a row's carries and borrows are +//! `sltu` too. + +use std::collections::BTreeMap; +use std::fmt::Write as _; + +/// A register, `x0` to `x31`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct R(pub u8); + +/// The ABI names, for the listing. +const NAMES: [&str; 32] = + ["zero", "ra", "sp", "gp", "tp", "t0", "t1", "t2", "s0", "s1", "a0", "a1", "a2", "a3", "a4", "a5", "a6", "a7", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10", "s11", "t3", "t4", "t5", "t6"]; + +impl std::fmt::Display for R { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(NAMES[self.0 as usize]) + } +} + +pub const ZERO: R = R(0); +pub const T0: R = R(5); +pub const T1: R = R(6); +pub const T2: R = R(7); +pub const S0: R = R(8); +pub const S1: R = R(9); +pub const A0: R = R(10); +pub const A1: R = R(11); +pub const A2: R = R(12); +pub const A3: R = R(13); +pub const A4: R = R(14); +pub const A5: R = R(15); +pub const A6: R = R(16); +pub const A7: R = R(17); +pub const S2: R = R(18); +pub const T3: R = R(28); +pub const T4: R = R(29); +pub const T5: R = R(30); +pub const T6: R = R(31); + +fn r_type(f7: u32, rs2: R, rs1: R, f3: u32, rd: R, op: u32) -> u32 { + (f7 << 25) | (u32::from(rs2.0) << 20) | (u32::from(rs1.0) << 15) | (f3 << 12) | (u32::from(rd.0) << 7) | op +} + +fn i_type(imm: i32, rs1: R, f3: u32, rd: R, op: u32) -> u32 { + assert!((-2048..2048).contains(&imm), "an I-type immediate is 12 bits: {imm}"); + (((imm as u32) & 0xfff) << 20) | (u32::from(rs1.0) << 15) | (f3 << 12) | (u32::from(rd.0) << 7) | op +} + +/// One branch waiting for its label. +struct Fixup { + at: usize, + label: &'static str, + f3: u32, + rs1: R, + rs2: R, +} + +/// A program being built: machine words and their listing. +#[derive(Default)] +pub struct Asm { + words: Vec, + lines: Vec, + labels: BTreeMap<&'static str, usize>, + fixups: Vec, +} + +macro_rules! r_ops { + ($($name:ident = $f7:literal, $f3:literal;)*) => {$( + pub fn $name(&mut self, rd: R, rs1: R, rs2: R) -> &mut Self { + self.emit(r_type($f7, rs2, rs1, $f3, rd, 0x33), format!("{} {rd}, {rs1}, {rs2}", stringify!($name))) + } + )*}; +} + +impl Asm { + fn emit(&mut self, word: u32, line: String) -> &mut Self { + self.words.push(word); + self.lines.push(line); + self + } + + r_ops! { + add = 0x00, 0; + sub = 0x20, 0; + sltu = 0x00, 3; + or = 0x00, 6; + mul = 0x01, 0; + mulh = 0x01, 1; + } + + pub fn addi(&mut self, rd: R, rs1: R, imm: i32) -> &mut Self { + self.emit(i_type(imm, rs1, 0, rd, 0x13), format!("addi {rd}, {rs1}, {imm}")) + } + + pub fn slli(&mut self, rd: R, rs1: R, sh: u32) -> &mut Self { + assert!(sh < 32); + self.emit(i_type(sh as i32, rs1, 1, rd, 0x13), format!("slli {rd}, {rs1}, {sh}")) + } + + pub fn srli(&mut self, rd: R, rs1: R, sh: u32) -> &mut Self { + assert!(sh < 32); + self.emit(i_type(sh as i32, rs1, 5, rd, 0x13), format!("srli {rd}, {rs1}, {sh}")) + } + + pub fn srai(&mut self, rd: R, rs1: R, sh: u32) -> &mut Self { + assert!(sh < 32); + self.emit(i_type(0x400 | sh as i32, rs1, 5, rd, 0x13), format!("srai {rd}, {rs1}, {sh}")) + } + + pub fn lw(&mut self, rd: R, off: i32, rs1: R) -> &mut Self { + self.emit(i_type(off, rs1, 2, rd, 0x03), format!("lw {rd}, {off}({rs1})")) + } + + pub fn sw(&mut self, rs2: R, off: i32, rs1: R) -> &mut Self { + assert!((-2048..2048).contains(&off), "an S-type offset is 12 bits: {off}"); + let imm = off as u32 & 0xfff; + let word = ((imm >> 5) << 25) | (u32::from(rs2.0) << 20) | (u32::from(rs1.0) << 15) | (2 << 12) | ((imm & 0x1f) << 7) | 0x23; + self.emit(word, format!("sw {rs2}, {off}({rs1})")) + } + + pub fn lui(&mut self, rd: R, upper: u32) -> &mut Self { + assert!(upper < 1 << 20); + self.emit((upper << 12) | (u32::from(rd.0) << 7) | 0x37, format!("lui {rd}, {upper:#x}")) + } + + /// `rd = value`, by `lui` and `addi` (or `addi` alone). + pub fn li(&mut self, rd: R, value: u32) -> &mut Self { + let low = ((value << 20) as i32) >> 20; // the low 12 bits, sign-extended + let upper = value.wrapping_sub(low as u32) >> 12; + if upper != 0 { + self.lui(rd, upper); + if low != 0 { + self.addi(rd, rd, low); + } + self + } else { + self.addi(rd, ZERO, low) + } + } + + pub fn label(&mut self, name: &'static str) -> &mut Self { + assert!(self.labels.insert(name, self.words.len()).is_none(), "label {name} twice"); + self.lines.push(format!("{name}:")); + self + } + + fn branch(&mut self, f3: u32, mnemonic: &str, rs1: R, rs2: R, label: &'static str) -> &mut Self { + self.fixups.push(Fixup { at: self.words.len(), label, f3, rs1, rs2 }); + self.emit(0, format!("{mnemonic} {rs1}, {rs2}, {label}")) + } + + pub fn beq(&mut self, rs1: R, rs2: R, label: &'static str) -> &mut Self { + self.branch(0, "beq", rs1, rs2, label) + } + + /// `jal zero, label`. + pub fn j(&mut self, label: &'static str) -> &mut Self { + self.fixups.push(Fixup { at: self.words.len(), label, f3: u32::MAX, rs1: ZERO, rs2: ZERO }); + self.emit(0, format!("j {label}")) + } + + /// The machine words, every branch resolved, and the listing. + pub fn finish(mut self) -> (Vec, String) { + for f in std::mem::take(&mut self.fixups) { + let target = *self.labels.get(f.label).unwrap_or_else(|| panic!("no label {}", f.label)); + let off = (target as i64 - f.at as i64) * 4; + let o = off as u32; + self.words[f.at] = if f.f3 == u32::MAX { + assert!((-(1 << 20)..(1 << 20)).contains(&off), "a jump reaches 1 MiB"); + (((o >> 20) & 1) << 31) | (((o >> 1) & 0x3ff) << 21) | (((o >> 11) & 1) << 20) | (((o >> 12) & 0xff) << 12) | 0x6f + } else { + assert!((-(1 << 12)..(1 << 12)).contains(&off), "a branch reaches 4 KiB"); + (((o >> 12) & 1) << 31) + | (((o >> 5) & 0x3f) << 25) + | (u32::from(f.rs2.0) << 20) + | (u32::from(f.rs1.0) << 15) + | (f.f3 << 12) + | (((o >> 1) & 0xf) << 8) + | (((o >> 11) & 1) << 7) + | 0x63 + }; + } + let mut text = String::new(); + for l in &self.lines { + let indent = if l.ends_with(':') { "" } else { " " }; + let _ = writeln!(text, "{indent}{l}"); + } + (self.words, text) + } +} + +/// Where a kernel's parameters and results live in the core's memory, from +/// `base`: the pair count, the gate's eight entries, the done word, four +/// columns of `capacity` operands, and the results. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Layout { + pub base: u32, + pub capacity: u32, +} + +impl Layout { + pub const COUNT: u32 = 0; + pub const ENTRIES: u32 = 4; + pub const DONE: u32 = 36; + const COLUMNS: u32 = 0x40; + + /// The address of column `k` (0 to 3). + pub fn column(&self, k: u32) -> u32 { + self.base + Self::COLUMNS + k * self.capacity * 4 + } + + /// The address of the results: for a gate, eight words per pair (each + /// row's low and high words, rows 0r, 0i, 1r, 1i); for the probe, four + /// (the product's low and high words, then `fx`'s). + pub fn results(&self) -> u32 { + self.column(4) + } + + /// The bytes from `base` the layout spans. + pub fn span(&self) -> u32 { + Self::COLUMNS + 4 * self.capacity * 4 + 8 * self.capacity * 4 + } +} + +/// `fx(m, a)` into (`T4` low, `T5` high): `A5` holds the rounding constant. +fn fx(p: &mut Asm, m: R, a: R, sh: u32) { + p.mul(T4, m, a).mulh(T5, m, a); + p.add(T4, T4, A5).sltu(T6, T4, A5).add(T5, T5, T6); + p.srli(T4, T4, sh).slli(T6, T5, 32 - sh).or(T4, T4, T6).srai(T5, T5, sh); +} + +/// The accumulator (`A6` low, `A7` high) plus or minus (`T4`, `T5`). +fn acc(p: &mut Asm, add: bool) { + if add { + p.add(A6, A6, T4).sltu(T6, A6, T4).add(A7, A7, T5).add(A7, A7, T6); + } else { + p.sltu(T6, A6, T4).sub(A6, A6, T4).sub(A7, A7, T5).sub(A7, A7, T6); + } +} + +/// The program that applies one gate to every pair: `rnd` the rounding +/// constant, `sh` the shift, `last_add` whether row 0's last real term is +/// added (a mutant). +pub fn rows_program(l: Layout, rnd: u32, sh: u32, last_add: bool) -> (Vec, String) { + // s0 base, s1 pairs left, s2..s9 the entries, a0..a3 the columns, a4 the + // results, a5 the rounding constant, t0..t3 one pair's operands. + const M: [R; 8] = [R(18), R(19), R(20), R(21), R(22), R(23), R(24), R(25)]; + let mut p = Asm::default(); + p.li(S0, l.base).lw(S1, Layout::COUNT as i32, S0); + for (k, &m) in M.iter().enumerate() { + p.lw(m, (Layout::ENTRIES + 4 * k as u32) as i32, S0); + } + p.li(A0, l.column(0)).li(A1, l.column(1)).li(A2, l.column(2)).li(A3, l.column(3)).li(A4, l.results()).li(A5, rnd); + p.label("next").beq(S1, ZERO, "done"); + p.lw(T0, 0, A0).lw(T1, 0, A1).lw(T2, 0, A2).lw(T3, 0, A3); + // Rows 0r, 0i, 1r, 1i: their terms (entry, operand, added) in order. + let rows: [[(usize, R, bool); 4]; 4] = [ + [(0, T0, true), (1, T1, false), (2, T2, true), (3, T3, last_add)], + [(0, T1, true), (1, T0, true), (2, T3, true), (3, T2, true)], + [(4, T0, true), (5, T1, false), (6, T2, true), (7, T3, false)], + [(4, T1, true), (5, T0, true), (6, T3, true), (7, T2, true)], + ]; + for (r, terms) in rows.iter().enumerate() { + for (i, &(e, a, add)) in terms.iter().enumerate() { + fx(&mut p, M[e], a, sh); + if i == 0 { + p.addi(A6, T4, 0).addi(A7, T5, 0); + } else { + acc(&mut p, add); + } + } + p.sw(A6, 8 * r as i32, A4).sw(A7, 8 * r as i32 + 4, A4); + } + p.addi(A0, A0, 4).addi(A1, A1, 4).addi(A2, A2, 4).addi(A3, A3, 4).addi(A4, A4, 32).addi(S1, S1, -1).j("next"); + p.label("done").addi(T6, ZERO, 1).sw(T6, Layout::DONE as i32, S0); + p.label("idle").j("idle"); + p.finish() +} + +/// The program that forms each pair's `a·b` and `fx(a, b)`: `a` in column 0, +/// `b` in column 1. +pub fn probe_program(l: Layout, rnd: u32, sh: u32) -> (Vec, String) { + let mut p = Asm::default(); + p.li(S0, l.base).lw(S1, Layout::COUNT as i32, S0); + p.li(A0, l.column(0)).li(A1, l.column(1)).li(A4, l.results()).li(A5, rnd); + p.label("next").beq(S1, ZERO, "done"); + p.lw(T0, 0, A0).lw(T1, 0, A1); + p.mul(T2, T0, T1).mulh(T3, T0, T1).sw(T2, 0, A4).sw(T3, 4, A4); + fx(&mut p, T1, T0, sh); + p.sw(T4, 8, A4).sw(T5, 12, A4); + p.addi(A0, A0, 4).addi(A1, A1, 4).addi(A4, A4, 16).addi(S1, S1, -1).j("next"); + p.label("done").addi(T6, ZERO, 1).sw(T6, Layout::DONE as i32, S0); + p.label("idle").j("idle"); + p.finish() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn one(f: impl FnOnce(&mut Asm)) -> u32 { + let mut p = Asm::default(); + f(&mut p); + p.finish().0[0] + } + + /// Encodings from the instruction set manual's formats, worked by hand. + #[test] + fn encodings_match_the_manual() { + assert_eq!(one(|p| { p.addi(R(1), ZERO, 1); }), 0x0010_0093); + assert_eq!(one(|p| { p.add(R(3), R(1), R(2)); }), 0x0020_81b3); + assert_eq!(one(|p| { p.sub(R(3), R(1), R(2)); }), 0x4020_81b3); + assert_eq!(one(|p| { p.mul(R(3), R(1), R(2)); }), 0x0220_81b3); + assert_eq!(one(|p| { p.mulh(R(3), R(1), R(2)); }), 0x0220_91b3); + assert_eq!(one(|p| { p.sltu(T1, T0, A5); }), 0x00f2_b333); + assert_eq!(one(|p| { p.or(R(3), R(1), R(2)); }), 0x0020_e1b3); + assert_eq!(one(|p| { p.lw(T0, 8, A0); }), 0x0085_2283); + assert_eq!(one(|p| { p.sw(T0, 8, A0); }), 0x0055_2423); + assert_eq!(one(|p| { p.lui(T0, 0x12345); }), 0x1234_52b7); + assert_eq!(one(|p| { p.slli(T0, T0, 2); }), 0x0022_9293); + assert_eq!(one(|p| { p.srli(T0, T0, 30); }), 0x01e2_d293); + assert_eq!(one(|p| { p.srai(T0, T0, 30); }), 0x41e2_d293); + assert_eq!(one(|p| { p.addi(T0, T0, -1); }), 0xfff2_8293); + assert_eq!(one(|p| { p.label("x").j("x"); }), 0x0000_006f); + assert_eq!(one(|p| { p.label("x").beq(ZERO, ZERO, "x"); }), 0x0000_0063); + } + + #[test] + fn branches_reach_forward_and_back() { + let mut p = Asm::default(); + p.label("top").beq(A0, A1, "end").addi(A0, A0, 1).j("top").label("end"); + let (w, text) = p.finish(); + // beq a0, a1, +12: imm[4:1] = 6. + assert_eq!(w[0], (6 << 8) | (11 << 20) | (10 << 15) | 0x63); + // j -8: imm = -8, 21 bits. + assert_eq!(w[2], 0xff9f_f06f); + assert!(text.contains("beq a0, a1, end") && text.contains("j top")); + } + + #[test] + fn li_builds_every_kind_of_constant() { + for v in [0u32, 1, 0x7ff, 0x800, 0xfff, 0x1000, 0x2000_0000, 0x1234_5678, 0x8000_0800, 0xffff_ffff, 0x1_0040] { + let mut p = Asm::default(); + p.li(T0, v); + let (w, _) = p.finish(); + // Evaluate: lui sets the upper bits, addi adds the sign-extended low. + let mut x = 0u32; + for word in w { + match word & 0x7f { + 0x37 => x = word & 0xffff_f000, + 0x13 => x = (if (word >> 15) & 0x1f == 0 { 0 } else { x }).wrapping_add(((word as i32) >> 20) as u32), + _ => unreachable!(), + } + } + assert_eq!(x, v, "{v:#x}"); + } + } + + #[test] + fn the_programs_and_their_mutants_differ_and_fit() { + let l = Layout { base: 0x1_0000, capacity: 1024 }; + let (w, text) = rows_program(l, 1 << 29, 30, false); + assert!(text.contains("mulh") && text.contains("srai") && w.len() > 200); + assert_ne!(rows_program(l, 1 << 28, 30, false).1, text); + assert_ne!(rows_program(l, 1 << 29, 29, false).1, text); + assert_ne!(rows_program(l, 1 << 29, 30, true).1, text); + assert_ne!(probe_program(l, 1 << 29, 30).1, probe_program(l, 1 << 28, 30).1); + assert_eq!(l.results(), l.base + 0x40 + 16 * 1024); + assert_eq!(l.span(), 0x40 + 48 * 1024); + } +} diff --git a/wai/wai-quantum/src/quantum_fabric_dataflow.rs b/wai/wai-quantum/src/quantum_fabric_dataflow.rs new file mode 100644 index 000000000..3b6c2b400 --- /dev/null +++ b/wai/wai-quantum/src/quantum_fabric_dataflow.rs @@ -0,0 +1,773 @@ +//! The fabric's dataflow member: the gate's fixed-point arithmetic on a RISC-V +//! core of a dataflow accelerator's compute tile, reached through the device's +//! own library (loaded at run time from `WAI_QUANTUM_DATAFLOW_LIB`), which +//! presents the device as a PCI endpoint: configuration space, memory windows +//! and clocks. +//! +//! **What runs where.** The program is built in this crate ([`rv32`]): plain +//! RV32IM, with every 64-bit value a pair of words (`mul` and `mulh` for the +//! product, `sltu` for carries and borrows, the arithmetic shift from word +//! shifts). It runs on the first core of each of [`TILES`] compute tiles, each +//! over a contiguous slice of a gate's pairs: the host holds every tile's cores +//! in reset, writes the program at the core's reset address and each tile's +//! parameters and slice beside it through that tile's memory window, releases +//! the first cores, advances the clock until each program writes its done word, +//! and reads the rows back in order. It checks each row's range and keeps +//! its low 32 bits, as [`lanes::apply_words`] does. +//! +//! **Provenance.** The library this member drives is a simulator of the device, +//! so every certificate says so (`WAI_QUANTUM_DATAFLOW_SIMULATOR` names it). +//! +//! The device is named by its PCI vendor and device ids. Three mutants are +//! compiled in and must each be caught. + +use crate::quantum::{Amp, Circuit, Gateset, MAX_QUBITS, ONE, QuantumError, StateVector}; +use crate::quantum_fabric::lanes::{self, GateWords, Lane}; +use crate::quantum_fabric::plan::{self, Config, LaneRun, Program, Step}; +use crate::quantum_fabric::rv32::{self, Layout}; +use crate::quantum_fabric::{Certificate, Certification, Corpus, FabricClass, KernelEntry, Member, MemberError, Presence, Provenance, RunFuture, RunMeter, Runner, apply_kernel}; +use std::sync::{Mutex, OnceLock}; + +/// One probe case: [`lanes::mul_wide`] and [`lanes::fxmul_wide`] as +/// `(high, low)` words. +pub type ProbeOutput = ((u32, u32), (u32, u32)); + +/// Why the device is not a member, or a run has no state. +#[derive(Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum DataflowError { + Unavailable(String), + Circuit(QuantumError), + TooWide { n_qubits: u8, max: u8 }, + OutOfRange { gate: u32 }, + Internal(&'static str), + Probe { a: i32, b: i32 }, + /// The core did not finish within the clocks its program needs. + Stalled { clocks: u64 }, +} + +impl std::fmt::Display for DataflowError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + DataflowError::Unavailable(why) => write!(f, "no dataflow device: {why}"), + DataflowError::Circuit(e) => write!(f, "{e}"), + DataflowError::TooWide { n_qubits, max } => write!(f, "{n_qubits} qubits; this member holds at most {max}"), + DataflowError::OutOfRange { gate } => write!(f, "the result of op {gate} leaves 32 bits"), + DataflowError::Internal(why) => write!(f, "host defect: {why}"), + DataflowError::Probe { a, b } => write!(f, "the kernel's arithmetic differs from the CPU's at ({a}, {b})"), + DataflowError::Stalled { clocks } => write!(f, "the core did not finish in {clocks} clocks"), + } + } +} + +impl std::error::Error for DataflowError {} + +impl From for MemberError { + fn from(e: DataflowError) -> MemberError { + match e { + DataflowError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max }, + DataflowError::OutOfRange { gate } => MemberError::OutOfRange { gate }, + DataflowError::Circuit(e) => MemberError::Circuit(e), + other => MemberError::Other(other.to_string()), + } + } +} + +/// The kernels a member runs: the conforming one, or a mutant. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Variant { + Conforming, + /// The rounding constant is half what it should be. + M1, + /// The product is shifted by one bit too few. + M2, + /// Row 0's real part adds its last term instead of subtracting it. + M3, +} + +impl Variant { + pub const MUTANTS: [Variant; 3] = [Variant::M1, Variant::M2, Variant::M3]; + + /// The kernel's constants: the rounding constant, the shift, and whether + /// row 0's last real term is added. + pub fn constants(self) -> (u32, u32, bool) { + match self { + Variant::Conforming => (1 << 29, 30, false), + Variant::M1 => (1 << 28, 30, false), + Variant::M2 => (1 << 29, 29, false), + Variant::M3 => (1 << 29, 30, true), + } + } +} + +/// Where the programs put their parameters and results in the tile's memory: +/// past the programs, a batch of up to `capacity` pairs. +pub const LAYOUT: Layout = Layout { base: 0x1_0000, capacity: 8192 }; + +/// The rows program and the probe program of a variant, as words and listing. +fn programs(v: Variant) -> ((Vec, String), (Vec, String)) { + let (rnd, sh, last_add) = v.constants(); + (rv32::rows_program(LAYOUT, rnd, sh, last_add), rv32::probe_program(LAYOUT, rnd, sh)) +} + +/// The implementation name the id starts with. +pub const IMPLEMENTATION: &str = "rv32-lanes64"; + +/// Version of the host layout: the gather of a gate's pairs into four columns, +/// the batch of [`LAYOUT`], the rows as word pairs, and the host's range check. +/// Part of the implementation id. +pub const HOST_LAYOUT: u32 = 1; + +/// `"rv32-lanes64/<16 hex>"` over a variant's two listings and the layout. +pub fn implementation_id_of(v: Variant) -> String { + let ((_, rows), (_, probe)) = programs(v); + let mut o = b"wai:quantum-fabric-dataflow\x01".to_vec(); + for text in [&rows, &probe] { + o.extend_from_slice(&(text.len() as u64).to_be_bytes()); + o.extend_from_slice(text.as_bytes()); + } + o.extend_from_slice(&HOST_LAYOUT.to_be_bytes()); + let h: String = blake3::hash(&o).as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect(); + format!("{IMPLEMENTATION}/{h}") +} + +pub fn implementation_id() -> String { + implementation_id_of(Variant::Conforming) +} + +/// The compute tiles a run spreads its pairs over, the first core of each. +pub const TILES: usize = 32; + +/// The device generations this member knows: the PCI device id, the compute +/// tiles it uses, and how a memory window is set up. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Chip { + pub device: u16, + /// The compute tiles' (x, y) on the device's network, as the host + /// addresses them. + pub tiles: &'static [(u32, u32); TILES], + /// A window's size, and the stride of its configuration registers. + window: u64, + cfg_stride: u64, +} + +/// The PCI vendor id of the devices this member knows. +pub const VENDOR: u16 = 0x1e52; + +/// Compute tiles in columns 1–4 and 6–9 of rows 1–4. +const TILES_401E: [(u32, u32); TILES] = { + let xs = [1, 2, 3, 4, 6, 7, 8, 9]; + let mut t = [(0, 0); TILES]; + let mut i = 0; + while i < TILES { + t[i] = (xs[i % 8], 1 + (i / 8) as u32); + i += 1; + } + t +}; + +/// Compute tiles in rows 2 and 3 (columns 1–7 and 10–14), then eight of row 4. +const TILES_B140: [(u32, u32); TILES] = { + let xs = [1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14]; + let mut t = [(0, 0); TILES]; + let mut i = 0; + while i < TILES { + t[i] = (xs[i % 12], 2 + (i / 12) as u32); + i += 1; + } + t +}; + +pub const CHIPS: [Chip; 2] = [ + // One 64-bit configuration register per 1 MiB window. + Chip { device: 0x401e, tiles: &TILES_401E, window: 1 << 20, cfg_stride: 8 }, + // Three 32-bit configuration registers per 2 MiB window. + Chip { device: 0xb140, tiles: &TILES_B140, window: 1 << 21, cfg_stride: 12 }, +]; + +/// The tile's soft-reset register: bit 11 holds the first core, 12–14 and 18 +/// the others; all five set holds every core. +const SOFT_RESET: u64 = 0xffb1_21b0; +const HOLD_ALL: u32 = 0x4_7800; +const RUN_FIRST: u32 = 0x4_7000; +/// Where the window configuration registers begin in the first memory window. +const TLB_CFG: u64 = 0x1fc0_0000; + +#[cfg(target_os = "linux")] +mod dl { + use std::ffi::{c_char, c_int, c_void}; + #[link(name = "dl")] + unsafe extern "C" { + pub fn dlopen(filename: *const c_char, flag: c_int) -> *mut c_void; + pub fn dlsym(handle: *mut c_void, symbol: *const c_char) -> *mut c_void; + pub fn dlerror() -> *const c_char; + } + pub const RTLD_NOW: c_int = 2; +} + +/// The device library's entry points. +struct Lib { + init: unsafe extern "C" fn(), + cfg_rd: unsafe extern "C" fn(u32, u32) -> u32, + mem_rd: unsafe extern "C" fn(u64, *mut std::ffi::c_void, u32), + mem_wr: unsafe extern "C" fn(u64, *const std::ffi::c_void, u32), + clock: unsafe extern "C" fn(u32), +} + +/// The device, open in this process: the library is one per process and runs +/// on the calling thread, so every access holds the lock. +struct Device { + lib: Lib, + chip: Chip, + bar0: u64, + /// The program now in every tile's memory. + loaded: Option>, + /// Clocks advanced so far. + clocks: u64, +} + +static DEVICE: OnceLock, String>> = OnceLock::new(); + +#[cfg(target_os = "linux")] +fn open_library(path: &str) -> Result { + use std::ffi::CString; + let c = CString::new(path).map_err(|_| format!("{path:?} holds a NUL"))?; + // SAFETY: dlopen and dlsym with valid C strings; each symbol is cast to the + // signature the library's interface documents. + unsafe { + let h = dl::dlopen(c.as_ptr(), dl::RTLD_NOW); + if h.is_null() { + let e = dl::dlerror(); + return Err(if e.is_null() { format!("{path}: cannot load") } else { format!("{path}: {}", std::ffi::CStr::from_ptr(e).to_string_lossy()) }); + } + let sym = |name: &str| -> Result<*mut std::ffi::c_void, String> { + let n = CString::new(name).expect("no NUL"); + let p = dl::dlsym(h, n.as_ptr()); + if p.is_null() { Err(format!("{path} has no {name}")) } else { Ok(p) } + }; + Ok(Lib { + init: std::mem::transmute::<*mut std::ffi::c_void, unsafe extern "C" fn()>(sym("libttsim_init")?), // allow-name: the device library's entry point + cfg_rd: std::mem::transmute::<*mut std::ffi::c_void, unsafe extern "C" fn(u32, u32) -> u32>(sym("libttsim_pci_config_rd32")?), // allow-name: the device library's entry point + mem_rd: std::mem::transmute::<*mut std::ffi::c_void, unsafe extern "C" fn(u64, *mut std::ffi::c_void, u32)>(sym("libttsim_pci_mem_rd_bytes")?), // allow-name: the device library's entry point + mem_wr: std::mem::transmute::<*mut std::ffi::c_void, unsafe extern "C" fn(u64, *const std::ffi::c_void, u32)>(sym("libttsim_pci_mem_wr_bytes")?), // allow-name: the device library's entry point + clock: std::mem::transmute::<*mut std::ffi::c_void, unsafe extern "C" fn(u32)>(sym("libttsim_clock")?), // allow-name: the device library's entry point + }) + } +} + +#[cfg(not(target_os = "linux"))] +fn open_library(_path: &str) -> Result { + Err(format!("the device library is loaded on Linux; this is {}", std::env::consts::OS)) +} + +impl Device { + fn open() -> Result { + let path = std::env::var("WAI_QUANTUM_DATAFLOW_LIB").ok().filter(|s| !s.trim().is_empty()).ok_or("no device library named (WAI_QUANTUM_DATAFLOW_LIB)")?; + let lib = open_library(path.trim())?; + // SAFETY: the library's documented sequence: initialise once per + // process (this runs once, under the OnceLock), then configuration + // reads of the one endpoint at bus 0, device 0, function 0. + let (id, lo, hi) = unsafe { + (lib.init)(); + ((lib.cfg_rd)(0, 0), (lib.cfg_rd)(0, 0x10), (lib.cfg_rd)(0, 0x14)) + }; + let (vendor, device) = (id as u16, (id >> 16) as u16); + if vendor != VENDOR { + return Err(format!("the endpoint is {vendor:04x}:{device:04x}, not vendor {VENDOR:04x}")); + } + let chip = CHIPS.into_iter().find(|c| c.device == device).ok_or(format!("device {vendor:04x}:{device:04x} is not one this member knows"))?; + let mut d = Device { lib, chip, bar0: (u64::from(hi) << 32) | u64::from(lo & !0xf), loaded: None, clocks: 0 }; + // Two windows per tile: its memory from 0, and its registers. + for (j, &tile) in chip.tiles.iter().enumerate() { + d.window(2 * j as u64, tile, 0); + d.window(2 * j as u64 + 1, tile, SOFT_RESET & !(chip.window - 1)); + d.reg_write(j, HOLD_ALL); + } + Ok(d) + } + + /// Points window `i` at `addr` (window-aligned) on `tile`. + fn window(&mut self, i: u64, tile: (u32, u32), addr: u64) { + let (x, y) = tile; + let at = self.bar0 + TLB_CFG + i * self.chip.cfg_stride; + if self.chip.window == 1 << 20 { + let cfg = (addr >> 20) | (u64::from(x) << 16) | (u64::from(y) << 22); + self.write(at, &(cfg as u32).to_le_bytes()); + self.write(at + 4, &((cfg >> 32) as u32).to_le_bytes()); + } else { + let coord = u64::from(x) | (u64::from(y) << 6); + self.write(at, &((addr >> 21) as u32).to_le_bytes()); + self.write(at + 4, &((((addr >> 53) & 0x7ff) | (coord << 11)) as u32).to_le_bytes()); + self.write(at + 8, &0u32.to_le_bytes()); + } + } + + fn write(&mut self, paddr: u64, bytes: &[u8]) { + // SAFETY: a write within a window the device decodes; the library + // copies `bytes.len()` bytes from the buffer. + unsafe { (self.lib.mem_wr)(paddr, bytes.as_ptr().cast(), bytes.len() as u32) } + } + + fn read(&mut self, paddr: u64, out: &mut [u8]) { + // SAFETY: as for `write`, into the buffer. + unsafe { (self.lib.mem_rd)(paddr, out.as_mut_ptr().cast(), out.len() as u32) } + } + + /// Writes tile `j`'s soft-reset register. + fn reg_write(&mut self, j: usize, v: u32) { + let at = self.bar0 + (2 * j as u64 + 1) * self.chip.window + (SOFT_RESET & (self.chip.window - 1)); + self.write(at, &v.to_le_bytes()); + } + + fn l1_write_words(&mut self, j: usize, addr: u32, words: &[u32]) { + let bytes: Vec = words.iter().flat_map(|w| w.to_le_bytes()).collect(); + self.write(self.bar0 + 2 * j as u64 * self.chip.window + u64::from(addr), &bytes); + } + + fn l1_read_words(&mut self, j: usize, addr: u32, n: usize) -> Vec { + let mut bytes = vec![0u8; 4 * n]; + self.read(self.bar0 + 2 * j as u64 * self.chip.window + u64::from(addr), &mut bytes); + bytes.as_chunks::<4>().0.iter().map(|&c| u32::from_le_bytes(c)).collect() + } + + /// Runs `program` on the first `counts.len()` tiles, tile `j` over + /// `counts[j]` pairs already written to it, until each writes its done + /// word; at most `budget` clocks. + fn run(&mut self, program: &[u32], counts: &[u32], budget: u64) -> Result<(), DataflowError> { + for j in 0..TILES { + self.reg_write(j, HOLD_ALL); + } + if self.loaded.as_deref() != Some(program) { + for j in 0..TILES { + self.l1_write_words(j, 0, program); + } + self.loaded = Some(program.to_vec()); + } + for (j, &count) in counts.iter().enumerate() { + self.l1_write_words(j, LAYOUT.base + Layout::COUNT, &[count]); + self.l1_write_words(j, LAYOUT.base + Layout::DONE, &[0]); + } + for j in 0..counts.len() { + self.reg_write(j, RUN_FIRST); + } + let mut spent = 0u64; + // A first stretch near what the program needs, then short ones. + let mut step = budget.div_ceil(2).clamp(1, u64::from(u32::MAX)); + let mut waiting: Vec = (0..counts.len()).collect(); + loop { + // SAFETY: advances the device's clock; nothing else is touched. + unsafe { (self.lib.clock)(step as u32) }; + spent += step; + self.clocks += step; + waiting.retain(|&j| self.l1_read_words(j, LAYOUT.base + Layout::DONE, 1)[0] != 1); + if waiting.is_empty() { + break; + } + if spent >= budget { + for j in 0..TILES { + self.reg_write(j, HOLD_ALL); + } + return Err(DataflowError::Stalled { clocks: spent }); + } + step = 4096.min(budget - spent).max(1); + } + for j in 0..counts.len() { + self.reg_write(j, HOLD_ALL); + } + Ok(()) + } +} + +fn device() -> Result<&'static Mutex, String> { + DEVICE.get_or_init(|| Device::open().map(Mutex::new)).as_ref().map_err(|e| e.clone()) +} + +/// What [`discover`] found. +#[derive(Debug)] +pub struct Discovery { + pub found: Vec, + pub refused: Vec<(String, DataflowError)>, + pub unavailable: Option, +} + +/// The device, opened and probed in this process, or why there is none. +pub fn discover() -> Discovery { + let mut d = Discovery { found: Vec::new(), refused: Vec::new(), unavailable: None }; + let chip = match device() { + Ok(dev) => dev.lock().unwrap_or_else(|p| p.into_inner()).chip, + Err(why) => { + d.unavailable = Some(why); + return d; + } + }; + match DataflowMember::open(chip, Variant::Conforming) { + Ok(m) => d.found.push(m), + Err(e) => d.refused.push((key_of(chip), e)), + } + d +} + +/// The key a runner declares: `dataflow--`. +pub fn key_of(chip: Chip) -> String { + format!("dataflow-{VENDOR:04x}-{:04x}", chip.device) +} + +/// The most qubits a run holds. +pub const MAX_DATAFLOW_QUBITS: u8 = if 20 < MAX_QUBITS { 20 } else { MAX_QUBITS }; + +/// Clocks a program needs for `count` pairs, with room to spare: the rows +/// program executes about 220 instructions a pair, the probe about 25. +fn budget(count: u32, per_pair: u64) -> u64 { + 1_000 + 2 * per_pair * u64::from(count) +} + +/// A compute tile's core as a fabric member. +#[derive(Clone, Debug)] +pub struct DataflowMember { + chip: Chip, + variant: Variant, + key: String, + id: String, + implementation: String, + detail: String, + provenance: Provenance, + max_qubits: u8, +} + +impl DataflowMember { + fn open(chip: Chip, variant: Variant) -> Result { + let label = std::env::var("WAI_QUANTUM_DATAFLOW_SIMULATOR").ok().filter(|s| !s.trim().is_empty()).unwrap_or_else(|| "the device library".into()); + let provenance = Provenance::Simulator(label.trim().to_string()); + let key = key_of(chip); + let mut o = b"wai:quantum-fabric-dataflow-member\x01".to_vec(); + o.extend_from_slice(key.as_bytes()); + for &(x, y) in chip.tiles { + o.extend_from_slice(&x.to_be_bytes()); + o.extend_from_slice(&y.to_be_bytes()); + } + o.extend_from_slice(provenance.label().as_bytes()); + let id = format!("local/dataflow-{}", blake3::hash(&o).as_bytes()[..6].iter().map(|b| format!("{b:02x}")).collect::()); + let member = DataflowMember { + chip, + variant, + detail: format!( + "dataflow; pci {VENDOR:04x}:{:04x}, the first RISC-V core (RV32IM) of {TILES} compute tiles; {}; up to {MAX_DATAFLOW_QUBITS} qubits; unmetered", + chip.device, + provenance.label() + ), + key, + id, + implementation: implementation_id_of(variant), + provenance, + max_qubits: MAX_DATAFLOW_QUBITS, + }; + if variant == Variant::Conforming { + let pairs = lanes::probe_pairs(1024, 0x5eed); + let out = member.probe(&pairs)?; + if let Some((a, b)) = lanes::first_mismatch(&pairs, &out) { + return Err(DataflowError::Probe { a, b }); + } + } + Ok(member) + } + + /// The same device running a mutant: for the mutation tests. + #[doc(hidden)] + pub fn with_variant(&self, v: Variant) -> DataflowMember { + DataflowMember { variant: v, implementation: implementation_id_of(v), ..self.clone() } + } + + #[doc(hidden)] + pub fn with_max_qubits(&self, max_qubits: u8) -> DataflowMember { + DataflowMember { max_qubits: max_qubits.min(self.max_qubits), ..self.clone() } + } + + /// The kernel this member runs: the conforming one, or a mutant. + pub fn variant(&self) -> Variant { + self.variant + } + + pub fn chip(&self) -> Chip { + self.chip + } + + /// The clocks the device has advanced in this process. + pub fn clocks(&self) -> u64 { + device().map(|d| d.lock().unwrap_or_else(|p| p.into_inner()).clocks).unwrap_or(0) + } + + pub fn id(&self) -> &str { + &self.id + } + + pub fn key(&self) -> &str { + &self.key + } + + pub fn class(&self) -> FabricClass { + FabricClass::Dataflow + } + + pub fn detail(&self) -> &str { + &self.detail + } + + pub fn implementation(&self) -> &str { + &self.implementation + } + + pub fn provenance(&self) -> &Provenance { + &self.provenance + } + + pub fn max_qubits(&self) -> u8 { + self.max_qubits + } + + pub fn config(&self) -> Config { + Config::PER_GATE + } + + fn device(&self) -> Result, DataflowError> { + Ok(device().map_err(DataflowError::Unavailable)?.lock().unwrap_or_else(|p| p.into_inner())) + } + + /// The kernel's [`lanes::mul_wide`] and [`lanes::fxmul_wide`] of each pair. + pub fn probe(&self, pairs: &[(i32, i32)]) -> Result, DataflowError> { + let (_, (program, _)) = programs(self.variant); + let mut dev = self.device()?; + let mut out = Vec::with_capacity(pairs.len()); + for round in pairs.chunks(TILES * LAYOUT.capacity as usize) { + let slices: Vec<&[(i32, i32)]> = round.chunks(round.len().div_ceil(TILES)).collect(); + for (j, slice) in slices.iter().enumerate() { + dev.l1_write_words(j, LAYOUT.column(0), &slice.iter().map(|p| p.0 as u32).collect::>()); + dev.l1_write_words(j, LAYOUT.column(1), &slice.iter().map(|p| p.1 as u32).collect::>()); + } + let counts: Vec = slices.iter().map(|s| s.len() as u32).collect(); + dev.run(&program, &counts, budget(counts[0], 25))?; + for (j, slice) in slices.iter().enumerate() { + let w = dev.l1_read_words(j, LAYOUT.results(), 4 * slice.len()); + out.extend(w.as_chunks::<4>().0.iter().map(|c| ((c[1], c[0]), (c[3], c[2])))); + } + } + Ok(out) + } + + /// The four rows of each pair, from the kernel. + fn rows(&self, m: &[i32; 8], cols: &[Vec; 4]) -> Result<[Vec; 4], DataflowError> { + let ((program, _), _) = programs(self.variant); + let mut dev = self.device()?; + let n = cols[0].len(); + let mut o: [Vec; 4] = Default::default(); + for start in (0..n).step_by(TILES * LAYOUT.capacity as usize) { + let end = (start + TILES * LAYOUT.capacity as usize).min(n); + // Contiguous slices, one per tile, in order. + let share = (end - start).div_ceil(TILES); + let slices: Vec<(usize, usize)> = (start..end).step_by(share).map(|s| (s, (s + share).min(end))).collect(); + for (j, &(s, e)) in slices.iter().enumerate() { + dev.l1_write_words(j, LAYOUT.base + Layout::ENTRIES, &m.map(|x| x as u32)); + for (k, col) in cols.iter().enumerate() { + dev.l1_write_words(j, LAYOUT.column(k as u32), &col[s..e].iter().map(|&v| v as u32).collect::>()); + } + } + let counts: Vec = slices.iter().map(|&(s, e)| (e - s) as u32).collect(); + dev.run(&program, &counts, budget(counts[0], 220))?; + for (j, &(s, e)) in slices.iter().enumerate() { + let w = dev.l1_read_words(j, LAYOUT.results(), 8 * (e - s)); + for pair in w.as_chunks::<8>().0 { + for (r, row) in o.iter_mut().enumerate() { + row.push((i64::from(pair[2 * r + 1] as i32) << 32) | i64::from(pair[2 * r])); + } + } + } + } + Ok(o) + } + + /// Gates in order from `state`, as [`lanes::run_words`] runs them. + pub fn run_lanes(&self, state: &[Lane], gates: &[GateWords]) -> Result { + let len = state.len(); + if len < 2 || !len.is_power_of_two() { + return Err(DataflowError::Internal("a state that is not 2^n lanes, n at least 1")); + } + let n = len.trailing_zeros() as u8; + if n > self.max_qubits { + return Err(DataflowError::TooWide { n_qubits: n, max: self.max_qubits }); + } + let planned: Vec<(&GateWords, u32)> = gates.iter().zip(0u32..).collect(); + self.execute(state.to_vec(), &planned) + } + + fn execute(&self, mut amps: Vec, gates: &[(&GateWords, u32)]) -> Result { + let len = amps.len(); + for (w, _) in gates { + let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize); + if tbit == 0 || !tbit.is_power_of_two() || tbit > len / 2 || ctrl & tbit != 0 || ctrl & !(len - 1) != 0 { + return Err(DataflowError::Internal("malformed gate words for this state")); + } + } + let mut report: Option = None; + for &(w, index) in gates { + let m: [i32; 8] = w.m.map(|x| x as i32); + let (tbit, ctrl) = (w.tbit as usize, w.ctrl as usize); + let mut at = Vec::new(); + let mut cols: [Vec; 4] = Default::default(); + for p in 0..len / 2 { + let low = p & (tbit - 1); + let i = ((p - low) << 1) | low; + if i & ctrl != ctrl { + continue; + } + at.push(i); + let (a0, a1) = (amps[i], amps[i | tbit]); + cols[0].push(a0[0]); + cols[1].push(a0[1]); + cols[2].push(a1[0]); + cols[3].push(a1[1]); + } + if at.is_empty() { + continue; + } + let [o0r, o0i, o1r, o1i] = self.rows(&m, &cols)?; + for (k, &i) in at.iter().enumerate() { + let rows = [o0r[k], o0i[k], o1r[k], o1i[k]]; + if rows.iter().any(|&v| i32::try_from(v).is_err()) { + report = Some(report.map_or(index, |r| r.min(index))); + } + let low = |v: i64| v as u64 as u32 as i32; + amps[i] = [low(rows[0]), low(rows[1])]; + amps[i | tbit] = [low(rows[2]), low(rows[3])]; + } + } + Ok(LaneRun { amps, out_of_range: report }) + } + + pub fn run_program(&self, p: &Program) -> Result { + let n = p.n_qubits; + if n == 0 { + return Ok(LaneRun { amps: vec![[ONE as i32, 0]], out_of_range: None }); + } + if n > self.max_qubits { + return Err(DataflowError::TooWide { n_qubits: n, max: self.max_qubits }); + } + if p.start >= 1usize << n { + return Err(DataflowError::Internal("the start is outside the state")); + } + let mut gates = Vec::with_capacity(p.steps.len()); + for step in &p.steps { + let Step::Gate { gate } = *step else { return Err(DataflowError::Internal("a tile step; this member runs gate steps only")) }; + let g = p.gates.get(gate as usize).ok_or(DataflowError::Internal("a step names a gate the program does not hold"))?; + if !g.is_well_formed() { + return Err(DataflowError::Internal("malformed gate words for this state")); + } + gates.push((&g.words, g.index)); + } + let mut amps = vec![[0i32, 0]; 1usize << n]; + amps[p.start] = [ONE as i32, 0]; + self.execute(amps, &gates) + } + + pub fn simulate_from(&self, c: &Circuit, start: usize) -> Result { + let p = plan::compile_with(c, start, &self.config()).map_err(DataflowError::Circuit)?; + if p.n_qubits > self.max_qubits { + return Err(DataflowError::TooWide { n_qubits: p.n_qubits, max: self.max_qubits }); + } + let run = self.run_program(&p)?; + if let Some(gate) = run.out_of_range { + return Err(DataflowError::OutOfRange { gate }); + } + Ok(StateVector { n_qubits: p.n_qubits, amps: run.amps.iter().map(|l| Amp { re: i64::from(l[0]), im: i64::from(l[1]) }).collect() }) + } + + pub fn simulate(&self, c: &Circuit) -> Result { + self.simulate_from(c, 0) + } + + /// The certificate of [`apply_kernel`]`(gs)` on this member, with its + /// provenance. + pub fn certify(&self, gs: Gateset, corpus: Corpus) -> Result { + let mut cert = Certification::new(); + for c in corpus.circuits(gs) { + let reference = lanes::reference_output(&c).map_err(DataflowError::Circuit)?; + let run = self.run_program(&plan::compile_with(&c, 0, &self.config()).map_err(DataflowError::Circuit)?)?; + if let Some(gate) = run.out_of_range { + return Err(DataflowError::OutOfRange { gate }); + } + cert.add(&lanes::circuit_bytes(&c), &reference, &lanes::output(&run.amps)); + } + Ok(cert.finish(apply_kernel(gs), &self.implementation, &self.id, &corpus.generator(gs)).with_provenance(self.provenance.clone())) + } + + pub fn member(&self, corpus: Corpus) -> Result { + let mut kernels = Vec::new(); + for gs in Gateset::ALL { + kernels.push(KernelEntry { kernel: apply_kernel(gs).into(), implementation: self.implementation.clone(), certificate: self.certify(gs, corpus)?, energy: None }); + } + Ok(Member { id: self.id.clone(), class: FabricClass::Dataflow, presence: Presence::Detected, kernels, detail: self.detail.clone() }) + } +} + +impl Runner for DataflowMember { + fn member_id(&self) -> String { + self.id.clone() + } + + fn run<'r>(&'r self, implementation: &'r str, c: &'r Circuit, start: usize, _meter: &'r mut dyn RunMeter) -> RunFuture<'r> { + Box::pin(async move { + if implementation != self.implementation { + return Err(MemberError::Other(format!("member {} runs implementation {}, not {implementation:?}", self.id, self.implementation))); + } + Ok((self.simulate_from(c, start)?, 0, None)) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_variant_has_its_own_implementation_id() { + let ids: std::collections::BTreeSet = [Variant::Conforming, Variant::M1, Variant::M2, Variant::M3].into_iter().map(implementation_id_of).collect(); + assert_eq!(ids.len(), 4); + assert!(implementation_id().starts_with("rv32-lanes64/")); + } + + #[test] + fn the_layout_fits_one_window_past_the_programs() { + let ((rows, _), (probe, _)) = programs(Variant::Conforming); + assert!(4 * rows.len().max(probe.len()) as u32 <= LAYOUT.base); + for chip in CHIPS { + assert!(u64::from(LAYOUT.base + LAYOUT.span()) <= chip.window); + } + } + + #[test] + fn each_chip_names_distinct_tiles_whose_windows_it_has() { + // Window registers: 156 of 1 MiB on the first, 202 of 2 MiB on the + // second; two per tile. + for (chip, windows) in CHIPS.into_iter().zip([156usize, 202]) { + let set: std::collections::BTreeSet<(u32, u32)> = chip.tiles.iter().copied().collect(); + assert_eq!(set.len(), TILES); + assert!(2 * TILES <= windows); + assert!(2 * TILES as u64 * chip.window <= TLB_CFG, "the windows end before their configuration registers"); + } + assert!(TILES_B140.iter().all(|&(x, y)| (2..=11).contains(&y) && !(8..=9).contains(&x) && x <= 14)); + assert!(TILES_401E.iter().all(|&(x, y)| x != 5 && x != 0 && (1..=5).contains(&y))); + } + + #[test] + fn the_keys_name_the_pci_ids() { + assert_eq!(key_of(CHIPS[0]), "dataflow-1e52-401e"); + assert_eq!(key_of(CHIPS[1]), "dataflow-1e52-b140"); + } + + #[test] + fn with_no_library_named_the_member_is_absent_and_says_why() { + if std::env::var_os("WAI_QUANTUM_DATAFLOW_LIB").is_none() { + let d = discover(); + assert!(d.found.is_empty() && d.refused.is_empty()); + assert!(d.unavailable.as_deref().is_some_and(|w| w.contains("WAI_QUANTUM_DATAFLOW_LIB") || w.contains("loaded on Linux")), "{:?}", d.unavailable); + } + } +} diff --git a/wai/wai-quantum/tests/dataflow_member.rs b/wai/wai-quantum/tests/dataflow_member.rs new file mode 100644 index 000000000..e357be9c6 --- /dev/null +++ b/wai/wai-quantum/tests/dataflow_member.rs @@ -0,0 +1,410 @@ +//! The fabric's dataflow member on the machine that runs this. +//! +//! The machine declares the devices it must have in `WAI_QUANTUM_DATAFLOW_EXPECT`: +//! their keys, comma-separated and sorted, or `none`. Every test that needs a +//! member fails unless the devices found are exactly the declared ones, so a +//! machine whose GPU or driver went missing, or a run that declared nothing, +//! fails instead of passing over no member. Each test prints what it +//! established and, when `WAI_QUANTUM_DATAFLOW_REPORT` names a file, appends the +//! same lines to it, which a CI step can count per device. +//! +//! ```text +//! WAI_QUANTUM_DATAFLOW_LIB= WAI_QUANTUM_DATAFLOW_EXPECT=dataflow-1e52-b140 \ +//! cargo test --release --features quantum_fabric_dataflow --test dataflow_member -- --test-threads=1 +//! ``` +#![cfg(not(target_arch = "wasm32"))] + +use std::io::Write; +use std::sync::OnceLock; +use wai_quantum::ed25519_dalek::SigningKey; +use wai_quantum::quantum::{BaseGate, Circuit, Gate, Gateset, MAX_QUBITS, QuantumError}; +use wai_quantum::quantum_fabric::lanes::{self, GateWords, Lane}; +use wai_quantum::quantum_fabric::plan::{Program, Step}; +use wai_quantum::quantum_fabric::{ + Corpus, CpuReference, FABRIC_DEFAULT, Fabric, FabricClass, FabricReceipt, MemberError, Parity, Presence, REFERENCE, apply_kernel, detect_local, place, ready, +}; +use wai_quantum::quantum_fabric_dataflow::{self as dataflow, DataflowError, DataflowMember, Discovery, Variant, discover}; + +/// The CPU's corpus digests: random-v1 (seed 7, 200 cases) and fabric-v1 +/// (seed 7, 200 + 24 cases), per gateset. A bit-exact member reproduces them. +const PINS: [(Gateset, &str, &str); 2] = [ + (Gateset::V1, "d18454c639acfd9c7b66450f4ded1aac4ab46cc10a7bedbcf4958a96e204fb36", "41c726a0ac80f7defef92e15ecf893bdb7a3076b5e7f670fa0b7142e5c033df7"), + (Gateset::V2, "3dbde276c1095bf827faec71d5980e050f8a55a5ba7d97cd3bbdaec3f43a13b2", "9a84b55af68f1a8b07390b234cf562dd8d85cef3185a697ab85690c8790a7809"), +]; + +const RANDOM_V1: Corpus = Corpus::Random { seed: 7, cases: 200 }; + + +static DISCOVERY: OnceLock = OnceLock::new(); + +fn discovery() -> &'static Discovery { + DISCOVERY.get_or_init(discover) +} + +/// Prints `line` and appends it to the file `WAI_QUANTUM_DATAFLOW_REPORT` names. +fn report(line: &str) { + println!("{line}"); + if let Some(path) = std::env::var_os("WAI_QUANTUM_DATAFLOW_REPORT") { + let mut f = std::fs::OpenOptions::new().create(true).append(true).open(&path).expect("the report file opens"); + writeln!(f, "{line}").expect("the report line is written"); + } +} + +/// The declared device set, as written, if it is exactly what was found; +/// otherwise why not, with what was found and refused and how to declare it. +fn declared(d: &Discovery) -> Result { + let mut found: Vec<&str> = d.found.iter().map(|m| m.key()).collect(); + found.sort_unstable(); + let found_list = if found.is_empty() { "none".to_string() } else { found.join(",") }; + let refused: Vec = d.refused.iter().map(|(k, e)| format!("{k}: {e}")).collect(); + let refused_list = if refused.is_empty() { "none".to_string() } else { refused.join("; ") }; + let unavailable = d.unavailable.as_deref().unwrap_or("no"); + let how = format!("found: {found_list}; refused: {refused_list}; unavailable: {unavailable}. Declare the devices this machine must have: WAI_QUANTUM_DATAFLOW_EXPECT={found_list}"); + let expect = match std::env::var("WAI_QUANTUM_DATAFLOW_EXPECT") { + Ok(v) if !v.is_empty() => v, + _ => return Err(format!("WAI_QUANTUM_DATAFLOW_EXPECT is not set. {how}")), + }; + let keys: Vec<&str> = if expect == "none" { Vec::new() } else { expect.split(',').collect() }; + let well_formed = keys.windows(2).all(|w| w[0] <= w[1]) && keys.iter().all(|k| !k.is_empty() && *k != "none" && k.trim() == *k); + if !well_formed { + return Err(format!("WAI_QUANTUM_DATAFLOW_EXPECT={expect:?} is not a sorted, comma-separated list of keys, nor none. {how}")); + } + for k in &keys { + if !found.contains(k) + && let Some((_, e)) = d.refused.iter().find(|(key, _)| key == k) + { + return Err(format!("the declared device {k} was refused: {e}. {how}")); + } + } + if found != keys { + return Err(format!("the devices found are not the declared ones ({expect}). {how}")); + } + Ok(expect) +} + +/// The members, once the declared devices are exactly the ones found. +fn members() -> &'static [DataflowMember] { + let d = discovery(); + if let Err(why) = declared(d) { + panic!("{why}"); + } + &d.found +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn next(s: &mut u64) -> u64 { + *s = s.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *s; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// A value in `[-bound, bound]`. +fn within(s: &mut u64, bound: i64) -> i32 { + ((next(s) % (2 * bound as u64 + 1)) as i64 - bound) as i32 +} + +/// A random circuit of `ops` gates on `n` qubits: every base gate, up to three +/// distinct controls, `P(1..=32)`. +fn random_circuit(n: u8, ops: usize, seed: u64, gs: Gateset) -> Circuit { + let bases = [BaseGate::I, BaseGate::X, BaseGate::Y, BaseGate::Z, BaseGate::H, BaseGate::S, BaseGate::Sdg, BaseGate::T, BaseGate::Tdg, BaseGate::P]; + let mut s = seed; + let mut c = Circuit::with_gateset(n, gs); + for _ in 0..ops { + let base = bases[(next(&mut s) % bases.len() as u64) as usize]; + let target = (next(&mut s) % u64::from(n)) as u8; + let mut controls = Vec::new(); + for _ in 0..next(&mut s) % 4 { + let q = (next(&mut s) % u64::from(n)) as u8; + if q != target && !controls.contains(&q) { + controls.push(q); + } + } + let param = if base == BaseGate::P { 1 + (next(&mut s) % 32) as u16 } else { 0 }; + c.ops.push(Gate { base, controls, target, param }); + } + c +} + +/// Five circuits the native simulator refuses, in each gateset. +fn refused_circuits() -> Vec { + let gate = |base, controls: Vec, target, param| Gate { base, controls, target, param }; + let bad = [ + (1, vec![gate(BaseGate::H, vec![], 0, 0), gate(BaseGate::P, vec![], 0, 0)]), + (1, vec![gate(BaseGate::P, vec![], 0, 33)]), + (1, vec![gate(BaseGate::X, vec![], 3, 0)]), + (2, vec![gate(BaseGate::X, vec![0], 0, 0)]), + (2, vec![gate(BaseGate::X, vec![5], 1, 0)]), + ]; + let mut out = Vec::new(); + for gs in Gateset::ALL { + for (n, ops) in bad.clone() { + let mut c = Circuit::with_gateset(n, gs); + c.ops = ops; + out.push(c); + } + } + out +} + +/// `Z` on qubit 0: on `[(0, 0), (i32::MIN, 0)]` its exact result is `2^31`, +/// which leaves 32 bits. +fn z_words() -> GateWords { + lanes::gate_words(&Gate { base: BaseGate::Z, controls: vec![], target: 0, param: 0 }, Gateset::V2) +} + +const Z_STATE: [Lane; 2] = [[0, 0], [i32::MIN, 0]]; + +#[test] +fn c0_devices_are_declared() { + let d = discovery(); + match declared(d) { + Ok(expect) => report(&format!("dataflow devices: as declared ({expect})")), + Err(why) => panic!("{why}"), + } + for m in &d.found { + assert_eq!(m.class(), FabricClass::Dataflow); + assert!(m.id().starts_with("local/dataflow-"), "{}", m.id()); + assert_eq!(m.implementation(), dataflow::implementation_id()); + assert!(m.max_qubits() >= 17, "{}: holds only {} qubits", m.key(), m.max_qubits()); + report(&format!("dataflow member: key={} id={} {}", m.key(), m.id(), m.detail())); + } +} + +#[test] +fn c1_probe_matches_lanes() { + for m in members() { + let pairs = lanes::probe_pairs(1 << 16, 0x0cda); + let out = m.probe(&pairs).unwrap_or_else(|e| panic!("{}: {e}", m.key())); + assert_eq!(lanes::first_mismatch(&pairs, &out), None, "{}", m.key()); + report(&format!("dataflow probe: key={} {} pairs agree", m.key(), pairs.len())); + } +} + +#[test] +fn c2_kernel_matches_run_words() { + for m in members() { + let mut s = 0x6b65_726e_656c_u64; + let (mut flagged, mut clean) = (0, 0); + for case in 0..2000 { + let n = 1 + (next(&mut s) % 12) as u32; + // Three regimes: the whole 32-bit range (nearly always out of + // range), small states and entries (never), and states and + // entries up to 2^30 (either). + let mode = case % 3; + let value = |s: &mut u64| -> i32 { + match mode { + 0 => next(s) as i32, + 1 => within(s, 1 << 22), + _ => within(s, 1 << 30), + } + }; + let entry = |s: &mut u64| -> u32 { + match mode { + 0 => next(s) as u32, + 1 => within(s, 1 << 29) as u32, + _ => within(s, 1 << 30) as u32, + } + }; + let state: Vec = (0..1usize << n).map(|_| [value(&mut s), value(&mut s)]).collect(); + let gates: Vec = (0..1 + next(&mut s) % 8) + .map(|_| { + let t = (next(&mut s) % u64::from(n)) as u32; + let ctrl = (0..n).filter(|&b| b != t && next(&mut s).is_multiple_of(4)).fold(0u32, |acc, b| acc | 1 << b); + let mut w = [0u32; 8]; + w.iter_mut().for_each(|x| *x = entry(&mut s)); + GateWords { m: w, tbit: 1 << t, ctrl } + }) + .collect(); + let want = lanes::run_words(&state, &gates).expect("valid shapes"); + let got = m.run_lanes(&state, &gates).unwrap_or_else(|e| panic!("{}: case {case}: {e}", m.key())); + assert_eq!(got.out_of_range, want.out_of_range, "{}: case {case}, n = {n}", m.key()); + assert!(got.amps == want.amps, "{}: case {case}, n = {n}: the lanes differ", m.key()); + if want.out_of_range.is_some() { + flagged += 1; + } else { + clean += 1; + } + } + assert!(flagged >= 100 && clean >= 100, "{}: {flagged} flagged, {clean} clean", m.key()); + report(&format!("dataflow kernel: key={} ok", m.key())); + } +} + +#[test] +fn c3_certificates_reproduce_the_pins() { + let ms = members(); + for (gs, random_pin, fabric_pin) in PINS { + let cpu = lanes::certify_corpus(gs, "local/test", RANDOM_V1); + assert_eq!(hex(&cpu.corpus), random_pin, "{gs:?}"); + let other = Corpus::Random { seed: 8, cases: 300 }; + let cpu_other = lanes::certify_corpus(gs, "local/test", other); + for m in ms { + for (corpus, cases, pin) in [(RANDOM_V1, 200, random_pin), (FABRIC_DEFAULT, 224, fabric_pin)] { + let c = m.certify(gs, corpus).unwrap_or_else(|e| panic!("{} {gs:?} {}: {e}", m.key(), corpus.name())); + assert_eq!(c.parity, Parity::BitExact, "{} {gs:?} {}", m.key(), corpus.name()); + assert_eq!(c.cases, cases); + assert_eq!(hex(&c.corpus), pin, "{} {gs:?} {}", m.key(), corpus.name()); + assert_eq!((c.kernel.as_str(), c.implementation.as_str(), c.member.as_str()), (apply_kernel(gs), m.implementation(), m.id())); + assert_eq!(c.generator, corpus.generator(gs)); + report(&format!( + "dataflow certified: key={} member={} kernel={} corpus={} implementation={} parity=bit-exact cases={} digest={}", + m.key(), + m.id(), + c.kernel, + corpus.name(), + c.implementation, + c.cases, + hex(&c.corpus) + )); + } + // A corpus with no pin: the member and the CPU agree on it. + let c = m.certify(gs, other).unwrap_or_else(|e| panic!("{} {gs:?} seed 8: {e}", m.key())); + assert_eq!((c.parity, c.cases), (Parity::BitExact, 300), "{} {gs:?}", m.key()); + assert_eq!(c.corpus, cpu_other.corpus, "{} {gs:?}", m.key()); + } + } +} + +#[test] +fn c4_wide_circuits_match_native() { + for m in members() { + let widest = m.max_qubits().min(20); + let mut s = 0x0077_6964_6500_u64; + for (k, n) in [12u8, 16, 18, widest].into_iter().enumerate() { + for gs in Gateset::ALL { + let c = random_circuit(n, 40, 0x1000 + k as u64, gs); + let start = (next(&mut s) as usize) % (1usize << n); + let want = c.simulate_from(start).expect("the native run"); + let got = m.simulate_from(&c, start).unwrap_or_else(|e| panic!("{}: n = {n} {gs:?}: {e}", m.key())); + assert!(got == want, "{}: n = {n} {gs:?}: the states differ", m.key()); + } + } + // The transforms the conformance corpus carries. + for n in [3u8, 5, 8, 12] { + let qft = Circuit::qft(n); + assert!(m.simulate(&qft).unwrap() == qft.simulate().unwrap(), "{}: QFT-{n}", m.key()); + } + report(&format!("dataflow wide: key={} up to {widest} qubits match native", m.key())); + } +} + +#[test] +fn c5_refusals_are_typed() { + for m in members() { + for c in refused_circuits() { + let native = c.simulate_from(0).unwrap_err(); + assert_eq!(m.simulate_from(&c, 0).unwrap_err(), DataflowError::Circuit(native), "{c:?}"); + } + assert_eq!(m.simulate(&Circuit::new(MAX_QUBITS + 1)).unwrap_err(), DataflowError::Circuit(QuantumError::TooManyQubits(MAX_QUBITS + 1))); + let small = m.with_max_qubits(4); + assert_eq!(small.simulate(&Circuit::new(5)).unwrap_err(), DataflowError::TooWide { n_qubits: 5, max: 4 }); + let wide = Program { n_qubits: 5, start: 0, gates: Vec::new(), steps: Vec::new() }; + assert_eq!(small.run_program(&wide).unwrap_err(), DataflowError::TooWide { n_qubits: 5, max: 4 }); + // A result that leaves 32 bits is reported at its op, and the lane + // holds its low 32 bits. + let r = m.run_lanes(&Z_STATE, &[z_words()]).unwrap(); + assert_eq!(r.out_of_range, Some(0), "{}", m.key()); + assert_eq!(r.amps[1][0] as u32, 0x8000_0000); + // Malformed words, a state the host cannot lay out and a tile step are + // host defects, never results. + let bad = GateWords { tbit: 3, ..z_words() }; + assert!(matches!(m.run_lanes(&[[0, 0]; 4], &[bad]), Err(DataflowError::Internal(_)))); + assert!(matches!(m.run_lanes(&[[0, 0]; 3], &[z_words()]), Err(DataflowError::Internal(_)))); + let tiled = Program { n_qubits: 2, start: 0, gates: Vec::new(), steps: vec![Step::Tile { first: 0, count: 1, size: 2 }] }; + assert!(matches!(m.run_program(&tiled), Err(DataflowError::Internal(_)))); + // The refusals that may still fall back to another member, do. + assert!(MemberError::from(DataflowError::TooWide { n_qubits: 5, max: 4 }).falls_back()); + report(&format!("dataflow refusals: key={} typed", m.key())); + } +} + +/// The kernel mutants, compiled in: each must be caught by the probe or by a +/// certificate. +#[test] +fn c6_mutants_are_killed() { + let edges = lanes::probe_pairs(0, 0); + for m in members() { + let mut killed = 0; + for v in Variant::MUTANTS { + let mm = m.with_variant(v); + assert_ne!(mm.implementation(), m.implementation()); + let out = mm.probe(&edges).expect("the mutant's probe runs"); + let differing = edges.iter().zip(&out).filter(|&(&(a, b), o)| *o != (lanes::mul_wide(a, b), lanes::fxmul_wide(a, b))).count(); + let caught = differing > 0 + || Gateset::ALL.into_iter().all(|gs| match mm.certify(gs, RANDOM_V1) { + Err(_) => true, + Ok(c) => c.parity != Parity::BitExact || c.corpus != lanes::certify_corpus(gs, "local/test", RANDOM_V1).corpus, + }); + println!("dataflow mutant {v:?} on {}: {differing} of {} edge pairs differ", m.key(), edges.len()); + assert!(caught, "{}: mutant {v:?} survived", m.key()); + killed += 1; + } + report(&format!("dataflow mutants: key={} killed {killed} of 3", m.key())); + } +} + +#[test] +fn c7_receipt_placement_and_the_fabric() { + for m in members() { + let member = m.member(FABRIC_DEFAULT).unwrap_or_else(|e| panic!("{}: {e}", m.key())); + assert_eq!((member.id.as_str(), member.class, member.presence), (m.id(), FabricClass::Dataflow, Presence::Detected)); + assert_eq!(member.kernels.len(), 2); + for (entry, gs) in member.kernels.iter().zip(Gateset::ALL) { + assert_eq!((entry.kernel.as_str(), entry.implementation.as_str()), (apply_kernel(gs), m.implementation())); + assert!(entry.energy.is_none()); + assert_eq!(entry.certificate.parity, Parity::BitExact, "{gs:?}"); + } + // A receipt over a member's run. + let qft = Circuit::qft(5); + let state = m.simulate(&qft).unwrap(); + assert_eq!(state, qft.simulate().unwrap()); + let entry = member.kernels.iter().find(|k| k.kernel == apply_kernel(qft.gateset)).unwrap(); + let signer = SigningKey::from_bytes(&[7; 32]); + let receipt = FabricReceipt::seal(&signer, "local/test", state.statevector_hash(), &member, entry, 0, None); + assert!(receipt.verify()); + assert!(receipt.certificate_matches(&entry.certificate)); + // Placement: beside the CPU reference with no energy figures, the + // reference; alone, the member. + let local = detect_local(); + for gs in Gateset::ALL { + let both = [local.clone(), member.clone()]; + let (placed, k) = place(&both, apply_kernel(gs), &Parity::BitExact).unwrap(); + assert_eq!((placed.id.as_str(), k.implementation.as_str()), (local.id.as_str(), REFERENCE)); + let (placed, k) = place(std::slice::from_ref(&member), apply_kernel(gs), &Parity::BitExact).unwrap(); + assert_eq!((placed.id.as_str(), k.implementation.as_str()), (m.id(), m.implementation())); + } + // On the fabric: the member alone runs the circuit; a pairing with + // another member's entries is refused. + let mut fabric = Fabric::new(); + fabric.add(member.clone(), m); + let run = ready(fabric.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs"); + assert_eq!(run.state, state); + assert_eq!(run.receipt.member_id, m.id()); + assert!(run.receipt.verify()); + let mut mismatched = Fabric::new(); + mismatched.add(local.clone(), m); + assert!(ready(mismatched.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").is_err()); + let mut both = Fabric::new(); + both.add(local.clone(), CpuReference).add(member.clone(), m); + let run = ready(both.run(&qft, 0, &Parity::BitExact, &signer, "local/test")).expect("resolves").expect("runs"); + assert_eq!(run.state, state); + report(&format!("dataflow receipt: key={} ok", m.key())); + } +} + +#[test] +fn c8_repeated_runs_agree() { + for m in members() { + let c = random_circuit(16, 60, 0xabc, Gateset::V2); + let first = m.simulate(&c).unwrap(); + for _ in 0..5 { + assert!(m.simulate(&c).unwrap() == first, "{}", m.key()); + } + report(&format!("dataflow repeat: key={} ok", m.key())); + } +} -- 2.54.0 (Apple Git-157) From e5191a6079d6b3a2fc4536e8df62dce6914a2f98 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Wed, 14 Oct 2026 19:56:41 -0400 Subject: [PATCH 08/11] ci: the 0.5.1 member workflows run only when the pull request's own diff touches them Neural processor, matrix extension, signal processor, vector units and the dataflow device gain the same gate as the other wai-quantum workflows. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wai-quantum-dataflow.yml | 27 ++++++++++++++++++++++ .github/workflows/wai-quantum-dsp.yml | 25 ++++++++++++++++++++ .github/workflows/wai-quantum-npu.yml | 27 ++++++++++++++++++++++ .github/workflows/wai-quantum-sme.yml | 27 ++++++++++++++++++++++ .github/workflows/wai-quantum-vector.yml | 27 ++++++++++++++++++++++ 5 files changed, 133 insertions(+) diff --git a/.github/workflows/wai-quantum-dataflow.yml b/.github/workflows/wai-quantum-dataflow.yml index e202e71f6..0ac0a7a8e 100644 --- a/.github/workflows/wai-quantum-dataflow.yml +++ b/.github/workflows/wai-quantum-dataflow.yml @@ -29,8 +29,33 @@ on: - '.github/workflows/wai-quantum-dataflow.yml' jobs: + # Does this pull request's own diff touch the paths below? This forge matches a + # workflow's `paths:` against the push, so a merge of main into a branch fires the + # workflow whatever the pull request changes; every job here waits on this answer. + # A push to main, a schedule, or anything the script cannot determine answers + # `true` (scripts/pr-touches-paths.py). + relevant: + name: does this change touch it + runs-on: linux-arm64 + outputs: + run: ${{ steps.gate.outputs.run }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - id: gate + run: | + set -euo pipefail + r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-dataflow.yml \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}") + echo "this change touches wai-quantum-dataflow: $r" + echo "run=$r" >> "$GITHUB_OUTPUT" + dataflow: name: member · dataflow device, simulated (pci ${{ matrix.pci }}) + needs: relevant + if: needs.relevant.outputs.run == 'true' runs-on: linux timeout-minutes: 60 strategy: @@ -148,6 +173,8 @@ jobs: dataflow-absent: name: member · dataflow device absent (macos) + needs: relevant + if: needs.relevant.outputs.run == 'true' runs-on: macos timeout-minutes: 60 env: diff --git a/.github/workflows/wai-quantum-dsp.yml b/.github/workflows/wai-quantum-dsp.yml index 8f41db676..57a841c11 100644 --- a/.github/workflows/wai-quantum-dsp.yml +++ b/.github/workflows/wai-quantum-dsp.yml @@ -46,8 +46,33 @@ on: - '.github/workflows/wai-quantum-dsp.yml' jobs: + # Does this pull request's own diff touch the paths below? This forge matches a + # workflow's `paths:` against the push, so a merge of main into a branch fires the + # workflow whatever the pull request changes; every job here waits on this answer. + # A push to main, a schedule, or anything the script cannot determine answers + # `true` (scripts/pr-touches-paths.py). + relevant: + name: does this change touch it + runs-on: linux-arm64 + outputs: + run: ${{ steps.gate.outputs.run }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - id: gate + run: | + set -euo pipefail + r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-dsp.yml \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}") + echo "this change touches wai-quantum-dsp: $r" + echo "run=$r" >> "$GITHUB_OUTPUT" + dsp-emulated: name: processor and member · signal processor, emulated + needs: relevant + if: needs.relevant.outputs.run == 'true' runs-on: linux timeout-minutes: 180 container: diff --git a/.github/workflows/wai-quantum-npu.yml b/.github/workflows/wai-quantum-npu.yml index 829e5158e..c1d9b9a82 100644 --- a/.github/workflows/wai-quantum-npu.yml +++ b/.github/workflows/wai-quantum-npu.yml @@ -31,8 +31,33 @@ on: - '.github/workflows/wai-quantum-npu.yml' jobs: + # Does this pull request's own diff touch the paths below? This forge matches a + # workflow's `paths:` against the push, so a merge of main into a branch fires the + # workflow whatever the pull request changes; every job here waits on this answer. + # A push to main, a schedule, or anything the script cannot determine answers + # `true` (scripts/pr-touches-paths.py). + relevant: + name: does this change touch it + runs-on: linux-arm64 + outputs: + run: ${{ steps.gate.outputs.run }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - id: gate + run: | + set -euo pipefail + r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-npu.yml \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}") + echo "this change touches wai-quantum-npu: $r" + echo "run=$r" >> "$GITHUB_OUTPUT" + npu-mac: name: member · neural processor (macos) + needs: relevant + if: needs.relevant.outputs.run == 'true' runs-on: macos timeout-minutes: 120 env: @@ -70,6 +95,8 @@ jobs: npu-absent: name: member · neural processor absent (linux) + needs: relevant + if: needs.relevant.outputs.run == 'true' runs-on: linux timeout-minutes: 60 env: diff --git a/.github/workflows/wai-quantum-sme.yml b/.github/workflows/wai-quantum-sme.yml index c8d8625b8..d02b563e9 100644 --- a/.github/workflows/wai-quantum-sme.yml +++ b/.github/workflows/wai-quantum-sme.yml @@ -32,8 +32,33 @@ on: - '.github/workflows/wai-quantum-sme.yml' jobs: + # Does this pull request's own diff touch the paths below? This forge matches a + # workflow's `paths:` against the push, so a merge of main into a branch fires the + # workflow whatever the pull request changes; every job here waits on this answer. + # A push to main, a schedule, or anything the script cannot determine answers + # `true` (scripts/pr-touches-paths.py). + relevant: + name: does this change touch it + runs-on: linux-arm64 + outputs: + run: ${{ steps.gate.outputs.run }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - id: gate + run: | + set -euo pipefail + r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-sme.yml \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}") + echo "this change touches wai-quantum-sme: $r" + echo "run=$r" >> "$GITHUB_OUTPUT" + sme-emulated: name: member · matrix extension, emulated (aarch64 Linux) + needs: relevant + if: needs.relevant.outputs.run == 'true' runs-on: linux timeout-minutes: 90 container: @@ -91,6 +116,8 @@ jobs: sme-absent: name: member · matrix extension absent (macos) + needs: relevant + if: needs.relevant.outputs.run == 'true' runs-on: macos timeout-minutes: 60 env: diff --git a/.github/workflows/wai-quantum-vector.yml b/.github/workflows/wai-quantum-vector.yml index 6268585fe..82e0a01d3 100644 --- a/.github/workflows/wai-quantum-vector.yml +++ b/.github/workflows/wai-quantum-vector.yml @@ -33,8 +33,33 @@ on: - '.github/workflows/wai-quantum-vector.yml' jobs: + # Does this pull request's own diff touch the paths below? This forge matches a + # workflow's `paths:` against the push, so a merge of main into a branch fires the + # workflow whatever the pull request changes; every job here waits on this answer. + # A push to main, a schedule, or anything the script cannot determine answers + # `true` (scripts/pr-touches-paths.py). + relevant: + name: does this change touch it + runs-on: linux-arm64 + outputs: + run: ${{ steps.gate.outputs.run }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - id: gate + run: | + set -euo pipefail + r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-quantum-vector.yml \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}") + echo "this change touches wai-quantum-vector: $r" + echo "run=$r" >> "$GITHUB_OUTPUT" + vector: name: member · vector unit, ${{ matrix.name }} + needs: relevant + if: needs.relevant.outputs.run == 'true' runs-on: linux timeout-minutes: 120 strategy: @@ -148,6 +173,8 @@ jobs: vector-absent: name: member · vector unit absent (macos) + needs: relevant + if: needs.relevant.outputs.run == 'true' runs-on: macos timeout-minutes: 60 env: -- 2.54.0 (Apple Git-157) From 02a45cbd3180d6287f9bb31c06887f5967d4ffde Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Wed, 14 Oct 2026 20:28:04 -0400 Subject: [PATCH 09/11] wai-quantum 0.5.1: review fixes - The DSP workflow's x86_64 step looked for "hvx devices" after the rename to dsp; it could never pass. - The neural-processor member's mutants now go through the placement check the conforming model does (a mutant placed elsewhere is refused, never credited to the processor), and carry their own placement in the detail. - Its input arrays are written through the strides the runtime gives them, as its outputs are read. - Its compiled model is kept per model text and system build and reused, instead of recompiled per process and left in the temp directory; the source file is removed after compiling. - Its per-pair arithmetic allocates nothing (an array, and a lazy error). - The matrix-extension kernels declare the predicate registers they write. - A dataflow core that runs past its clock budget is a timeout, which may fall back to another member, not a defect. - The encoder's check against the assembler now covers the programs the device runs: the member's own layout, and every mutant's rows and probe. Verified: neural processor 9/9 on the M3 Ultra twice over one cache (both mutants placed and killed; one compiled model per text, nothing left in the temp directory); matrix extension 9/9 on M4 Max silicon; all seven device programs identical under the independent assembler. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wai-quantum-dataflow.yml | 2 +- .github/workflows/wai-quantum-dsp.yml | 2 +- wai/wai-quantum/examples/rv32_listing.rs | 29 ++-- .../src/quantum_fabric_dataflow.rs | 7 +- wai/wai-quantum/src/quantum_fabric_npu.rs | 125 ++++++++++++++---- wai/wai-quantum/src/quantum_fabric_sme.rs | 8 +- 6 files changed, 129 insertions(+), 44 deletions(-) diff --git a/.github/workflows/wai-quantum-dataflow.yml b/.github/workflows/wai-quantum-dataflow.yml index 0ac0a7a8e..c0f66523b 100644 --- a/.github/workflows/wai-quantum-dataflow.yml +++ b/.github/workflows/wai-quantum-dataflow.yml @@ -145,7 +145,7 @@ jobs: cmp "$p.bin" "$p.as.bin" || { echo "::error::$(basename "$p"): the encoder's words differ from the assembler's"; exit 1; } n=$((n + 1)) done - [ "$n" = 5 ] || { echo "::error::$n of 5 programs were compared"; exit 1; } + [ "$n" = 7 ] || { echo "::error::$n of 7 programs were compared"; exit 1; } echo "all $n programs: the encoder's words are the assembler's" - name: The member's tests shell: bash diff --git a/.github/workflows/wai-quantum-dsp.yml b/.github/workflows/wai-quantum-dsp.yml index 57a841c11..858621c3d 100644 --- a/.github/workflows/wai-quantum-dsp.yml +++ b/.github/workflows/wai-quantum-dsp.yml @@ -229,4 +229,4 @@ jobs: report="$RUNNER_TEMP/dsp-absent.txt"; : > "$report" WAI_QUANTUM_DSP_EXPECT=none WAI_QUANTUM_DSP_REPORT="$report" cargo test --locked -j 6 --features quantum_fabric_dsp --test dsp_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/host-device.log" grep -qE '^test result: ok\. 9 passed; 0 failed' "$RUNNER_TEMP/host-device.log" || { echo "::error::not every device test ran and passed"; exit 1; } - grep -qx 'hvx devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; } + grep -qx 'dsp devices: as declared (none)' "$report" || { echo "::error::the absent case was not established"; exit 1; } diff --git a/wai/wai-quantum/examples/rv32_listing.rs b/wai/wai-quantum/examples/rv32_listing.rs index b49b22607..cd5eb2256 100644 --- a/wai/wai-quantum/examples/rv32_listing.rs +++ b/wai/wai-quantum/examples/rv32_listing.rs @@ -1,6 +1,7 @@ -//! Writes the dataflow member's RV32IM programs (conforming and mutants) as -//! assembly listings and as the machine words this crate encodes, so an -//! independent assembler can be checked against the encoder: +//! Writes the dataflow member's RV32IM programs, exactly as the device runs +//! them (conforming and mutants), as assembly listings and as the machine words +//! this crate encodes, so an independent assembler can be checked against the +//! encoder: //! //! ```text //! cargo run --example rv32_listing --features quantum_fabric_dataflow -- @@ -8,20 +9,22 @@ //! //! In ``, for each program ``: `.s` (the listing, with a //! `.text` header) and `.bin` (the words, little-endian). -use wai_quantum::quantum_fabric::rv32::{Layout, probe_program, rows_program}; +use wai_quantum::quantum_fabric_dataflow::{Variant, programs}; fn main() { let dir = std::path::PathBuf::from(std::env::args().nth(1).expect("an output directory")); std::fs::create_dir_all(&dir).expect("the directory"); - let l = Layout { base: 0x0001_0000, capacity: 1024 }; - let programs = [ - ("rows", rows_program(l, 1 << 29, 30, false)), - ("rows_m1", rows_program(l, 1 << 28, 30, false)), - ("rows_m2", rows_program(l, 1 << 29, 29, false)), - ("rows_m3", rows_program(l, 1 << 29, 30, true)), - ("probe", probe_program(l, 1 << 29, 30)), - ]; - for (name, (words, text)) in programs { + // The programs the device runs: the rows and probe programs of the + // conforming kernel and of each mutant (M3's probe is the conforming one). + let mut all = Vec::new(); + for (tag, v) in [("", Variant::Conforming), ("_m1", Variant::M1), ("_m2", Variant::M2), ("_m3", Variant::M3)] { + let (rows, probe) = programs(v); + all.push((format!("rows{tag}"), rows)); + if v != Variant::M3 { + all.push((format!("probe{tag}"), probe)); + } + } + for (name, (words, text)) in all { std::fs::write(dir.join(format!("{name}.s")), format!(".text\n.option norvc\n{text}")).expect("the listing"); std::fs::write(dir.join(format!("{name}.bin")), words.iter().flat_map(|w| w.to_le_bytes()).collect::>()).expect("the words"); println!("{name}: {} words", words.len()); diff --git a/wai/wai-quantum/src/quantum_fabric_dataflow.rs b/wai/wai-quantum/src/quantum_fabric_dataflow.rs index 3b6c2b400..c93fb30ca 100644 --- a/wai/wai-quantum/src/quantum_fabric_dataflow.rs +++ b/wai/wai-quantum/src/quantum_fabric_dataflow.rs @@ -68,6 +68,8 @@ impl From for MemberError { DataflowError::TooWide { n_qubits, max } => MemberError::TooWide { n_qubits, max }, DataflowError::OutOfRange { gate } => MemberError::OutOfRange { gate }, DataflowError::Circuit(e) => MemberError::Circuit(e), + // A core that ran past its clock budget: a wait past its limit. + DataflowError::Stalled { .. } => MemberError::Timeout, other => MemberError::Other(other.to_string()), } } @@ -104,8 +106,9 @@ impl Variant { /// past the programs, a batch of up to `capacity` pairs. pub const LAYOUT: Layout = Layout { base: 0x1_0000, capacity: 8192 }; -/// The rows program and the probe program of a variant, as words and listing. -fn programs(v: Variant) -> ((Vec, String), (Vec, String)) { +/// The rows program and the probe program of a variant, as words and listing: +/// exactly what the device runs. +pub fn programs(v: Variant) -> ((Vec, String), (Vec, String)) { let (rnd, sh, last_add) = v.constants(); (rv32::rows_program(LAYOUT, rnd, sh, last_add), rv32::probe_program(LAYOUT, rnd, sh)) } diff --git a/wai/wai-quantum/src/quantum_fabric_npu.rs b/wai/wai-quantum/src/quantum_fabric_npu.rs index cabc92b92..ffedbe1b4 100644 --- a/wai/wai-quantum/src/quantum_fabric_npu.rs +++ b/wai/wai-quantum/src/quantum_fabric_npu.rs @@ -137,7 +137,8 @@ fn fits32(v: i64) -> bool { /// 32 bits: row 0 is `(m0 + i·m1)·a0 + (m2 + i·m3)·a1`, row 1 likewise with /// `m4..m7`, as [`lanes::apply_words`] computes them. fn apply_pair(p: &[i64]) -> (Lane, Lane, bool) { - let f: Vec = p.iter().map(|&x| fx(x)).collect(); + debug_assert!(p.len() >= 16); + let f: [i64; 16] = std::array::from_fn(|k| fx(p[k])); let o0r = f[0] - f[1] + f[2] - f[3]; let o0i = f[4] + f[5] + f[6] + f[7]; let o1r = f[8] - f[9] + f[10] - f[11]; @@ -194,12 +195,7 @@ mod coreml { /// neural processor, and reads the runtime's compute plan for it. pub fn load(model: &[u8], dir: &Path) -> Result { std::fs::create_dir_all(dir).map_err(|e| NpuError::Runtime(format!("{}: {e}", dir.display())))?; - let src = dir.join("limbs.mlmodel"); - std::fs::write(&src, model).map_err(|e| NpuError::Runtime(e.to_string()))?; - let url = NSURL::fileURLWithPath(&NSString::from_str(&src.to_string_lossy())); - // SAFETY: a file URL to a model description just written. - #[allow(deprecated)] // The synchronous form: discovery is synchronous. - let compiled = unsafe { MLModel::compileModelAtURL_error(&url) }.map_err(|e| NpuError::Runtime(format!("compiling the model: {}", ns_error(&e))))?; + let compiled = compiled_in(model, dir)?; let cfg = config(); // SAFETY: the compiled model's URL and a configuration. let model = unsafe { MLModel::modelWithContentsOfURL_configuration_error(&compiled, &cfg) }.map_err(|e| NpuError::Runtime(format!("loading the model: {}", ns_error(&e))))?; @@ -207,6 +203,49 @@ mod coreml { Ok(Loaded { model, placement }) } + /// The compiled form of `model`, kept in `dir` (`limbs.mlmodelc`): compiled + /// once, and loaded by later processes without compiling again. Two + /// processes compiling at once each move their copy in; the first stays. + fn compiled_in(model: &[u8], dir: &Path) -> Result, NpuError> { + let kept = dir.join("limbs.mlmodelc"); + let url_of = |p: &Path| NSURL::fileURLWithPath(&NSString::from_str(&p.to_string_lossy())); + if kept.is_dir() { + return Ok(url_of(&kept)); + } + let io = |e: std::io::Error| NpuError::Runtime(format!("{}: {e}", dir.display())); + let src = dir.join(format!("limbs-{}.mlmodel", std::process::id())); + std::fs::write(&src, model).map_err(io)?; + // SAFETY: a file URL to a model description just written. + #[allow(deprecated)] // The synchronous form: discovery is synchronous. + let compiled = unsafe { MLModel::compileModelAtURL_error(&url_of(&src)) }; + let _ = std::fs::remove_file(&src); + let compiled = compiled.map_err(|e| NpuError::Runtime(format!("compiling the model: {}", ns_error(&e))))?; + let out = compiled.path().ok_or_else(|| NpuError::Runtime("the compiled model has no path".into()))?.to_string(); + let out = std::path::PathBuf::from(out); + if std::fs::rename(&out, &kept).is_err() { + if !kept.is_dir() { + // Another volume: copy it in under a private name, then move. + let part = dir.join(format!("limbs.mlmodelc.{}", std::process::id())); + copy_dir(&out, &part).map_err(io)?; + if std::fs::rename(&part, &kept).is_err() { + let _ = std::fs::remove_dir_all(&part); + } + } + let _ = std::fs::remove_dir_all(&out); + } + if kept.is_dir() { Ok(url_of(&kept)) } else { Err(NpuError::Runtime(format!("{}: the compiled model was not kept", kept.display()))) } + } + + fn copy_dir(from: &Path, to: &Path) -> std::io::Result<()> { + std::fs::create_dir_all(to)?; + for e in std::fs::read_dir(from)? { + let e = e?; + let t = to.join(e.file_name()); + if e.file_type()?.is_dir() { copy_dir(&e.path(), &t)? } else { std::fs::copy(e.path(), t).map(|_| ())? } + } + Ok(()) + } + /// The runtime's compute plan for the compiled model: each operation's /// preferred device. fn plan(compiled: &NSURL, cfg: &MLModelConfiguration) -> Result, NpuError> { @@ -261,16 +300,31 @@ mod coreml { Ok(out) } - /// The fp16 array `[BATCH, SHIFTS, LIMBS]` holding `bits`. + /// The fp16 array `[BATCH, SHIFTS, LIMBS]` holding `bits`, written + /// through the strides the runtime gives it (which may pad a dimension). fn array(bits: &[u16]) -> Result, NpuError> { + assert_eq!(bits.len(), BATCH * SHIFTS * LIMBS); let shape: Retained> = NSArray::from_retained_slice(&[NSNumber::new_usize(BATCH), NSNumber::new_usize(SHIFTS), NSNumber::new_usize(LIMBS)]); - // SAFETY: a contiguous fp16 array of the model's input shape, then - // filled with exactly that many values. + // SAFETY: an fp16 array of the model's input shape; every write lands + // at an index its own strides give, inside the array. unsafe { let a = MLMultiArray::initWithShape_dataType_error(MLMultiArray::alloc(), &shape, MLMultiArrayDataType::Float16).map_err(|e| NpuError::Runtime(ns_error(&e)))?; + let strides: Vec = a.strides().iter().map(|n| n.as_usize()).collect(); + let [s0, s1, s2] = strides[..] else { return Err(NpuError::Runtime(format!("the input has strides {strides:?}"))) }; + if s2 != 1 || s1 < LIMBS || s0 < SHIFTS * s1 { + return Err(NpuError::Runtime(format!("the input's strides {strides:?} do not hold [{BATCH}, {SHIFTS}, {LIMBS}]"))); + } #[allow(deprecated)] let p = a.dataPointer().as_ptr().cast::(); - std::ptr::copy_nonoverlapping(bits.as_ptr(), p, BATCH * SHIFTS * LIMBS); + if (s0, s1) == (SHIFTS * LIMBS, LIMBS) { + std::ptr::copy_nonoverlapping(bits.as_ptr(), p, BATCH * SHIFTS * LIMBS); + } else { + for i in 0..BATCH { + for s in 0..SHIFTS { + std::ptr::copy_nonoverlapping(bits.as_ptr().add((i * SHIFTS + s) * LIMBS), p.add(i * s0 + s * s1), LIMBS); + } + } + } Ok(a) } } @@ -365,10 +419,34 @@ impl std::fmt::Debug for NpuMember { } #[cfg(target_os = "macos")] +/// Where a model's compiled form is kept: by the model's text and the +/// system's build, whose compiler made it. fn cache_dir(model: &[u8]) -> std::path::PathBuf { let base = std::env::var_os("WAI_QUANTUM_NPU_CACHE").map(std::path::PathBuf::from).unwrap_or_else(|| std::env::temp_dir().join("wai-quantum-npu")); - let tag: String = blake3::hash(model).as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect(); - base.join(format!("{tag}-{}", std::process::id())) + let os = std::process::Command::new("sysctl").args(["-n", "kern.osversion"]).output().map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()).unwrap_or_default(); + let mut h = blake3::Hasher::new(); + h.update(model); + h.update(os.as_bytes()); + let tag: String = h.finalize().as_bytes()[..8].iter().map(|b| format!("{b:02x}")).collect(); + base.join(tag) +} + +/// The placement as text, if the runtime put every operation on the neural +/// processor; otherwise why not. +#[cfg(target_os = "macos")] +fn placed_on_the_processor(placement: &[(String, String)]) -> Result { + let elsewhere: Vec = placement.iter().filter(|(_, d)| d != NEURAL_PROCESSOR).map(|(op, d)| format!("{op} on {d}")).collect(); + if placement.is_empty() || !elsewhere.is_empty() { + return Err(NpuError::Placement(if elsewhere.is_empty() { "no operation was reported".into() } else { elsewhere.join("; ") })); + } + Ok(placement.iter().map(|(op, d)| format!("{op}: {d}")).collect::>().join(", ")) +} + +#[cfg(target_os = "macos")] +fn detail_of(placement_text: &str) -> String { + format!( + "npu; the platform's neural processor through its model runtime; placement by the runtime's compute plan: {placement_text}; fp16 limb products, the host assembling; up to {MAX_NPU_QUBITS} qubits; unmetered" + ) } /// The neural processor running `model`: loaded, its placement checked, and @@ -376,11 +454,7 @@ fn cache_dir(model: &[u8]) -> std::path::PathBuf { #[cfg(target_os = "macos")] fn open(model: &[u8]) -> Result { let loaded = coreml::load(model, &cache_dir(model))?; - let elsewhere: Vec = loaded.placement.iter().filter(|(_, d)| d != NEURAL_PROCESSOR).map(|(op, d)| format!("{op} on {d}")).collect(); - let placement_text = loaded.placement.iter().map(|(op, d)| format!("{op}: {d}")).collect::>().join(", "); - if loaded.placement.is_empty() || !elsewhere.is_empty() { - return Err(NpuError::Placement(if elsewhere.is_empty() { "no operation was reported".into() } else { elsewhere.join("; ") })); - } + let placement_text = placed_on_the_processor(&loaded.placement)?; let mut o = b"wai:quantum-fabric-npu-member\x01".to_vec(); o.extend_from_slice(std::env::consts::ARCH.as_bytes()); o.extend_from_slice(placement_text.as_bytes()); @@ -392,9 +466,7 @@ fn open(model: &[u8]) -> Result { id, key: "npu-0".into(), implementation: npu::implementation_id_of(model), - detail: format!( - "npu; the platform's neural processor through its model runtime; placement by the runtime's compute plan: {placement_text}; fp16 limb products, the host assembling; up to {MAX_NPU_QUBITS} qubits; unmetered" - ), + detail: detail_of(&placement_text), placement: loaded.placement, max_qubits: MAX_NPU_QUBITS, }; @@ -448,7 +520,7 @@ impl NpuMember { for p in 0..chunk.len() { let mut c = [0i64; SHIFTS]; for (s, slot) in c.iter_mut().enumerate() { - *slot = npu::int_of_f16(z[p * SHIFTS + s]).ok_or(NpuError::Runtime("a coefficient that is no integer of fp16's exact range".into()))?; + *slot = npu::int_of_f16(z[p * SHIFTS + s]).ok_or_else(|| NpuError::Runtime("a coefficient that is no integer of fp16's exact range".into()))?; } out.push(npu::assemble(&c)); } @@ -607,7 +679,14 @@ impl NpuMember { #[cfg(target_os = "macos")] { let loaded = coreml::load(model, &cache_dir(model))?; - Ok(NpuMember { model: std::sync::Arc::new(std::sync::Mutex::new(coreml::Handed(loaded.model))), implementation: npu::implementation_id_of(model), placement: loaded.placement, ..self.clone() }) + let placement_text = placed_on_the_processor(&loaded.placement)?; + Ok(NpuMember { + model: std::sync::Arc::new(std::sync::Mutex::new(coreml::Handed(loaded.model))), + implementation: npu::implementation_id_of(model), + detail: detail_of(&placement_text), + placement: loaded.placement, + ..self.clone() + }) } #[cfg(not(target_os = "macos"))] { diff --git a/wai/wai-quantum/src/quantum_fabric_sme.rs b/wai/wai-quantum/src/quantum_fabric_sme.rs index dafbbbdf6..6661401e0 100644 --- a/wai/wai-quantum/src/quantum_fabric_sme.rs +++ b/wai/wai-quantum/src/quantum_fabric_sme.rs @@ -180,8 +180,8 @@ macro_rules! rows_fn { unsafe fn $name(m: &[i64; 8], n: usize, a0r: &[i32], a0i: &[i32], a1r: &[i32], a1i: &[i32], o0r: &mut [i64], o0i: &mut [i64], o1r: &mut [i64], o1i: &mut [i64]) { debug_assert!([a0r.len(), a0i.len(), a1r.len(), a1i.len(), o0r.len(), o0i.len(), o1r.len(), o1i.len()].iter().all(|&l| l >= n)); // SAFETY: the caller's contract; streaming mode is entered and left - // inside the block, and every vector register it writes is declared - // clobbered (leaving streaming mode resets them). + // inside the block, and every vector and predicate register it + // writes is declared clobbered (leaving streaming mode resets them). unsafe { std::arch::asm!( rows_text!($rnd, $sh, $s), @@ -195,7 +195,7 @@ macro_rules! rows_fn { o0i = in(reg) o0i.as_mut_ptr(), o1r = in(reg) o1r.as_mut_ptr(), o1i = in(reg) o1i.as_mut_ptr(), - out("x9") _, out("x10") _, + out("x9") _, out("x10") _, out("p0") _, out("p1") _, out("v0") _, out("v1") _, out("v2") _, out("v3") _, out("v4") _, out("v5") _, out("v6") _, out("v7") _, out("v8") _, out("v9") _, out("v10") _, out("v11") _, out("v12") _, out("v13") _, out("v14") _, out("v15") _, out("v16") _, out("v17") _, out("v18") _, out("v19") _, out("v20") _, out("v21") _, out("v22") _, out("v23") _, @@ -224,7 +224,7 @@ macro_rules! probe_fn { b = in(reg) b.as_ptr(), prod = in(reg) prod.as_mut_ptr(), fx = in(reg) fx.as_mut_ptr(), - out("x9") _, out("x10") _, + out("x9") _, out("x10") _, out("p0") _, out("p1") _, out("v0") _, out("v1") _, out("v2") _, out("v3") _, out("v4") _, out("v5") _, out("v6") _, out("v7") _, out("v8") _, out("v9") _, out("v10") _, out("v11") _, out("v12") _, out("v13") _, out("v14") _, out("v15") _, out("v16") _, out("v17") _, out("v18") _, out("v19") _, out("v20") _, out("v21") _, out("v22") _, out("v23") _, -- 2.54.0 (Apple Git-157) From e2392f920a6af3d7e179294b5a1c640c61e9138c Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 15 Oct 2026 01:52:03 -0400 Subject: [PATCH 10/11] scripts: no pipe into grep -q under pipefail `printf ... | grep -q` under `set -o pipefail` reports no match when grep exits at its first match while printf is still writing: printf dies of SIGPIPE and the pipeline's status is 141. The tests-in-CI self-test failed that way on the repository checks job (main shows the same job red on one commit and green on its neighbours), and check-lockfiles.sh classifies cargo's message the same way, so a timing accident could misread it. Each now reads a here-string, with nothing to break. Co-Authored-By: Claude Opus 5.5 --- scripts/check-lockfiles-selftest.sh | 2 +- scripts/check-lockfiles.sh | 2 +- scripts/check-tests-in-ci-selftest.sh | 2 +- scripts/standards-touched-selftest.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/check-lockfiles-selftest.sh b/scripts/check-lockfiles-selftest.sh index a113adb23..0b9e4885c 100755 --- a/scripts/check-lockfiles-selftest.sh +++ b/scripts/check-lockfiles-selftest.sh @@ -36,7 +36,7 @@ expect() { # expect [text the output must hold] out=$("$check" "$tmp/ws" 2>&1); rc=$? if [ "$want" = pass ] && [ $rc -ne 0 ]; then echo "FAIL $what: the check failed"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi if [ "$want" = fail ] && [ $rc -eq 0 ]; then echo "FAIL $what: the check passed"; fails=$((fails+1)); return; fi - if [ -n "$needle" ] && ! printf '%s' "$out" | grep -q -- "$needle"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi + if [ -n "$needle" ] && ! grep -q -- "$needle" <<<"$out"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi echo "ok $what" } diff --git a/scripts/check-lockfiles.sh b/scripts/check-lockfiles.sh index 0117406c7..30dfb8403 100755 --- a/scripts/check-lockfiles.sh +++ b/scripts/check-lockfiles.sh @@ -64,7 +64,7 @@ for d in "${dirs[@]}"; do out=$( (cd "$d" && cargo update --workspace --locked) 2>&1 ); rc=$? if [ $rc -eq 0 ]; then continue - elif printf '%s' "$out" | grep -qE -- '--locked was passed|lock file .* (is missing|needs to be updated)'; then + elif grep -qE -- '--locked was passed|lock file .* (is missing|needs to be updated)' <<<"$out"; then stale+=("$d") echo "STALE $d" printf '%s\n' "$out" | grep -E '^error|^Caused by|^ ' | head -6 | sed 's/^/ /' diff --git a/scripts/check-tests-in-ci-selftest.sh b/scripts/check-tests-in-ci-selftest.sh index a9f6dbb9f..8f63033d6 100755 --- a/scripts/check-tests-in-ci-selftest.sh +++ b/scripts/check-tests-in-ci-selftest.sh @@ -31,7 +31,7 @@ expect() { # expect [text the output must hold] out=$(python3 -I "$check" --root "$tmp" 2>&1); rc=$? if [ "$want" = pass ] && [ $rc -ne 0 ]; then echo "FAIL $what: the check failed"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi if [ "$want" = fail ] && [ $rc -eq 0 ]; then echo "FAIL $what: the check passed"; fails=$((fails+1)); return; fi - if [ -n "$needle" ] && ! printf '%s' "$out" | grep -q -- "$needle"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi + if [ -n "$needle" ] && ! grep -q -- "$needle" <<<"$out"; then echo "FAIL $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/ /'; fails=$((fails+1)); return; fi echo "ok $what" } diff --git a/scripts/standards-touched-selftest.sh b/scripts/standards-touched-selftest.sh index 039eaf17e..646f18764 100755 --- a/scripts/standards-touched-selftest.sh +++ b/scripts/standards-touched-selftest.sh @@ -49,7 +49,7 @@ got=$(python3 -I scripts/standards-touched.py "" "" | python3 -c 'import json,sy got=$(python3 -I scripts/standards-touched.py "$base" "0000000000000000000000000000000000000bad" 2>/dev/null | python3 -c 'import json,sys; print(",".join(sorted(json.load(sys.stdin)["run"])))') [ "$got" = "a-rs,b-rs,c-rs" ] && echo "ok a diff it cannot compute: runs everything" || { echo "FAIL an unknown head ran [$got]"; fails=$((fails+1)); } out=$(GITHUB_OUTPUT="$tmp/out" python3 -I scripts/standards-touched.py --github-output "" "" >/dev/null; cat "$tmp/out") -printf '%s\n' "$out" | grep -q '"z3":true' && printf '%s\n' "$out" | grep -q '"feature_tests":"-p x --features y,z"' && printf '%s\n' "$out" | grep -q '^energy=false$' \ +grep -q '"z3":true' <<<"$out" && grep -q '"feature_tests":"-p x --features y,z"' <<<"$out" && grep -q '^energy=false$' <<<"$out" \ && echo "ok the matrix entries keep their booleans and quoted values" || { echo "FAIL the emitted matrix: $out"; fails=$((fails+1)); } if [ $fails -ne 0 ]; then echo "standards-touched-selftest: $fails failure(s)"; exit 1; fi echo "standards-touched-selftest: the plan builds what a change can affect, and everything when unsure." -- 2.54.0 (Apple Git-157) From 43cead017edf00fea37f20d5f2f25011335d33b7 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Thu, 15 Oct 2026 09:20:13 -0400 Subject: [PATCH 11/11] ci: the signal-processor workflow's emulator variables are not QEMU_* qemu-user reads every QEMU_