From 9c3c42ee28519d51acf6edbce117a37655369ae0 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Fri, 6 Nov 2026 03:09:57 -0500 Subject: [PATCH] ci: the wai-quantum legs build the commit they test, and fail when they did not Seven wai-quantum workflows (processors, dsp, dataflow, sme, tensor-engine, tensor-program, vector) take the forge's archive of the commit, extract it to the same path on every run, and build into a target directory kept in the runner's cache. tar gave every file its commit's time, and cargo takes a crate as built when no source file is newer than its last build. So a commit older than the cache's last build ran the binary another commit had built there, and its tests said nothing about its own code. On 5 Nov, 16 emulated legs went green on another commit's binary, and #662's three reds were its legs running #660's two frame_reference tests against #662's check for three. - Fetch the source extracts with tar -m: every file gets the time it was extracted. - That alone does not close it. Each build waits on the host-wide lock after its extraction, and another run can build in the cache meanwhile, after which this one's files are again the older. So every step that builds touches the extracted tree once it holds the lock. - The first build in each such step must print "Compiling wai-quantum v..."; one that compiled nothing fails the leg, saying its tests would run a binary already in the cache. The host-runner legs (macOS, linux-hw, linux-amd, linux-arm64) check out with git into a fresh tree, and are not changed here. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wai-quantum-dataflow.yml | 18 ++++++++++++++++-- .github/workflows/wai-quantum-dsp.yml | 18 +++++++++++++++++- .github/workflows/wai-quantum-processors.yml | 13 ++++++++++++- .github/workflows/wai-quantum-sme.yml | 13 ++++++++++++- .../workflows/wai-quantum-tensor-engine.yml | 13 ++++++++++++- .../workflows/wai-quantum-tensor-program.yml | 13 ++++++++++++- .github/workflows/wai-quantum-vector.yml | 13 ++++++++++++- 7 files changed, 93 insertions(+), 8 deletions(-) diff --git a/.github/workflows/wai-quantum-dataflow.yml b/.github/workflows/wai-quantum-dataflow.yml index a61e9d6ab..08e14a028 100644 --- a/.github/workflows/wai-quantum-dataflow.yml +++ b/.github/workflows/wai-quantum-dataflow.yml @@ -97,7 +97,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The builds + # reuse a target directory kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The simulator, built from source at the pinned commit (cached), and Rust shell: bash @@ -138,10 +142,18 @@ jobs: cache=/ci-cache/wai-quantum-dataflow exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) The next step takes the + # lock afresh, and refreshes again. + find ../.. -type f -exec touch {} + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" export PATH="$CARGO_HOME/bin:$PATH" out="$RUNNER_TEMP/rv32" - cargo run --locked --release -q --example rv32_listing --features quantum_fabric_dataflow -- "$out" + # Not -q: the build's own lines show it compiled wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + cargo run --locked --release --example rv32_listing --features quantum_fabric_dataflow -- "$out" 2>&1 | tee "$RUNNER_TEMP/rv32-build.log" + grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/rv32-build.log" || { echo "::error::the build compiled no wai-quantum: the encoder that ran is one already in the cache"; exit 1; } riscv64-linux-gnu-as --version | head -1 n=0 for s in "$out"/*.s; do @@ -161,6 +173,7 @@ jobs: cache=/ci-cache/wai-quantum-dataflow exec 9>"$cache/.lock" flock 9 + find ../.. -type f -exec touch {} + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" export PATH="$CARGO_HOME/bin:$PATH" export WAI_QUANTUM_DATAFLOW_LIB="$cache/sim-$SIM_COMMIT/${{ matrix.build }}/libttsim.so" @@ -168,6 +181,7 @@ jobs: # table; another generation lays some instructions out differently. if [ -n "${{ matrix.vector }}" ]; then export WAI_QUANTUM_TENSIX_ISA="$cache/sim-$SIM_COMMIT/${{ matrix.build }}/tensix_isa.json"; fi cargo test --locked --release -j 6 --features quantum_fabric_dataflow --lib -- quantum_fabric_dataflow:: quantum_fabric::rv32:: quantum_fabric::tensix:: quantum_fabric::vector_engine:: --nocapture 2>&1 | tee "$RUNNER_TEMP/unit.log" + grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } # The vector engine's encoder against the simulator's own table (the # test returns early without the table, so require its line). Tests diff --git a/.github/workflows/wai-quantum-dsp.yml b/.github/workflows/wai-quantum-dsp.yml index 253598676..902c8fca5 100644 --- a/.github/workflows/wai-quantum-dsp.yml +++ b/.github/workflows/wai-quantum-dsp.yml @@ -103,7 +103,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The builds + # reuse target directories kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The cross toolchain, the emulator and Rust (cached, pinned) shell: bash @@ -170,6 +174,11 @@ jobs: cache=/ci-cache/wai-quantum-hexagon exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) Each step below that + # builds takes the lock afresh, and refreshes again. + find ../.. -type f -exec touch {} + tc="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu" export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" export PATH="$CARGO_HOME/bin:$tc/bin:$PATH" RUSTC_BOOTSTRAP=1 @@ -184,6 +193,9 @@ jobs: --target hexagon-unknown-linux-musl --features "${FEATURES:-full}" "$@" 2>&1 | tee "$logs/$name.log" } t bit_identity --test bit_identity -- --test-threads=1 + # The first build after the refresh compiles wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + grep -qE '^ *Compiling wai-quantum v[0-9]' "$logs/bit_identity.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. 23 passed; 0 failed' "$logs/bit_identity.log" || { echo "::error::the 23 bit-identity digests did not all pass"; exit 1; } t frame --test frame_reference -- --test-threads=1 grep -qE '^test result: ok\. 2 passed; 0 failed' "$logs/frame.log" || { echo "::error::the fixtures did not pass"; exit 1; } @@ -200,6 +212,7 @@ jobs: cache=/ci-cache/wai-quantum-hexagon exec 9>"$cache/.lock" flock 9 + find ../.. -type f -exec touch {} + tc="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu" # The vector unit in 128-byte mode, for the whole build (the standard # library included): a separate target directory. @@ -210,6 +223,7 @@ jobs: export CC_hexagon_unknown_linux_musl=hexagon-linux-musl-clang AR_hexagon_unknown_linux_musl=hexagon-linux-musl-ar b="-Zbuild-std -Zbuild-std-features=system-llvm-libunwind --target hexagon-unknown-linux-musl --features quantum_fabric_dsp" timeout 1800 cargo test --locked --release -j 6 $b --lib -- quantum_fabric_dsp:: 2>&1 | tee "$RUNNER_TEMP/unit.log" + grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } report="$RUNNER_TEMP/dsp-report.txt"; : > "$report" WAI_QUANTUM_DSP_REPORT="$report" timeout 5400 cargo test --locked --release -j 6 $b --test dsp_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log" @@ -229,9 +243,11 @@ jobs: cache=/ci-cache/wai-quantum-hexagon exec 9>"$cache/.lock" flock 9 + find ../.. -type f -exec touch {} + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target-host" export PATH="$CARGO_HOME/bin:$PATH" cargo test --locked -j 6 --features quantum_fabric_dsp --lib -- quantum_fabric_dsp:: 2>&1 | tee "$RUNNER_TEMP/host-unit.log" + grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/host-unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/host-unit.log" || { echo "::error::no unit test ran"; exit 1; } report="$RUNNER_TEMP/dsp-absent.txt"; : > "$report" WAI_QUANTUM_DSP_EXPECT=none WAI_QUANTUM_DSP_REPORT="$report" cargo test --locked -j 6 --features quantum_fabric_dsp --test dsp_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/host-device.log" diff --git a/.github/workflows/wai-quantum-processors.yml b/.github/workflows/wai-quantum-processors.yml index 14679f927..8f514c22a 100644 --- a/.github/workflows/wai-quantum-processors.yml +++ b/.github/workflows/wai-quantum-processors.yml @@ -101,7 +101,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The build + # reuses a target directory kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The byte-pinning tests shell: bash @@ -115,6 +119,10 @@ jobs: # emulated tests are heavy, and the toolchain cache is shared. exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) + find ../.. -type f -exec touch {} + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" export PATH="$CARGO_HOME/bin:$PATH" if ! command -v rustup >/dev/null; then @@ -129,6 +137,9 @@ jobs: timeout 2700 cargo test --locked --release -j 6 --target "$t" --features full "$@" 2>&1 | tee "$logs/$name.log" } run bit_identity --test bit_identity + # The first build after the refresh compiles wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + grep -qE '^ *Compiling wai-quantum v[0-9]' "$logs/bit_identity.log" || { echo "::error::$t: the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. 23 passed; 0 failed' "$logs/bit_identity.log" || { echo "::error::$t: the 23 bit-identity digests did not all pass"; exit 1; } run frame --test frame_reference grep -qE '^test result: ok\. 2 passed; 0 failed' "$logs/frame.log" || { echo "::error::$t: the fixtures did not pass"; exit 1; } diff --git a/.github/workflows/wai-quantum-sme.yml b/.github/workflows/wai-quantum-sme.yml index 4f49399dd..ec94ca974 100644 --- a/.github/workflows/wai-quantum-sme.yml +++ b/.github/workflows/wai-quantum-sme.yml @@ -81,7 +81,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The build + # reuses a target directory kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The member's device tests under the emulator shell: bash @@ -92,6 +96,10 @@ jobs: mkdir -p "$cache" exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) + find ../.. -type f -exec touch {} + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" export PATH="$CARGO_HOME/bin:$PATH" if ! command -v rustup >/dev/null; then @@ -101,6 +109,9 @@ jobs: rustup target add aarch64-unknown-linux-gnu report="$(mktemp)" WAI_QUANTUM_SME_REPORT="$report" cargo test --locked --release -j 6 --target aarch64-unknown-linux-gnu --features quantum_fabric_sme --test sme_member -- --test-threads=1 2>&1 | tee "$cache/device.log" + # The first build after the refresh compiles wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + grep -qE '^ *Compiling wai-quantum v[0-9]' "$cache/device.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. 9 passed; 0 failed' "$cache/device.log" || { echo "::error::not every device test ran and passed"; exit 1; } cat "$report" key="$WAI_QUANTUM_SME_EXPECT" diff --git a/.github/workflows/wai-quantum-tensor-engine.yml b/.github/workflows/wai-quantum-tensor-engine.yml index ecae048f8..543189c8c 100644 --- a/.github/workflows/wai-quantum-tensor-engine.yml +++ b/.github/workflows/wai-quantum-tensor-engine.yml @@ -85,7 +85,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The build + # reuses a target directory kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The member's tests on the simulator shell: bash @@ -96,6 +100,10 @@ jobs: mkdir -p "$cache" exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) + find ../.. -type f -exec touch {} + venv="$cache/kernel-toolchain" if ! "$venv/bin/python" -I -c 'import nki, ml_dtypes' 2>/dev/null || [ "$(cat "$venv/pinned" 2>/dev/null)" != "$KERNEL_TOOLCHAIN" ]; then rm -rf "$venv" @@ -110,6 +118,9 @@ jobs: fi rustup toolchain install cargo test --locked --features quantum_fabric_tensor_engine --lib -- quantum_fabric_tensor_engine:: quantum_fabric::tensor_engine:: 2>&1 | tee "$RUNNER_TEMP/unit.log" + # The first build after the refresh compiles wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } key="$WAI_QUANTUM_TENSOR_ENGINE_EXPECT" report="$RUNNER_TEMP/tensor-engine-report.txt"; : > "$report" diff --git a/.github/workflows/wai-quantum-tensor-program.yml b/.github/workflows/wai-quantum-tensor-program.yml index 9528e9b11..3b7603db8 100644 --- a/.github/workflows/wai-quantum-tensor-program.yml +++ b/.github/workflows/wai-quantum-tensor-program.yml @@ -98,7 +98,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The build + # reuses a target directory kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The program and its mutants on the CPU, both compilers; the GPU builds shell: bash @@ -109,6 +113,10 @@ jobs: mkdir -p "$cache" exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) + find ../.. -type f -exec touch {} + venv="$cache/compilers-iree-$IREE_VERSION-jax-$JAX_VERSION" [ -x "$venv/bin/iree-compile" ] || { rm -rf "$venv"; python3 -m venv "$venv"; "$venv/bin/pip" install -q "iree-base-compiler==$IREE_VERSION" "iree-base-runtime==$IREE_VERSION" "jax==$JAX_VERSION"; } export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" @@ -118,6 +126,9 @@ jobs: fi rustup toolchain install cargo test --locked --features quantum_fabric --lib -- quantum_fabric::stablehlo:: 2>&1 | tee "$RUNNER_TEMP/unit.log" + # The first build after the refresh compiles wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } check="bash $PWD/ci/tensor-program/check.sh" d="$RUNNER_TEMP/program" diff --git a/.github/workflows/wai-quantum-vector.yml b/.github/workflows/wai-quantum-vector.yml index 268560eb3..80589328a 100644 --- a/.github/workflows/wai-quantum-vector.yml +++ b/.github/workflows/wai-quantum-vector.yml @@ -133,7 +133,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The build + # reuses a target directory kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The member's tests shell: bash @@ -144,6 +148,10 @@ jobs: mkdir -p "$cache" exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) + find ../.. -type f -exec touch {} + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target-${{ matrix.target }}" export PATH="$CARGO_HOME/bin:$PATH" if ! command -v rustup >/dev/null; then @@ -160,6 +168,9 @@ jobs: fi t="--target ${{ matrix.target }} --features quantum_fabric_vector" cargo test --locked --release -j 6 $t --lib -- quantum_fabric_vector:: 2>&1 | tee "$cache/unit.log" + # The first build after the refresh compiles wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + grep -qE '^ *Compiling wai-quantum v[0-9]' "$cache/unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$cache/unit.log" || { echo "::error::no unit test ran"; exit 1; } report="$(mktemp)" WAI_QUANTUM_VECTOR_REPORT="$report" cargo test --locked --release -j 6 $t --test vector_member -- --test-threads=1 2>&1 | tee "$cache/device.log" -- 2.54.0 (Apple Git-157)