diff --git a/.github/workflows/wai-quantum-dataflow.yml b/.github/workflows/wai-quantum-dataflow.yml index a61e9d6ab..08e14a028 100644 --- a/.github/workflows/wai-quantum-dataflow.yml +++ b/.github/workflows/wai-quantum-dataflow.yml @@ -97,7 +97,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The builds + # reuse a target directory kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The simulator, built from source at the pinned commit (cached), and Rust shell: bash @@ -138,10 +142,18 @@ jobs: cache=/ci-cache/wai-quantum-dataflow exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) The next step takes the + # lock afresh, and refreshes again. + find ../.. -type f -exec touch {} + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" export PATH="$CARGO_HOME/bin:$PATH" out="$RUNNER_TEMP/rv32" - cargo run --locked --release -q --example rv32_listing --features quantum_fabric_dataflow -- "$out" + # Not -q: the build's own lines show it compiled wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + cargo run --locked --release --example rv32_listing --features quantum_fabric_dataflow -- "$out" 2>&1 | tee "$RUNNER_TEMP/rv32-build.log" + grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/rv32-build.log" || { echo "::error::the build compiled no wai-quantum: the encoder that ran is one already in the cache"; exit 1; } riscv64-linux-gnu-as --version | head -1 n=0 for s in "$out"/*.s; do @@ -161,6 +173,7 @@ jobs: cache=/ci-cache/wai-quantum-dataflow exec 9>"$cache/.lock" flock 9 + find ../.. -type f -exec touch {} + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" export PATH="$CARGO_HOME/bin:$PATH" export WAI_QUANTUM_DATAFLOW_LIB="$cache/sim-$SIM_COMMIT/${{ matrix.build }}/libttsim.so" @@ -168,6 +181,7 @@ jobs: # table; another generation lays some instructions out differently. if [ -n "${{ matrix.vector }}" ]; then export WAI_QUANTUM_TENSIX_ISA="$cache/sim-$SIM_COMMIT/${{ matrix.build }}/tensix_isa.json"; fi cargo test --locked --release -j 6 --features quantum_fabric_dataflow --lib -- quantum_fabric_dataflow:: quantum_fabric::rv32:: quantum_fabric::tensix:: quantum_fabric::vector_engine:: --nocapture 2>&1 | tee "$RUNNER_TEMP/unit.log" + grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } # The vector engine's encoder against the simulator's own table (the # test returns early without the table, so require its line). Tests diff --git a/.github/workflows/wai-quantum-dsp.yml b/.github/workflows/wai-quantum-dsp.yml index 253598676..902c8fca5 100644 --- a/.github/workflows/wai-quantum-dsp.yml +++ b/.github/workflows/wai-quantum-dsp.yml @@ -103,7 +103,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The builds + # reuse target directories kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The cross toolchain, the emulator and Rust (cached, pinned) shell: bash @@ -170,6 +174,11 @@ jobs: cache=/ci-cache/wai-quantum-hexagon exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) Each step below that + # builds takes the lock afresh, and refreshes again. + find ../.. -type f -exec touch {} + tc="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu" export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" export PATH="$CARGO_HOME/bin:$tc/bin:$PATH" RUSTC_BOOTSTRAP=1 @@ -184,6 +193,9 @@ jobs: --target hexagon-unknown-linux-musl --features "${FEATURES:-full}" "$@" 2>&1 | tee "$logs/$name.log" } t bit_identity --test bit_identity -- --test-threads=1 + # The first build after the refresh compiles wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + grep -qE '^ *Compiling wai-quantum v[0-9]' "$logs/bit_identity.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. 23 passed; 0 failed' "$logs/bit_identity.log" || { echo "::error::the 23 bit-identity digests did not all pass"; exit 1; } t frame --test frame_reference -- --test-threads=1 grep -qE '^test result: ok\. 2 passed; 0 failed' "$logs/frame.log" || { echo "::error::the fixtures did not pass"; exit 1; } @@ -200,6 +212,7 @@ jobs: cache=/ci-cache/wai-quantum-hexagon exec 9>"$cache/.lock" flock 9 + find ../.. -type f -exec touch {} + tc="$cache/toolchain-$TOOLCHAIN_VERSION/clang+llvm-$TOOLCHAIN_VERSION-cross-hexagon-unknown-linux-musl/x86_64-linux-gnu" # The vector unit in 128-byte mode, for the whole build (the standard # library included): a separate target directory. @@ -210,6 +223,7 @@ jobs: export CC_hexagon_unknown_linux_musl=hexagon-linux-musl-clang AR_hexagon_unknown_linux_musl=hexagon-linux-musl-ar b="-Zbuild-std -Zbuild-std-features=system-llvm-libunwind --target hexagon-unknown-linux-musl --features quantum_fabric_dsp" timeout 1800 cargo test --locked --release -j 6 $b --lib -- quantum_fabric_dsp:: 2>&1 | tee "$RUNNER_TEMP/unit.log" + grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } report="$RUNNER_TEMP/dsp-report.txt"; : > "$report" WAI_QUANTUM_DSP_REPORT="$report" timeout 5400 cargo test --locked --release -j 6 $b --test dsp_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/device.log" @@ -229,9 +243,11 @@ jobs: cache=/ci-cache/wai-quantum-hexagon exec 9>"$cache/.lock" flock 9 + find ../.. -type f -exec touch {} + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target-host" export PATH="$CARGO_HOME/bin:$PATH" cargo test --locked -j 6 --features quantum_fabric_dsp --lib -- quantum_fabric_dsp:: 2>&1 | tee "$RUNNER_TEMP/host-unit.log" + grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/host-unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/host-unit.log" || { echo "::error::no unit test ran"; exit 1; } report="$RUNNER_TEMP/dsp-absent.txt"; : > "$report" WAI_QUANTUM_DSP_EXPECT=none WAI_QUANTUM_DSP_REPORT="$report" cargo test --locked -j 6 --features quantum_fabric_dsp --test dsp_member -- --test-threads=1 2>&1 | tee "$RUNNER_TEMP/host-device.log" diff --git a/.github/workflows/wai-quantum-processors.yml b/.github/workflows/wai-quantum-processors.yml index 14679f927..8f514c22a 100644 --- a/.github/workflows/wai-quantum-processors.yml +++ b/.github/workflows/wai-quantum-processors.yml @@ -101,7 +101,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The build + # reuses a target directory kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The byte-pinning tests shell: bash @@ -115,6 +119,10 @@ jobs: # emulated tests are heavy, and the toolchain cache is shared. exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) + find ../.. -type f -exec touch {} + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" export PATH="$CARGO_HOME/bin:$PATH" if ! command -v rustup >/dev/null; then @@ -129,6 +137,9 @@ jobs: timeout 2700 cargo test --locked --release -j 6 --target "$t" --features full "$@" 2>&1 | tee "$logs/$name.log" } run bit_identity --test bit_identity + # The first build after the refresh compiles wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + grep -qE '^ *Compiling wai-quantum v[0-9]' "$logs/bit_identity.log" || { echo "::error::$t: the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. 23 passed; 0 failed' "$logs/bit_identity.log" || { echo "::error::$t: the 23 bit-identity digests did not all pass"; exit 1; } run frame --test frame_reference grep -qE '^test result: ok\. 2 passed; 0 failed' "$logs/frame.log" || { echo "::error::$t: the fixtures did not pass"; exit 1; } diff --git a/.github/workflows/wai-quantum-sme.yml b/.github/workflows/wai-quantum-sme.yml index aaf1cdbe7..913917ecb 100644 --- a/.github/workflows/wai-quantum-sme.yml +++ b/.github/workflows/wai-quantum-sme.yml @@ -87,7 +87,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The build + # reuses a target directory kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The member's device tests under the emulator shell: bash @@ -98,6 +102,10 @@ jobs: mkdir -p "$cache" exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) + find ../.. -type f -exec touch {} + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" export PATH="$CARGO_HOME/bin:$PATH" if ! command -v rustup >/dev/null; then @@ -134,6 +142,9 @@ jobs: echo "emulator: $("$q/qemu-aarch64" --version | head -1); an aarch64 program runs under it" report="$(mktemp)" WAI_QUANTUM_SME_REPORT="$report" cargo test --locked --release -j 6 --target aarch64-unknown-linux-gnu --features quantum_fabric_sme --test sme_member -- --test-threads=1 2>&1 | tee "$cache/device.log" + # The first build after the refresh compiles wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + grep -qE '^ *Compiling wai-quantum v[0-9]' "$cache/device.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. 9 passed; 0 failed' "$cache/device.log" || { echo "::error::not every device test ran and passed"; exit 1; } cat "$report" key="$WAI_QUANTUM_SME_EXPECT" diff --git a/.github/workflows/wai-quantum-tensor-engine.yml b/.github/workflows/wai-quantum-tensor-engine.yml index ecae048f8..543189c8c 100644 --- a/.github/workflows/wai-quantum-tensor-engine.yml +++ b/.github/workflows/wai-quantum-tensor-engine.yml @@ -85,7 +85,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The build + # reuses a target directory kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The member's tests on the simulator shell: bash @@ -96,6 +100,10 @@ jobs: mkdir -p "$cache" exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) + find ../.. -type f -exec touch {} + venv="$cache/kernel-toolchain" if ! "$venv/bin/python" -I -c 'import nki, ml_dtypes' 2>/dev/null || [ "$(cat "$venv/pinned" 2>/dev/null)" != "$KERNEL_TOOLCHAIN" ]; then rm -rf "$venv" @@ -110,6 +118,9 @@ jobs: fi rustup toolchain install cargo test --locked --features quantum_fabric_tensor_engine --lib -- quantum_fabric_tensor_engine:: quantum_fabric::tensor_engine:: 2>&1 | tee "$RUNNER_TEMP/unit.log" + # The first build after the refresh compiles wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } key="$WAI_QUANTUM_TENSOR_ENGINE_EXPECT" report="$RUNNER_TEMP/tensor-engine-report.txt"; : > "$report" diff --git a/.github/workflows/wai-quantum-tensor-program.yml b/.github/workflows/wai-quantum-tensor-program.yml index 9528e9b11..3b7603db8 100644 --- a/.github/workflows/wai-quantum-tensor-program.yml +++ b/.github/workflows/wai-quantum-tensor-program.yml @@ -98,7 +98,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The build + # reuses a target directory kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The program and its mutants on the CPU, both compilers; the GPU builds shell: bash @@ -109,6 +113,10 @@ jobs: mkdir -p "$cache" exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) + find ../.. -type f -exec touch {} + venv="$cache/compilers-iree-$IREE_VERSION-jax-$JAX_VERSION" [ -x "$venv/bin/iree-compile" ] || { rm -rf "$venv"; python3 -m venv "$venv"; "$venv/bin/pip" install -q "iree-base-compiler==$IREE_VERSION" "iree-base-runtime==$IREE_VERSION" "jax==$JAX_VERSION"; } export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target" @@ -118,6 +126,9 @@ jobs: fi rustup toolchain install cargo test --locked --features quantum_fabric --lib -- quantum_fabric::stablehlo:: 2>&1 | tee "$RUNNER_TEMP/unit.log" + # The first build after the refresh compiles wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + grep -qE '^ *Compiling wai-quantum v[0-9]' "$RUNNER_TEMP/unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$RUNNER_TEMP/unit.log" || { echo "::error::no unit test ran"; exit 1; } check="bash $PWD/ci/tensor-program/check.sh" d="$RUNNER_TEMP/program" diff --git a/.github/workflows/wai-quantum-vector.yml b/.github/workflows/wai-quantum-vector.yml index 83669449e..c4772c2e0 100644 --- a/.github/workflows/wai-quantum-vector.yml +++ b/.github/workflows/wai-quantum-vector.yml @@ -146,7 +146,11 @@ jobs: gzip -dc src.tar.gz > src.tar got="$(git get-tar-commit-id < src.tar)" [ "$got" = "$want" ] || { echo "::error::the archive is of $got, not $want"; exit 1; } - tar -xf src.tar -C src --strip-components=1 + # -m: every file gets the time it was extracted, not its commit's. The build + # reuses a target directory kept between runs, and cargo takes a crate as + # built when no source file is newer than its last build: with the commit's + # times, an older commit ran the binary another commit had built there. + tar -xmf src.tar -C src --strip-components=1 rm src.tar.gz src.tar - name: The member's tests shell: bash @@ -157,6 +161,10 @@ jobs: mkdir -p "$cache" exec 9>"$cache/.lock" flock 9 + # Every source file newer than every build in the cache, now that the lock is + # held: another run may have built here while this one waited for it, after + # this one's extraction. (../.. is the extracted tree.) + find ../.. -type f -exec touch {} + export RUSTUP_HOME="$cache/rustup" CARGO_HOME="$cache/cargo" CARGO_TARGET_DIR="$cache/target-${{ matrix.target }}" export PATH="$CARGO_HOME/bin:$PATH" if ! command -v rustup >/dev/null; then @@ -203,6 +211,9 @@ jobs: fi t="--target ${{ matrix.target }} --features quantum_fabric_vector" cargo test --locked --release -j 6 $t --lib -- quantum_fabric_vector:: 2>&1 | tee "$cache/unit.log" + # The first build after the refresh compiles wai-quantum; one that compiled + # none ran a binary another commit left in the cache. + grep -qE '^ *Compiling wai-quantum v[0-9]' "$cache/unit.log" || { echo "::error::the build compiled no wai-quantum: its tests ran a binary already in the cache"; exit 1; } grep -qE '^test result: ok\. [1-9][0-9]* passed; 0 failed' "$cache/unit.log" || { echo "::error::no unit test ran"; exit 1; } report="$(mktemp)" WAI_QUANTUM_VECTOR_REPORT="$report" cargo test --locked --release -j 6 $t --test vector_member -- --test-threads=1 2>&1 | tee "$cache/device.log"