diff --git a/.github/workflows/wai-byte-equality.yml b/.github/workflows/wai-byte-equality.yml index 53036065e..bae9986b3 100644 --- a/.github/workflows/wai-byte-equality.yml +++ b/.github/workflows/wai-byte-equality.yml @@ -159,26 +159,31 @@ jobs: # Self-hosted fleet labels, not hosted images. macos-x86_64 is # dropped: no Intel-Mac runner exists in the fleet, and by the same # argument the cargo-classes matrix already makes, it adds no arch - # coverage — x86_64 is held by linux-x86_64, arm64 by linux-aarch64 - # and the macos-arm64 reference. - # Windows x86_64 leg removed 2026-09-30: runner offline. Restore by - # re-adding `- { os: windows, label: windows-x86_64 }` here once the - # runner is back online. - # The two disk steps below are bash, and skip on Windows. + # coverage — x86_64 is held by linux-x86_64 and windows-x86_64, arm64 + # by linux-aarch64 and the macos-arm64 reference. + # windows-x86_64 was removed on 2026-09-30 while its runner was offline, and + # is back now that the runner is (a `windows` job passed in run 19310). Not a + # required status, as wai-quantum-webgpu.yml's member-windows is not: like every + # leg here it runs only when the `relevant` gate above answers true. A job on an + # offline runner stays pending and never settles, so if the runner goes offline + # again the leg comes out of this list until it is back. + # The two disk steps below come in a bash form for the unix legs and a pwsh form + # for Windows, whose `bash` is the WSL stub. include: - { os: linux, label: linux-x86_64 } - { os: linux-arm64, label: linux-aarch64 } - { os: macos, label: macos-arm64 (reference) } + - { os: windows, label: windows-x86_64 } env: # The floor of the disk check below, in GiB. The leg writes under 1 GiB (a # shallow checkout of about 0.3 GiB plus its pack, and a release target under # 0.1 GiB, measured 2026-11-03), and a cold toolchain install about 1 GiB more # under HOME; the rest is room for the jobs that share the disk. Measured with the - # step's own loop on 2026-11-04, every label had 50 GiB or more free under all - # four directories, so the floor fails a leg only on a host within a few GiB of - # full. The macOS runner's HOME and TMPDIR sit on its internal disk, which has - # gone from 7 GiB free to 118 MiB within hours, and at that level its jobs failed - # writing under HOME. + # step's own loop on 2026-11-04, every unix label had 50 GiB or more free under all + # four directories (the Windows host was not measured), so the floor fails a leg + # only on a host within a few GiB of full. The macOS runner's HOME and TMPDIR sit + # on its internal disk, which has gone from 7 GiB free to 118 MiB within hours, and + # at that level its jobs failed writing under HOME. DISK_FLOOR_GIB: '4' steps: # Infrastructure first, before the checkout writes anything. Main's run 17676 @@ -231,6 +236,61 @@ jobs: rm -f "$f" done exit $short + # The same check on the Windows leg, in pwsh: its `bash` is the WSL stub, which + # would measure another system's disk. The directories are the workspace, the + # runner's temporary directory, TEMP, and the profile directory, where rustup keeps + # the toolchain. The free space is what each one's volume leaves the runner's user, + # quotas included, and the probe writes 32 MiB through to each volume once. The + # write errors named infrastructure are the Win32 codes for a full disk (39, 112), + # a write-protected one (19), an I/O device error (1117) and a quota (1295). + - name: "infrastructure: this runner's disk has room for the leg (Windows)" + id: disk_windows + if: matrix.os == 'windows' + shell: pwsh + run: | + Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force + $floor = [int64]$env:DISK_FLOOR_GIB * 1GB + $short = 0; $seen = @() + $tmp = [System.IO.Path]::GetTempPath() + $dirs = @( + $(if ($env:GITHUB_WORKSPACE) { $env:GITHUB_WORKSPACE } else { $PWD.Path }), + $(if ($env:RUNNER_TEMP) { $env:RUNNER_TEMP } else { $tmp }), + $(if ($env:TEMP) { $env:TEMP } else { $tmp }), + $HOME + ) + foreach ($d in $dirs) { + $p = [System.IO.Path]::GetFullPath($d) + while (-not (Test-Path -LiteralPath $p)) { $p = Split-Path -Parent $p } + $vol = [System.IO.Path]::GetPathRoot($p) + $free = [System.IO.DriveInfo]::new($vol).AvailableFreeSpace + Write-Output "${d}: $([math]::Floor($free / 1GB)) GiB free on $vol" + if ($free -lt $floor) { + Write-Output "::error::infrastructure: ${{ matrix.label }}: $d is on $vol, which has $([math]::Floor($free / 1MB)) MiB free, below the floor of $env:DISK_FLOOR_GIB GiB. The runner host's disk is full: no fixture was decoded, so this is not a byte-equality failure." + $short = 1; continue + } + if ($seen -contains $vol) { continue } + $seen += $vol; $f = Join-Path $p ".wai-disk-probe.$PID" + try { + $s = [System.IO.FileStream]::new($f, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None, 1048576, [System.IO.FileOptions]::WriteThrough) + try { + $buf = [byte[]]::new(1048576) + for ($i = 0; $i -lt 32; $i++) { $s.Write($buf, 0, $buf.Length) } + $s.Flush($true) + } finally { $s.Dispose() } + } catch { + $e = $_.Exception; while ($e.InnerException) { $e = $e.InnerException } + $msg = (($e.Message -split "`r?`n")[0]).TrimEnd('.', ' ') + if (($e.HResult -band 0xFFFF) -in 19, 39, 112, 1117, 1295) { + Write-Output "::error::infrastructure: ${{ matrix.label }}: a 32 MiB write under $p, on $vol, failed: $msg. The runner host's disk cannot take the leg's writes: no fixture was decoded, so this is not a byte-equality failure." + $short = 1 + } else { + Write-Output "::warning::a 32 MiB write under $p failed for another reason: $msg" + } + } finally { + Remove-Item -LiteralPath $f -Force -ErrorAction SilentlyContinue + } + } + exit $short - name: Checkout uses: actions/checkout@v4 @@ -292,3 +352,52 @@ jobs: done [ "$short" = 1 ] || echo "Every disk this leg writes to has room: the failure above is not the disk." exit $short + # The same measurement and write on the Windows leg, in pwsh, as its first check. + - name: "infrastructure: was the disk full when the leg failed? (Windows)" + if: failure() && matrix.os == 'windows' && steps.disk_windows.outcome == 'success' + shell: pwsh + run: | + Set-ExecutionPolicy -Scope Process -ExecutionPolicy Bypass -Force + $floor = [int64]$env:DISK_FLOOR_GIB * 1GB + $short = 0; $seen = @() + $tmp = [System.IO.Path]::GetTempPath() + $dirs = @( + $(if ($env:GITHUB_WORKSPACE) { $env:GITHUB_WORKSPACE } else { $PWD.Path }), + $(if ($env:RUNNER_TEMP) { $env:RUNNER_TEMP } else { $tmp }), + $(if ($env:TEMP) { $env:TEMP } else { $tmp }), + $HOME + ) + foreach ($d in $dirs) { + $p = [System.IO.Path]::GetFullPath($d) + while (-not (Test-Path -LiteralPath $p)) { $p = Split-Path -Parent $p } + $vol = [System.IO.Path]::GetPathRoot($p) + $free = [System.IO.DriveInfo]::new($vol).AvailableFreeSpace + Write-Output "${d}: $([math]::Floor($free / 1GB)) GiB free on $vol" + if ($free -lt $floor) { + Write-Output "::error::infrastructure: ${{ matrix.label }}: $d is on $vol, which has $([math]::Floor($free / 1MB)) MiB free, below the floor of $env:DISK_FLOOR_GIB GiB. The disk filled while the leg ran: read the failure above as the runner host's disk, not as a byte-equality failure." + $short = 1; continue + } + if ($seen -contains $vol) { continue } + $seen += $vol; $f = Join-Path $p ".wai-disk-probe.$PID" + try { + $s = [System.IO.FileStream]::new($f, [System.IO.FileMode]::Create, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None, 1048576, [System.IO.FileOptions]::WriteThrough) + try { + $buf = [byte[]]::new(1048576) + for ($i = 0; $i -lt 32; $i++) { $s.Write($buf, 0, $buf.Length) } + $s.Flush($true) + } finally { $s.Dispose() } + } catch { + $e = $_.Exception; while ($e.InnerException) { $e = $e.InnerException } + $msg = (($e.Message -split "`r?`n")[0]).TrimEnd('.', ' ') + if (($e.HResult -band 0xFFFF) -in 19, 39, 112, 1117, 1295) { + Write-Output "::error::infrastructure: ${{ matrix.label }}: a 32 MiB write under $p, on $vol, failed: $msg. The disk filled while the leg ran: read the failure above as the runner host's disk, not as a byte-equality failure." + $short = 1 + } else { + Write-Output "::warning::a 32 MiB write under $p failed for another reason: $msg" + } + } finally { + Remove-Item -LiteralPath $f -Force -ErrorAction SilentlyContinue + } + } + if ($short -eq 0) { Write-Output "Every disk this leg writes to has room: the failure above is not the disk." } + exit $short diff --git a/.github/workflows/wai-neural-gates.yml b/.github/workflows/wai-neural-gates.yml index 28be8f8e6..191e36ffb 100644 --- a/.github/workflows/wai-neural-gates.yml +++ b/.github/workflows/wai-neural-gates.yml @@ -284,15 +284,18 @@ jobs: WAI_REQUIRE_SOTA_VECTORS: '1' steps: - uses: actions/checkout@v5 - # Bootstrap rustup: the docker runner image ships no Rust. - - name: Install Rust 1.98.0 + # Bootstrap rustup: the docker runner image ships no Rust. The toolchain is the + # repository's pin: `rustup toolchain install` with no name installs the one + # rust-toolchain.toml names, as the wai-quantum member jobs do, so a change of the + # pin needs no edit here. The last line logs the toolchain cargo will use. + - name: Install Rust (the pin in rust-toolchain.toml) run: | export PATH="$HOME/.cargo/bin:$PATH" command -v rustup >/dev/null 2>&1 || \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.98.0 - rustup toolchain install 1.98.0 --profile minimal - rustup default 1.98.0 + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain none + rustup toolchain install echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + rustup show active-toolchain - name: Fetch the JPEG-AI inputs and check each against the pinned manifest working-directory: wai shell: bash