ci: guard the concurrency groups, and rebalance the runners by evidence #163

Merged
dcharlot merged 1 commit from ci/workflow-guard into main 2026-10-05 16:18:57 -04:00
4 changed files with 194 additions and 4 deletions

View file

@ -78,14 +78,29 @@ jobs:
- { name: map-rs, dir: map/map-rs, runner: linux-arm64, conformance: map-conformance }
- { name: aac-rs, dir: aac/aac-rs, runner: linux-arm64, conformance: aac-conformance }
- { name: joulecontract-rs, dir: joulecontract/joulecontract-rs, runner: linux-arm64, z3: true }
# Runner placement is by what a workspace actually needs, not by
# habit. The dev Mac hosts BOTH the `macos` runner and the
# `linux-arm64` colima VM, so it served 13 of this repo's 21
# contexts while jetson-hub (`linux`) sat idle — which is why a PR's
# macOS contexts could wait indefinitely while its `linux` ones
# returned in minutes.
#
# `arl-rs`, `sense-rs`, `joulehook-rs` and `joule-ui-rs` moved to
# `linux` on that basis: each has zero `target_os = "macos"` cfgs,
# zero Apple dependencies, and zero Apple crates in its Cargo.lock,
# so running them on macOS asserted nothing the repo relies on.
# `jouledesk-rs` stays, because it does need it — `libwebrtc` and 19
# Apple crates in its lockfile. `linux-arm64` placements stay too:
# arm64 is the deployment target, and `jouleclaw-rs` needs
# `+fp16`.
- { name: jouledesk-rs, dir: jouledesk/jouledesk-rs, runner: macos }
- { name: arl-rs, dir: arl/arl-rs, runner: macos }
- { name: sense-rs, dir: sense/sense-rs, runner: macos }
- { name: arl-rs, dir: arl/arl-rs, runner: linux }
- { name: sense-rs, dir: sense/sense-rs, runner: linux }
- { name: proof-rs, dir: proof/proof-rs, runner: linux, z3: true, conformance: proof-conformance }
- { name: joulehook-rs, dir: joulehook/joulehook-rs, runner: macos }
- { name: joulehook-rs, dir: joulehook/joulehook-rs, runner: linux }
- { name: eoc-rs, dir: eoc/eoc-rs, runner: linux, feature_tests: "-p eoc-meter --features cuda" }
- { name: mesh-rs, dir: mesh/mesh-rs, runner: linux-arm64 }
- { name: joule-ui-rs, dir: joule-ui/joule-ui-rs, runner: macos }
- { name: joule-ui-rs, dir: joule-ui/joule-ui-rs, runner: linux }
# jouleclaw-rs is the one entry whose runner IS a correctness choice.
# It reaches candle → ug → gemm, which uses inline aarch64 assembly
# (`fmla {0:v}.8h, ...`) requiring the ARMv8.2 FP16 extension.

47
.github/workflows/workflow-hygiene.yml vendored Normal file
View file

@ -0,0 +1,47 @@
name: workflow hygiene
# Properties of the workflows themselves, checked rather than remembered.
#
# Today that is one property: every workflow declares a concurrency group that
# varies by ref. None of them did until 2026-10-05, and the consequence was not
# a slow build — it was that CI stopped being usable for anyone. A session
# pushing four versions of a branch queued four complete sets of runs, all of
# which had to drain; one PR's 21 contexts settled none in 70 minutes behind a
# backlog of 25, and two PRs were merged on locally-executed equivalents
# because the gate could not run. One omitted block reintroduces that for the
# whole repo, invisibly: the workflow still passes, it just queues forever
# under load.
#
# This job runs on `linux`, which is jetson-hub — deliberately not the dev Mac,
# which already serves 13 of this repo's 21 contexts through two runners. It
# needs no toolchain and finishes in seconds.
on:
push:
branches: [main]
pull_request:
paths:
- '.github/workflows/**'
- 'scripts/check-workflow-concurrency.sh'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
concurrency-groups:
name: every workflow supersedes its own queue
runs-on: linux
steps:
- uses: actions/checkout@v4
# The check must be able to fail, or it is decoration. The self-test
# breaks the property both ways against a scratch copy and asserts the
# check rejects each. It lives in a script because embedding its Python
# in this `run:` block made the file invalid YAML, which would have meant
# no CI at all.
- name: The check fails closed (self-test)
run: ./scripts/check-workflow-concurrency-selftest.sh
- name: Every workflow is grouped per ref
run: ./scripts/check-workflow-concurrency.sh

View file

@ -0,0 +1,61 @@
#!/usr/bin/env bash
#
# check-workflow-concurrency-selftest.sh
#
# Proves `check-workflow-concurrency.sh` can fail. A check that cannot fail is
# decoration, and this repo has a documented history of gates that passed
# because nothing they guarded was ever read.
#
# Both ways the property can break are exercised against a scratch copy, never
# the working tree:
#
# 1. a workflow with no top-level `concurrency:` block;
# 2. a group that does not vary by `github.ref` — which would serialise every
# branch against every other, worse than having no group at all.
#
# In a script rather than inline in the workflow on purpose. The first version
# embedded a Python heredoc in a `run:` block, and the heredoc body sat at
# column zero, which made the workflow file invalid YAML. An unparseable
# workflow means no CI at all — the exact failure this change exists to
# prevent, reintroduced by the fix for it.
set -euo pipefail
cd "$(dirname "$0")/.."
root=$(pwd)
target=.github/workflows/workflow-hygiene.yml
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
cp -R .github scripts "$tmp/"
# ── 1. remove the block entirely ────────────────────────────────────────────
python3 - "$tmp/$target" <<'PY'
import re, sys
p = sys.argv[1]
s = open(p).read()
out = re.sub(r"^concurrency:\n( .*\n)+", "", s, count=1, flags=re.M)
assert out != s, "fixture unchanged: there was no concurrency block to remove"
open(p, "w").write(out)
PY
if ( cd "$tmp" && ./scripts/check-workflow-concurrency.sh ) >/dev/null 2>&1; then
echo "::error::the check passed a workflow with no concurrency block"
exit 1
fi
# ── 2. a group that does not vary by ref ────────────────────────────────────
cp -f "$root/$target" "$tmp/$target"
python3 - "$tmp/$target" <<'PY'
import re, sys
p = sys.argv[1]
s = open(p).read()
out = re.sub(r"^( group:).*$", r"\1 fixed-for-the-whole-repo", s, count=1, flags=re.M)
assert out != s, "fixture unchanged: there was no group line to rewrite"
open(p, "w").write(out)
PY
if ( cd "$tmp" && ./scripts/check-workflow-concurrency.sh ) >/dev/null 2>&1; then
echo "::error::the check passed a group that does not vary by ref"
exit 1
fi
echo "self-test: the check rejects both a missing block and a fixed group"

View file

@ -0,0 +1,67 @@
#!/usr/bin/env bash
#
# check-workflow-concurrency.sh
#
# Every workflow must declare a concurrency group, so a push supersedes the run
# before it instead of queueing behind it.
#
# Why this is a gate and not a convention. None of the 15 workflows declared
# `concurrency:` until 2026-10-05. A session pushing four versions of a branch
# therefore queued four complete sets of runs, all of which had to drain: one
# PR's 21 contexts settled none in 70 minutes behind a backlog of 25 runs that
# grew faster than it cleared. The repo's CI became unusable for every other
# session, and two PRs were merged on locally-executed equivalents instead of
# on the gate. A gate that cannot run is not a gate.
#
# One omitted `concurrency:` block reintroduces that for the whole repo, and it
# is invisible: the workflow still passes, it just queues forever under load.
# So it is checked rather than remembered.
#
# Runs in seconds with no toolchain, as a step inside an existing job — adding
# a job of its own would spend runner capacity to protect runner capacity.
set -euo pipefail
cd "$(dirname "$0")/.."
fail=0
count=0
for f in .github/workflows/*.yml; do
count=$((count + 1))
name=$(basename "$f")
# A top-level key, so match at column zero: a `concurrency:` nested under a
# job is a different (and legal) thing that does not bound the workflow.
if ! grep -qE '^concurrency:' "$f"; then
echo "::error file=$f::$name declares no top-level \`concurrency:\` block."
echo " Add:"
echo " concurrency:"
echo " group: \${{ github.workflow }}-\${{ github.ref }}"
echo " cancel-in-progress: true"
fail=1
continue
fi
# The group must vary by ref. A fixed group serialises every branch against
# every other, which is worse than no group at all: one busy branch then
# blocks the repo instead of only itself.
if ! grep -A3 -E '^concurrency:' "$f" | grep -qE 'group:.*github\.ref'; then
echo "::error file=$f::$name has a concurrency group that does not vary by \`github.ref\`."
echo " A fixed group serialises unrelated branches against each other."
fail=1
fi
done
if [ "$count" -eq 0 ]; then
echo "::error::no workflows found — this check looked in the wrong place."
exit 1
fi
if [ "$fail" -ne 0 ]; then
echo
echo "checked $count workflow(s): at least one would queue instead of superseding."
exit 1
fi
echo "workflow concurrency: $count workflow(s), each grouped per ref."