ci: guard the concurrency groups, and rebalance the runners by evidence #163
4 changed files with 194 additions and 4 deletions
23
.github/workflows/standards-build.yml
vendored
23
.github/workflows/standards-build.yml
vendored
|
|
@ -78,14 +78,29 @@ jobs:
|
|||
- { name: map-rs, dir: map/map-rs, runner: linux-arm64, conformance: map-conformance }
|
||||
- { name: aac-rs, dir: aac/aac-rs, runner: linux-arm64, conformance: aac-conformance }
|
||||
- { name: joulecontract-rs, dir: joulecontract/joulecontract-rs, runner: linux-arm64, z3: true }
|
||||
# Runner placement is by what a workspace actually needs, not by
|
||||
# habit. The dev Mac hosts BOTH the `macos` runner and the
|
||||
# `linux-arm64` colima VM, so it served 13 of this repo's 21
|
||||
# contexts while jetson-hub (`linux`) sat idle — which is why a PR's
|
||||
# macOS contexts could wait indefinitely while its `linux` ones
|
||||
# returned in minutes.
|
||||
#
|
||||
# `arl-rs`, `sense-rs`, `joulehook-rs` and `joule-ui-rs` moved to
|
||||
# `linux` on that basis: each has zero `target_os = "macos"` cfgs,
|
||||
# zero Apple dependencies, and zero Apple crates in its Cargo.lock,
|
||||
# so running them on macOS asserted nothing the repo relies on.
|
||||
# `jouledesk-rs` stays, because it does need it — `libwebrtc` and 19
|
||||
# Apple crates in its lockfile. `linux-arm64` placements stay too:
|
||||
# arm64 is the deployment target, and `jouleclaw-rs` needs
|
||||
# `+fp16`.
|
||||
- { name: jouledesk-rs, dir: jouledesk/jouledesk-rs, runner: macos }
|
||||
- { name: arl-rs, dir: arl/arl-rs, runner: macos }
|
||||
- { name: sense-rs, dir: sense/sense-rs, runner: macos }
|
||||
- { name: arl-rs, dir: arl/arl-rs, runner: linux }
|
||||
- { name: sense-rs, dir: sense/sense-rs, runner: linux }
|
||||
- { name: proof-rs, dir: proof/proof-rs, runner: linux, z3: true, conformance: proof-conformance }
|
||||
- { name: joulehook-rs, dir: joulehook/joulehook-rs, runner: macos }
|
||||
- { name: joulehook-rs, dir: joulehook/joulehook-rs, runner: linux }
|
||||
- { name: eoc-rs, dir: eoc/eoc-rs, runner: linux, feature_tests: "-p eoc-meter --features cuda" }
|
||||
- { name: mesh-rs, dir: mesh/mesh-rs, runner: linux-arm64 }
|
||||
- { name: joule-ui-rs, dir: joule-ui/joule-ui-rs, runner: macos }
|
||||
- { name: joule-ui-rs, dir: joule-ui/joule-ui-rs, runner: linux }
|
||||
# jouleclaw-rs is the one entry whose runner IS a correctness choice.
|
||||
# It reaches candle → ug → gemm, which uses inline aarch64 assembly
|
||||
# (`fmla {0:v}.8h, ...`) requiring the ARMv8.2 FP16 extension.
|
||||
|
|
|
|||
47
.github/workflows/workflow-hygiene.yml
vendored
Normal file
47
.github/workflows/workflow-hygiene.yml
vendored
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
name: workflow hygiene
|
||||
|
||||
# Properties of the workflows themselves, checked rather than remembered.
|
||||
#
|
||||
# Today that is one property: every workflow declares a concurrency group that
|
||||
# varies by ref. None of them did until 2026-10-05, and the consequence was not
|
||||
# a slow build — it was that CI stopped being usable for anyone. A session
|
||||
# pushing four versions of a branch queued four complete sets of runs, all of
|
||||
# which had to drain; one PR's 21 contexts settled none in 70 minutes behind a
|
||||
# backlog of 25, and two PRs were merged on locally-executed equivalents
|
||||
# because the gate could not run. One omitted block reintroduces that for the
|
||||
# whole repo, invisibly: the workflow still passes, it just queues forever
|
||||
# under load.
|
||||
#
|
||||
# This job runs on `linux`, which is jetson-hub — deliberately not the dev Mac,
|
||||
# which already serves 13 of this repo's 21 contexts through two runners. It
|
||||
# needs no toolchain and finishes in seconds.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/workflows/**'
|
||||
- 'scripts/check-workflow-concurrency.sh'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
concurrency-groups:
|
||||
name: every workflow supersedes its own queue
|
||||
runs-on: linux
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# The check must be able to fail, or it is decoration. The self-test
|
||||
# breaks the property both ways against a scratch copy and asserts the
|
||||
# check rejects each. It lives in a script because embedding its Python
|
||||
# in this `run:` block made the file invalid YAML, which would have meant
|
||||
# no CI at all.
|
||||
- name: The check fails closed (self-test)
|
||||
run: ./scripts/check-workflow-concurrency-selftest.sh
|
||||
|
||||
- name: Every workflow is grouped per ref
|
||||
run: ./scripts/check-workflow-concurrency.sh
|
||||
61
scripts/check-workflow-concurrency-selftest.sh
Executable file
61
scripts/check-workflow-concurrency-selftest.sh
Executable file
|
|
@ -0,0 +1,61 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# check-workflow-concurrency-selftest.sh
|
||||
#
|
||||
# Proves `check-workflow-concurrency.sh` can fail. A check that cannot fail is
|
||||
# decoration, and this repo has a documented history of gates that passed
|
||||
# because nothing they guarded was ever read.
|
||||
#
|
||||
# Both ways the property can break are exercised against a scratch copy, never
|
||||
# the working tree:
|
||||
#
|
||||
# 1. a workflow with no top-level `concurrency:` block;
|
||||
# 2. a group that does not vary by `github.ref` — which would serialise every
|
||||
# branch against every other, worse than having no group at all.
|
||||
#
|
||||
# In a script rather than inline in the workflow on purpose. The first version
|
||||
# embedded a Python heredoc in a `run:` block, and the heredoc body sat at
|
||||
# column zero, which made the workflow file invalid YAML. An unparseable
|
||||
# workflow means no CI at all — the exact failure this change exists to
|
||||
# prevent, reintroduced by the fix for it.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
root=$(pwd)
|
||||
target=.github/workflows/workflow-hygiene.yml
|
||||
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
cp -R .github scripts "$tmp/"
|
||||
|
||||
# ── 1. remove the block entirely ────────────────────────────────────────────
|
||||
python3 - "$tmp/$target" <<'PY'
|
||||
import re, sys
|
||||
p = sys.argv[1]
|
||||
s = open(p).read()
|
||||
out = re.sub(r"^concurrency:\n( .*\n)+", "", s, count=1, flags=re.M)
|
||||
assert out != s, "fixture unchanged: there was no concurrency block to remove"
|
||||
open(p, "w").write(out)
|
||||
PY
|
||||
if ( cd "$tmp" && ./scripts/check-workflow-concurrency.sh ) >/dev/null 2>&1; then
|
||||
echo "::error::the check passed a workflow with no concurrency block"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── 2. a group that does not vary by ref ────────────────────────────────────
|
||||
cp -f "$root/$target" "$tmp/$target"
|
||||
python3 - "$tmp/$target" <<'PY'
|
||||
import re, sys
|
||||
p = sys.argv[1]
|
||||
s = open(p).read()
|
||||
out = re.sub(r"^( group:).*$", r"\1 fixed-for-the-whole-repo", s, count=1, flags=re.M)
|
||||
assert out != s, "fixture unchanged: there was no group line to rewrite"
|
||||
open(p, "w").write(out)
|
||||
PY
|
||||
if ( cd "$tmp" && ./scripts/check-workflow-concurrency.sh ) >/dev/null 2>&1; then
|
||||
echo "::error::the check passed a group that does not vary by ref"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "self-test: the check rejects both a missing block and a fixed group"
|
||||
67
scripts/check-workflow-concurrency.sh
Executable file
67
scripts/check-workflow-concurrency.sh
Executable file
|
|
@ -0,0 +1,67 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# check-workflow-concurrency.sh
|
||||
#
|
||||
# Every workflow must declare a concurrency group, so a push supersedes the run
|
||||
# before it instead of queueing behind it.
|
||||
#
|
||||
# Why this is a gate and not a convention. None of the 15 workflows declared
|
||||
# `concurrency:` until 2026-10-05. A session pushing four versions of a branch
|
||||
# therefore queued four complete sets of runs, all of which had to drain: one
|
||||
# PR's 21 contexts settled none in 70 minutes behind a backlog of 25 runs that
|
||||
# grew faster than it cleared. The repo's CI became unusable for every other
|
||||
# session, and two PRs were merged on locally-executed equivalents instead of
|
||||
# on the gate. A gate that cannot run is not a gate.
|
||||
#
|
||||
# One omitted `concurrency:` block reintroduces that for the whole repo, and it
|
||||
# is invisible: the workflow still passes, it just queues forever under load.
|
||||
# So it is checked rather than remembered.
|
||||
#
|
||||
# Runs in seconds with no toolchain, as a step inside an existing job — adding
|
||||
# a job of its own would spend runner capacity to protect runner capacity.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
fail=0
|
||||
count=0
|
||||
|
||||
for f in .github/workflows/*.yml; do
|
||||
count=$((count + 1))
|
||||
name=$(basename "$f")
|
||||
|
||||
# A top-level key, so match at column zero: a `concurrency:` nested under a
|
||||
# job is a different (and legal) thing that does not bound the workflow.
|
||||
if ! grep -qE '^concurrency:' "$f"; then
|
||||
echo "::error file=$f::$name declares no top-level \`concurrency:\` block."
|
||||
echo " Add:"
|
||||
echo " concurrency:"
|
||||
echo " group: \${{ github.workflow }}-\${{ github.ref }}"
|
||||
echo " cancel-in-progress: true"
|
||||
fail=1
|
||||
continue
|
||||
fi
|
||||
|
||||
# The group must vary by ref. A fixed group serialises every branch against
|
||||
# every other, which is worse than no group at all: one busy branch then
|
||||
# blocks the repo instead of only itself.
|
||||
if ! grep -A3 -E '^concurrency:' "$f" | grep -qE 'group:.*github\.ref'; then
|
||||
echo "::error file=$f::$name has a concurrency group that does not vary by \`github.ref\`."
|
||||
echo " A fixed group serialises unrelated branches against each other."
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$count" -eq 0 ]; then
|
||||
echo "::error::no workflows found — this check looked in the wrong place."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$fail" -ne 0 ]; then
|
||||
echo
|
||||
echo "checked $count workflow(s): at least one would queue instead of superseding."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "workflow concurrency: $count workflow(s), each grouped per ref."
|
||||
Loading…
Reference in a new issue