open-standards/scripts/changelog-fragments-selftest.sh

#!/usr/bin/env bash
#
# changelog-fragments-selftest.sh — the changelog's entry files are checked, and rendered the
# same way every time.
#
# scripts/check-changelog-fragment.py decides whether a pull request carries its changelog
# entry, and scripts/changelog-assemble.py renders the entry files into the changelog for a
# release. This builds a scratch repository laid out as this one is and requires: a change to
# wai/ with no entry fails, and passes with an entry file directly in wai/changes/ or with the
# trailer line `Changelog: none` (that line exactly, in a message's trailer block); an entry in
# a subdirectory, or a file there that is no entry, fails; a change outside wai/ needs none; a
# malformed entry fails, saying why; a pull request opened before the check landed passes as
# it is, with an edit of CHANGELOG.md or with nothing, and one opened since that adds its entry
# to the head of Unreleased fails, saying to add the file; the check lands when it reaches the
# base branch, by the merge, not when the commit that adds it was made, and before then every
# pull request counts as opened before it; a release (the entries rendered, their files
# removed) passes; commits the clone cannot compare, a shallow clone and an event payload with
# no creation time fail, saying why; and the assembler renders the files at the head of
# Unreleased in a stable order, the same bytes twice, writes them and removes the files, and
# refuses a malformed one.
set -uo pipefail

# The job that runs this is a pull request's, whose own event payload must not decide here.
unset GITHUB_EVENT_PATH

here=$(cd "$(dirname "$0")" && pwd)
check="$here/check-changelog-fragment.py"
asm="$here/changelog-assemble.py"
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
R="$tmp/r"
mkdir -p "$R/scripts" "$R/wai/changes" "$R/wai/wai-rs" "$R/jcp"
cp "$check" "$asm" "$R/scripts/"
git -C "$R" init -q
git -C "$R" config user.email [email protected]
git -C "$R" config user.name t
printf '# changelog\n\nIntro.\n\n## Unreleased\n\n### The entry before\n\n- It was.\n\n## v0.1.0\n\n### Old\n\n- Old.\n' > "$R/wai/CHANGELOG.md"
printf '# Entries\n' > "$R/wai/changes/README.md"
echo a > "$R/wai/wai-rs/lib.rs"; echo b > "$R/jcp/SPEC.md"
# The commit that adds the check: it lands at 2026-11-04 12:00 UTC.
git -C "$R" add -A; GIT_COMMITTER_DATE="2026-11-04T12:00:00+00:00" git -C "$R" commit -qm base
BASE=$(git -C "$R" rev-parse HEAD)
# Event payloads of pull requests opened before and after it.
printf '{"pull_request": {"number": 1, "created_at": "2026-11-01T09:30:00Z"}}\n' > "$tmp/before.json"
printf '{"pull_request": {"number": 2, "created_at": "2026-11-04T08:00:01-04:00"}}\n' > "$tmp/after.json"
printf '{"pull_request": {"number": 3}}\n' > "$tmp/no-time.json"

fails=0
W="$R" # the repository the checks below run in
expect() { # expect <0|1> <what> <text the output must hold or ""> -- <command...>
  local want=$1 what=$2 needle=$3 out rc
  shift 4
  out=$(cd "$W" && "$@" 2>&1); rc=$?
  if [ "$want" = 0 ] && [ $rc -ne 0 ]; then echo "FAIL  $what: exit $rc"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  if [ "$want" = 1 ] && [ $rc -ne 1 ]; then echo "FAIL  $what: exit $rc, not 1"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  if [ -n "$needle" ] && ! grep -qF -- "$needle" <<<"$out"; then echo "FAIL  $what: the output does not say '$needle'"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); return; fi
  echo "ok    $what"
}
branch() { git -C "$R" checkout -q -B "$1" "$BASE"; }
commit() { git -C "$R" add -A; git -C "$R" commit -q -m "${1:-change}"; git -C "$R" rev-parse HEAD; }
entry='### A thing changed\n\n- **What.** It changed.\n'
C="python3 -I scripts/check-changelog-fragment.py"
opened() { env GITHUB_EVENT_PATH="$tmp/$1.json" python3 -I scripts/check-changelog-fragment.py "${@:2}"; } # opened <before|after|no-time> <args>
by_hand() { perl -pi -e 's/^### The entry before$/### A thing changed\n\n- It did.\n\n### The entry before/' "${1:-$R}/wai/CHANGELOG.md"; } # by_hand [<repository>]

# The pull request check.
branch outside; echo c > "$R/jcp/SPEC.md"; H=$(commit)
expect 0 "a change outside wai/ needs no entry" "touches nothing under wai/" -- $C "$BASE" "$H"
branch bare; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit)
expect 1 "a change to wai/ with no entry fails" "adds no entry" -- $C "$BASE" "$H"
printf "$entry" > "$R/wai/changes/2026-11-04-a-thing.md"; H=$(commit)
expect 0 "and passes once it adds an entry file" "2026-11-04-a-thing.md" -- $C "$BASE" "$H"
expect 0 "an entry file passes whenever the pull request was opened" "2026-11-04-a-thing.md" -- opened after "$BASE" "$H"

# Only directly in wai/changes/, where the assembler and wai breaking read.
branch subdir; echo c > "$R/wai/wai-rs/lib.rs"; mkdir -p "$R/wai/changes/sub"; printf "$entry" > "$R/wai/changes/sub/2026-11-04-planted.md"; H=$(commit)
expect 1 "an entry file in a subdirectory of wai/changes/ fails" "wai/changes/sub/2026-11-04-planted.md: an entry file goes directly in wai/changes/, not in a subdirectory" -- $C "$BASE" "$H"
printf "$entry" > "$R/wai/changes/2026-11-04-a-thing.md"; H=$(commit)
expect 1 "and fails beside a good entry too, which the subdirectory's would never join" "not in a subdirectory" -- $C "$BASE" "$H"
branch notmd; echo c > "$R/wai/wai-rs/lib.rs"; printf "$entry" > "$R/wai/changes/2026-11-04-a-thing.txt"; H=$(commit)
expect 1 "a file of wai/changes/ that is no .md entry fails" "not an entry file" -- $C "$BASE" "$H"

# `Changelog: none`: that line exactly, in the trailer block of a commit message.
branch none; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'lockfile only\n\nChangelog: none')
expect 0 "a commit that says Changelog: none passes" "Changelog: none" -- $C "$BASE" "$H"
branch none-trailers; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'lockfile only\n\nWhy, in a paragraph.\n\nChangelog: none\nCo-Authored-By: T <[email protected]>')
expect 0 "among the other trailers too" "Changelog: none" -- $C "$BASE" "$H"
branch none-in-text; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'the word Changelog: none mid-line does not count')
expect 1 "the trailer starts its line" "adds no entry" -- $C "$BASE" "$H"
branch none-of; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'a subject\n\nChangelog: none of the tests changed')
expect 1 "'Changelog: none of ...' is not the trailer" "adds no entry" -- $C "$BASE" "$H"
branch none-case; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'a subject\n\nchangelog: None')
expect 1 "nor is it in another case" "adds no entry" -- $C "$BASE" "$H"
branch none-mid; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'a subject\n\nChangelog: none\n\nA paragraph after it.')
expect 1 "a trailer line before the last paragraph is no trailer" "adds no entry" -- $C "$BASE" "$H"
branch none-body; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit $'a subject\n\nA body line, then no blank line.\nChangelog: none')
expect 1 "nor is one in a paragraph of prose" "in its last paragraph, after a blank line" -- $C "$BASE" "$H"
branch none-subject; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit 'Changelog: none')
expect 1 "nor is a subject line" "adds no entry" -- $C "$BASE" "$H"

branch readme; echo c > "$R/wai/wai-rs/lib.rs"; echo more >> "$R/wai/changes/README.md"; H=$(commit)
expect 1 "the entries' README is no entry" "adds no entry" -- $C "$BASE" "$H"
branch badname; echo c > "$R/wai/wai-rs/lib.rs"; printf "$entry" > "$R/wai/changes/A Thing.md"; H=$(commit)
expect 1 "an entry file named otherwise fails" "not named <yyyy-mm-dd>-<slug>.md" -- $C "$BASE" "$H"
branch baddate; echo c > "$R/wai/wai-rs/lib.rs"; printf "$entry" > "$R/wai/changes/2026-02-30-a-thing.md"; H=$(commit)
expect 1 "an entry file whose date is no date fails" "is not a date" -- $C "$BASE" "$H"
branch notitle; echo c > "$R/wai/wai-rs/lib.rs"; printf -- '- **What.** It changed.\n' > "$R/wai/changes/2026-11-04-a-thing.md"; H=$(commit)
expect 1 "an entry with no title fails" "is not the entry's title" -- $C "$BASE" "$H"
branch two; echo c > "$R/wai/wai-rs/lib.rs"; printf "$entry\n### Another\n\n- And.\n" > "$R/wai/changes/2026-11-04-a-thing.md"; H=$(commit)
expect 1 "a file holding two entries fails" "a heading inside the entry" -- $C "$BASE" "$H"
branch fence; echo c > "$R/wai/wai-rs/lib.rs"; printf '### Code\n\n- Run:\n\n```bash\n# a comment, not a heading\nwai hello\n```\n' > "$R/wai/changes/2026-11-04-code.md"; H=$(commit)
expect 0 "a # line inside a code block is no heading" "2026-11-04-code.md" -- $C "$BASE" "$H"

# The transition: opened before the check landed, a pull request passes as it is.
branch old-style; echo c > "$R/wai/wai-rs/lib.rs"; by_hand; H=$(commit)
expect 0 "opened before the check landed, an edit of CHANGELOG.md's head passes" "before this check landed" -- opened before "$BASE" "$H"
expect 1 "opened since, the same edit fails, saying to add the entry file" "move each to wai/changes/<yyyy-mm-dd>-<slug>.md" -- opened after "$BASE" "$H"
expect 1 "and it names the entry it found" "(\`### A thing changed\`)" -- opened after "$BASE" "$H"
expect 1 "with no event payload, a change is taken as opened now" "taken as opened now" -- $C "$BASE" "$H"
branch bare-old; echo c > "$R/wai/wai-rs/lib.rs"; H=$(commit)
expect 0 "opened before, a change with no changelog edit at all passes too" "passes as it is" -- opened before "$BASE" "$H"
expect 1 "and fails when opened since" "adds no entry" -- opened after "$BASE" "$H"
branch both; echo c > "$R/wai/wai-rs/lib.rs"; by_hand; printf "$entry" > "$R/wai/changes/2026-11-04-a-thing.md"; H=$(commit)
expect 1 "opened since, an entry file does not excuse an entry added to the head as well" "to the head of \`## Unreleased\`" -- opened after "$BASE" "$H"
expect 1 "an event payload with no creation time fails, saying so" "cannot be read from the event payload" -- opened no-time "$BASE" "$(git -C "$R" rev-parse bare-old)"

# The check lands when it reaches the base branch: the first commit of main's first-parent
# history whose tree holds it, here the merge of the branch that adds it, at that merge's own
# time. A second repository, whose main starts without the check: the branch that adds it is
# committed on 11-05 at 03:09 and merged on 11-06 at 10:00; a pull request opened on 11-05 at
# 12:00, between the two, edits the head of Unreleased and then merges main in.
T="$tmp/t"
mkdir -p "$T/wai/changes" "$T/wai/wai-rs"
git -C "$T" init -q; git -C "$T" symbolic-ref HEAD refs/heads/main
git -C "$T" config user.email [email protected]; git -C "$T" config user.name t
cp "$R/wai/changes/README.md" "$T/wai/changes/"
printf '# changelog\n\nIntro.\n\n## Unreleased\n\n### The entry before\n\n- It was.\n' > "$T/wai/CHANGELOG.md"
echo a > "$T/wai/wai-rs/lib.rs"
at() { local d=$1; shift; GIT_AUTHOR_DATE="$d" GIT_COMMITTER_DATE="$d" git -C "$T" "$@"; } # at <time> <git args>
tcommit() { git -C "$T" add -A; at "$1" commit -q -m "$2"; git -C "$T" rev-parse HEAD; } # tcommit <time> <message>
M0=$(tcommit 2026-11-04T00:00:00+00:00 "main, before the check")
git -C "$T" checkout -q -b adds-check; mkdir -p "$T/scripts"; cp "$check" "$asm" "$T/scripts/"
A=$(tcommit 2026-11-05T03:09:34+00:00 "adds the check")
git -C "$T" checkout -q -b early "$M0"; echo e > "$T/wai/wai-rs/lib.rs"; tcommit 2026-11-05T04:00:00+00:00 "on main before the check" >/dev/null
git -C "$T" checkout -q -b on-check "$A"; echo q > "$T/wai/wai-rs/lib.rs"; by_hand "$T"; Q=$(tcommit 2026-11-05T13:00:00+00:00 "built on the check's branch")
git -C "$T" checkout -q -b window "$M0"; echo w > "$T/wai/wai-rs/lib.rs"; by_hand "$T"; tcommit 2026-11-05T12:00:00+00:00 "opened between" >/dev/null
git -C "$T" checkout -q -b window-bare "$M0"; echo v > "$T/wai/wai-rs/lib.rs"; tcommit 2026-11-05T12:00:00+00:00 "opened between, no entry" >/dev/null
git -C "$T" checkout -q main; at 2026-11-06T10:00:00+00:00 merge -q --no-ff -m "merge the check" adds-check; M1=$(git -C "$T" rev-parse HEAD)
for b in window window-bare; do git -C "$T" checkout -q "$b"; at 2026-11-06T11:00:00+00:00 merge -q --no-ff -m "merge main" main; done
P1=$(git -C "$T" rev-parse window); P2=$(git -C "$T" rev-parse window-bare)
git -C "$T" checkout -q -b after main; echo f > "$T/wai/wai-rs/lib.rs"; by_hand "$T"; P3=$(tcommit 2026-11-06T12:30:00+00:00 "opened after the merge")
git -C "$T" checkout -q main
printf '{"pull_request": {"number": 4, "created_at": "2026-11-05T12:00:00Z"}}\n' > "$tmp/window.json"
printf '{"pull_request": {"number": 5, "created_at": "2026-11-05T13:00:00Z"}}\n' > "$tmp/on-check.json"
printf '{"pull_request": {"number": 6, "created_at": "2026-11-06T12:00:00Z"}}\n' > "$tmp/merged.json"
W="$T"
expect 0 "opened after the commit that adds the check but before its merge, an edit of the head passes" \
  "before this check landed on the base branch (2026-11-06 10:00:00 UTC, ${M1:0:12})" -- opened window "$M1" "$P1"
expect 0 "and so does one with no entry at all" "passes as it is" -- opened window "$M1" "$P2"
expect 1 "opened after the merge, an edit of the head fails" \
  "Since this check landed on the base branch (2026-11-06 10:00:00 UTC, ${M1:0:12}; this pull request: 2026-11-06 12:00:00 UTC)" -- opened merged "$M1" "$P3"
expect 0 "before the check is on the base branch, a pull request counts as opened before it landed" \
  "this check is not on the base branch yet" -- opened on-check "$M0" "$Q"
expect 0 "and so does a local run, with no event payload" "not on the base branch yet" -- $C "$M0" "$Q"
S3="$tmp/shallow3"; git clone -q --no-local --depth 2 --branch early "file://$T" "$S3" 2>/dev/null
out=$(cd "$S3" && env GITHUB_EVENT_PATH="$tmp/window.json" python3 -I "$check" "$M0" "$(git -C "$T" rev-parse early)" 2>&1); rc=$?
if [ $rc -eq 1 ] && grep -qF "this shallow clone's history stops at ${M0:0:12} before any commit of it holds scripts/check-changelog-fragment.py" <<<"$out"; then
  echo "ok    a shallow clone whose base's history stops before the check fails, saying so"
else echo "FAIL  a shallow clone with the base's history cut short: exit $rc"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); fi
W="$R"

# A release renders the entries into the changelog and removes their files.
branch release; printf '### First\n\n- A.\n' > "$R/wai/changes/2026-11-04-first.md"; printf '### Second\n\n- B.\n' > "$R/wai/changes/2026-11-03-second.md"; P=$(commit)
(cd "$R" && python3 -I scripts/changelog-assemble.py --write >/dev/null); H=$(commit "release")
expect 0 "a release (entries rendered, their files removed) passes" "renders 2 entries of wai/changes/ into wai/CHANGELOG.md" -- opened after "$P" "$H"
git -C "$R" checkout -q release; perl -pi -e 's/^### The entry before$/### Slipped in\n\n- By hand.\n\n### The entry before/' "$R/wai/CHANGELOG.md"; H2=$(commit "release, and one by hand")
expect 1 "but an entry it adds by hand beside them fails" "(\`### Slipped in\`)" -- opened after "$P" "$H2"
branch unrendered; printf '### First\n\n- A.\n' > "$R/wai/changes/2026-11-04-first.md"; P=$(commit)
rm "$R/wai/changes/2026-11-04-first.md"; perl -pi -e 's/^Intro\.$/Intro, reworded./' "$R/wai/CHANGELOG.md"; H=$(commit "drops an entry")
expect 1 "and removing entry files beside an edit that does not render them is no release" "adds no entry" -- opened after "$P" "$H"

# It never passes unchecked.
expect 1 "commits the clone cannot compare fail, saying so" "cannot be compared" -- $C "$BASE" 0000000000000000000000000000000000000000
S1="$tmp/shallow1"; git clone -q --no-local --depth 1 --branch bare-old "file://$R" "$S1" 2>/dev/null; git -C "$S1" remote remove origin
out=$(cd "$S1" && python3 -I scripts/check-changelog-fragment.py "$BASE" "$(git -C "$R" rev-parse bare-old)" 2>&1); rc=$?
if [ $rc -eq 1 ] && grep -qF "cannot be compared in this clone, even after fetching its history" <<<"$out"; then echo "ok    a shallow clone that cannot fetch the base fails, saying so"
else echo "FAIL  a shallow clone with no way to the base: exit $rc"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); fi
S2="$tmp/shallow2"; git clone -q --no-local --depth 2 --branch bare-old "file://$R" "$S2" 2>/dev/null
out=$(cd "$S2" && env GITHUB_EVENT_PATH="$tmp/before.json" python3 -I scripts/check-changelog-fragment.py "$BASE" "$(git -C "$R" rev-parse bare-old)" 2>&1); rc=$?
if [ $rc -eq 1 ] && grep -qF "is where this shallow clone's history stops, so the time this check landed is unknown" <<<"$out"; then echo "ok    a shallow clone cannot tell when the check landed, and fails, saying so"
else echo "FAIL  a shallow clone and the transition: exit $rc"; printf '%s\n' "$out" | sed 's/^/      /'; fails=$((fails+1)); fi

# The assembler.
branch assemble
printf '### Second of the day\n\n- B.\n' > "$R/wai/changes/2026-11-04-b-second.md"
printf '### First of the day\n\n- A.\n' > "$R/wai/changes/2026-11-04-a-first.md"
printf '### Older\n\n- Before.\n' > "$R/wai/changes/2026-10-30-older.md"
want=$(printf '## Unreleased\n\n### First of the day\n\n- A.\n\n### Second of the day\n\n- B.\n\n### Older\n\n- Before.\n\n### The entry before\n\n- It was.\n')
got=$(cd "$R" && python3 -I scripts/changelog-assemble.py); again=$(cd "$R" && python3 -I scripts/changelog-assemble.py)
if [ "$got" = "$want" ] && [ "$again" = "$got" ]; then echo "ok    entries render newest date first, by name within a date, the same twice"
else echo "FAIL  the rendered section"; diff <(printf '%s\n' "$want") <(printf '%s\n' "$got") | sed 's/^/      /'; fails=$((fails+1)); fi
expect 0 "--check passes well-formed entries" "one entry in the changelog's form" -- python3 -I scripts/changelog-assemble.py --check
expect 0 "--write writes the changelog" "3 entries written" -- python3 -I scripts/changelog-assemble.py --write
full=$(printf '# changelog\n\nIntro.\n\n%s\n\n## v0.1.0\n\n### Old\n\n- Old.\n' "$want")
if [ "$(cat "$R/wai/CHANGELOG.md")" = "$full" ]; then echo "ok    and the changelog is the rendered section in place, the rest unchanged"
else echo "FAIL  the written changelog"; diff <(printf '%s\n' "$full") "$R/wai/CHANGELOG.md" | sed 's/^/      /'; fails=$((fails+1)); fi
left=$(cd "$R/wai/changes" && ls)
[ "$left" = "README.md" ] && echo "ok    and the entry files are removed, the README kept" || { echo "FAIL  files left: $left"; fails=$((fails+1)); }
expect 0 "with no entry files, it changes nothing" "" -- python3 -I scripts/changelog-assemble.py --write
[ "$(cat "$R/wai/CHANGELOG.md")" = "$full" ] && echo "ok    and the changelog stays as it was" || { echo "FAIL  the changelog changed with no entries"; fails=$((fails+1)); }
printf '### Trailing\n\n- Space. \n' > "$R/wai/changes/2026-11-04-trailing.md"
expect 1 "a malformed entry fails --check" "trailing white space" -- python3 -I scripts/changelog-assemble.py --check
expect 1 "and --write writes nothing" "nothing was written" -- python3 -I scripts/changelog-assemble.py --write
[ -f "$R/wai/changes/2026-11-04-trailing.md" ] && [ "$(cat "$R/wai/CHANGELOG.md")" = "$full" ] \
  && echo "ok    the file and the changelog are left as they were" || { echo "FAIL  a refused --write changed something"; fails=$((fails+1)); }
rm "$R/wai/changes/2026-11-04-trailing.md"
printf '### No newline\n\n- At the end.' > "$R/wai/changes/2026-11-04-end.md"
expect 1 "an entry with no newline at its end fails" "exactly one newline" -- python3 -I scripts/changelog-assemble.py --check

if [ $fails -ne 0 ]; then echo "changelog-fragments-selftest: $fails failure(s)"; exit 1; fi
echo "changelog-fragments-selftest: the check and the assembler pass what they should and fail each way they should."