open-standards/scripts/check-frozen-rules.py

#!/usr/bin/env python3
"""check-frozen-rules.py: a decision rule committed before the runs it judges stays as it
first reached the history.

    check-frozen-rules.py <head-sha> [<base-sha>]

A research record of WAI may hold a rule committed before any run it judges: a file
`wai/research/<record>/<name>-rule<suffix>.json` (FROZEN), whose SHA-256 the record's text
quotes and its verifier holds. The rule is worth what it is because it was fixed before
the data. A rule changed after the results, together with every copy of its digest, passes
every check that reads the tree alone, so this one reads the history.

It walks head's first-parent history from its first commit. Each rule file's pin is the mode
and blob it had in the first commit there that added it, against that commit's first parent,
and head must hold every pinned path with that mode and blob. So a rule file changed, removed
or renamed after it first reached the history fails on the first run that sees it and on
every run after, whichever pushes came between or were cancelled, until the file is restored.
A later rule is a new file.

This is a check after the fact. It stops no commit; a red check and the history are what
record the change. scripts/check-frozen-rules.sh runs it as it first reached the history as
well as it stands, since a change to this file could otherwise hide a change to a rule.

A commit reaches main as it is, so a branch's own commits are this history once they land:
a rule revised by a later commit of its branch fails here, and the branch's commits are
rewritten so that one commit adds the rule in its final form. With a merge commit the first
parent sees only the merge, and the rule's pin is what the merge brought.

With <base-sha> (a pull request's base), every rule file base holds must also have base's
mode and blob wherever head holds that path: a branch made before a rule reached main can add
another file at the rule's path, which head's own history cannot show.

Exit 0 when every rule file is as pinned; 1 when one is not, naming it, and 1 when head or
base cannot be read, the clone is shallow, or git fails: the check never passes unchecked.
2 on a wrong command line.

Standard library only; runs on Python 3.9.
"""
from __future__ import annotations

import re
import subprocess
import sys

FROZEN = re.compile(r"^wai/research/[^/]+/[^/]*-rule[^/]*\.json$")
ROOT = "wai/research"
RAW_ADD = re.compile(r"^:000000 (\d{6}) 0{40,64} ([0-9a-f]{40,64}) A\t(.*)$")


class Fault(Exception):
    """Something the check needs and cannot read: it fails, saying what."""


def git(*args):
    return subprocess.run(["git", "-c", "core.quotePath=false", "-c", "log.showRoot=true", *args],
                          capture_output=True, text=True, encoding="utf-8", errors="surrogateescape")


def must(*args):
    r = git(*args)
    if r.returncode != 0:
        raise Fault(f"`git {' '.join(args)}` failed: {r.stderr.strip() or 'exit ' + str(r.returncode)}")
    return r.stdout


def commit_of(rev):
    """rev as a full commit id, fetching it once if the clone lacks it."""
    for attempt in (1, 2):
        r = git("rev-parse", "--verify", "--quiet", f"{rev}^{{commit}}")
        if r.returncode == 0:
            return r.stdout.strip()
        if attempt == 1:
            git("fetch", "-q", "--no-tags", "origin", rev)
    raise Fault(f"{rev[:12]} is not a commit of this clone, even after fetching it")


def first_added(head):
    """{path: (mode, blob, commit)} for each rule file head's first-parent history ever held,
    as the first commit there that added it had it."""
    if must("rev-parse", "--is-shallow-repository").strip() != "false":
        raise Fault("this clone is shallow, so its history does not reach back to where a rule first appears")
    out = {}
    commit = None
    for line in must("log", "--first-parent", "-m", "--reverse", "--no-renames", "--diff-filter=A", "--raw",
                     "--no-abbrev", "--format=commit %H", head, "--", ROOT).splitlines():
        if line.startswith("commit "):
            commit = line.split(" ", 1)[1]
            continue
        add = RAW_ADD.match(line)
        if add and FROZEN.match(add.group(3)):
            out.setdefault(add.group(3), (add.group(1), add.group(2), commit))
    return out


def rules(commit):
    """Each rule file of the tree of commit, as {path: (mode, blob)}."""
    out = {}
    for line in must("ls-tree", "-r", "-z", "--full-tree", commit, "--", ROOT).split("\0"):
        if not line:
            continue
        meta, path = line.split("\t", 1)
        mode, kind, blob = meta.split(" ")
        if FROZEN.match(path):
            out[path] = (mode, blob)
    return out


def check(head, base=None):
    head = commit_of(head)
    pins = first_added(head)
    now = rules(head)
    bad = []
    for path, (mode, blob, commit) in sorted(pins.items()):
        at = f"as it first reached this history in {commit[:12]}"
        if path not in now:
            bad.append(f"{path} is removed or renamed, {at}")
        elif now[path][1] != blob:
            bad.append(f"{path} is changed: blob {blob[:12]} {at}, {now[path][1][:12]} here")
        elif now[path][0] != mode:
            bad.append(f"{path} changes its mode from {mode}, {at}, to {now[path][0]}")
    if base is not None:
        for path, (mode, blob) in sorted(rules(commit_of(base)).items()):
            if path in now and now[path] != (mode, blob):
                bad.append(f"{path} is another file than the one the base {base[:12]} holds at that path")
    if bad:
        for b in bad:
            print(f"::error::frozen rules: {b}. A rule committed before the runs it judges stays as it first reached "
                  f"main: restore it, and put a later rule in a new file, with its own SHA-256 in the record's text "
                  f"and verifier. A rule still on its branch enters the history once: rewrite the branch so that one "
                  f"commit adds it in its final form.")
        return 1
    print(f"frozen rules: the {len(pins)} rule file(s) of {head[:12]}'s first-parent history are as they first reached it")
    return 0


def main(argv):
    if len(argv) not in (1, 2) or not all(argv):
        print("usage: check-frozen-rules.py <head-sha> [<base-sha>]")
        return 2
    try:
        return check(*argv)
    except Fault as e:
        print(f"::error::frozen rules: {e}")
        return 1


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))