open-standards/scripts/check-tests-in-ci.py

#!/usr/bin/env python3
"""check-tests-in-ci.py: every integration test of a crate is run by a job, and none
can run empty.

The conformance workflows name the tests they run, one `--test NAME` at a time: the
crate is too large to run whole on every leg. So a test file nobody adds to a step never
runs, and nothing says so. And a file that is `#![cfg(feature = "...")]` compiles, in a
step that lacks the feature, to a binary with no tests in it that reports `ok`.

A step may also take its tests from elsewhere: the steps of wai-conformance.yml's
independent verifiers are the files of `.github/wai-conformance/verifiers/`, and a step that
runs `scripts/wai-conformance-lists.py tests TAG` or `test-tagged TAG` runs every test file
whose line `// wai-conformance: TAG...` names TAG. Both count as naming the test.

For each crate listed in CRATES this requires that

  1. every `tests/*.rs` is named by `--test NAME` in a workflow or a verifier step's file,
     or carries a tag a workflow's step takes, or is listed in the crate's
     `tests/NOT-IN-CI.txt` with the reason no job runs it;
  2. every `tests/*.rs` whose crate-level `#![cfg(...)]` names features has a `[[test]]`
     entry in Cargo.toml whose `required-features` are exactly those features, so that
     naming it in a step without them stops cargo with an error;
  3. `NOT-IN-CI.txt` names only test files that exist and that no workflow names, each
     with a reason;
  4. every `[[test]]` entry names a file that exists.

A crate whose workflow runs `cargo test` over all its targets needs no entry in CRATES:
cargo finds its tests itself.

    check-tests-in-ci.py [--root DIR]

Exit 0 when all four hold, 1 with one line a finding otherwise. Standard library only.
"""
import os
import re
import sys

# Crates whose workflows name their tests one by one.
CRATES = ["wai/wai-rs"]

NOT_IN_CI = "NOT-IN-CI.txt"


# The verifier steps wai-conformance.yml runs from their own files.
STEP_DIRS = [os.path.join(".github", "wai-conformance", "verifiers")]


def workflow_test_names(root):
    """Every NAME of a `--test NAME` in a workflow file or a verifier step's file."""
    names = set()
    wf = os.path.join(root, ".github", "workflows")
    files = [os.path.join(wf, f) for f in sorted(os.listdir(wf)) if f.endswith((".yml", ".yaml"))]
    for d in STEP_DIRS:
        d = os.path.join(root, d)
        if os.path.isdir(d):
            files += [os.path.join(d, f) for f in sorted(os.listdir(d)) if f.endswith(".sh")]
    for path in files:
        with open(path, encoding="utf-8") as fh:
            names.update(re.findall(r"--test[ =]([A-Za-z0-9_-]+)", fh.read()))
    return names


def workflow_tags(root):
    """Every tag a workflow step takes its tests by: `wai-conformance-lists.py tests TAG...`
    or `test-tagged TAG...`."""
    tags = set()
    wf = os.path.join(root, ".github", "workflows")
    for f in sorted(os.listdir(wf)):
        if f.endswith((".yml", ".yaml")):
            with open(os.path.join(wf, f), encoding="utf-8") as fh:
                for m in re.finditer(r"wai-conformance-lists\.py (?:tests|test-tagged)((?: [a-z0-9-]+)+)", fh.read()):
                    tags.update(m.group(1).split())
    return tags


def file_tags(path):
    """The tags of a test file's `// wai-conformance:` lines."""
    with open(path, encoding="utf-8") as fh:
        return {t for line in re.findall(r"^//\s*wai-conformance:\s*(.*?)\s*$", fh.read(), re.M) for t in line.split()}


def crate_cfg_features(path):
    """The features a test file's crate-level `#![cfg(...)]` requires, as a sorted list;
    [] when it has none. None when the attribute is a form this check does not read
    (`any`, `not`, or something that is not a feature), which is a finding."""
    with open(path, encoding="utf-8") as fh:
        src = fh.read()
    feats = set()
    for m in re.finditer(r"^#!\[cfg\((.*)\)\]\s*$", src, re.M):
        inner = m.group(1).strip()
        body = inner
        am = re.fullmatch(r"all\((.*)\)", inner, re.S)
        if am:
            body = am.group(1)
        parts = [p.strip() for p in body.split(",") if p.strip()]
        for p in parts:
            fm = re.fullmatch(r'feature\s*=\s*"([A-Za-z0-9_-]+)"', p)
            if not fm:
                return None
            feats.add(fm.group(1))
    return sorted(feats)


def cargo_tests(path):
    """The `[[test]]` entries of a Cargo.toml: {name: (path, required-features or None)}."""
    with open(path, encoding="utf-8") as fh:
        text = fh.read()
    out = {}
    for block in re.split(r"^\[\[", text, flags=re.M)[1:]:
        if not block.startswith("test]]"):
            continue
        body = re.split(r"^\[", block[len("test]]"):], flags=re.M)[0]
        name = re.search(r'^name\s*=\s*"([^"]+)"', body, re.M)
        if not name:
            continue
        p = re.search(r'^path\s*=\s*"([^"]+)"', body, re.M)
        rf = re.search(r"^required-features\s*=\s*\[(.*?)\]", body, re.M | re.S)
        feats = sorted(re.findall(r'"([^"]+)"', rf.group(1))) if rf else None
        out[name.group(1)] = (p.group(1) if p else f"tests/{name.group(1)}.rs", feats)
    return out


def not_in_ci(path):
    """`NOT-IN-CI.txt`: {name: reason}. A line is a name, blank space, and the reason;
    `#` starts a comment line."""
    out, bad = {}, []
    if not os.path.exists(path):
        return out, bad
    with open(path, encoding="utf-8") as fh:
        for n, line in enumerate(fh, 1):
            line = line.rstrip("\n")
            if not line.strip() or line.lstrip().startswith("#"):
                continue
            m = re.fullmatch(r"([A-Za-z0-9_-]+)\s+(\S.*)", line)
            if not m:
                bad.append(f"{path}:{n}: not `name  reason`")
                continue
            if m.group(1) in out:
                bad.append(f"{path}:{n}: {m.group(1)} is listed twice")
            out[m.group(1)] = m.group(2).strip()
    return out, bad


def check(root):
    findings = []
    named = workflow_test_names(root)
    tags = workflow_tags(root)
    for crate in CRATES:
        tdir = os.path.join(root, crate, "tests")
        files = sorted(f[:-3] for f in os.listdir(tdir) if f.endswith(".rs") and os.path.isfile(os.path.join(tdir, f)))
        declared = cargo_tests(os.path.join(root, crate, "Cargo.toml"))
        listed, bad = not_in_ci(os.path.join(tdir, NOT_IN_CI))
        findings += bad
        named = named | {t for t in files if file_tags(os.path.join(tdir, t + ".rs")) & tags}
        for t in files:
            if t not in named and t not in listed:
                findings.append(f"{crate}/tests/{t}.rs: no workflow step names it (`--test {t}`) or takes a tag it carries, and {NOT_IN_CI} does not list it")
            feats = crate_cfg_features(os.path.join(tdir, t + ".rs"))
            if feats is None:
                findings.append(f"{crate}/tests/{t}.rs: its #![cfg(...)] is not a feature or an all(...) of features; this check cannot tell what it needs")
            elif feats:
                entry = declared.get(t)
                if entry is None or entry[1] is None:
                    findings.append(f'{crate}/tests/{t}.rs: gated on {", ".join(feats)} with no [[test]] required-features in Cargo.toml, so a step without the feature runs it empty')
                elif entry[1] != feats:
                    findings.append(f'{crate}/Cargo.toml: [[test]] {t} requires {entry[1]}, and the file is gated on {feats}')
        for t, why in sorted(listed.items()):
            if t not in files:
                findings.append(f"{crate}/tests/{NOT_IN_CI}: {t} is not a test file")
            elif t in named:
                findings.append(f"{crate}/tests/{NOT_IN_CI}: {t} is listed as not run, and a workflow names it; remove the line")
        for t, (path, _) in sorted(declared.items()):
            if not os.path.isfile(os.path.join(root, crate, path)):
                findings.append(f"{crate}/Cargo.toml: [[test]] {t} names {path}, which does not exist")
    return findings


def main(argv):
    root = "."
    if len(argv) >= 2 and argv[0] == "--root":
        root = argv[1]
    findings = check(root)
    for f in findings:
        print(f)
    if findings:
        print(f"check-tests-in-ci: {len(findings)} finding(s)")
        return 1
    print("check-tests-in-ci: every test file is run by a job or listed with its reason, and none can run empty.")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))