open-standards/aac/aac-rs/tools/pagesets/catalog.py

#!/usr/bin/env python3
"""The page-set catalogue: every family of page sets, built and named by content.

    python3 tools/pagesets/catalog.py --write        # rebuild; update data/pagesets/index.json
    python3 tools/pagesets/catalog.py --check        # rebuild; fail if data/pagesets/index.json differs
    python3 tools/pagesets/catalog.py --out DIR      # rebuild; write the site's copy into DIR
    python3 tools/pagesets/catalog.py --check --out DIR   # both (the site's build does this)

The packages (.obz) are never committed: they are rebuilt from their sources
wherever they are needed. What is committed is the catalogue,
data/pagesets/index.json, which names every package by its content (SHA-256)
with its size, so the app knows each package before it downloads it, checks
it when it does, and a build whose packages differ from the committed
catalogue — stale sources, or a generator that is not reproducible — fails.

Families
--------
A family of page sets comes from a generator listed in generators.json (beside
this file): `{"generators": [{"script": "build.py", "args": [], "families":
["growing-core", "fixed-paths"]}, ...]}` (a phrase-set generator registers as
`{"script": "phrases/build.py", "args": [], "families": [...]}`). The contract
a generator keeps:

    python3 <script> <args...> --out <dir>

writes its packages anywhere under <dir> and <dir>/families.json:

    {"families": [{
        "key": "growing-core",            # unique across the catalogue
        "name", "summary", "description", # what the chooser says
        "chooser": "growing-core",        # how the app offers it: growing-core, fixed-paths,
                                          # phrases (a package per language and speaker form), list
        "offer": "...",                   # optional: who the set is for, in a sentence
        "version", "licence", "attribution", "sources": [{"name", "licence"}],
        "languages": {"<code>": {"status": "GlossOnly" | ..., "glossed": 0.87}},
        "language_note": "...",           # what a person in another language should know
        "packages": [{
            "key": "growing-core-3x3",    # unique within the family
            "file": "growing-core/growing-core-3x3.obz",   # under <dir>
            "lang": "*",                  # "*": English keys the app glosses; else the package's language
                                          # ("es", "pt-br"): the app offers the person's (exact tag,
                                          # then base), else the "*" or "en" one
            "gender": null,               # "f" / "m" where the words change with the speaker's: the
                                          # app asks the person which, never assumes
            "variant": "full", "size": "3x3", "rows": 3, "cols": 3,
            "boards", "cells", "cells_in_english",   # optional counts (how far a language goes)
            "label", "description", ...   # anything else the chooser shows, kept as written
        }]
    }]}

The catalogue copies each family and gives every package `file` (its name in
the catalogue's directory: "<key>.<sha256-12>.obz"), `bytes` and `sha256`.
Needs Python 3.11+ (and whatever each generator needs: PyYAML for these).
"""

from __future__ import annotations

import argparse
import hashlib
import json
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path

HERE = Path(__file__).resolve().parent
AAC_RS = HERE.parent.parent
COMMITTED = AAC_RS / "data" / "pagesets" / "index.json"
FORMAT = "aac-pagesets"
VERSION = 1

LICENSE_HEAD = """# Page sets — licence

Generated by `aac/aac-rs/tools/pagesets/catalog.py`; do not edit by hand.

The page sets in this directory (`*.obz`) and their catalogue (`index.json`)
are **data, not code**, each licensed as its family states below (all
CC BY-SA 4.0 so far: https://creativecommons.org/licenses/by-sa/4.0/), not the
Apache-2.0 licence of the code around them. Each symbol a board names carries
its set's own licence and attribution in its `images` entry.
"""


def run_generators(work: Path) -> list[tuple[dict, Path]]:
    """Run every registered generator into its own directory under `work`."""
    registry = json.loads((HERE / "generators.json").read_text(encoding="utf-8"))
    out = []
    for n, gen in enumerate(registry["generators"]):
        target = work / f"g{n}"
        cmd = [sys.executable, str(HERE / gen["script"]), *gen.get("args", []), "--out", str(target)]
        res = subprocess.run(cmd, capture_output=True, text=True)
        if res.returncode != 0:
            raise SystemExit(f"{gen['script']} failed:\n{res.stdout}{res.stderr}")
        fams = json.loads((target / "families.json").read_text(encoding="utf-8"))
        for fam in fams["families"]:
            out.append((fam, target))
    return out


def catalogue(work: Path) -> tuple[dict, dict[str, Path]]:
    """The catalogue, and the file behind each package name."""
    families = []
    files: dict[str, Path] = {}
    keys: set[str] = set()
    for fam, base in run_generators(work):
        if fam["key"] in keys:
            raise SystemExit(f"two families are called {fam['key']!r}")
        keys.add(fam["key"])
        packages = []
        for pkg in fam["packages"]:
            src = base / pkg["file"]
            data = src.read_bytes()
            sha = hashlib.sha256(data).hexdigest()
            name = f"{pkg['key']}.{sha[:12]}.obz"
            if name in files:
                raise SystemExit(f"two packages are called {name}")
            files[name] = src
            packages.append({**pkg, "file": name, "bytes": len(data), "sha256": sha})
        families.append({**fam, "packages": packages})
    index = {
        "format": FORMAT,
        "version": VERSION,
        "about": (
            "Page sets the app offers, built from their sources by aac/aac-rs/tools/pagesets "
            "(catalog.py). Each package is named by its content and listed with its size and SHA-256; "
            "a client downloads one only when it is chosen, and checks it."
        ),
        "families": families,
    }
    return index, files


def dumps(index: dict) -> str:
    return json.dumps(index, ensure_ascii=False, sort_keys=True, indent=1) + "\n"


def license_md(index: dict) -> str:
    parts = [LICENSE_HEAD]
    for fam in index["families"]:
        parts.append(f"\n## {fam['name']} ({fam['licence']})\n\n{fam.get('attribution', '').strip()}\n")
    return "".join(parts)


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--write", action="store_true", help="update the committed catalogue")
    ap.add_argument("--check", action="store_true", help="fail if the committed catalogue differs")
    ap.add_argument("--out", type=Path, help="write the packages, index.json and LICENSE.md here")
    args = ap.parse_args()
    if not (args.write or args.check or args.out):
        ap.error("say --write, --check or --out")
    with tempfile.TemporaryDirectory(prefix="aac-pagesets-") as tmp:
        index, files = catalogue(Path(tmp))
        text = dumps(index)
        if args.check:
            committed = COMMITTED.read_text(encoding="utf-8") if COMMITTED.exists() else ""
            if committed != text:
                raise SystemExit(
                    "data/pagesets/index.json is out of date with the page sets' sources, or a generator is not "
                    "reproducible: run tools/pagesets/catalog.py --write, and commit the catalogue with its sources"
                )
            print(f"catalogue up to date: {sum(len(f['packages']) for f in index['families'])} packages")
        if args.write:
            COMMITTED.parent.mkdir(parents=True, exist_ok=True)
            COMMITTED.write_text(text, encoding="utf-8")
            (COMMITTED.parent / "LICENSE.md").write_text(license_md(index), encoding="utf-8")
            print(f"wrote {COMMITTED.relative_to(AAC_RS)}")
        if args.out:
            if args.out.exists():
                shutil.rmtree(args.out)
            args.out.mkdir(parents=True)
            for name, src in files.items():
                shutil.copyfile(src, args.out / name)
            (args.out / "index.json").write_text(text, encoding="utf-8")
            (args.out / "LICENSE.md").write_text(license_md(index), encoding="utf-8")
            total = sum(p.stat().st_size for p in args.out.glob("*.obz"))
            print(f"wrote {len(files)} packages ({total / 1e6:.1f} MB), index.json and LICENSE.md to {args.out}")


if __name__ == "__main__":
    main()