255 lines
12 KiB
Python
255 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""The page-set catalogue: every family of page sets, built and named by content.
|
|
|
|
python3 tools/pagesets/catalog.py --write # rebuild; update data/pagesets/index.json
|
|
python3 tools/pagesets/catalog.py --check # rebuild; fail if data/pagesets/index.json differs
|
|
python3 tools/pagesets/catalog.py --out DIR # rebuild; write the site's copy into DIR
|
|
python3 tools/pagesets/catalog.py --check --out DIR # both (the site's build does this)
|
|
|
|
The packages (.obz) are never committed: they are rebuilt from their sources
|
|
wherever they are needed. What is committed is the catalogue,
|
|
data/pagesets/index.json, which names every package by its content (SHA-256)
|
|
with its size, so the app knows each package before it downloads it, checks
|
|
it when it does, and a build whose packages differ from the committed
|
|
catalogue — stale sources, or a generator that is not reproducible — fails.
|
|
|
|
Families
|
|
--------
|
|
A family of page sets comes from a generator listed in generators.json (beside
|
|
this file): `{"generators": [{"script": "build.py", "args": [], "families":
|
|
["growing-core", "fixed-paths"]}, ...]}` (a phrase-set generator registers as
|
|
`{"script": "phrases/build.py", "args": [], "families": [...]}`). The contract
|
|
a generator keeps:
|
|
|
|
python3 <script> <args...> --out <dir>
|
|
|
|
writes its packages anywhere under <dir> and <dir>/families.json:
|
|
|
|
{"families": [{
|
|
"key": "growing-core", # unique across the catalogue
|
|
"name", "summary", "description", # what the chooser says
|
|
"chooser": "growing-core", # how the app offers it: growing-core, fixed-paths,
|
|
# phrases (a package per language and speaker form), list
|
|
"offer": "...", # optional: who the set is for, in a sentence
|
|
"version", "licence", "attribution", "sources": [{"name", "licence"}],
|
|
"languages": {"<code>": {"status": "GlossOnly" | ..., "glossed": 0.87}},
|
|
"language_note": "...", # what a person in another language should know
|
|
"packages": [{
|
|
"key": "growing-core-3x3", # unique within the family
|
|
"file": "growing-core/growing-core-3x3.obz", # under <dir>
|
|
"lang": "*", # "*": English keys the app glosses; else the package's language
|
|
# ("es", "pt-br"): the app offers the person's (exact tag,
|
|
# then base), else the "*" or "en" one
|
|
"gender": null, # "f" / "m" where the words change with the speaker's: the
|
|
# app asks the person which, never assumes
|
|
"variant": "full", "size": "3x3", "rows": 3, "cols": 3,
|
|
"boards", "cells", "cells_in_english", # optional counts (how far a language goes)
|
|
"label", "description", ... # anything else the chooser shows, kept as written
|
|
}]
|
|
}]}
|
|
|
|
Every build checks that every word of the app's own boards is translated, or
|
|
marked as one the language has no word for, in every language the app offers;
|
|
and that no board of any package shows the same words on two
|
|
of its cells, in any language (pslib/samewords.py), and fails if one does; and
|
|
that on a set that grows, at every step, no word is left out for a key that is
|
|
not shown with it (the key shown first keeps a word two keys share).
|
|
|
|
The catalogue copies each family and gives every package `file` (its name in
|
|
the catalogue's directory: "<key>.<sha256-12>.obz"), `bytes` and `sha256`.
|
|
Needs Python 3.11+ (and whatever each generator needs: PyYAML for these).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
AAC_RS = HERE.parent.parent
|
|
COMMITTED = AAC_RS / "data" / "pagesets" / "index.json"
|
|
FORMAT = "aac-pagesets"
|
|
VERSION = 1
|
|
|
|
LICENSE_HEAD = """# Page sets — licence
|
|
|
|
Generated by `aac/aac-rs/tools/pagesets/catalog.py`; do not edit by hand.
|
|
|
|
The page sets in this directory (`*.obz`) and their catalogue (`index.json`)
|
|
are **data, not code**, each licensed as its family states below (all
|
|
CC BY-SA 4.0 so far: https://creativecommons.org/licenses/by-sa/4.0/), not the
|
|
Apache-2.0 licence of the code around them. Each symbol a board names carries
|
|
its set's own licence and attribution in its `images` entry.
|
|
"""
|
|
|
|
|
|
def run_generators(work: Path) -> list[tuple[dict, Path]]:
|
|
"""Run every registered generator into its own directory under `work`."""
|
|
registry = json.loads((HERE / "generators.json").read_text(encoding="utf-8"))
|
|
out = []
|
|
for n, gen in enumerate(registry["generators"]):
|
|
target = work / f"g{n}"
|
|
cmd = [sys.executable, str(HERE / gen["script"]), *gen.get("args", []), "--out", str(target)]
|
|
res = subprocess.run(cmd, capture_output=True, text=True)
|
|
if res.returncode != 0:
|
|
raise SystemExit(f"{gen['script']} failed:\n{res.stdout}{res.stderr}")
|
|
fams = json.loads((target / "families.json").read_text(encoding="utf-8"))
|
|
for fam in fams["families"]:
|
|
out.append((fam, target))
|
|
return out
|
|
|
|
|
|
def catalogue(work: Path) -> tuple[dict, dict[str, Path]]:
|
|
"""The catalogue, and the file behind each package name."""
|
|
families = []
|
|
files: dict[str, Path] = {}
|
|
keys: set[str] = set()
|
|
for fam, base in run_generators(work):
|
|
if fam["key"] in keys:
|
|
raise SystemExit(f"two families are called {fam['key']!r}")
|
|
keys.add(fam["key"])
|
|
packages = []
|
|
for pkg in fam["packages"]:
|
|
src = base / pkg["file"]
|
|
data = src.read_bytes()
|
|
sha = hashlib.sha256(data).hexdigest()
|
|
name = f"{pkg['key']}.{sha[:12]}.obz"
|
|
if name in files:
|
|
raise SystemExit(f"two packages are called {name}")
|
|
files[name] = src
|
|
packages.append({**pkg, "file": name, "bytes": len(data), "sha256": sha})
|
|
families.append({**fam, "packages": packages})
|
|
index = {
|
|
"format": FORMAT,
|
|
"version": VERSION,
|
|
"about": (
|
|
"Page sets the app offers, built from their sources by aac/aac-rs/tools/pagesets "
|
|
"(catalog.py). Each package is named by its content and listed with its size and SHA-256; "
|
|
"a client downloads one only when it is chosen, and checks it."
|
|
),
|
|
"families": families,
|
|
}
|
|
return index, files
|
|
|
|
|
|
def own_words_check() -> None:
|
|
"""Every word of the app's own boards (aac-web own_board_words.json,
|
|
kept by its test from defaults.rs) has, in every language the app offers,
|
|
a gloss or a mark that the language has no such word (aac-core
|
|
no_word.json; proposed SPEC §4.1 item 4). A word neither translated nor
|
|
marked would show in English on the app's own boards: the build fails."""
|
|
sys.path.insert(0, str(HERE))
|
|
from pslib.data import CORE_SRC, WEB_SRC, Data # noqa: E402
|
|
|
|
data = Data()
|
|
keys = json.loads((WEB_SRC / "own_board_words.json").read_text(encoding="utf-8"))
|
|
none = json.loads((CORE_SRC / "no_word.json").read_text(encoding="utf-8"))["keys"]
|
|
gaps = []
|
|
for key in keys:
|
|
glosses = data.glosses(key)
|
|
for lang in (lang["code"] for lang in data.languages if lang["code"] != "en"):
|
|
if lang not in glosses and lang not in none.get(key, []):
|
|
gaps.append(f"{key} in {lang}")
|
|
if lang in glosses and lang in none.get(key, []):
|
|
gaps.append(f"{key} in {lang}: marked as no word, yet glossed")
|
|
if gaps:
|
|
raise SystemExit(
|
|
f"{len(gaps)} words of the app's own boards are neither translated nor marked as none: "
|
|
+ ", ".join(gaps[:40]) + ("…" if len(gaps) > 40 else "")
|
|
+ "\nTranslate each (translations_data.json) or, where the language has no such word, mark it (no_word.json)."
|
|
)
|
|
|
|
|
|
def same_words_check(index: dict, files: dict[str, Path], allow: Path | None) -> None:
|
|
"""No board of any package shows the same words on two cells, in any
|
|
language (pslib/samewords.py; proposed SPEC §4.1 item 5). A clash fails the
|
|
build: it is never a warning. (`allow`: write what is found there
|
|
instead, for the gloss audit that settles each clash.)"""
|
|
sys.path.insert(0, str(HERE))
|
|
from pslib import samewords # noqa: E402
|
|
from pslib.data import Data # noqa: E402
|
|
|
|
langs = ["en"] + [lang["code"] for lang in Data().languages if lang["code"] != "en"]
|
|
# On a set that grows, a word is never left out for a key hidden for
|
|
# later: at every step, every word shown has its words somewhere.
|
|
left = samewords.gaps(index, files)
|
|
if left:
|
|
shown = "\n".join(" " + samewords.describe_gap(g) for g in left[:40])
|
|
more = f"\n … and {len(left) - 40} more" if len(left) > 40 else ""
|
|
raise SystemExit(
|
|
f"{len(left)} words are left out for a key that is not shown with them:\n{shown}{more}\n"
|
|
"The key shown first keeps a word two keys share (`shown_from`, aac-core samewords; pslib/obf.py)."
|
|
)
|
|
found = samewords.check(index, files, langs)
|
|
if allow is not None:
|
|
allow.write_text(json.dumps(found, ensure_ascii=False, indent=1), encoding="utf-8")
|
|
print(f"{len(found)} boards show the same words twice: written to {allow}")
|
|
return
|
|
if found:
|
|
shown = "\n".join(" " + samewords.describe(c) for c in found[:40])
|
|
more = f"\n … and {len(found) - 40} more" if len(found) > 40 else ""
|
|
raise SystemExit(
|
|
f"{len(found)} boards show the same words on two cells:\n{shown}{more}\n"
|
|
"Each needs a gloss audit: a word of its own for one key (translations_data.json), or, where the "
|
|
"language says both with one word, the pair in aac-core same_words.json (the word then shows once)."
|
|
)
|
|
|
|
|
|
def dumps(index: dict) -> str:
|
|
return json.dumps(index, ensure_ascii=False, sort_keys=True, indent=1) + "\n"
|
|
|
|
|
|
def license_md(index: dict) -> str:
|
|
parts = [LICENSE_HEAD]
|
|
for fam in index["families"]:
|
|
parts.append(f"\n## {fam['name']} ({fam['licence']})\n\n{fam.get('attribution', '').strip()}\n")
|
|
return "".join(parts)
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("--write", action="store_true", help="update the committed catalogue")
|
|
ap.add_argument("--check", action="store_true", help="fail if the committed catalogue differs")
|
|
ap.add_argument("--out", type=Path, help="write the packages, index.json and LICENSE.md here")
|
|
ap.add_argument("--same-words", type=Path, help="write the boards that show the same words twice here, and do not fail on them (for a gloss audit)")
|
|
args = ap.parse_args()
|
|
if not (args.write or args.check or args.out):
|
|
ap.error("say --write, --check or --out")
|
|
with tempfile.TemporaryDirectory(prefix="aac-pagesets-") as tmp:
|
|
index, files = catalogue(Path(tmp))
|
|
own_words_check()
|
|
same_words_check(index, files, args.same_words)
|
|
text = dumps(index)
|
|
if args.check:
|
|
committed = COMMITTED.read_text(encoding="utf-8") if COMMITTED.exists() else ""
|
|
if committed != text:
|
|
raise SystemExit(
|
|
"data/pagesets/index.json is out of date with the page sets' sources, or a generator is not "
|
|
"reproducible: run tools/pagesets/catalog.py --write, and commit the catalogue with its sources"
|
|
)
|
|
print(f"catalogue up to date: {sum(len(f['packages']) for f in index['families'])} packages")
|
|
if args.write:
|
|
COMMITTED.parent.mkdir(parents=True, exist_ok=True)
|
|
COMMITTED.write_text(text, encoding="utf-8")
|
|
(COMMITTED.parent / "LICENSE.md").write_text(license_md(index), encoding="utf-8")
|
|
print(f"wrote {COMMITTED.relative_to(AAC_RS)}")
|
|
if args.out:
|
|
if args.out.exists():
|
|
shutil.rmtree(args.out)
|
|
args.out.mkdir(parents=True)
|
|
for name, src in files.items():
|
|
shutil.copyfile(src, args.out / name)
|
|
(args.out / "index.json").write_text(text, encoding="utf-8")
|
|
(args.out / "LICENSE.md").write_text(license_md(index), encoding="utf-8")
|
|
total = sum(p.stat().st_size for p in args.out.glob("*.obz"))
|
|
print(f"wrote {len(files)} packages ({total / 1e6:.1f} MB), index.json and LICENSE.md to {args.out}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|