#!/usr/bin/env python3 """Build the open page sets: Growing Core and Fixed Paths. python3 tools/pagesets/build.py # update the design documents' tables python3 tools/pagesets/build.py --check # fail if they are out of date python3 tools/pagesets/build.py --out DIR # write the packages into DIR The packages are not committed: they are built where they are needed (the site's build, through tools/pagesets/catalog.py, and aac-core's tests), from these sources, byte for byte the same every time. What is committed is the catalogue (data/pagesets/index.json, written by catalog.py), which names each package with its size and SHA-256, so a build that does not reproduce them fails. Reads tools/pagesets/data (the design and its committed layouts), sources/ (the cited word lists) and the app's own data (glosses, symbols, word classes); checks every rule of DESIGN-core.md and DESIGN-motor.md; with --out: growing-core/*.obz one package per size, and reveal examples fixed-paths/*.obz one package per size, and filter examples families.json both families, for the catalogue (catalog.py) cells.json every cell of every board (for aac-core's tests) metrics.json the metrics of both designs languages.json what each language gets, and the native work it needs words.json every word, with its evidence and always the metric tables inside DESIGN-core.md and DESIGN-motor.md. Reproducible: fixed ids (UUID v5), a fixed date, sorted keys. Needs Python 3.11+ and PyYAML; nothing else, no network. """ from __future__ import annotations import argparse import json import re import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from pslib import costs, langs, metrics, obf # noqa: E402 from pslib.boards import core_packages, motor_packages # noqa: E402 from pslib.data import HERE, WORD_CLASS, Data # noqa: E402 from pslib.layout import load_core, load_motor # noqa: E402 def class_exceptions(data: Data, words: set[str]) -> dict: out = {} for w in sorted(words): e = data.lexicon[w] ours = e.word_class app = data.app_vocabulary.get(w.lower()) if app and ours and app[0] != ours: out[w] = {"ours": ours, "app": app[0], "why": e.sense or f"used as a {e.pos}"} return out def index_json(pkgs, exceptions) -> dict: out = {"version": 1, "packages": [], "class_exceptions": exceptions} for p in pkgs: boards = [] for b in p.boards: cells = [] for btn in sorted(b.buttons, key=lambda x: (x.row, x.col)): extra = btn.target if btn.action == "navigate" else btn.modifier if btn.action == "modifier" else None cells.append([btn.key, btn.row, btn.col, btn.action, extra, btn.hidden, btn.control]) boards.append({"id": b.id, "name": b.name, "rows": b.rows, "cols": b.cols, "board": b.pageset.get("board"), "cells": cells}) out["packages"].append({"file": p.file, "set": p.set, "size": p.size, "variant": p.variant, "root": p.root, "boards": boards}) return out def words_json(data: Data, pkgs) -> dict: out = {} for set_name in ("growing-core", "fixed-paths"): words = sorted({btn.key for p in pkgs if p.set == set_name and p.variant == "full" for b in p.boards for btn in b.buttons if btn.action == "speak"}, key=str.lower) rows = {} for w in words: e = data.lexicon[w] sym = obf.lookup_for_cell(data, w, "speak") rows[w] = { "pos": e.pos, "word_class": WORD_CLASS[e.pos], "app_class": (data.app_vocabulary.get(w.lower()) or [None])[0], "role": e.role, "sense": e.sense, "lists": data.in_aac_lists(w), "spoken_rank": data.spoken_rank.get(w.lower()), "symbol": f"{sym[0]}:{sym[1]}" if sym else None, "glosses": len(data.glosses(w)), } out[set_name] = rows return out # ── tables for the design documents ────────────────────────────────────── def md_table(head: list[str], rows: list[list]) -> str: out = ["| " + " | ".join(head) + " |", "|" + "|".join("---" for _ in head) + "|"] for r in rows: out.append("| " + " | ".join(str(x) for x in r) + " |") return "\n".join(out) def mm(t: dict, key: str) -> str: x = t[key] return f"{x['width_mm']:.0f}×{x['height_mm']:.0f}" def core_tables(m: dict, lay, data: Data) -> dict[str, str]: t = {} rows = [] for size, s in m["sizes"].items(): c = s["coverage"] rows.append([ size, s["cells"], s["core_words"], s["pages"], s["words_reachable"], f"{c['top25']['direct']}/25", f"{c['top50']['direct']}/50", f"{c['top100']['direct']}/100", f"{c['top200']['direct']}/200", f"{c['token_share_direct']:.0%}", f"{s['lists']['universal_core']['one_hit']}/{s['lists']['universal_core']['of']}", f"{s['lists']['banajee_2003']['one_hit']}/{s['lists']['banajee_2003']['of']}", ]) t["core-reach"] = md_table( ["Size", "Cells", "Core words (1 hit)", "Pages", "Words in reach", "Top 25 spoken", "Top 50", "Top 100", "Top 200", "Spoken tokens covered", "Universal Core on the core", "Banajee on the core"], rows) rows = [] for size, s in m["sizes"].items(): w = s["weighted"] rows.append([size, w.get("hits", "–"), w.get("scan", "–"), w.get("touch", "–"), w.get("gaze", "–")]) t["core-costs"] = md_table(["Size", "Hits per word", "Row-column scan steps", "Touch (Fitts ID, bits)", "Gaze (eccentricity)"], rows) rows = [] for size, s in m["sizes"].items(): tg = s["targets"] rows.append([size, mm(tg, "tablet-landscape"), mm(tg, "tablet-portrait"), mm(tg, "phone-portrait"), mm(tg, "phone-landscape"), ", ".join(tg["tablet-landscape"]["meets"]) or "none", ", ".join(tg["phone-portrait"]["meets"]) or "none"]) t["core-targets"] = md_table(["Size", "10.2\" tablet, landscape (mm)", "tablet, portrait", "6.1\" phone, portrait", "phone, landscape", "Meets on the tablet (landscape)", "Meets on the phone (portrait)"], rows) st = m["stability_core"] w = st["worst"] t["core-stability"] = "\n".join([ f"- Pairs of sizes compared (an item present at both): **{st['pairs']}**.", f"- Mean shift of a button's centre: **{st['mean_shift']:.3f}** of the board's width or height.", f"- Largest shift: **{w['cells']:.2f} cells of the smaller board** (`{w['item']}`, {w['from']} → {w['to']}: " f"{w['du']:.3f} across, {w['dv']:.3f} down). Threshold: less than one cell of the smaller board — " f"**{'met' if st['within_one_cell_of_smaller'] else 'NOT met'}**.", f"- Largest shift between neighbouring sizes of the ladder: **{st['worst_shift_one_step']:.3f}** of the board.", f"- Every item's buttons, at every size, share one point of the screen: **{'yes' if st['every_item_keeps_a_common_point'] else 'NO'}**.", "- Pages: " + "; ".join(f"{n} {'ok' if s['within_one_cell_of_smaller'] and s['every_item_keeps_a_common_point'] else 'FAILS'}" for n, s in m["stability_pages"].items()) + ".", ]) rows = [] names = [s.name for s in lay.sizes] for k, size in enumerate(names): words = [] for p in lay.core: if p.stage != k: continue if p.kind == "page": words.append(f"**{p.key}** ▸") continue ev = [] lists = data.in_aac_lists(p.key) if "universal_core" in lists: ev.append("U") if "banajee_2003" in lists: ev.append("B") rank = data.spoken_rank.get(p.key.lower()) ev.append(f"#{rank}" if rank else "#–") words.append(f"{p.key} ({', '.join(ev)})") rows.append([size, len(words), "; ".join(words)]) t["core-stages"] = md_table(["Stage", "New", "Words (U: Universal Core, B: Banajee et al. 2003, #: spoken rank)"], rows) rows = [] for name, page in lay.pages.items(): words = [p.key for p in page.items if p.kind in ("word", "modifier")] rows.append([name, page.sizes[0].name, len(words), ", ".join(words[:12]) + (", …" if len(words) > 12 else "")]) t["core-pages"] = md_table(["Page", "Opens from", "Words", "First words (they appear at the page's first size)"], rows) grids = [] for s in lay.sizes: from pslib.layout import core_grid grids.append(f"**{s.name}**\n\n```\n{core_grid(lay, s)}\n```") t["core-grids"] = "\n\n".join(grids) return t def motor_tables(m: dict, lay, data: Data) -> dict[str, str]: t = {} rows = [] for size, s in m["sizes"].items(): h = s["hits"] c = s["coverage"] rows.append([size, s["grid"], s["home_words"], s["pages"], s["groups"], s["words"], h.get("1", 0), h.get("2", 0), h.get("3", 0), s["mean_hits_unweighted"], s["weighted_spoken"]["hits"], s["weighted_design"]["hits"], f"{c['top25']['one_hit']}/25", f"{c['top100']['one_hit']}/100", f"{c['top100']['direct']}/100", f"{c['top200']['direct']}/200", f"{c['token_share_direct']:.0%}"]) t["motor-reach"] = md_table( ["Locations", "Grid", "Words on Home", "Pages", "Groups", "Words", "1 hit", "2 hits", "3 hits", "Mean hits", "Hits per word, spoken weight", "Hits, design weight", "Top 25 in 1 hit", "Top 100 in 1 hit", "Top 100 in reach", "Top 200 in reach", "Spoken tokens covered"], rows) rows = [] for size, s in m["sizes"].items(): w = s["weighted_spoken"] rows.append([size, w["hits"], w["scan"], w["touch"], w["gaze"]]) t["motor-costs"] = md_table(["Locations", "Hits per word", "Row-column scan steps", "Touch (Fitts ID, bits)", "Gaze (eccentricity)"], rows) rows = [] for size, s in m["sizes"].items(): for name, v in s["tradeoffs"].items(): rows.append([size, name.replace("_", " "), v["hits"], v["scan"], v["touch"], v["gaze"]]) t["motor-tradeoffs"] = md_table(["Locations", "Layout", "Hits", "Scan steps", "Touch (bits)", "Gaze"], rows) rows = [] for size, s in m["sizes"].items(): tg = s["targets"] rows.append([size, s["grid"], mm(tg, "tablet-landscape"), mm(tg, "tablet-portrait"), mm(tg, "phone-portrait"), mm(tg, "phone-landscape"), ", ".join(tg["tablet-landscape"]["meets"]) or "none", ", ".join(tg["phone-portrait"]["meets"]) or "none"]) t["motor-targets"] = md_table(["Locations", "Grid", "10.2\" tablet, landscape (mm)", "tablet, portrait", "6.1\" phone, portrait", "phone, landscape", "Meets on the tablet (landscape)", "Meets on the phone (portrait)"], rows) st = m["stability_home"] w = st["worst"] t["motor-stability"] = ( f"Home items present at two sizes: {st['pairs']} pairs; mean shift {st['mean_shift']:.3f} of the board; " f"largest {w['cells']:.2f} cells of the smaller board (`{w['item']}`, {w['from']} → {w['to']}); " f"every item's buttons share a point: {'yes' if st['every_item_keeps_a_common_point'] else 'NO'}." ) rows = [] d = data.fixed_paths for page, spec in d["pages"].items(): groups = spec.get("groups", {}) used = {ms.label: sorted(g for g in ms.pages[page]["groups"]) for ms in lay.sizes} n = len(spec.get("words", [])) + sum(len(v) for v in groups.values()) + len(spec.get("modifiers", [])) rows.append([page, n, ", ".join(groups) or "–", "; ".join(f"{k}: {', '.join(v) or 'none'}" for k, v in used.items())]) t["motor-pages"] = md_table(["Page", "Words", "Groups (by meaning)", "Groups in use, by size"], rows) grids = [] for ms in lay.sizes: from pslib.layout import _grid_text cells = {cell: key + ("▸" if key in ms.pages else "") for key, cell in ms.home.items()} grids.append(f"**{ms.label} ({ms.size.name})**\n\n```\n{_grid_text(ms.size, cells, 11)}\n```") t["motor-grids"] = "\n\n".join(grids) return t def lang_table(rep: dict) -> str: rows = [] for l in rep["languages"]: if l["code"] == "en": rows.append(["en", "English", "ValidatedNativeCorpus", l["prediction_pack"], "source", "source", "–"]) continue gc, fp = l["sets"]["growing-core"], l["sets"]["fixed-paths"] rows.append([l["code"], l["name"], l["core_status"], l["prediction_pack"] or "none", f"{gc['glossed_share']:.0%} ({gc['glossed_share_by_spoken_frequency']:.0%} by use)", f"{fp['glossed_share']:.0%} ({fp['glossed_share_by_spoken_frequency']:.0%} by use)", "yes" if any("published core" in n for n in l["needs"]) else ""]) return md_table(["Code", "Language", "Core status", "Prediction pack", "Growing Core words glossed", "Fixed Paths words glossed", "Published native core list exists"], rows) def fill(doc: Path, tables: dict[str, str]) -> str: text = doc.read_text(encoding="utf-8") for key, body in tables.items(): pat = re.compile(rf"(\n).*?()", re.S) if not pat.search(text): raise SystemExit(f"{doc.name}: no block") text = pat.sub(lambda mt: mt.group(1) + body + "\n" + mt.group(2), text) return text FAMILY_TEXT = { "growing-core": { "name": "Growing Core", "summary": "Core words that grow from 1 to 80 on a board, and no word ever moves.", "description": ( "Our own core vocabulary of the words people say most, in eleven sizes from one button to 80. " "A word learned on a small board is in the same place on every bigger one, so reaching for it " "carries over. Pages of more words open from the right edge, each with Back in its top-left corner. " "Words are coloured by the Fitzgerald Key. You can also start on a bigger board with only the " "first words shown, and show more in their places later." ), "chooser": "growing-core", }, "fixed-paths": { "name": "Fixed Paths", "summary": "Every word in one place, reached the same way every time, in 1 to 3 touches.", "description": ( "A motor-plan vocabulary: every word is one, two or three touches from Home, always the same ones, " "so the movements become automatic. Made to go back to Home after each word. 30 places (the " "largest targets, for eye gaze and head pointers), 60 or 84. You can start with 45 first words " "shown and the rest hidden in their places." ), "chooser": "fixed-paths", }, } SOURCES_SHORT = [ {"name": "Universal Core vocabulary (UNC CLDS / Project Core)", "licence": "CC-BY-SA-4.0"}, {"name": "Banajee, DiCarlo & Buras Stricklin (2003), core vocabulary for toddlers", "licence": "cited"}, {"name": "FrequencyWords 2018 (OpenSubtitles), Hermit Dave", "licence": "CC-BY-SA-4.0"}, {"name": "Mulberry Symbols, OpenMoji, Blissymbolics", "licence": "CC-BY-SA-4.0"}, ] def families_json(data: Data, pkgs, rep: dict) -> dict: """This generator's families, for tools/pagesets/catalog.py.""" langs_by_set: dict[str, dict] = {} for set_name in ("growing-core", "fixed-paths"): out = {} for lang in rep["languages"]: if lang["code"] == "en": out["en"] = {"status": "ValidatedNativeCorpus"} else: st = lang["sets"][set_name] out[lang["code"]] = {"status": lang["core_status"], "glossed": st["glossed_share"]} langs_by_set[set_name] = out fams = [] for set_name in ("growing-core", "fixed-paths"): d = data.growing_core if set_name == "growing-core" else data.fixed_paths packages = [] for p in (q for q in pkgs if q.set == set_name): root = next(b for b in p.boards if b.id == p.root) home_words = sum(1 for b in root.buttons if b.action == "speak" and not b.hidden) pages = sum(1 for b in root.buttons if b.action == "navigate" and not b.control) words = len({btn.key for b in p.boards for btn in b.buttons if btn.action == "speak" and not btn.hidden}) entry = { "key": p.file.rsplit("/", 1)[-1].removesuffix(".obz"), "file": p.file, "lang": "*", "variant": p.variant, "size": p.size, "rows": root.rows, "cols": root.cols, "boards": len(p.boards), "home": p.root, "words": words, "home_words": home_words, "pages": pages, } if set_name == "growing-core": entry["label"] = p.size.replace("x", " \u00d7 ") entry["description"] = f"{home_words} word{'' if home_words == 1 else 's'}" + ( f" and {pages} pages of more" if pages else "" ) if p.variant != "full": entry["reveal_stage"] = p.manifest["reveal_stage"] else: entry["label"] = f"{p.size} places" entry["description"] = f"{home_words} words on Home, {pages} pages, {words} words in all" if p.manifest.get("filter"): entry["filter"] = p.manifest["filter"] packages.append(entry) fam = { "key": set_name, **FAMILY_TEXT[set_name], "version": d["version"], "licence": "CC-BY-SA-4.0", "attribution": data.catalog["attribution"].strip(), "sources": SOURCES_SHORT, "languages": langs_by_set[set_name], "language_note": ( "The words are English, shown in your language through the app's translations, which are not " "your language's own core words. Words without a translation in your language are hidden in " "their places; a partner can add their own words there in Edit mode." ), "packages": packages, } if set_name == "growing-core": fam["ladder"] = d["ladder"] else: fam["recommends"] = {"return_home_after_word": True, "make_home": True} fams.append(fam) return {"families": fams} def main() -> None: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--check", action="store_true", help="fail if the design documents' tables are out of date") ap.add_argument("--out", type=Path, help="write the packages, families.json (for catalog.py), cells.json (for " "aac-core's tests) and the reports into this directory") args = ap.parse_args() data = Data() core = load_core(data) motor = load_motor(data) pkgs = core_packages(data, core) + motor_packages(data, motor) all_words = {btn.key for p in pkgs for b in p.boards for btn in b.buttons if btn.action == "speak"} exceptions = class_exceptions(data, all_words) cm = metrics.core_metrics(data, core, [p for p in pkgs if p.set == "growing-core"]) mm_ = metrics.motor_metrics(data, motor, [p for p in pkgs if p.set == "fixed-paths"]) mx = { "about": "Generated by tools/pagesets/build.py. Models and reference lists: DESIGN-core.md and DESIGN-motor.md.", "screens": {k: vars(v) for k, v in costs.SCREENS.items()}, "growing_core": cm, "fixed_paths": mm_, "endings_the_app_gets_wrong": metrics.endings_report(data, all_words), "word_class_differences": exceptions, } rep = langs.report(data, pkgs) def js(v) -> bytes: return (json.dumps(v, ensure_ascii=False, sort_keys=True, indent=1) + "\n").encode() # The design documents' tables: committed, and checked. docs: dict[Path, bytes] = {} ct = core_tables(cm, core, data) ct["languages"] = lang_table(rep) ct["endings"] = md_table(["Word", "Ending", "The app says", "Correct"], [[e["word"], e["ending"], e["app_says"], e["correct"]] for e in mx["endings_the_app_gets_wrong"]]) ct["class-differences"] = md_table(["Word", "These sets", "The app's vocabulary.rs", "Why"], [[w, v["ours"], v["app"], v["why"]] for w, v in exceptions.items()]) if exceptions else "None." no_sym = sorted({btn.key for p in pkgs if p.variant == "full" for b in p.boards for btn in b.buttons if not btn.control and not btn.symbol and not obf.lookup_for_cell(data, btn.key, btn.action)}, key=str.lower) ct["no-symbol"] = ", ".join(f"`{w}`" for w in no_sym) + f" ({len(no_sym)})" for doc, tables in ((HERE / "DESIGN-core.md", ct), (HERE / "DESIGN-motor.md", motor_tables(mm_, motor, data))): if doc.exists(): docs[doc] = fill(doc, tables).encode() if args.out: out: dict[Path, bytes] = {} for p in pkgs: out[args.out / p.file] = obf.obz(data, p.boards, p.root, p.manifest) out[args.out / "families.json"] = js(families_json(data, pkgs, rep)) out[args.out / "cells.json"] = js(index_json(pkgs, exceptions)) out[args.out / "metrics.json"] = js(mx) out[args.out / "languages.json"] = js(rep) out[args.out / "words.json"] = js(words_json(data, pkgs)) for path, content in out.items(): path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(content) total = sum(len(c) for p, c in out.items() if p.suffix == ".obz") print(f"wrote {len(out)} files to {args.out} ({len(pkgs)} packages, {total / 1e6:.1f} MB)") stale = [] for path, content in docs.items(): if args.check: if path.read_bytes() != content: stale.append(str(path.relative_to(HERE.parent.parent))) else: path.write_bytes(content) if args.check: if stale: print("out of date (run tools/pagesets/build.py):\n " + "\n ".join(stale)) sys.exit(1) print("the design documents' tables are up to date") if __name__ == "__main__": main()