#!/usr/bin/env python3 """Build the phrase-based page sets as Open Board Format packages (.obz). Four page sets, each written by hand as a human-readable source in sources/*.yaml and built here into OBF boards packed in .obz files: adult.yaml adults with acquired conditions (aphasia, ALS/MND, PPA, TBI, Parkinson's): one package, adult.obz intent.yaml intent-first vocabulary in 12, 20 and 40 cells: intent-12.obz, intent-20.obz, intent-40.obz themes.yaml themed phrase grids, 6 x 3: themes.obz emergent.yaml first switches and single messages: emergent.obz Generated packages are not committed: the site build makes them. What `--out DIR` writes (deterministic: the same sources give the same bytes): DIR/.obz English, with the pictures and every agreed translation on each button DIR//.obz the set in one language; .f.obz and .m.obz where the speaker's gender changes the words DIR/index.json the catalogue a page-set chooser reads: sets, sizes, descriptions, licence, every file with its size and sha256, per-language status DIR/families.json the same sets as families of the page-set catalogue (tools/pagesets/catalog.py's generator contract), so the catalogue can list them beside the core sets DIR/LICENSE.md CC BY-SA 4.0, with the symbols' attribution Commands: python3 build.py --out DIR [--hash-names] build into DIR (--hash-names: ..obz, for a cache that keeps files for hours) python3 build.py --check [--out DIR] validate; fail if two builds differ, if reports/ is stale, or (with --out) if DIR is stale python3 build.py --reports rewrite reports/*.md (committed review aids: contents, symbols wanted, translation status) python3 build.py --strings work/strings.json for translators Requires Python >= 3.11 and PyYAML. Reads the symbol index (crates/aac-core/src/symbols/index.json) and sprites (crates/aac-web/public/symbols/*.svg) the app ships; never the network. The data model, the extensions and the translation process are described in tools/pagesets/DESIGN-phrases-common.md. """ from __future__ import annotations import argparse import hashlib import html import io import json import re import sys import uuid import zipfile from dataclasses import dataclass, field from pathlib import Path from typing import Any import yaml HERE = Path(__file__).resolve().parent AAC_RS = HERE.parents[2] SOURCES = HERE / "sources" TRANSLATIONS = HERE / "translations" REPORTS = HERE / "reports" WORK = HERE / "work" SYMBOL_INDEX = AAC_RS / "crates" / "aac-core" / "src" / "symbols" / "index.json" SYMBOL_INDEX_NC = AAC_RS / "crates" / "aac-core" / "src" / "symbols" / "index-nc.json" SPRITES = AAC_RS / "crates" / "aac-web" / "public" / "symbols" OBF_FORMAT = "open-board-0.1" GENERATOR = "aac-rs/tools/pagesets/phrases/build.py" # Ids are UUIDv5 under this namespace, so a rebuilt board keeps its id. NAMESPACE = uuid.UUID("6f0f3a52-8a4e-5b55-9a43-7b1d2c0e5a11") # 1980-01-01, the earliest DOS date: fixed so a build is reproducible. ZIP_DATE = (1980, 1, 1, 0, 0, 0) # The ten largest app languages, in the order the work was done. The other # 34 are listed as pending in the catalogue. LANGUAGES = { "es": ("Spanish", "es-ES"), "fr": ("French", "fr-FR"), "de": ("German", "de-DE"), "pt": ("Portuguese (Brazil)", "pt-BR"), "it": ("Italian", "it-IT"), "zh": ("Chinese (Simplified)", "zh-CN"), "ja": ("Japanese", "ja-JP"), "ar": ("Arabic", "ar-SA"), "hi": ("Hindi", "hi-IN"), "ru": ("Russian", "ru-RU"), } PENDING_LANGUAGES = [ "nl", "ko", "he", "ur", "fa", "bn", "ta", "te", "ml", "mr", "kn", "gu", "pa", "th", "vi", "id", "ms", "tl", "pl", "uk", "ro", "cs", "sv", "da", "nb", "fi", "sw", "am", "ha", "yo", "zu", "tr", "el", "hu", ] AGREED = "agreed" STATUS_TEXT = "agreed by two model readers; native review pending" # Names that belong to other people's products. None may appear in a board's # words (the design notes may name them, to say what we did not copy). PROPRIETARY_NAMES = [ "PODD", "Pragmatic Organisation", "Pragmatic Organization", "TD Snap", "Snap Core", "Core First", "Tobii", "Dynavox", "Proloquo", "Crescendo", "LAMP", "Words for Life", "Unity", "Minspeak", "Gateway", "WordPower", "TouchChat", "Lingraphica", "Aphasia Duo", "Grid 3", "Super Core", "BIGmack", "Big Mack", "Step-by-Step", "Step by Step", "QuickFire", "Communication Journey", "PECS", "Boardmaker", "PCS", "SymbolStix", "Widgit", ] # Words that would read as diagnosis or treatment. The sets are for # wellness, research and clinical-trial use, not a medical device. CLAIM_WORDS = [ r"\bdiagnos\w*", r"\btreatments?\b", r"\btherap\w*", r"\bcure[sd]?\b", r"\bheal(s|ing|ed)?\b", r"\brehabilitat\w*", r"\brecover(y|s|ing|ed)?\b", r"\bclinically\b", r"\bimproves? your\b", r"\bslows? (the|your|down the)\b", r"\bprevents?\b", ] ACTIONS = {"clear": ":clear", "backspace": ":backspace", "speak": ":speak"} class SourceError(Exception): pass class PhraseLoader(yaml.SafeLoader): """YAML without YAML 1.1's other booleans: in a phrase source, yes, no, on and off are words (a cell named `no`, the message "Yes"), and only true and false are booleans.""" PhraseLoader.yaml_implicit_resolvers = { first: [(tag, regexp) for tag, regexp in resolvers if tag != "tag:yaml.org,2002:bool"] for first, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() } PhraseLoader.add_implicit_resolver( "tag:yaml.org,2002:bool", re.compile(r"^(?:true|false)$"), list("tf")) # ─── Symbols ──────────────────────────────────────────────────────────────── class Symbols: """The app's concept -> symbol index and its sprites.""" def __init__(self) -> None: index = json.loads(SYMBOL_INDEX.read_text()) self.sets: dict[str, dict] = index["sets"] self.concepts: dict[str, dict[str, str]] = index["concepts"] self._sprites: dict[str, str] = {} def resolve(self, concept: str, prefer: list[str]) -> tuple[str, str] | None: """(set, symbol id) for a concept, from the first preferred set that draws it.""" sets = self.concepts.get(concept) if not sets: return None for s in prefer: if s in sets: return s, sets[s] return None def svg(self, set_name: str, symbol_id: str) -> tuple[bytes, int, int]: """The symbol cut out of its sprite as a standalone SVG document (as aac_core::obf::extract_symbol_svg does), with its attribution.""" if set_name not in self._sprites: self._sprites[set_name] = (SPRITES / self.sets[set_name]["file"]).read_text() sprite = self._sprites[set_name] start = sprite.find(f'", start) tag = sprite[start:tag_end] body_end = sprite.index("", tag_end) body = sprite[tag_end + 1:body_end] view_box = re.search(r'viewBox="([^"]+)"', tag).group(1) dims = [float(n) for n in re.split(r"[ ,]+", view_box.strip())] width, height = max(1, int(-(-dims[2] // 1))), max(1, int(-(-dims[3] // 1))) note = f'{self.sets[set_name]["attribution"]} Adapted: optimised and cut from a sprite.'.replace("--", "-") doc = (f'{body}') return doc.encode(), width, height def license(self, set_name: str) -> dict: info = self.sets[set_name] return { "type": info["license"]["spdx"], "copyright_notice_url": info["license"]["url"], "source_url": info["source"]["url"], "author_name": info["name"], "ext_aurasense_attribution": info["attribution"], "ext_aurasense_conditions": info["conditions"], } # ─── Model ────────────────────────────────────────────────────────────────── @dataclass class Cell: row: int col: int label: str = "" say: str | None = None # Speak add: str | None = None # Buffer: add to the message without speaking go: str | None = None # board key do: str | None = None # clear | backspace | speak symbol: str | None = None # concept in the symbol index want_symbol: str | None = None # a concept the index does not have yet color: str | None = None border: str | None = None text_color: str | None = None fill: dict | None = None # ext_aurasense_fill_in seq: dict | None = None # ext_aurasense_sequence effect: str | None = None # ext_aurasense_effect src: str | None = None # source of the words, if not original note: str | None = None # context for translators origin: str = "" # where in the sources this cell came from @property def blank(self) -> bool: return not (self.label or self.say or self.add or self.go or self.do) @dataclass class Board: key: str name: str rows: int cols: int cells: list[Cell] = field(default_factory=list) parent: str | None = None popup: bool = False speak_only: bool = False description: str = "" page: int | None = None page_no: int | None = None # 2, 3 … on a section's continuation pages origin: str = "" def display(self, name: str) -> str: """The board's name (in any language) with its page number.""" return name if self.page_no is None else f"{name} {self.page_no}" @dataclass class PageSet: key: str # file name stem, e.g. adult, intent-20 family: str # source file stem, e.g. adult, intent name: str description: str version: str license: dict prefer: list[str] root: str boards: dict[str, Board] size: int | None = None # ─── Reading the sources ──────────────────────────────────────────────────── def hex_to_obf(color: str) -> str: c = color.strip().lstrip("#") if len(c) == 3: c = "".join(ch * 2 for ch in c) if not re.fullmatch(r"[0-9A-Fa-f]{6}", c): raise SourceError(f"not a #RRGGBB colour: {color!r}") r, g, b = (int(c[i:i + 2], 16) for i in (0, 2, 4)) return f"rgb({r}, {g}, {b})" class Reader: """Turns one source file into page sets.""" def __init__(self, path: Path) -> None: self.path = path self.doc = yaml.load(path.read_text(), Loader=PhraseLoader) self.meta = self.doc["set"] self.palette: dict[str, str] = self.doc.get("palette", {}) self.named: dict[str, Any] = self.doc.get("cells", {}) def color(self, value: str | None) -> str | None: if value is None: return None return self.palette.get(value, value) def cell(self, spec: Any, row: int, col: int, origin: str, inherited: dict | None = None) -> Cell: """A cell from its source form: None (blank), a string (a message: shown and said), "$name" (a named cell), or a mapping.""" if spec is None: return Cell(row, col, origin=origin) if isinstance(spec, str) and spec.startswith("$"): name = spec[1:] if name not in self.named: raise SourceError(f"{origin}: no named cell ${name}") return self.cell(self.named[name], row, col, origin, inherited) if isinstance(spec, str): spec = {"say": spec} if not isinstance(spec, dict): raise SourceError(f"{origin}: a cell is a string, $name, ~ or a mapping, not {spec!r}") if "use" in spec: base = self.named.get(spec["use"]) if base is None: raise SourceError(f"{origin}: no named cell {spec['use']}") merged = dict(base if isinstance(base, dict) else {"say": base}) merged.update({k: v for k, v in spec.items() if k != "use"}) spec = merged known = {"label", "say", "add", "go", "do", "symbol", "want_symbol", "color", "border", "text_color", "fill", "seq", "effect", "src", "note", "min_size"} unknown = set(spec) - known if unknown: raise SourceError(f"{origin}: unknown cell keys {sorted(unknown)}") style = dict(inherited or {}) c = Cell( row, col, label=str(spec.get("label") or spec.get("say") or spec.get("add") or ""), say=spec.get("say"), add=spec.get("add"), go=spec.get("go"), do=spec.get("do"), symbol=spec.get("symbol"), want_symbol=spec.get("want_symbol"), color=self.color(spec.get("color", style.get("color"))), border=self.color(spec.get("border", style.get("border"))), text_color=self.color(spec.get("text_color", style.get("text_color"))), fill=spec.get("fill"), seq=spec.get("seq"), effect=spec.get("effect"), src=spec.get("src"), note=spec.get("note"), origin=origin, ) if c.seq is not None and not c.label: raise SourceError(f"{origin}: a sequence cell needs a label") if sum(x is not None for x in (c.say, c.add, c.do)) > 1: raise SourceError(f"{origin}: a cell says, adds or acts — one of them") if c.do is not None and c.do not in ACTIONS: raise SourceError(f"{origin}: unknown action {c.do!r}") if c.go is None and c.do is None and c.add is None and c.say is None and c.label and c.seq is None: c.say = c.label if c.seq is not None and c.say is None: # Fallback for a client without the extension: say the first line. first = c.seq["lines"][0] c.say = first["say"] if isinstance(first, dict) else (None if first.startswith("@") else first) return c def board(self, key: str, spec: dict, rows: int, cols: int, frame: list[dict] | None = None) -> Board: origin = f"{self.path.name}#{key}" b = Board( key=key, name=spec["name"], rows=rows, cols=cols, parent=spec.get("parent"), popup=bool(spec.get("popup", False)), speak_only=bool(spec.get("speak_only", False)), description=spec.get("description", ""), origin=origin, ) style = spec.get("style") grid: list[list[Any]] = [[None] * cols for _ in range(rows)] for f in frame or []: r, c = f["at"] grid[r][c] = f["cell"] if "rows" in spec: if len(spec["rows"]) > rows: raise SourceError(f"{origin}: {len(spec['rows'])} rows in a {rows}-row grid") first = 0 if frame and spec.get("frame_row", True) and any(f["at"][0] == 0 for f in frame): first = 1 if len(spec["rows"]) < rows else 0 for r, row in enumerate(spec["rows"], start=first): if len(row) > cols: raise SourceError(f"{origin}: row {r} has {len(row)} cells in a {cols}-column grid") for c, item in enumerate(row): if item is not None: if grid[r][c] is not None: raise SourceError(f"{origin}: ({r},{c}) is the frame's") grid[r][c] = item for r in range(rows): for c in range(cols): cell = self.cell(grid[r][c], r, c, f"{origin}({r},{c})", style) if not cell.blank: b.cells.append(cell) return b def page_set(self, key: str, boards: dict[str, Board], size: int | None = None) -> PageSet: m = self.meta # A page set is shared under CC BY-SA and may be offered anywhere: it # draws only the open sets, never one licensed for non-commercial use # only (index-nc.json: ARASAAC, Sclera). nc = json.loads(SYMBOL_INDEX_NC.read_text())["sets"] if SYMBOL_INDEX_NC.exists() else {} open_sets = json.loads(SYMBOL_INDEX.read_text())["sets"] if bad := [s for s in m["symbols"] if s in nc or s not in open_sets]: raise SourceError(f"{self.path.name}: symbols {bad}: a page set draws only the open sets {sorted(open_sets)}") return PageSet( key=key, family=self.path.stem, name=m["name"] if size is None else m["name"].format(size=size), description=m["description"].strip() if size is None else m["description"].strip().format(size=size), version=str(m["version"]), license=m["license"], prefer=m["symbols"], root=m["root"], boards=boards, size=size, ) def read_plain(path: Path) -> list[PageSet]: """A set whose boards are each laid out by hand (adult, themes, emergent).""" rd = Reader(path) frames = rd.doc.get("frames", {}) boards: dict[str, Board] = {} for key, spec in rd.doc["boards"].items(): rows, cols = spec.get("grid", rd.meta.get("grid")) frame_name = spec.get("frame", rd.meta.get("frame")) frame = frames.get(frame_name) if frame_name else None if key in boards: raise SourceError(f"{path.name}: board {key} twice") boards[key] = rd.board(key, spec, rows, cols, frame) finish(boards, rd.meta["root"], path.name) return [rd.page_set(rd.path.stem, boards)] def read_intent(path: Path) -> list[PageSet]: """The intent-first set: sections of items flowed into pages, for each grid size, inside that size's frame of operational commands.""" rd = Reader(path) out = [] for size, layout in rd.doc["sizes"].items(): size = int(size) rows, cols = layout["grid"] frame: list[dict] = layout["frame"] turn = tuple(layout["turn"]) taken = {tuple(f["at"]) for f in frame} content = [(r, c) for r in range(rows) for c in range(cols) if (r, c) not in taken] boards: dict[str, Board] = {} sections = rd.doc["sections"] for key, sec in sections.items(): items = sec.get("items_by_size", {}).get(size, sec.get("items_by_size", {}).get(str(size))) if items is None: items = [it for it in sec["items"] if not (isinstance(it, dict) and it.get("min_size", 0) > size)] pages = paginate(items, content, turn) for n, page_items in enumerate(pages): bkey = key if n == 0 else f"{key}~{n + 1}" b = Board(key=bkey, name=sec["name"], rows=rows, cols=cols, page_no=n + 1 if n else None, parent=sec.get("parent", rd.meta["root"]), popup=bool(sec.get("popup", False)), description=sec.get("description", ""), origin=f"{path.name}#{key}@{size}") if key == rd.meta["root"]: b.parent = None for f in frame: cell = rd.cell(f["cell"], *f["at"], origin=f"{path.name}#frame@{size}") if cell.go == "@back" and b.parent is None: continue if cell.go == "@back": cell.go = b.parent if cell.go == bkey: # A board does not link to itself: the place stays empty. continue b.cells.append(cell) for (r, c), item in zip(content, page_items): cell = rd.cell(item, r, c, f"{path.name}#{key}@{size}", sec.get("style")) if not cell.blank: b.cells.append(cell) if len(pages) > 1: nxt = key if n + 1 == len(pages) else f"{key}~{n + 2}" t = rd.cell("$turn", *turn, origin=f"{path.name}#turn@{size}") t.go = nxt b.cells.append(t) boards[bkey] = b finish(boards, rd.meta["root"], path.name) ps = rd.page_set(f"{path.stem}-{size}", boards, size) out.append(ps) return out def finish(boards: dict[str, Board], root: str, source: str) -> None: """Settle what a board can only know once the whole set is read: its parent (declared, or the first board that links to it), what "@back" links to, and the cells a sequence's "@row,col" lines name.""" first_linker: dict[str, str] = {} for key, b in boards.items(): for c in sorted(b.cells, key=lambda c: (c.row, c.col)): if c.go and c.go != "@back" and c.go != key: first_linker.setdefault(c.go, key) for key, b in boards.items(): if b.parent is None and key != root: b.parent = first_linker.get(key.split("~")[0], first_linker.get(key)) kept = [] for c in b.cells: if c.go == "@back": if b.parent is None: continue c.go = b.parent kept.append(c) b.cells = kept for c in b.cells: if not c.seq: continue lines = [] for line in c.seq["lines"]: if isinstance(line, str) and line.startswith("@"): r, col = (int(x) for x in line[1:].split(",")) target = next((x for x in b.cells if (x.row, x.col) == (r, col)), None) if target is None or target.say is None: raise SourceError(f"{c.origin}: sequence line {line} names no speaking cell") lines.append({"say": target.say, "cell": [r, col]}) elif isinstance(line, str): lines.append({"say": line}) else: lines.append(dict(line)) c.seq = {**c.seq, "lines": lines} c.say = lines[0]["say"] def paginate(items: list, content: list[tuple[int, int]], turn: tuple[int, int]) -> list[list]: """Split items into pages. A page that continues gives up its turn-the-page cell; every page keeps the same positions for the same slots.""" if len(items) <= len(content): return [items] per_page = [pos for pos in content if pos != turn] pages = [] for i in range(0, len(items), len(per_page)): chunk = items[i:i + len(per_page)] # Re-insert a hole at the turn cell's slot so positions stay fixed. slotted = [] it = iter(chunk) for pos in content: if pos == turn: slotted.append(None) else: slotted.append(next(it, None)) pages.append(slotted) return pages def read_all() -> list[PageSet]: sets: list[PageSet] = [] for name in ("adult", "intent", "themes", "emergent"): path = SOURCES / f"{name}.yaml" sets.extend(read_intent(path) if name == "intent" else read_plain(path)) return sets # ─── Validation ───────────────────────────────────────────────────────────── def texts_of(cell: Cell) -> list[str]: out = [cell.label] for t in (cell.say, cell.add): if t and t != cell.label: out.append(t) if cell.seq: for line in cell.seq["lines"]: out.append(line["say"]) if cell.fill and cell.fill.get("hint"): out.append(cell.fill["hint"]) return [t for t in out if t] def validate(sets: list[PageSet], symbols: Symbols) -> list[str]: """Every problem found, as text. Empty means the sources are sound.""" errs: list[str] = [] for ps in sets: if ps.root not in ps.boards: errs.append(f"{ps.key}: root board {ps.root} is missing") linked: set[str] = set() for b in ps.boards.values(): seen: set[tuple[int, int]] = set() for c in b.cells: if not (0 <= c.row < b.rows and 0 <= c.col < b.cols): errs.append(f"{c.origin}: outside the {b.rows}x{b.cols} grid") if (c.row, c.col) in seen: errs.append(f"{c.origin}: two cells at ({c.row},{c.col})") seen.add((c.row, c.col)) if c.go is not None: if c.go not in ps.boards: errs.append(f"{c.origin}: links to {c.go}, which is not in {ps.key}") linked.add(c.go) if c.symbol is not None and symbols.resolve(c.symbol, ps.prefer) is None: errs.append(f"{c.origin}: symbol concept {c.symbol!r} is not drawn by {ps.prefer}") if c.seq is not None: lines = c.seq.get("lines") or [] if len(lines) < 2: errs.append(f"{c.origin}: a sequence needs two lines or more") if c.seq.get("order", "in_order") not in ("in_order", "shuffle"): errs.append(f"{c.origin}: sequence order is in_order or shuffle") if c.seq.get("at_end", "start_again") not in ("start_again", "stay"): errs.append(f"{c.origin}: sequence at_end is start_again or stay") for text in texts_of(c): for name in PROPRIETARY_NAMES: if re.search(rf"(?.json: only entries both readers agreed on are used.""" def __init__(self) -> None: self.by_lang: dict[str, dict[str, dict]] = {} for code in LANGUAGES: path = TRANSLATIONS / f"{code}.json" if path.exists(): self.by_lang[code] = json.loads(path.read_text())["strings"] else: self.by_lang[code] = {} def get(self, lang: str, english: str) -> Translation | None: e = self.by_lang.get(lang, {}).get(english) if not e or e.get("status") != AGREED: return None if e.get("forms"): return Translation(None, dict(e["forms"])) return Translation(e["text"], None) def status(self, lang: str, english: str) -> str: e = self.by_lang.get(lang, {}).get(english) return e.get("status", "pending") if e else "pending" def gendered(self, lang: str, texts: list[str]) -> bool: return any((t := self.get(lang, x)) is not None and t.forms for x in texts) def words(self, lang: str, english: str, speaker: str | None) -> str | None: t = self.get(lang, english) if t is None: return None if t.forms: return t.forms.get(speaker) if speaker else None return t.text # Board descriptions in a localized package: a visible note that some words # are still in English. Its translation is in each language file under this key. NOTE_KEY = "Some messages on this board are still in English: their translation has not yet been agreed by two readers." # ─── Writing OBF ──────────────────────────────────────────────────────────── def bid(ps: PageSet, board: str) -> str: return str(uuid.uuid5(NAMESPACE, f"{ps.key}/{board}")) def cid(ps: PageSet, board: str, cell: Cell) -> str: return str(uuid.uuid5(NAMESPACE, f"{ps.key}/{board}/{cell.row},{cell.col}")) def board_path(ps: PageSet, board: str) -> str: return f"boards/{bid(ps, board)}.obf" def book_pages(ps: PageSet) -> dict[str, int]: """Page numbers for a printed book: breadth-first from the root in the order links appear (row by row), continuation pages straight after their first page.""" order: list[str] = [] queue = [ps.root] while queue: key = queue.pop(0) if key in order: continue order.append(key) base = key.split("~")[0] conts = sorted((k for k in ps.boards if k.startswith(base + "~")), key=lambda k: int(k.split("~")[1])) for k in conts: if k not in order: order.append(k) b = ps.boards[key] for c in sorted(b.cells, key=lambda c: (c.row, c.col)): if c.go and c.go not in order and c.go not in queue: queue.append(c.go) for k in sorted(ps.boards): if k not in order: order.append(k) return {k: n + 1 for n, k in enumerate(order)} def same_words(text: str) -> str: """Words as a person reads them, to tell whether two cells show the same (pslib/samewords.py `normal`).""" import unicodedata return " ".join(unicodedata.normalize("NFKC", text).casefold().split()) class Package: """One .obz: boards in one language (or English with translations).""" def __init__(self, ps: PageSet, symbols: Symbols, tr: Translations, lang: str | None, speaker: str | None): self.ps = ps self.symbols = symbols self.tr = tr self.lang = lang # None: the English package with every agreed translation self.speaker = speaker self.images: dict[str, bytes] = {} self.pages = book_pages(ps) self.counts = {"cells": 0, "texts": 0, "in_english": 0} @property def locale(self) -> str: return "en-US" if self.lang is None else LANGUAGES[self.lang][1] def words(self, english: str) -> tuple[str, bool]: """The words in this package's language, and whether they had to stay in English.""" if self.lang is None: return english, False t = self.tr.words(self.lang, english, self.speaker) return (english, True) if t is None else (t, False) @staticmethod def cell_texts(cell: Cell) -> list[str]: """The texts of a cell that must all be in one language: what it shows, what it says, and the lines of its sequence.""" texts = [cell.label] spoken = cell.say if cell.say is not None else cell.add if spoken is not None and spoken != cell.label: texts.append(spoken) if cell.seq: texts += [line["say"] for line in cell.seq["lines"]] return texts def localize(self, cell: Cell, lang: str, speaker: str | None) -> dict[str, str] | None: """Every text of the cell in `lang`, or None if any is not agreed: a cell is shown and said in one language, never half in English.""" out = {} for t in self.cell_texts(cell): w = self.tr.words(lang, t, speaker) if w is None: return None if w == "" and (t == cell.label or cell.go is None): # Empty words are only for a link that needs no words in # this language; a button never shows a blank label. return None out[t] = w return out def form_entry(self, cell: Cell, loc: dict[str, str]) -> dict: """A cell's words in one speaker form: what it shows, says, and the lines of its sequence.""" spoken = cell.say if cell.say is not None else cell.add e: dict[str, Any] = {"label": loc[cell.label]} if spoken is not None and spoken != cell.label: e["vocalization"] = loc[spoken] if cell.seq: e["lines"] = [loc[line["say"]] for line in cell.seq["lines"]] return e def button_translations(self, cell: Cell) -> dict: """`translations` for the English package: the agreed words of each language, with the speaker's two forms where they differ.""" spoken = cell.say if cell.say is not None else cell.add def entry(loc: dict[str, str]) -> dict: e: dict[str, Any] = {"label": loc[cell.label]} if spoken is not None and spoken != cell.label: e["vocalization"] = loc[spoken] if cell.seq: e["ext_aurasense_sequence_lines"] = [loc[line["say"]] for line in cell.seq["lines"]] return e out: dict[str, Any] = {"en": entry({t: t for t in self.cell_texts(cell)})} for code in LANGUAGES: plain = self.localize(cell, code, None) if plain is not None: out[code] = entry(plain) continue f, m = self.localize(cell, code, "f"), self.localize(cell, code, "m") if f is not None and m is not None: out[code] = {"ext_aurasense_speaker_forms": {"f": entry(f), "m": entry(m)}} return out def button(self, b: Board, cell: Cell, n: int) -> tuple[dict, tuple[str, dict] | None]: ps = self.ps spoken_src = cell.say if cell.say is not None else cell.add loc = None if self.lang is None else self.localize(cell, self.lang, self.speaker) def tx(text: str) -> str: return loc[text] if loc else text self.counts["cells"] += 1 btn: dict[str, Any] = {"id": cid(ps, b.key, cell), "label": tx(cell.label)} if spoken_src is not None and cell.go is not None and tx(spoken_src) == "": # A link's words that this language does not need (the English # "to" of "I want" + "to" + "watch TV"): the link alone. spoken_src = None if spoken_src is not None: btn["vocalization"] = tx(spoken_src) prov: dict[str, Any] = {"content": "text", "source": cell.origin, "source_label": btn["label"]} if spoken_src is not None and btn["vocalization"] != btn["label"]: prov["source_vocalization"] = btn["vocalization"] if self.lang is not None: if loc: # The words of this language, kept exactly as written (the # app's own-words cell, Cell::lang): never glossed. prov["words_locale"] = self.locale prov["translation"] = STATUS_TEXT prov["english"] = " | ".join(dict.fromkeys(self.cell_texts(cell))) else: self.counts["in_english"] += 1 btn["label_locale"] = "en" if spoken_src is not None: btn["vocalization_locale"] = "en" prov["translation"] = "pending: shown and said in English" if loc and self.speaker is not None: # Words that change with who says them carry both forms, so # the app can put the person's there, or leave the word out # while they would rather not say (aac-web pagesets.rs). f, m = self.localize(cell, self.lang, "f"), self.localize(cell, self.lang, "m") if f is not None and m is not None and f != m: btn["ext_aurasense_speaker_forms"] = {"f": self.form_entry(cell, f), "m": self.form_entry(cell, m)} if cell.src: prov["words_source"] = cell.src if cell.add is not None: btn["ext_aurasense_action"] = "buffer" if cell.go is not None: target = ps.boards[cell.go] tname = target.display(self.words(target.name)[0]) btn["load_board"] = {"id": bid(ps, cell.go), "name": tname, "path": board_path(ps, cell.go)} if spoken_src is not None and cell.say is not None: # Words and a link: the client adds these words, then opens the # board. The words themselves, not a flag: a reader that maps the # button to a link keeps no vocalization, but keeps this field. btn["ext_aurasense_link_says"] = btn["vocalization"] btn["ext_aurasense_book_goto"] = self.pages[cell.go] if cell.do is not None: btn["action"] = ACTIONS[cell.do] image = None if cell.symbol is not None: btn["ext_aurasense_symbol"] = cell.symbol if cell.symbol is not None and self.lang is None: # The pictures travel in the English package only (any OBF app # can show them); a language's package names each symbol by its # concept, which the app draws from its own symbol sets. set_name, sym_id = self.symbols.resolve(cell.symbol, ps.prefer) image_id = f"{sym_id}.{n}" path = f"images/{sym_id}.svg" data, w, h = self.symbols.svg(set_name, sym_id) self.images[path] = data image = (image_id, { "id": image_id, "content_type": "image/svg+xml", "width": w, "height": h, "path": path, "symbol": {"set": set_name, "filename": f"{sym_id}.svg"}, "license": self.symbols.license(set_name), }) btn["image_id"] = image_id elif cell.symbol is None and cell.want_symbol: prov["symbol_wanted"] = cell.want_symbol if cell.color: btn["background_color"] = hex_to_obf(cell.color) if cell.border: btn["border_color"] = hex_to_obf(cell.border) btn["ext_aurasense_text_color"] = cell.text_color or "#000000" if cell.fill: fill = {k: v for k, v in cell.fill.items() if k != "hint"} if "hint" in cell.fill: hint, hint_en = self.words(cell.fill["hint"]) fill["hint"] = hint if hint_en and self.lang is not None: fill["hint_locale"] = "en" btn["ext_aurasense_fill_in"] = fill if cell.seq: lines = [] for line in cell.seq["lines"]: out_line: dict[str, Any] = {"say": tx(line["say"])} if "cell" in line: # The cell by its place, which never changes (SPEC §3); # its id may (a set added twice gets new ids). out_line["at"] = list(line["cell"]) lines.append(out_line) seq = { "version": 1, "lines": lines, "order": cell.seq.get("order", "in_order"), "at_end": cell.seq.get("at_end", "start_again"), "add_to_message": bool(cell.seq.get("add_to_message", False)), } if "reset_after_minutes" in cell.seq: seq["reset_after_minutes"] = cell.seq["reset_after_minutes"] btn["ext_aurasense_sequence"] = seq if cell.effect: btn["ext_aurasense_effect"] = cell.effect if self.lang is None: tr = self.button_translations(cell) if len(tr) > 1: btn["translations"] = tr btn["ext_aurasense_locked"] = False btn["ext_aurasense_provenance"] = prov return btn, image def board(self, b: Board, n: int) -> dict: ps = self.ps order: list[list[str | None]] = [[None] * b.cols for _ in range(b.rows)] buttons: list[dict] = [] images: dict[str, dict] = {} before = self.counts["in_english"] # One word, one place (pslib/samewords.py; proposed SPEC §4.1 item 5): where # this language says two of the board's phrases with the same words # ("No" and "not" are both "no" in Spanish; "I feel" and "I'm # feeling" both "Me siento"), they show once, at the first's place, # and the other is hidden in its place (`ext_aurasense_same_word`: # whatever language the app is in, since these words are written in # the package's): no cell moves. A slot a partner fills in is told # apart by what is filled in; a link is never hidden (the build fails # instead, for a person to settle). shown: dict[str, str] = {} for cell in sorted(b.cells, key=lambda c: (c.row, c.col)): btn, image = self.button(b, cell, n) words = same_words(btn.get("label") or "") if self.lang is not None and words and "ext_aurasense_fill_in" not in btn: if words in shown: if cell.go is not None or "load_board" in btn: raise SourceError(f"{b.origin}: in {self.lang}, the link “{btn['label']}” shows the same words as “{shown[words]}”") btn["ext_aurasense_same_word"] = {"*": shown[words]} else: shown[words] = btn["label"] order[cell.row][cell.col] = btn["id"] buttons.append(btn) if image: images.setdefault(image[0], image[1]) name, name_en = self.words(b.name) name = b.display(name) parts = [] if b.description: parts.append(self.words(b.description)[0]) if self.lang is not None and (name_en or self.counts["in_english"] > before): # The visible note: some of this board is still in English. parts.append(self.words(NOTE_KEY)[0]) prov: dict[str, Any] = { "generator": GENERATOR, "source_locale": "en", "source_name": name, "board_type": "Grid", "created_at": f"{ps.version}T00:00:00+00:00", "set": ps.key, "source": b.origin, } if self.lang is not None: if name_en: prov["name_locale"] = "en" else: prov["words_locale"] = self.locale prov["english_name"] = b.name out: dict[str, Any] = { "format": OBF_FORMAT, "id": bid(ps, b.key), "locale": self.locale, "name": name, "license": ps.license, "grid": {"rows": b.rows, "columns": b.cols, "order": order}, "buttons": buttons, "images": [images[k] for k in sorted(images)], "sounds": [], "ext_aurasense_locked": False, "ext_aurasense_provenance": prov, } if parts: out["description_html"] = "".join(f"

{html.escape(p)}

" for p in parts) behaviour = {} if b.popup: behaviour["after_choice"] = "return" if b.speak_only: behaviour["message"] = "speak_only" if behaviour: out["ext_aurasense_behaviour"] = behaviour out["ext_aurasense_book"] = {"page": self.pages[b.key]} return out def build(self) -> bytes: ps = self.ps keys = [ps.root] + sorted((k for k in ps.boards if k != ps.root), key=lambda k: self.pages[k]) files: list[tuple[str, bytes]] = [] board_paths = {} image_paths: dict[str, str] = {} for n, key in enumerate(keys, start=1): v = self.board(ps.boards[key], n) for img in v["images"]: image_paths[img["id"]] = img["path"] board_paths[bid(ps, key)] = board_path(ps, key) files.append((board_path(ps, key), json.dumps(v, ensure_ascii=False, separators=(",", ":")).encode())) for path in sorted(self.images): files.append((path, self.images[path])) manifest = { "format": OBF_FORMAT, "root": board_path(ps, ps.root), "paths": {"boards": board_paths, "images": dict(sorted(image_paths.items())), "sounds": {}}, "ext_aurasense_page_set": { "key": ps.key, "name": ps.name, "version": ps.version, "locale": self.locale, "speaker_forms": self.speaker, "generator": GENERATOR, }, } files.insert(0, ("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=1).encode())) return stored_zip(files) def stored_zip(files: list[tuple[str, bytes]]) -> bytes: """A zip of uncompressed entries with a fixed date (what the app's own importer reads without inflating).""" buf = io.BytesIO() with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_STORED) as z: for name, data in files: info = zipfile.ZipInfo(name, date_time=ZIP_DATE) info.compress_type = zipfile.ZIP_STORED info.external_attr = 0o644 << 16 info.create_system = 0 z.writestr(info, data) return buf.getvalue() # ─── Strings, catalogue and reports ───────────────────────────────────────── def all_strings(sets: list[PageSet]) -> dict[str, dict]: """Every English text a translator must see, with where it is used.""" out: dict[str, dict] = {} def add(text: str, where: str, role: str, note: str | None = None) -> None: e = out.setdefault(text, {"uses": [], "roles": [], "notes": []}) if where not in e["uses"] and len(e["uses"]) < 4: e["uses"].append(where) if role not in e["roles"]: e["roles"].append(role) if note and note not in e["notes"]: e["notes"].append(note) for ps in sets: for b in ps.boards.values(): add(b.name, f"{ps.family}: board name", "board name") if b.description: add(b.description, f"{ps.family} > {b.name}", "a note to the partner, shown with the board") for c in b.cells: where = f"{ps.family} > {b.name}" add(c.label, where, "label (shown on the cell)" if (c.say or c.add) != c.label else "label and spoken message", c.note) spoken = c.say if c.say is not None else c.add if spoken and spoken != c.label: add(spoken, where, "spoken message" if c.say else "words added to the message", c.note) if c.seq: for line in c.seq["lines"]: add(line["say"], f"{where} > {c.label}", "a line of a sequence said one press at a time", c.note) if c.fill and c.fill.get("hint"): add(c.fill["hint"], where, "hint to the partner who sets the cell up") add(NOTE_KEY, "every board", "a note shown to the partner") return dict(sorted(out.items())) FAMILIES = { "adult": { "style": "Adults with acquired conditions", "offer": "Adults whose talking has changed (aphasia, ALS/MND, primary progressive aphasia, brain injury, " "Parkinson's). Offer \"Make Simple page my home\" as a later choice.", }, "intent": { "style": "Intent-first vocabulary", "offer": "Anyone who starts from what they want to do; choose 12, 20 or 40 cells a page. Also printable " "as a book a partner reads from.", }, "themes": { "style": "Themed phrase grids", "offer": "Readers who want the likely messages of a situation on one page.", }, "emergent": { "style": "First switches and single messages", "offer": "First communicators and first switch users; the index pages are for the partner.", }, } def catalogue(sets: list[PageSet], built: dict[str, dict], tr: Translations, strings: dict) -> dict: """The catalogue a page-set chooser reads: every set, its files in every language (with size and checksum), and how far each language goes.""" langs = {} for code, (name, locale) in LANGUAGES.items(): agreed = sum(1 for s in strings if tr.status(code, s) == AGREED) gendered = any((t := tr.get(code, s)) is not None and t.forms for s in strings) langs[code] = {"language": name, "locale": locale, "agreed_texts": agreed, "texts": len(strings), "speaker_forms": gendered, "status": STATUS_TEXT} families = [] for fam, meta in FAMILIES.items(): members = [ps for ps in sets if ps.family == fam] families.append({ "family": fam, "style": meta["style"], "name": members[0].name.replace(" ({size})", "").split(" (")[0], "description": (members[0].description.replace( f"{members[0].size} cells", ", ".join(str(ps.size) for ps in members[:-1]) + f" or {members[-1].size} cells") if members[0].size else members[0].description), "sets": [ps.key for ps in members], "sizes": [ps.size for ps in members] if members[0].size else None, "offer": meta["offer"], }) return { "version": 1, "generator": GENERATOR, "about": ("Phrase page sets for aac.aurasense.ai/app and any Open Board Format app, built from " "tools/pagesets/phrases/sources. English is the source; a translation is used only where two " "independent model readers agreed on it, and a native speaker's review is still to come. A cell " "without an agreed translation stays in English and says so (label_locale \"en\"). Where " "files[lang] has f and m, choose by the person's \"words about myself\" setting."), "license": {"type": "CC-BY-SA-4.0", "url": "https://creativecommons.org/licenses/by-sa/4.0/", "author": "AuraSense Tech Corporation and the open-standards AAC contributors", "file": "LICENSE.md"}, "families": families, "sets": [ { "key": ps.key, "family": ps.family, "name": ps.name, "description": ps.description, "size": ps.size, "boards": len(ps.boards), "cells": sum(len(b.cells) for b in ps.boards.values()), "files": built[ps.key], } for ps in sets ], "languages": langs, "pending_languages": PENDING_LANGUAGES, } # The families as the page-set catalogue (tools/pagesets/catalog.py) names # them: its generator contract is `build.py --out DIR` writing # DIR/families.json beside the packages. CATALOGUE_FAMILIES = { "adult": {"key": "phrases-adult", "name": "Adult conversation"}, "intent": {"key": "phrases-intent", "name": "Intent first"}, "themes": {"key": "phrases-themes", "name": "Themed phrases"}, "emergent": {"key": "phrases-emergent", "name": "First switches and messages"}, } ATTRIBUTION = ("Phrase page sets \u00a9 2026 AuraSense Tech Corporation and the open-standards AAC contributors, " "CC BY-SA 4.0. Symbols are named from Mulberry Symbols (Steve Lee; designed by Claire Barge) and " "OpenMoji, each CC BY-SA 4.0, with their own attributions on each image. The 40-cell intent-first " "start page uses 21 words of the Universal Core vocabulary (Center for Literacy and Disability " "Studies, UNC Chapel Hill; Project Core), words only.") LANGUAGE_NOTE = ("Written in English. In Spanish, French, German, Portuguese, Italian, Chinese, Japanese, Arabic, Hindi " "and Russian the set comes in that language, where two independent AI models agreed on each " "translation (a native speaker's review is still to come); anything not agreed stays in English " "and says so. In other languages it is in English for now.") def families(sets: list[PageSet], built: dict[str, dict]) -> dict: """The sets as families of the page-set catalogue (catalog.py's generator contract): one package per set, language and speaker form, each with its language (`lang`), and `gender` "f" / "m" where the words change with the speaker's; `size` is the cells a page for the intent-first sizes.""" out = [] for fam, meta in FAMILIES.items(): cat = CATALOGUE_FAMILIES[fam] members = [ps for ps in sets if ps.family == fam] packages = [] cells: dict[str, list[int]] = {} for ps in members: root = ps.boards[ps.root] for code, entry in built[ps.key].items(): forms = entry.items() if "file" not in entry else [(None, entry)] for gender, f in forms: key = f"{cat['key']}-{ps.key.removeprefix(fam).lstrip('-') or 'set'}-{code}" + (f"-{gender}" if gender else "") packages.append({ "key": key.replace("-set-", "-"), "file": f["file"], "lang": code, "gender": gender, "variant": "full", "size": str(ps.size) if ps.size else "", "rows": root.rows, "cols": root.cols, "label": f"{ps.size} cells a page" if ps.size else ps.name, "description": ps.description, "home": bid(ps, ps.root), "boards": len(ps.boards), "cells": f["cells"], "cells_in_english": f["cells_in_english"], }) if code != "en" and gender in (None, "f"): c = cells.setdefault(code, [0, 0]) c[0] += f["cells"] c[1] += f["cells_in_english"] languages = {code: {"status": "GlossOnly", "glossed": round(1 - english / total, 3)} for code, (total, english) in sorted(cells.items()) if total} sources = [{"name": "Phrases written for this project (AuraSense; open-standards AAC)", "licence": "CC-BY-SA-4.0"}, {"name": "Mulberry Symbols, OpenMoji", "licence": "CC-BY-SA-4.0"}] if fam == "intent": sources.insert(1, {"name": "Universal Core vocabulary (UNC CLDS / Project Core), 21 words, words only", "licence": "CC-BY-SA-4.0"}) description = (members[0].description.replace( f"{members[0].size} cells", ", ".join(str(ps.size) for ps in members[:-1]) + f" or {members[-1].size} cells") if members[0].size else members[0].description) out.append({ "key": cat["key"], "name": cat["name"], "summary": meta["style"], "description": description, "chooser": "phrases", "version": members[0].version, "licence": "CC-BY-SA-4.0", "attribution": ATTRIBUTION, "sources": sources, "languages": languages, "language_note": LANGUAGE_NOTE, "offer": meta["offer"], "packages": packages, }) return {"families": out} def report_symbols(sets: list[PageSet], symbols: Symbols) -> str: wanted: dict[str, list[str]] = {} used: dict[str, set[str]] = {} for ps in sets: for b in ps.boards.values(): for c in b.cells: if c.want_symbol: wanted.setdefault(c.want_symbol, []) where = f"{ps.family}: {c.label}" if where not in wanted[c.want_symbol]: wanted[c.want_symbol].append(where) if c.symbol: s, _ = symbols.resolve(c.symbol, ps.prefer) used.setdefault(c.symbol, set()).add(s) lines = [ "# Symbols for the phrase page sets", "", "Generated by `build.py --reports`; do not edit.", "", f"{len(used)} concepts of the app's symbol index are used. {len(wanted)} concepts are wanted but the " "index has no symbol for them yet; those cells show their words alone (an adult set may use the " "person's own photo instead). Add them to `tools/symbols` (a confident match only: a wrong picture " "is worse than none) and name them in the sources with `symbol:` once they exist.", "", "## Wanted (no symbol yet)", "", "| Concept | Used for |", "|---|---|", ] for concept in sorted(wanted): uses = wanted[concept] lines.append(f"| `{concept}` | {'; '.join(uses[:4])}{' …' if len(uses) > 4 else ''} |") lines += ["", "## Used", "", "| Concept | Set |", "|---|---|"] for concept in sorted(used): lines.append(f"| `{concept}` | {', '.join(sorted(used[concept]))} |") return "\n".join(lines) + "\n" def report_translations(strings: dict, tr: Translations) -> str: lines = [ "# Translation status of the phrase page sets", "", "Generated by `build.py --reports`; do not edit. Each text was drafted, then back-translated by a " "second reader who never saw the English, and the two compared. Only texts both agreed on are used " f"(\"{STATUS_TEXT}\"); the rest stay in English, marked as such in every package.", "", "| Language | Agreed | Disputed | Not yet translated | Speaker-gendered |", "|---|---|---|---|---|", ] for code, (name, _) in LANGUAGES.items(): st = [tr.status(code, s) for s in strings] gendered = sum(1 for s in strings if (t := tr.get(code, s)) is not None and t.forms) lines.append(f"| {name} (`{code}`) | {st.count(AGREED)} / {len(st)} | {st.count('disputed')} | " f"{st.count('pending')} | {gendered} |") lines += ["", f"Pending (English only for now): {', '.join(f'`{c}`' for c in PENDING_LANGUAGES)}.", ""] for code, (name, _) in LANGUAGES.items(): disputed = [s for s in strings if tr.status(code, s) == "disputed"] if disputed: lines += [f"## {name}: disputed, shown in English", ""] for s in disputed: e = tr.by_lang[code][s] reason = (e.get("notes") or "").replace("\n", " ") lines.append(f"- {s!r}: {reason}") lines.append("") return "\n".join(lines) + "\n" def report_sets(sets: list[PageSet], built: dict[str, dict]) -> str: lines = ["# Phrase page sets: contents", "", "Generated by `build.py --reports`; do not edit.", ""] for ps in sets: cells = sum(len(b.cells) for b in ps.boards.values()) lines += [f"## {ps.name} (`{ps.key}.obz`)", "", ps.description, "", f"{len(ps.boards)} boards, {cells} cells.", "", "| Page | Board | Grid | Cells | Behaviour |", "|---|---|---|---|---|"] pages = book_pages(ps) for key in sorted(ps.boards, key=lambda k: pages[k]): b = ps.boards[key] beh = ", ".join(x for x in ("closes after a choice" if b.popup else "", "speaks without building a message" if b.speak_only else "") if x) lines.append(f"| {pages[key]} | {b.display(b.name)} | {b.rows}×{b.cols} | {len(b.cells)} | {beh} |") lines.append("") return "\n".join(lines) + "\n" # ─── Commands ─────────────────────────────────────────────────────────────── def file_entry(name: str, data: bytes, pkg: "Package") -> dict: return { "file": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest(), "cells": pkg.counts["cells"], "cells_in_english": pkg.counts["in_english"] if pkg.lang else 0, } def build_outputs(hash_names: bool = False) -> tuple[dict[str, bytes], dict[Path, bytes]]: """Every file of a build, deterministic: (the packages, catalogue and licence, by path relative to the output directory; the review reports, by path in the repository).""" symbols = Symbols() sets = read_all() errs = validate(sets, symbols) if errs: raise SourceError("\n".join(errs)) tr = Translations() strings = all_strings(sets) out: dict[str, bytes] = {} built: dict[str, dict] = {} def put(stem: str, pkg: Package) -> dict: data = pkg.build() name = f"{stem}.{hashlib.sha256(data).hexdigest()[:12]}.obz" if hash_names else f"{stem}.obz" out[name] = data return file_entry(name, data, pkg) for ps in sets: files: dict[str, Any] = {"en": put(ps.key, Package(ps, symbols, tr, None, None))} texts = [t for b in ps.boards.values() for c in b.cells for t in texts_of(c)] + [b.name for b in ps.boards.values()] texts += [b.description for b in ps.boards.values() if b.description] for code in LANGUAGES: if tr.gendered(code, texts): files[code] = {g: put(f"{code}/{ps.key}.{g}", Package(ps, symbols, tr, code, g)) for g in ("f", "m")} else: files[code] = put(f"{code}/{ps.key}", Package(ps, symbols, tr, code, None)) built[ps.key] = files out["index.json"] = (json.dumps(catalogue(sets, built, tr, strings), ensure_ascii=False, indent=1) + "\n").encode() out["families.json"] = (json.dumps(families(sets, built), ensure_ascii=False, indent=1) + "\n").encode() out["LICENSE.md"] = (HERE / "LICENSE.md").read_bytes() reports = { REPORTS / "symbols.md": report_symbols(sets, symbols).encode(), REPORTS / "translation-status.md": report_translations(strings, tr).encode(), REPORTS / "sets.md": report_sets(sets, built).encode(), } return out, reports def differences(want: dict, have_dir: Path | None, have: dict | None = None) -> list[str]: """Files of `want` that are missing or different in `have_dir` (or the in-memory `have`), and .obz files there that `want` does not have.""" diffs = [] for name, data in want.items(): if have is not None: got = have.get(name) else: path = Path(name) if isinstance(name, Path) else have_dir / name got = path.read_bytes() if path.exists() else None if got != data: diffs.append(str(name)) if have is None and have_dir is not None and have_dir.exists(): for old in have_dir.rglob("*.obz"): if str(old.relative_to(have_dir)) not in want: diffs.append(f"{old.relative_to(have_dir)} (not built any more)") return diffs def write(files: dict, base: Path | None) -> None: for name, data in files.items(): path = Path(name) if base is None else base / name path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) def main() -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--out", type=Path, help="write the packages, index.json, families.json and LICENSE.md into this directory") ap.add_argument("--hash-names", action="store_true", help="name packages ..obz") ap.add_argument("--check", action="store_true", help="fail if the sources are invalid, two builds differ, reports/ is stale, or (with --out) " "the directory differs from a fresh build; writes nothing") ap.add_argument("--reports", action="store_true", help="rewrite reports/ (committed review aids)") ap.add_argument("--strings", action="store_true", help="write work/strings.json for translators") args = ap.parse_args() if not (args.out or args.check or args.reports or args.strings): ap.print_help() return 2 try: if args.strings: sets = read_all() errs = validate(sets, Symbols()) if errs: raise SourceError("\n".join(errs)) WORK.mkdir(exist_ok=True) strings = all_strings(sets) (WORK / "strings.json").write_text(json.dumps(strings, ensure_ascii=False, indent=1) + "\n") print(f"{len(strings)} strings -> {WORK / 'strings.json'}") if not (args.out or args.check or args.reports): return 0 out, reports = build_outputs(args.hash_names) except SourceError as e: print(f"error:\n{e}", file=sys.stderr) return 1 if args.check: again, _ = build_outputs(args.hash_names) problems = [f"not reproducible: {d}" for d in differences(out, None, again)] problems += [f"stale (run build.py --reports): {d}" for d in differences(reports, None)] if args.out: problems += [f"stale in {args.out}: {d}" for d in differences(out, args.out)] if problems: print("\n".join(problems), file=sys.stderr) return 1 print(f"ok: {sum(1 for n in out if n.endswith('.obz'))} packages reproducible" + (f", {args.out} up to date" if args.out else "") + ", reports up to date") return 0 if args.reports: write(reports, None) print(f"reports written: {', '.join(str(p.relative_to(AAC_RS)) for p in reports)}") if args.out: if args.out.exists(): for old in args.out.rglob("*.obz"): if str(old.relative_to(args.out)) not in out: old.unlink() write(out, args.out) total = sum(len(d) for d in out.values()) print(f"{sum(1 for n in out if n.endswith('.obz'))} packages, index.json, families.json and LICENSE.md -> {args.out} " f"({total / 1e6:.1f} MB)") return 0 if __name__ == "__main__": sys.exit(main())