8ce1532b64
"Two independent model readers" read as people who model (the Turkish judge caught "two exemplary readers"). catalog.py --write: the four phrase families' language_note in data/pagesets/index.json (no package changes); its message is named by a hash of the English, so pageset-catalogue-8712848ae6d2 gives way to pageset-catalogue-8c8211ab1b93. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1460 lines
66 KiB
Python
1460 lines
66 KiB
Python
#!/usr/bin/env python3
|
||
"""Build the phrase-based page sets as Open Board Format packages (.obz).
|
||
|
||
Four page sets, each written by hand as a human-readable source in
|
||
sources/*.yaml and built here into OBF boards packed in .obz files:
|
||
|
||
adult.yaml adults with acquired conditions (aphasia, ALS/MND, PPA,
|
||
TBI, Parkinson's): one package, adult.obz
|
||
intent.yaml intent-first vocabulary in 12, 20 and 40 cells:
|
||
intent-12.obz, intent-20.obz, intent-40.obz
|
||
themes.yaml themed phrase grids, 6 x 3: themes.obz
|
||
emergent.yaml first switches and single messages: emergent.obz
|
||
|
||
Generated packages are not committed: the site build makes them. What
|
||
`--out DIR` writes (deterministic: the same sources give the same bytes):
|
||
|
||
DIR/<set>.obz English, with the pictures and every agreed
|
||
translation on each button
|
||
DIR/<lang>/<set>.obz the set in one language; <set>.f.obz and
|
||
<set>.m.obz where the speaker's gender changes
|
||
the words
|
||
DIR/index.json the catalogue a page-set chooser reads: sets,
|
||
sizes, descriptions, licence, every file with
|
||
its size and sha256, per-language status
|
||
DIR/families.json the same sets as families of the page-set
|
||
catalogue (tools/pagesets/catalog.py's
|
||
generator contract), so the catalogue can list
|
||
them beside the core sets
|
||
DIR/LICENSE.md CC BY-SA 4.0, with the symbols' attribution
|
||
|
||
Commands:
|
||
|
||
python3 build.py --out DIR [--hash-names] build into DIR (--hash-names:
|
||
<set>.<sha256-12>.obz, for a
|
||
cache that keeps files for hours)
|
||
python3 build.py --check [--out DIR] validate; fail if two builds
|
||
differ, if reports/ is stale, or
|
||
(with --out) if DIR is stale
|
||
python3 build.py --reports rewrite reports/*.md (committed
|
||
review aids: contents, symbols
|
||
wanted, translation status)
|
||
python3 build.py --strings work/strings.json for translators
|
||
|
||
Requires Python >= 3.11 and PyYAML. Reads the symbol index
|
||
(crates/aac-core/src/symbols/index.json) and sprites
|
||
(crates/aac-web/public/symbols/*.svg) the app ships; never the network.
|
||
|
||
The data model, the extensions and the translation process are described in
|
||
tools/pagesets/DESIGN-phrases-common.md.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import html
|
||
import io
|
||
import json
|
||
import re
|
||
import sys
|
||
import uuid
|
||
import zipfile
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import yaml
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
AAC_RS = HERE.parents[2]
|
||
SOURCES = HERE / "sources"
|
||
TRANSLATIONS = HERE / "translations"
|
||
REPORTS = HERE / "reports"
|
||
WORK = HERE / "work"
|
||
SYMBOL_INDEX = AAC_RS / "crates" / "aac-core" / "src" / "symbols" / "index.json"
|
||
SYMBOL_INDEX_NC = AAC_RS / "crates" / "aac-core" / "src" / "symbols" / "index-nc.json"
|
||
SPRITES = AAC_RS / "crates" / "aac-web" / "public" / "symbols"
|
||
|
||
OBF_FORMAT = "open-board-0.1"
|
||
GENERATOR = "aac-rs/tools/pagesets/phrases/build.py"
|
||
# Ids are UUIDv5 under this namespace, so a rebuilt board keeps its id.
|
||
NAMESPACE = uuid.UUID("6f0f3a52-8a4e-5b55-9a43-7b1d2c0e5a11")
|
||
# 1980-01-01, the earliest DOS date: fixed so a build is reproducible.
|
||
ZIP_DATE = (1980, 1, 1, 0, 0, 0)
|
||
|
||
# The ten largest app languages, in the order the work was done. The other
|
||
# 34 are listed as pending in the catalogue.
|
||
LANGUAGES = {
|
||
"es": ("Spanish", "es-ES"),
|
||
"fr": ("French", "fr-FR"),
|
||
"de": ("German", "de-DE"),
|
||
"pt": ("Portuguese (Brazil)", "pt-BR"),
|
||
"it": ("Italian", "it-IT"),
|
||
"zh": ("Chinese (Simplified)", "zh-CN"),
|
||
"ja": ("Japanese", "ja-JP"),
|
||
"ar": ("Arabic", "ar-SA"),
|
||
"hi": ("Hindi", "hi-IN"),
|
||
"ru": ("Russian", "ru-RU"),
|
||
}
|
||
PENDING_LANGUAGES = [
|
||
"nl", "ko", "he", "ur", "fa", "bn", "ta", "te", "ml", "mr", "kn", "gu", "pa", "th", "vi", "id",
|
||
"ms", "tl", "pl", "uk", "ro", "cs", "sv", "da", "nb", "fi", "sw", "am", "ha", "yo", "zu", "tr",
|
||
"el", "hu",
|
||
]
|
||
|
||
AGREED = "agreed"
|
||
STATUS_TEXT = "agreed by two model readers; native review pending"
|
||
|
||
# Names that belong to other people's products. None may appear in a board's
|
||
# words (the design notes may name them, to say what we did not copy).
|
||
PROPRIETARY_NAMES = [
|
||
"PODD", "Pragmatic Organisation", "Pragmatic Organization", "TD Snap", "Snap Core", "Core First",
|
||
"Tobii", "Dynavox", "Proloquo", "Crescendo", "LAMP", "Words for Life", "Unity", "Minspeak",
|
||
"Gateway", "WordPower", "TouchChat", "Lingraphica", "Aphasia Duo", "Grid 3", "Super Core",
|
||
"BIGmack", "Big Mack", "Step-by-Step", "Step by Step", "QuickFire", "Communication Journey",
|
||
"PECS", "Boardmaker", "PCS", "SymbolStix", "Widgit",
|
||
]
|
||
# Words that would read as diagnosis or treatment. The sets are for
|
||
# wellness, research and clinical-trial use, not a medical device.
|
||
CLAIM_WORDS = [
|
||
r"\bdiagnos\w*", r"\btreatments?\b", r"\btherap\w*", r"\bcure[sd]?\b", r"\bheal(s|ing|ed)?\b",
|
||
r"\brehabilitat\w*", r"\brecover(y|s|ing|ed)?\b", r"\bclinically\b", r"\bimproves? your\b",
|
||
r"\bslows? (the|your|down the)\b", r"\bprevents?\b",
|
||
]
|
||
|
||
ACTIONS = {"clear": ":clear", "backspace": ":backspace", "speak": ":speak"}
|
||
|
||
|
||
class SourceError(Exception):
|
||
pass
|
||
|
||
|
||
class PhraseLoader(yaml.SafeLoader):
|
||
"""YAML without YAML 1.1's other booleans: in a phrase source, yes, no,
|
||
on and off are words (a cell named `no`, the message "Yes"), and only
|
||
true and false are booleans."""
|
||
|
||
|
||
PhraseLoader.yaml_implicit_resolvers = {
|
||
first: [(tag, regexp) for tag, regexp in resolvers if tag != "tag:yaml.org,2002:bool"]
|
||
for first, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items()
|
||
}
|
||
PhraseLoader.add_implicit_resolver(
|
||
"tag:yaml.org,2002:bool", re.compile(r"^(?:true|false)$"), list("tf"))
|
||
|
||
|
||
# ─── Symbols ────────────────────────────────────────────────────────────────
|
||
|
||
|
||
class Symbols:
|
||
"""The app's concept -> symbol index and its sprites."""
|
||
|
||
def __init__(self) -> None:
|
||
index = json.loads(SYMBOL_INDEX.read_text())
|
||
self.sets: dict[str, dict] = index["sets"]
|
||
self.concepts: dict[str, dict[str, str]] = index["concepts"]
|
||
self._sprites: dict[str, str] = {}
|
||
|
||
def resolve(self, concept: str, prefer: list[str]) -> tuple[str, str] | None:
|
||
"""(set, symbol id) for a concept, from the first preferred set that
|
||
draws it."""
|
||
sets = self.concepts.get(concept)
|
||
if not sets:
|
||
return None
|
||
for s in prefer:
|
||
if s in sets:
|
||
return s, sets[s]
|
||
return None
|
||
|
||
def svg(self, set_name: str, symbol_id: str) -> tuple[bytes, int, int]:
|
||
"""The symbol cut out of its sprite as a standalone SVG document
|
||
(as aac_core::obf::extract_symbol_svg does), with its attribution."""
|
||
if set_name not in self._sprites:
|
||
self._sprites[set_name] = (SPRITES / self.sets[set_name]["file"]).read_text()
|
||
sprite = self._sprites[set_name]
|
||
start = sprite.find(f'<symbol id="{symbol_id}"')
|
||
if start < 0:
|
||
raise SourceError(f"symbol {symbol_id} is not in the {set_name} sprite")
|
||
tag_end = sprite.index(">", start)
|
||
tag = sprite[start:tag_end]
|
||
body_end = sprite.index("</symbol>", tag_end)
|
||
body = sprite[tag_end + 1:body_end]
|
||
view_box = re.search(r'viewBox="([^"]+)"', tag).group(1)
|
||
dims = [float(n) for n in re.split(r"[ ,]+", view_box.strip())]
|
||
width, height = max(1, int(-(-dims[2] // 1))), max(1, int(-(-dims[3] // 1)))
|
||
note = f'{self.sets[set_name]["attribution"]} Adapted: optimised and cut from a sprite.'.replace("--", "-")
|
||
doc = (f'<svg xmlns="http://www.w3.org/2000/svg" viewBox="{view_box}" width="{width}" '
|
||
f'height="{height}"><!-- {note} -->{body}</svg>')
|
||
return doc.encode(), width, height
|
||
|
||
def license(self, set_name: str) -> dict:
|
||
info = self.sets[set_name]
|
||
return {
|
||
"type": info["license"]["spdx"],
|
||
"copyright_notice_url": info["license"]["url"],
|
||
"source_url": info["source"]["url"],
|
||
"author_name": info["name"],
|
||
"ext_aurasense_attribution": info["attribution"],
|
||
"ext_aurasense_conditions": info["conditions"],
|
||
}
|
||
|
||
|
||
# ─── Model ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
@dataclass
|
||
class Cell:
|
||
row: int
|
||
col: int
|
||
label: str = ""
|
||
say: str | None = None # Speak
|
||
add: str | None = None # Buffer: add to the message without speaking
|
||
go: str | None = None # board key
|
||
do: str | None = None # clear | backspace | speak
|
||
symbol: str | None = None # concept in the symbol index
|
||
want_symbol: str | None = None # a concept the index does not have yet
|
||
color: str | None = None
|
||
border: str | None = None
|
||
text_color: str | None = None
|
||
fill: dict | None = None # ext_aurasense_fill_in
|
||
seq: dict | None = None # ext_aurasense_sequence
|
||
effect: str | None = None # ext_aurasense_effect
|
||
src: str | None = None # source of the words, if not original
|
||
note: str | None = None # context for translators
|
||
origin: str = "" # where in the sources this cell came from
|
||
|
||
@property
|
||
def blank(self) -> bool:
|
||
return not (self.label or self.say or self.add or self.go or self.do)
|
||
|
||
|
||
@dataclass
|
||
class Board:
|
||
key: str
|
||
name: str
|
||
rows: int
|
||
cols: int
|
||
cells: list[Cell] = field(default_factory=list)
|
||
parent: str | None = None
|
||
popup: bool = False
|
||
speak_only: bool = False
|
||
description: str = ""
|
||
page: int | None = None
|
||
page_no: int | None = None # 2, 3 … on a section's continuation pages
|
||
origin: str = ""
|
||
|
||
def display(self, name: str) -> str:
|
||
"""The board's name (in any language) with its page number."""
|
||
return name if self.page_no is None else f"{name} {self.page_no}"
|
||
|
||
|
||
@dataclass
|
||
class PageSet:
|
||
key: str # file name stem, e.g. adult, intent-20
|
||
family: str # source file stem, e.g. adult, intent
|
||
name: str
|
||
description: str
|
||
version: str
|
||
license: dict
|
||
prefer: list[str]
|
||
root: str
|
||
boards: dict[str, Board]
|
||
size: int | None = None
|
||
|
||
|
||
# ─── Reading the sources ────────────────────────────────────────────────────
|
||
|
||
|
||
def hex_to_obf(color: str) -> str:
|
||
c = color.strip().lstrip("#")
|
||
if len(c) == 3:
|
||
c = "".join(ch * 2 for ch in c)
|
||
if not re.fullmatch(r"[0-9A-Fa-f]{6}", c):
|
||
raise SourceError(f"not a #RRGGBB colour: {color!r}")
|
||
r, g, b = (int(c[i:i + 2], 16) for i in (0, 2, 4))
|
||
return f"rgb({r}, {g}, {b})"
|
||
|
||
|
||
class Reader:
|
||
"""Turns one source file into page sets."""
|
||
|
||
def __init__(self, path: Path) -> None:
|
||
self.path = path
|
||
self.doc = yaml.load(path.read_text(), Loader=PhraseLoader)
|
||
self.meta = self.doc["set"]
|
||
self.palette: dict[str, str] = self.doc.get("palette", {})
|
||
self.named: dict[str, Any] = self.doc.get("cells", {})
|
||
|
||
def color(self, value: str | None) -> str | None:
|
||
if value is None:
|
||
return None
|
||
return self.palette.get(value, value)
|
||
|
||
def cell(self, spec: Any, row: int, col: int, origin: str, inherited: dict | None = None) -> Cell:
|
||
"""A cell from its source form: None (blank), a string (a message:
|
||
shown and said), "$name" (a named cell), or a mapping."""
|
||
if spec is None:
|
||
return Cell(row, col, origin=origin)
|
||
if isinstance(spec, str) and spec.startswith("$"):
|
||
name = spec[1:]
|
||
if name not in self.named:
|
||
raise SourceError(f"{origin}: no named cell ${name}")
|
||
return self.cell(self.named[name], row, col, origin, inherited)
|
||
if isinstance(spec, str):
|
||
spec = {"say": spec}
|
||
if not isinstance(spec, dict):
|
||
raise SourceError(f"{origin}: a cell is a string, $name, ~ or a mapping, not {spec!r}")
|
||
if "use" in spec:
|
||
base = self.named.get(spec["use"])
|
||
if base is None:
|
||
raise SourceError(f"{origin}: no named cell {spec['use']}")
|
||
merged = dict(base if isinstance(base, dict) else {"say": base})
|
||
merged.update({k: v for k, v in spec.items() if k != "use"})
|
||
spec = merged
|
||
known = {"label", "say", "add", "go", "do", "symbol", "want_symbol", "color", "border", "text_color",
|
||
"fill", "seq", "effect", "src", "note", "min_size"}
|
||
unknown = set(spec) - known
|
||
if unknown:
|
||
raise SourceError(f"{origin}: unknown cell keys {sorted(unknown)}")
|
||
style = dict(inherited or {})
|
||
c = Cell(
|
||
row, col,
|
||
label=str(spec.get("label") or spec.get("say") or spec.get("add") or ""),
|
||
say=spec.get("say"),
|
||
add=spec.get("add"),
|
||
go=spec.get("go"),
|
||
do=spec.get("do"),
|
||
symbol=spec.get("symbol"),
|
||
want_symbol=spec.get("want_symbol"),
|
||
color=self.color(spec.get("color", style.get("color"))),
|
||
border=self.color(spec.get("border", style.get("border"))),
|
||
text_color=self.color(spec.get("text_color", style.get("text_color"))),
|
||
fill=spec.get("fill"),
|
||
seq=spec.get("seq"),
|
||
effect=spec.get("effect"),
|
||
src=spec.get("src"),
|
||
note=spec.get("note"),
|
||
origin=origin,
|
||
)
|
||
if c.seq is not None and not c.label:
|
||
raise SourceError(f"{origin}: a sequence cell needs a label")
|
||
if sum(x is not None for x in (c.say, c.add, c.do)) > 1:
|
||
raise SourceError(f"{origin}: a cell says, adds or acts — one of them")
|
||
if c.do is not None and c.do not in ACTIONS:
|
||
raise SourceError(f"{origin}: unknown action {c.do!r}")
|
||
if c.go is None and c.do is None and c.add is None and c.say is None and c.label and c.seq is None:
|
||
c.say = c.label
|
||
if c.seq is not None and c.say is None:
|
||
# Fallback for a client without the extension: say the first line.
|
||
first = c.seq["lines"][0]
|
||
c.say = first["say"] if isinstance(first, dict) else (None if first.startswith("@") else first)
|
||
return c
|
||
|
||
def board(self, key: str, spec: dict, rows: int, cols: int, frame: list[dict] | None = None) -> Board:
|
||
origin = f"{self.path.name}#{key}"
|
||
b = Board(
|
||
key=key,
|
||
name=spec["name"],
|
||
rows=rows,
|
||
cols=cols,
|
||
parent=spec.get("parent"),
|
||
popup=bool(spec.get("popup", False)),
|
||
speak_only=bool(spec.get("speak_only", False)),
|
||
description=spec.get("description", ""),
|
||
origin=origin,
|
||
)
|
||
style = spec.get("style")
|
||
grid: list[list[Any]] = [[None] * cols for _ in range(rows)]
|
||
for f in frame or []:
|
||
r, c = f["at"]
|
||
grid[r][c] = f["cell"]
|
||
if "rows" in spec:
|
||
if len(spec["rows"]) > rows:
|
||
raise SourceError(f"{origin}: {len(spec['rows'])} rows in a {rows}-row grid")
|
||
first = 0
|
||
if frame and spec.get("frame_row", True) and any(f["at"][0] == 0 for f in frame):
|
||
first = 1 if len(spec["rows"]) < rows else 0
|
||
for r, row in enumerate(spec["rows"], start=first):
|
||
if len(row) > cols:
|
||
raise SourceError(f"{origin}: row {r} has {len(row)} cells in a {cols}-column grid")
|
||
for c, item in enumerate(row):
|
||
if item is not None:
|
||
if grid[r][c] is not None:
|
||
raise SourceError(f"{origin}: ({r},{c}) is the frame's")
|
||
grid[r][c] = item
|
||
for r in range(rows):
|
||
for c in range(cols):
|
||
cell = self.cell(grid[r][c], r, c, f"{origin}({r},{c})", style)
|
||
if not cell.blank:
|
||
b.cells.append(cell)
|
||
return b
|
||
|
||
def page_set(self, key: str, boards: dict[str, Board], size: int | None = None) -> PageSet:
|
||
m = self.meta
|
||
# A page set is shared under CC BY-SA and may be offered anywhere: it
|
||
# draws only the open sets, never one licensed for non-commercial use
|
||
# only (index-nc.json: ARASAAC, Sclera).
|
||
nc = json.loads(SYMBOL_INDEX_NC.read_text())["sets"] if SYMBOL_INDEX_NC.exists() else {}
|
||
open_sets = json.loads(SYMBOL_INDEX.read_text())["sets"]
|
||
if bad := [s for s in m["symbols"] if s in nc or s not in open_sets]:
|
||
raise SourceError(f"{self.path.name}: symbols {bad}: a page set draws only the open sets {sorted(open_sets)}")
|
||
return PageSet(
|
||
key=key,
|
||
family=self.path.stem,
|
||
name=m["name"] if size is None else m["name"].format(size=size),
|
||
description=m["description"].strip() if size is None else m["description"].strip().format(size=size),
|
||
version=str(m["version"]),
|
||
license=m["license"],
|
||
prefer=m["symbols"],
|
||
root=m["root"],
|
||
boards=boards,
|
||
size=size,
|
||
)
|
||
|
||
|
||
def read_plain(path: Path) -> list[PageSet]:
|
||
"""A set whose boards are each laid out by hand (adult, themes, emergent)."""
|
||
rd = Reader(path)
|
||
frames = rd.doc.get("frames", {})
|
||
boards: dict[str, Board] = {}
|
||
for key, spec in rd.doc["boards"].items():
|
||
rows, cols = spec.get("grid", rd.meta.get("grid"))
|
||
frame_name = spec.get("frame", rd.meta.get("frame"))
|
||
frame = frames.get(frame_name) if frame_name else None
|
||
if key in boards:
|
||
raise SourceError(f"{path.name}: board {key} twice")
|
||
boards[key] = rd.board(key, spec, rows, cols, frame)
|
||
finish(boards, rd.meta["root"], path.name)
|
||
return [rd.page_set(rd.path.stem, boards)]
|
||
|
||
|
||
def read_intent(path: Path) -> list[PageSet]:
|
||
"""The intent-first set: sections of items flowed into pages, for each
|
||
grid size, inside that size's frame of operational commands."""
|
||
rd = Reader(path)
|
||
out = []
|
||
for size, layout in rd.doc["sizes"].items():
|
||
size = int(size)
|
||
rows, cols = layout["grid"]
|
||
frame: list[dict] = layout["frame"]
|
||
turn = tuple(layout["turn"])
|
||
taken = {tuple(f["at"]) for f in frame}
|
||
content = [(r, c) for r in range(rows) for c in range(cols) if (r, c) not in taken]
|
||
boards: dict[str, Board] = {}
|
||
sections = rd.doc["sections"]
|
||
for key, sec in sections.items():
|
||
items = sec.get("items_by_size", {}).get(size, sec.get("items_by_size", {}).get(str(size)))
|
||
if items is None:
|
||
items = [it for it in sec["items"] if not (isinstance(it, dict) and it.get("min_size", 0) > size)]
|
||
pages = paginate(items, content, turn)
|
||
for n, page_items in enumerate(pages):
|
||
bkey = key if n == 0 else f"{key}~{n + 1}"
|
||
b = Board(key=bkey, name=sec["name"], rows=rows, cols=cols, page_no=n + 1 if n else None, parent=sec.get("parent", rd.meta["root"]),
|
||
popup=bool(sec.get("popup", False)), description=sec.get("description", ""),
|
||
origin=f"{path.name}#{key}@{size}")
|
||
if key == rd.meta["root"]:
|
||
b.parent = None
|
||
for f in frame:
|
||
cell = rd.cell(f["cell"], *f["at"], origin=f"{path.name}#frame@{size}")
|
||
if cell.go == "@back" and b.parent is None:
|
||
continue
|
||
if cell.go == "@back":
|
||
cell.go = b.parent
|
||
if cell.go == bkey:
|
||
# A board does not link to itself: the place stays empty.
|
||
continue
|
||
b.cells.append(cell)
|
||
for (r, c), item in zip(content, page_items):
|
||
cell = rd.cell(item, r, c, f"{path.name}#{key}@{size}", sec.get("style"))
|
||
if not cell.blank:
|
||
b.cells.append(cell)
|
||
if len(pages) > 1:
|
||
nxt = key if n + 1 == len(pages) else f"{key}~{n + 2}"
|
||
t = rd.cell("$turn", *turn, origin=f"{path.name}#turn@{size}")
|
||
t.go = nxt
|
||
b.cells.append(t)
|
||
boards[bkey] = b
|
||
finish(boards, rd.meta["root"], path.name)
|
||
ps = rd.page_set(f"{path.stem}-{size}", boards, size)
|
||
out.append(ps)
|
||
return out
|
||
|
||
|
||
def finish(boards: dict[str, Board], root: str, source: str) -> None:
|
||
"""Settle what a board can only know once the whole set is read: its
|
||
parent (declared, or the first board that links to it), what "@back"
|
||
links to, and the cells a sequence's "@row,col" lines name."""
|
||
first_linker: dict[str, str] = {}
|
||
for key, b in boards.items():
|
||
for c in sorted(b.cells, key=lambda c: (c.row, c.col)):
|
||
if c.go and c.go != "@back" and c.go != key:
|
||
first_linker.setdefault(c.go, key)
|
||
for key, b in boards.items():
|
||
if b.parent is None and key != root:
|
||
b.parent = first_linker.get(key.split("~")[0], first_linker.get(key))
|
||
kept = []
|
||
for c in b.cells:
|
||
if c.go == "@back":
|
||
if b.parent is None:
|
||
continue
|
||
c.go = b.parent
|
||
kept.append(c)
|
||
b.cells = kept
|
||
for c in b.cells:
|
||
if not c.seq:
|
||
continue
|
||
lines = []
|
||
for line in c.seq["lines"]:
|
||
if isinstance(line, str) and line.startswith("@"):
|
||
r, col = (int(x) for x in line[1:].split(","))
|
||
target = next((x for x in b.cells if (x.row, x.col) == (r, col)), None)
|
||
if target is None or target.say is None:
|
||
raise SourceError(f"{c.origin}: sequence line {line} names no speaking cell")
|
||
lines.append({"say": target.say, "cell": [r, col]})
|
||
elif isinstance(line, str):
|
||
lines.append({"say": line})
|
||
else:
|
||
lines.append(dict(line))
|
||
c.seq = {**c.seq, "lines": lines}
|
||
c.say = lines[0]["say"]
|
||
|
||
|
||
def paginate(items: list, content: list[tuple[int, int]], turn: tuple[int, int]) -> list[list]:
|
||
"""Split items into pages. A page that continues gives up its turn-the-page
|
||
cell; every page keeps the same positions for the same slots."""
|
||
if len(items) <= len(content):
|
||
return [items]
|
||
per_page = [pos for pos in content if pos != turn]
|
||
pages = []
|
||
for i in range(0, len(items), len(per_page)):
|
||
chunk = items[i:i + len(per_page)]
|
||
# Re-insert a hole at the turn cell's slot so positions stay fixed.
|
||
slotted = []
|
||
it = iter(chunk)
|
||
for pos in content:
|
||
if pos == turn:
|
||
slotted.append(None)
|
||
else:
|
||
slotted.append(next(it, None))
|
||
pages.append(slotted)
|
||
return pages
|
||
|
||
|
||
def read_all() -> list[PageSet]:
|
||
sets: list[PageSet] = []
|
||
for name in ("adult", "intent", "themes", "emergent"):
|
||
path = SOURCES / f"{name}.yaml"
|
||
sets.extend(read_intent(path) if name == "intent" else read_plain(path))
|
||
return sets
|
||
|
||
|
||
# ─── Validation ─────────────────────────────────────────────────────────────
|
||
|
||
|
||
def texts_of(cell: Cell) -> list[str]:
|
||
out = [cell.label]
|
||
for t in (cell.say, cell.add):
|
||
if t and t != cell.label:
|
||
out.append(t)
|
||
if cell.seq:
|
||
for line in cell.seq["lines"]:
|
||
out.append(line["say"])
|
||
if cell.fill and cell.fill.get("hint"):
|
||
out.append(cell.fill["hint"])
|
||
return [t for t in out if t]
|
||
|
||
|
||
def validate(sets: list[PageSet], symbols: Symbols) -> list[str]:
|
||
"""Every problem found, as text. Empty means the sources are sound."""
|
||
errs: list[str] = []
|
||
for ps in sets:
|
||
if ps.root not in ps.boards:
|
||
errs.append(f"{ps.key}: root board {ps.root} is missing")
|
||
linked: set[str] = set()
|
||
for b in ps.boards.values():
|
||
seen: set[tuple[int, int]] = set()
|
||
for c in b.cells:
|
||
if not (0 <= c.row < b.rows and 0 <= c.col < b.cols):
|
||
errs.append(f"{c.origin}: outside the {b.rows}x{b.cols} grid")
|
||
if (c.row, c.col) in seen:
|
||
errs.append(f"{c.origin}: two cells at ({c.row},{c.col})")
|
||
seen.add((c.row, c.col))
|
||
if c.go is not None:
|
||
if c.go not in ps.boards:
|
||
errs.append(f"{c.origin}: links to {c.go}, which is not in {ps.key}")
|
||
linked.add(c.go)
|
||
if c.symbol is not None and symbols.resolve(c.symbol, ps.prefer) is None:
|
||
errs.append(f"{c.origin}: symbol concept {c.symbol!r} is not drawn by {ps.prefer}")
|
||
if c.seq is not None:
|
||
lines = c.seq.get("lines") or []
|
||
if len(lines) < 2:
|
||
errs.append(f"{c.origin}: a sequence needs two lines or more")
|
||
if c.seq.get("order", "in_order") not in ("in_order", "shuffle"):
|
||
errs.append(f"{c.origin}: sequence order is in_order or shuffle")
|
||
if c.seq.get("at_end", "start_again") not in ("start_again", "stay"):
|
||
errs.append(f"{c.origin}: sequence at_end is start_again or stay")
|
||
for text in texts_of(c):
|
||
for name in PROPRIETARY_NAMES:
|
||
if re.search(rf"(?<![A-Za-z]){re.escape(name)}(?![A-Za-z])", text):
|
||
errs.append(f"{c.origin}: {text!r} names {name!r}")
|
||
for w in CLAIM_WORDS:
|
||
if re.search(w, text, re.IGNORECASE):
|
||
errs.append(f"{c.origin}: {text!r} reads as a medical claim ({w!r})")
|
||
if b.speak_only and any(c.add for c in b.cells):
|
||
errs.append(f"{b.origin}: a speak-only board cannot add to the message")
|
||
unreachable = set(ps.boards) - linked - {ps.root}
|
||
for key in sorted(unreachable):
|
||
errs.append(f"{ps.key}: board {key} is not linked from any board")
|
||
return errs
|
||
|
||
|
||
# ─── Translations ───────────────────────────────────────────────────────────
|
||
|
||
|
||
@dataclass
|
||
class Translation:
|
||
text: str | None # the words, when they do not depend on the speaker
|
||
forms: dict[str, str] | None # {"f": ..., "m": ...} when they do
|
||
|
||
|
||
class Translations:
|
||
"""translations/<lang>.json: only entries both readers agreed on are used."""
|
||
|
||
def __init__(self) -> None:
|
||
self.by_lang: dict[str, dict[str, dict]] = {}
|
||
for code in LANGUAGES:
|
||
path = TRANSLATIONS / f"{code}.json"
|
||
if path.exists():
|
||
self.by_lang[code] = json.loads(path.read_text())["strings"]
|
||
else:
|
||
self.by_lang[code] = {}
|
||
|
||
def get(self, lang: str, english: str) -> Translation | None:
|
||
e = self.by_lang.get(lang, {}).get(english)
|
||
if not e or e.get("status") != AGREED:
|
||
return None
|
||
if e.get("forms"):
|
||
return Translation(None, dict(e["forms"]))
|
||
return Translation(e["text"], None)
|
||
|
||
def status(self, lang: str, english: str) -> str:
|
||
e = self.by_lang.get(lang, {}).get(english)
|
||
return e.get("status", "pending") if e else "pending"
|
||
|
||
def gendered(self, lang: str, texts: list[str]) -> bool:
|
||
return any((t := self.get(lang, x)) is not None and t.forms for x in texts)
|
||
|
||
def words(self, lang: str, english: str, speaker: str | None) -> str | None:
|
||
t = self.get(lang, english)
|
||
if t is None:
|
||
return None
|
||
if t.forms:
|
||
return t.forms.get(speaker) if speaker else None
|
||
return t.text
|
||
|
||
|
||
# Board descriptions in a localized package: a visible note that some words
|
||
# are still in English. Its translation is in each language file under this key.
|
||
NOTE_KEY = "Some messages on this board are still in English: their translation has not yet been agreed by two readers."
|
||
|
||
|
||
# ─── Writing OBF ────────────────────────────────────────────────────────────
|
||
|
||
|
||
def bid(ps: PageSet, board: str) -> str:
|
||
return str(uuid.uuid5(NAMESPACE, f"{ps.key}/{board}"))
|
||
|
||
|
||
def cid(ps: PageSet, board: str, cell: Cell) -> str:
|
||
return str(uuid.uuid5(NAMESPACE, f"{ps.key}/{board}/{cell.row},{cell.col}"))
|
||
|
||
|
||
def board_path(ps: PageSet, board: str) -> str:
|
||
return f"boards/{bid(ps, board)}.obf"
|
||
|
||
|
||
def book_pages(ps: PageSet) -> dict[str, int]:
|
||
"""Page numbers for a printed book: breadth-first from the root in the
|
||
order links appear (row by row), continuation pages straight after their
|
||
first page."""
|
||
order: list[str] = []
|
||
queue = [ps.root]
|
||
while queue:
|
||
key = queue.pop(0)
|
||
if key in order:
|
||
continue
|
||
order.append(key)
|
||
base = key.split("~")[0]
|
||
conts = sorted((k for k in ps.boards if k.startswith(base + "~")), key=lambda k: int(k.split("~")[1]))
|
||
for k in conts:
|
||
if k not in order:
|
||
order.append(k)
|
||
b = ps.boards[key]
|
||
for c in sorted(b.cells, key=lambda c: (c.row, c.col)):
|
||
if c.go and c.go not in order and c.go not in queue:
|
||
queue.append(c.go)
|
||
for k in sorted(ps.boards):
|
||
if k not in order:
|
||
order.append(k)
|
||
return {k: n + 1 for n, k in enumerate(order)}
|
||
|
||
|
||
def same_words(text: str) -> str:
|
||
"""Words as a person reads them, to tell whether two cells show the
|
||
same (pslib/samewords.py `normal`)."""
|
||
import unicodedata
|
||
|
||
return " ".join(unicodedata.normalize("NFKC", text).casefold().split())
|
||
|
||
|
||
class Package:
|
||
"""One .obz: boards in one language (or English with translations)."""
|
||
|
||
def __init__(self, ps: PageSet, symbols: Symbols, tr: Translations, lang: str | None, speaker: str | None):
|
||
self.ps = ps
|
||
self.symbols = symbols
|
||
self.tr = tr
|
||
self.lang = lang # None: the English package with every agreed translation
|
||
self.speaker = speaker
|
||
self.images: dict[str, bytes] = {}
|
||
self.pages = book_pages(ps)
|
||
self.counts = {"cells": 0, "texts": 0, "in_english": 0}
|
||
|
||
@property
|
||
def locale(self) -> str:
|
||
return "en-US" if self.lang is None else LANGUAGES[self.lang][1]
|
||
|
||
def words(self, english: str) -> tuple[str, bool]:
|
||
"""The words in this package's language, and whether they had to stay
|
||
in English."""
|
||
if self.lang is None:
|
||
return english, False
|
||
t = self.tr.words(self.lang, english, self.speaker)
|
||
return (english, True) if t is None else (t, False)
|
||
|
||
@staticmethod
|
||
def cell_texts(cell: Cell) -> list[str]:
|
||
"""The texts of a cell that must all be in one language: what it
|
||
shows, what it says, and the lines of its sequence."""
|
||
texts = [cell.label]
|
||
spoken = cell.say if cell.say is not None else cell.add
|
||
if spoken is not None and spoken != cell.label:
|
||
texts.append(spoken)
|
||
if cell.seq:
|
||
texts += [line["say"] for line in cell.seq["lines"]]
|
||
return texts
|
||
|
||
def localize(self, cell: Cell, lang: str, speaker: str | None) -> dict[str, str] | None:
|
||
"""Every text of the cell in `lang`, or None if any is not agreed: a
|
||
cell is shown and said in one language, never half in English."""
|
||
out = {}
|
||
for t in self.cell_texts(cell):
|
||
w = self.tr.words(lang, t, speaker)
|
||
if w is None:
|
||
return None
|
||
if w == "" and (t == cell.label or cell.go is None):
|
||
# Empty words are only for a link that needs no words in
|
||
# this language; a button never shows a blank label.
|
||
return None
|
||
out[t] = w
|
||
return out
|
||
|
||
def form_entry(self, cell: Cell, loc: dict[str, str]) -> dict:
|
||
"""A cell's words in one speaker form: what it shows, says, and the
|
||
lines of its sequence."""
|
||
spoken = cell.say if cell.say is not None else cell.add
|
||
e: dict[str, Any] = {"label": loc[cell.label]}
|
||
if spoken is not None and spoken != cell.label:
|
||
e["vocalization"] = loc[spoken]
|
||
if cell.seq:
|
||
e["lines"] = [loc[line["say"]] for line in cell.seq["lines"]]
|
||
return e
|
||
|
||
def button_translations(self, cell: Cell) -> dict:
|
||
"""`translations` for the English package: the agreed words of each
|
||
language, with the speaker's two forms where they differ."""
|
||
spoken = cell.say if cell.say is not None else cell.add
|
||
|
||
def entry(loc: dict[str, str]) -> dict:
|
||
e: dict[str, Any] = {"label": loc[cell.label]}
|
||
if spoken is not None and spoken != cell.label:
|
||
e["vocalization"] = loc[spoken]
|
||
if cell.seq:
|
||
e["ext_aurasense_sequence_lines"] = [loc[line["say"]] for line in cell.seq["lines"]]
|
||
return e
|
||
|
||
out: dict[str, Any] = {"en": entry({t: t for t in self.cell_texts(cell)})}
|
||
for code in LANGUAGES:
|
||
plain = self.localize(cell, code, None)
|
||
if plain is not None:
|
||
out[code] = entry(plain)
|
||
continue
|
||
f, m = self.localize(cell, code, "f"), self.localize(cell, code, "m")
|
||
if f is not None and m is not None:
|
||
out[code] = {"ext_aurasense_speaker_forms": {"f": entry(f), "m": entry(m)}}
|
||
return out
|
||
|
||
def button(self, b: Board, cell: Cell, n: int) -> tuple[dict, tuple[str, dict] | None]:
|
||
ps = self.ps
|
||
spoken_src = cell.say if cell.say is not None else cell.add
|
||
loc = None if self.lang is None else self.localize(cell, self.lang, self.speaker)
|
||
|
||
def tx(text: str) -> str:
|
||
return loc[text] if loc else text
|
||
|
||
self.counts["cells"] += 1
|
||
btn: dict[str, Any] = {"id": cid(ps, b.key, cell), "label": tx(cell.label)}
|
||
if spoken_src is not None and cell.go is not None and tx(spoken_src) == "":
|
||
# A link's words that this language does not need (the English
|
||
# "to" of "I want" + "to" + "watch TV"): the link alone.
|
||
spoken_src = None
|
||
if spoken_src is not None:
|
||
btn["vocalization"] = tx(spoken_src)
|
||
prov: dict[str, Any] = {"content": "text", "source": cell.origin, "source_label": btn["label"]}
|
||
if spoken_src is not None and btn["vocalization"] != btn["label"]:
|
||
prov["source_vocalization"] = btn["vocalization"]
|
||
if self.lang is not None:
|
||
if loc:
|
||
# The words of this language, kept exactly as written (the
|
||
# app's own-words cell, Cell::lang): never glossed.
|
||
prov["words_locale"] = self.locale
|
||
prov["translation"] = STATUS_TEXT
|
||
prov["english"] = " | ".join(dict.fromkeys(self.cell_texts(cell)))
|
||
else:
|
||
self.counts["in_english"] += 1
|
||
btn["label_locale"] = "en"
|
||
if spoken_src is not None:
|
||
btn["vocalization_locale"] = "en"
|
||
prov["translation"] = "pending: shown and said in English"
|
||
if loc and self.speaker is not None:
|
||
# Words that change with who says them carry both forms, so
|
||
# the app can put the person's there, or leave the word out
|
||
# while they would rather not say (aac-web pagesets.rs).
|
||
f, m = self.localize(cell, self.lang, "f"), self.localize(cell, self.lang, "m")
|
||
if f is not None and m is not None and f != m:
|
||
btn["ext_aurasense_speaker_forms"] = {"f": self.form_entry(cell, f), "m": self.form_entry(cell, m)}
|
||
if cell.src:
|
||
prov["words_source"] = cell.src
|
||
if cell.add is not None:
|
||
btn["ext_aurasense_action"] = "buffer"
|
||
if cell.go is not None:
|
||
target = ps.boards[cell.go]
|
||
tname = target.display(self.words(target.name)[0])
|
||
btn["load_board"] = {"id": bid(ps, cell.go), "name": tname, "path": board_path(ps, cell.go)}
|
||
if spoken_src is not None and cell.say is not None:
|
||
# Words and a link: the client adds these words, then opens the
|
||
# board. The words themselves, not a flag: a reader that maps the
|
||
# button to a link keeps no vocalization, but keeps this field.
|
||
btn["ext_aurasense_link_says"] = btn["vocalization"]
|
||
btn["ext_aurasense_book_goto"] = self.pages[cell.go]
|
||
if cell.do is not None:
|
||
btn["action"] = ACTIONS[cell.do]
|
||
image = None
|
||
if cell.symbol is not None:
|
||
btn["ext_aurasense_symbol"] = cell.symbol
|
||
if cell.symbol is not None and self.lang is None:
|
||
# The pictures travel in the English package only (any OBF app
|
||
# can show them); a language's package names each symbol by its
|
||
# concept, which the app draws from its own symbol sets.
|
||
set_name, sym_id = self.symbols.resolve(cell.symbol, ps.prefer)
|
||
image_id = f"{sym_id}.{n}"
|
||
path = f"images/{sym_id}.svg"
|
||
data, w, h = self.symbols.svg(set_name, sym_id)
|
||
self.images[path] = data
|
||
image = (image_id, {
|
||
"id": image_id,
|
||
"content_type": "image/svg+xml",
|
||
"width": w,
|
||
"height": h,
|
||
"path": path,
|
||
"symbol": {"set": set_name, "filename": f"{sym_id}.svg"},
|
||
"license": self.symbols.license(set_name),
|
||
})
|
||
btn["image_id"] = image_id
|
||
elif cell.symbol is None and cell.want_symbol:
|
||
prov["symbol_wanted"] = cell.want_symbol
|
||
if cell.color:
|
||
btn["background_color"] = hex_to_obf(cell.color)
|
||
if cell.border:
|
||
btn["border_color"] = hex_to_obf(cell.border)
|
||
btn["ext_aurasense_text_color"] = cell.text_color or "#000000"
|
||
if cell.fill:
|
||
fill = {k: v for k, v in cell.fill.items() if k != "hint"}
|
||
if "hint" in cell.fill:
|
||
hint, hint_en = self.words(cell.fill["hint"])
|
||
fill["hint"] = hint
|
||
if hint_en and self.lang is not None:
|
||
fill["hint_locale"] = "en"
|
||
btn["ext_aurasense_fill_in"] = fill
|
||
if cell.seq:
|
||
lines = []
|
||
for line in cell.seq["lines"]:
|
||
out_line: dict[str, Any] = {"say": tx(line["say"])}
|
||
if "cell" in line:
|
||
# The cell by its place, which never changes (SPEC §3);
|
||
# its id may (a set added twice gets new ids).
|
||
out_line["at"] = list(line["cell"])
|
||
lines.append(out_line)
|
||
seq = {
|
||
"version": 1,
|
||
"lines": lines,
|
||
"order": cell.seq.get("order", "in_order"),
|
||
"at_end": cell.seq.get("at_end", "start_again"),
|
||
"add_to_message": bool(cell.seq.get("add_to_message", False)),
|
||
}
|
||
if "reset_after_minutes" in cell.seq:
|
||
seq["reset_after_minutes"] = cell.seq["reset_after_minutes"]
|
||
btn["ext_aurasense_sequence"] = seq
|
||
if cell.effect:
|
||
btn["ext_aurasense_effect"] = cell.effect
|
||
if self.lang is None:
|
||
tr = self.button_translations(cell)
|
||
if len(tr) > 1:
|
||
btn["translations"] = tr
|
||
btn["ext_aurasense_locked"] = False
|
||
btn["ext_aurasense_provenance"] = prov
|
||
return btn, image
|
||
|
||
def board(self, b: Board, n: int) -> dict:
|
||
ps = self.ps
|
||
order: list[list[str | None]] = [[None] * b.cols for _ in range(b.rows)]
|
||
buttons: list[dict] = []
|
||
images: dict[str, dict] = {}
|
||
before = self.counts["in_english"]
|
||
# One word, one place (pslib/samewords.py; proposed SPEC §4.1 item 5): where
|
||
# this language says two of the board's phrases with the same words
|
||
# ("No" and "not" are both "no" in Spanish; "I feel" and "I'm
|
||
# feeling" both "Me siento"), they show once, at the first's place,
|
||
# and the other is hidden in its place (`ext_aurasense_same_word`:
|
||
# whatever language the app is in, since these words are written in
|
||
# the package's): no cell moves. A slot a partner fills in is told
|
||
# apart by what is filled in; a link is never hidden (the build fails
|
||
# instead, for a person to settle).
|
||
shown: dict[str, str] = {}
|
||
for cell in sorted(b.cells, key=lambda c: (c.row, c.col)):
|
||
btn, image = self.button(b, cell, n)
|
||
words = same_words(btn.get("label") or "")
|
||
if self.lang is not None and words and "ext_aurasense_fill_in" not in btn:
|
||
if words in shown:
|
||
if cell.go is not None or "load_board" in btn:
|
||
raise SourceError(f"{b.origin}: in {self.lang}, the link “{btn['label']}” shows the same words as “{shown[words]}”")
|
||
btn["ext_aurasense_same_word"] = {"*": shown[words]}
|
||
else:
|
||
shown[words] = btn["label"]
|
||
order[cell.row][cell.col] = btn["id"]
|
||
buttons.append(btn)
|
||
if image:
|
||
images.setdefault(image[0], image[1])
|
||
name, name_en = self.words(b.name)
|
||
name = b.display(name)
|
||
parts = []
|
||
if b.description:
|
||
parts.append(self.words(b.description)[0])
|
||
if self.lang is not None and (name_en or self.counts["in_english"] > before):
|
||
# The visible note: some of this board is still in English.
|
||
parts.append(self.words(NOTE_KEY)[0])
|
||
prov: dict[str, Any] = {
|
||
"generator": GENERATOR,
|
||
"source_locale": "en",
|
||
"source_name": name,
|
||
"board_type": "Grid",
|
||
"created_at": f"{ps.version}T00:00:00+00:00",
|
||
"set": ps.key,
|
||
"source": b.origin,
|
||
}
|
||
if self.lang is not None:
|
||
if name_en:
|
||
prov["name_locale"] = "en"
|
||
else:
|
||
prov["words_locale"] = self.locale
|
||
prov["english_name"] = b.name
|
||
out: dict[str, Any] = {
|
||
"format": OBF_FORMAT,
|
||
"id": bid(ps, b.key),
|
||
"locale": self.locale,
|
||
"name": name,
|
||
"license": ps.license,
|
||
"grid": {"rows": b.rows, "columns": b.cols, "order": order},
|
||
"buttons": buttons,
|
||
"images": [images[k] for k in sorted(images)],
|
||
"sounds": [],
|
||
"ext_aurasense_locked": False,
|
||
"ext_aurasense_provenance": prov,
|
||
}
|
||
if parts:
|
||
out["description_html"] = "".join(f"<p>{html.escape(p)}</p>" for p in parts)
|
||
behaviour = {}
|
||
if b.popup:
|
||
behaviour["after_choice"] = "return"
|
||
if b.speak_only:
|
||
behaviour["message"] = "speak_only"
|
||
if behaviour:
|
||
out["ext_aurasense_behaviour"] = behaviour
|
||
out["ext_aurasense_book"] = {"page": self.pages[b.key]}
|
||
return out
|
||
|
||
def build(self) -> bytes:
|
||
ps = self.ps
|
||
keys = [ps.root] + sorted((k for k in ps.boards if k != ps.root), key=lambda k: self.pages[k])
|
||
files: list[tuple[str, bytes]] = []
|
||
board_paths = {}
|
||
image_paths: dict[str, str] = {}
|
||
for n, key in enumerate(keys, start=1):
|
||
v = self.board(ps.boards[key], n)
|
||
for img in v["images"]:
|
||
image_paths[img["id"]] = img["path"]
|
||
board_paths[bid(ps, key)] = board_path(ps, key)
|
||
files.append((board_path(ps, key), json.dumps(v, ensure_ascii=False, separators=(",", ":")).encode()))
|
||
for path in sorted(self.images):
|
||
files.append((path, self.images[path]))
|
||
manifest = {
|
||
"format": OBF_FORMAT,
|
||
"root": board_path(ps, ps.root),
|
||
"paths": {"boards": board_paths, "images": dict(sorted(image_paths.items())), "sounds": {}},
|
||
"ext_aurasense_page_set": {
|
||
"key": ps.key,
|
||
"name": ps.name,
|
||
"version": ps.version,
|
||
"locale": self.locale,
|
||
"speaker_forms": self.speaker,
|
||
"generator": GENERATOR,
|
||
},
|
||
}
|
||
files.insert(0, ("manifest.json", json.dumps(manifest, ensure_ascii=False, indent=1).encode()))
|
||
return stored_zip(files)
|
||
|
||
|
||
def stored_zip(files: list[tuple[str, bytes]]) -> bytes:
|
||
"""A zip of uncompressed entries with a fixed date (what the app's own
|
||
importer reads without inflating)."""
|
||
buf = io.BytesIO()
|
||
with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_STORED) as z:
|
||
for name, data in files:
|
||
info = zipfile.ZipInfo(name, date_time=ZIP_DATE)
|
||
info.compress_type = zipfile.ZIP_STORED
|
||
info.external_attr = 0o644 << 16
|
||
info.create_system = 0
|
||
z.writestr(info, data)
|
||
return buf.getvalue()
|
||
|
||
|
||
# ─── Strings, catalogue and reports ─────────────────────────────────────────
|
||
|
||
|
||
def all_strings(sets: list[PageSet]) -> dict[str, dict]:
|
||
"""Every English text a translator must see, with where it is used."""
|
||
out: dict[str, dict] = {}
|
||
|
||
def add(text: str, where: str, role: str, note: str | None = None) -> None:
|
||
e = out.setdefault(text, {"uses": [], "roles": [], "notes": []})
|
||
if where not in e["uses"] and len(e["uses"]) < 4:
|
||
e["uses"].append(where)
|
||
if role not in e["roles"]:
|
||
e["roles"].append(role)
|
||
if note and note not in e["notes"]:
|
||
e["notes"].append(note)
|
||
|
||
for ps in sets:
|
||
for b in ps.boards.values():
|
||
add(b.name, f"{ps.family}: board name", "board name")
|
||
if b.description:
|
||
add(b.description, f"{ps.family} > {b.name}", "a note to the partner, shown with the board")
|
||
for c in b.cells:
|
||
where = f"{ps.family} > {b.name}"
|
||
add(c.label, where, "label (shown on the cell)" if (c.say or c.add) != c.label else "label and spoken message", c.note)
|
||
spoken = c.say if c.say is not None else c.add
|
||
if spoken and spoken != c.label:
|
||
add(spoken, where, "spoken message" if c.say else "words added to the message", c.note)
|
||
if c.seq:
|
||
for line in c.seq["lines"]:
|
||
add(line["say"], f"{where} > {c.label}", "a line of a sequence said one press at a time", c.note)
|
||
if c.fill and c.fill.get("hint"):
|
||
add(c.fill["hint"], where, "hint to the partner who sets the cell up")
|
||
add(NOTE_KEY, "every board", "a note shown to the partner")
|
||
return dict(sorted(out.items()))
|
||
|
||
|
||
FAMILIES = {
|
||
"adult": {
|
||
"style": "Adults with acquired conditions",
|
||
"offer": "Adults whose talking has changed (aphasia, ALS/MND, primary progressive aphasia, brain injury, "
|
||
"Parkinson's). Offer \"Make Simple page my home\" as a later choice.",
|
||
},
|
||
"intent": {
|
||
"style": "Intent-first vocabulary",
|
||
"offer": "Anyone who starts from what they want to do; choose 12, 20 or 40 cells a page. Also printable "
|
||
"as a book a partner reads from.",
|
||
},
|
||
"themes": {
|
||
"style": "Themed phrase grids",
|
||
"offer": "Readers who want the likely messages of a situation on one page.",
|
||
},
|
||
"emergent": {
|
||
"style": "First switches and single messages",
|
||
"offer": "First communicators and first switch users; the index pages are for the partner.",
|
||
},
|
||
}
|
||
|
||
|
||
def catalogue(sets: list[PageSet], built: dict[str, dict], tr: Translations, strings: dict) -> dict:
|
||
"""The catalogue a page-set chooser reads: every set, its files in every
|
||
language (with size and checksum), and how far each language goes."""
|
||
langs = {}
|
||
for code, (name, locale) in LANGUAGES.items():
|
||
agreed = sum(1 for s in strings if tr.status(code, s) == AGREED)
|
||
gendered = any((t := tr.get(code, s)) is not None and t.forms for s in strings)
|
||
langs[code] = {"language": name, "locale": locale, "agreed_texts": agreed, "texts": len(strings),
|
||
"speaker_forms": gendered, "status": STATUS_TEXT}
|
||
families = []
|
||
for fam, meta in FAMILIES.items():
|
||
members = [ps for ps in sets if ps.family == fam]
|
||
families.append({
|
||
"family": fam,
|
||
"style": meta["style"],
|
||
"name": members[0].name.replace(" ({size})", "").split(" (")[0],
|
||
"description": (members[0].description.replace(
|
||
f"{members[0].size} cells", ", ".join(str(ps.size) for ps in members[:-1]) + f" or {members[-1].size} cells")
|
||
if members[0].size else members[0].description),
|
||
"sets": [ps.key for ps in members],
|
||
"sizes": [ps.size for ps in members] if members[0].size else None,
|
||
"offer": meta["offer"],
|
||
})
|
||
return {
|
||
"version": 1,
|
||
"generator": GENERATOR,
|
||
"about": ("Phrase page sets for aac.aurasense.ai/app and any Open Board Format app, built from "
|
||
"tools/pagesets/phrases/sources. English is the source; a translation is used only where two "
|
||
"independent model readers agreed on it, and a native speaker's review is still to come. A cell "
|
||
"without an agreed translation stays in English and says so (label_locale \"en\"). Where "
|
||
"files[lang] has f and m, choose by the person's \"words about myself\" setting."),
|
||
"license": {"type": "CC-BY-SA-4.0", "url": "https://creativecommons.org/licenses/by-sa/4.0/",
|
||
"author": "AuraSense Tech Corporation and the open-standards AAC contributors",
|
||
"file": "LICENSE.md"},
|
||
"families": families,
|
||
"sets": [
|
||
{
|
||
"key": ps.key,
|
||
"family": ps.family,
|
||
"name": ps.name,
|
||
"description": ps.description,
|
||
"size": ps.size,
|
||
"boards": len(ps.boards),
|
||
"cells": sum(len(b.cells) for b in ps.boards.values()),
|
||
"files": built[ps.key],
|
||
}
|
||
for ps in sets
|
||
],
|
||
"languages": langs,
|
||
"pending_languages": PENDING_LANGUAGES,
|
||
}
|
||
|
||
|
||
# The families as the page-set catalogue (tools/pagesets/catalog.py) names
|
||
# them: its generator contract is `build.py --out DIR` writing
|
||
# DIR/families.json beside the packages.
|
||
CATALOGUE_FAMILIES = {
|
||
"adult": {"key": "phrases-adult", "name": "Adult conversation"},
|
||
"intent": {"key": "phrases-intent", "name": "Intent first"},
|
||
"themes": {"key": "phrases-themes", "name": "Themed phrases"},
|
||
"emergent": {"key": "phrases-emergent", "name": "First switches and messages"},
|
||
}
|
||
ATTRIBUTION = ("Phrase page sets \u00a9 2026 AuraSense Tech Corporation and the open-standards AAC contributors, "
|
||
"CC BY-SA 4.0. Symbols are named from Mulberry Symbols (Steve Lee; designed by Claire Barge) and "
|
||
"OpenMoji, each CC BY-SA 4.0, with their own attributions on each image. The 40-cell intent-first "
|
||
"start page uses 21 words of the Universal Core vocabulary (Center for Literacy and Disability "
|
||
"Studies, UNC Chapel Hill; Project Core), words only.")
|
||
LANGUAGE_NOTE = ("Written in English. In Spanish, French, German, Portuguese, Italian, Chinese, Japanese, Arabic, Hindi "
|
||
"and Russian the set comes in that language, where two independent AI models agreed on each "
|
||
"translation (a native speaker's review is still to come); anything not agreed stays in English "
|
||
"and says so. In other languages it is in English for now.")
|
||
|
||
|
||
def families(sets: list[PageSet], built: dict[str, dict]) -> dict:
|
||
"""The sets as families of the page-set catalogue (catalog.py's generator
|
||
contract): one package per set, language and speaker form, each with its
|
||
language (`lang`), and `gender` "f" / "m" where the words change with the
|
||
speaker's; `size` is the cells a page for the intent-first sizes."""
|
||
out = []
|
||
for fam, meta in FAMILIES.items():
|
||
cat = CATALOGUE_FAMILIES[fam]
|
||
members = [ps for ps in sets if ps.family == fam]
|
||
packages = []
|
||
cells: dict[str, list[int]] = {}
|
||
for ps in members:
|
||
root = ps.boards[ps.root]
|
||
for code, entry in built[ps.key].items():
|
||
forms = entry.items() if "file" not in entry else [(None, entry)]
|
||
for gender, f in forms:
|
||
key = f"{cat['key']}-{ps.key.removeprefix(fam).lstrip('-') or 'set'}-{code}" + (f"-{gender}" if gender else "")
|
||
packages.append({
|
||
"key": key.replace("-set-", "-"),
|
||
"file": f["file"],
|
||
"lang": code,
|
||
"gender": gender,
|
||
"variant": "full",
|
||
"size": str(ps.size) if ps.size else "",
|
||
"rows": root.rows,
|
||
"cols": root.cols,
|
||
"label": f"{ps.size} cells a page" if ps.size else ps.name,
|
||
"description": ps.description,
|
||
"home": bid(ps, ps.root),
|
||
"boards": len(ps.boards),
|
||
"cells": f["cells"],
|
||
"cells_in_english": f["cells_in_english"],
|
||
})
|
||
if code != "en" and gender in (None, "f"):
|
||
c = cells.setdefault(code, [0, 0])
|
||
c[0] += f["cells"]
|
||
c[1] += f["cells_in_english"]
|
||
languages = {code: {"status": "GlossOnly", "glossed": round(1 - english / total, 3)}
|
||
for code, (total, english) in sorted(cells.items()) if total}
|
||
sources = [{"name": "Phrases written for this project (AuraSense; open-standards AAC)", "licence": "CC-BY-SA-4.0"},
|
||
{"name": "Mulberry Symbols, OpenMoji", "licence": "CC-BY-SA-4.0"}]
|
||
if fam == "intent":
|
||
sources.insert(1, {"name": "Universal Core vocabulary (UNC CLDS / Project Core), 21 words, words only",
|
||
"licence": "CC-BY-SA-4.0"})
|
||
description = (members[0].description.replace(
|
||
f"{members[0].size} cells", ", ".join(str(ps.size) for ps in members[:-1]) + f" or {members[-1].size} cells")
|
||
if members[0].size else members[0].description)
|
||
out.append({
|
||
"key": cat["key"],
|
||
"name": cat["name"],
|
||
"summary": meta["style"],
|
||
"description": description,
|
||
"chooser": "phrases",
|
||
"version": members[0].version,
|
||
"licence": "CC-BY-SA-4.0",
|
||
"attribution": ATTRIBUTION,
|
||
"sources": sources,
|
||
"languages": languages,
|
||
"language_note": LANGUAGE_NOTE,
|
||
"offer": meta["offer"],
|
||
"packages": packages,
|
||
})
|
||
return {"families": out}
|
||
|
||
|
||
def report_symbols(sets: list[PageSet], symbols: Symbols) -> str:
|
||
wanted: dict[str, list[str]] = {}
|
||
used: dict[str, set[str]] = {}
|
||
for ps in sets:
|
||
for b in ps.boards.values():
|
||
for c in b.cells:
|
||
if c.want_symbol:
|
||
wanted.setdefault(c.want_symbol, [])
|
||
where = f"{ps.family}: {c.label}"
|
||
if where not in wanted[c.want_symbol]:
|
||
wanted[c.want_symbol].append(where)
|
||
if c.symbol:
|
||
s, _ = symbols.resolve(c.symbol, ps.prefer)
|
||
used.setdefault(c.symbol, set()).add(s)
|
||
lines = [
|
||
"# Symbols for the phrase page sets",
|
||
"",
|
||
"Generated by `build.py --reports`; do not edit.",
|
||
"",
|
||
f"{len(used)} concepts of the app's symbol index are used. {len(wanted)} concepts are wanted but the "
|
||
"index has no symbol for them yet; those cells show their words alone (an adult set may use the "
|
||
"person's own photo instead). Add them to `tools/symbols` (a confident match only: a wrong picture "
|
||
"is worse than none) and name them in the sources with `symbol:` once they exist.",
|
||
"",
|
||
"## Wanted (no symbol yet)",
|
||
"",
|
||
"| Concept | Used for |",
|
||
"|---|---|",
|
||
]
|
||
for concept in sorted(wanted):
|
||
uses = wanted[concept]
|
||
lines.append(f"| `{concept}` | {'; '.join(uses[:4])}{' …' if len(uses) > 4 else ''} |")
|
||
lines += ["", "## Used", "", "| Concept | Set |", "|---|---|"]
|
||
for concept in sorted(used):
|
||
lines.append(f"| `{concept}` | {', '.join(sorted(used[concept]))} |")
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
def report_translations(strings: dict, tr: Translations) -> str:
|
||
lines = [
|
||
"# Translation status of the phrase page sets",
|
||
"",
|
||
"Generated by `build.py --reports`; do not edit. Each text was drafted, then back-translated by a "
|
||
"second reader who never saw the English, and the two compared. Only texts both agreed on are used "
|
||
f"(\"{STATUS_TEXT}\"); the rest stay in English, marked as such in every package.",
|
||
"",
|
||
"| Language | Agreed | Disputed | Not yet translated | Speaker-gendered |",
|
||
"|---|---|---|---|---|",
|
||
]
|
||
for code, (name, _) in LANGUAGES.items():
|
||
st = [tr.status(code, s) for s in strings]
|
||
gendered = sum(1 for s in strings if (t := tr.get(code, s)) is not None and t.forms)
|
||
lines.append(f"| {name} (`{code}`) | {st.count(AGREED)} / {len(st)} | {st.count('disputed')} | "
|
||
f"{st.count('pending')} | {gendered} |")
|
||
lines += ["", f"Pending (English only for now): {', '.join(f'`{c}`' for c in PENDING_LANGUAGES)}.", ""]
|
||
for code, (name, _) in LANGUAGES.items():
|
||
disputed = [s for s in strings if tr.status(code, s) == "disputed"]
|
||
if disputed:
|
||
lines += [f"## {name}: disputed, shown in English", ""]
|
||
for s in disputed:
|
||
e = tr.by_lang[code][s]
|
||
reason = (e.get("notes") or "").replace("\n", " ")
|
||
lines.append(f"- {s!r}: {reason}")
|
||
lines.append("")
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
def report_sets(sets: list[PageSet], built: dict[str, dict]) -> str:
|
||
lines = ["# Phrase page sets: contents", "", "Generated by `build.py --reports`; do not edit.", ""]
|
||
for ps in sets:
|
||
cells = sum(len(b.cells) for b in ps.boards.values())
|
||
lines += [f"## {ps.name} (`{ps.key}.obz`)", "", ps.description, "",
|
||
f"{len(ps.boards)} boards, {cells} cells.", "", "| Page | Board | Grid | Cells | Behaviour |",
|
||
"|---|---|---|---|---|"]
|
||
pages = book_pages(ps)
|
||
for key in sorted(ps.boards, key=lambda k: pages[k]):
|
||
b = ps.boards[key]
|
||
beh = ", ".join(x for x in ("closes after a choice" if b.popup else "",
|
||
"speaks without building a message" if b.speak_only else "") if x)
|
||
lines.append(f"| {pages[key]} | {b.display(b.name)} | {b.rows}×{b.cols} | {len(b.cells)} | {beh} |")
|
||
lines.append("")
|
||
return "\n".join(lines) + "\n"
|
||
|
||
|
||
# ─── Commands ───────────────────────────────────────────────────────────────
|
||
|
||
|
||
def file_entry(name: str, data: bytes, pkg: "Package") -> dict:
|
||
return {
|
||
"file": name,
|
||
"bytes": len(data),
|
||
"sha256": hashlib.sha256(data).hexdigest(),
|
||
"cells": pkg.counts["cells"],
|
||
"cells_in_english": pkg.counts["in_english"] if pkg.lang else 0,
|
||
}
|
||
|
||
|
||
def build_outputs(hash_names: bool = False) -> tuple[dict[str, bytes], dict[Path, bytes]]:
|
||
"""Every file of a build, deterministic: (the packages, catalogue and
|
||
licence, by path relative to the output directory; the review reports,
|
||
by path in the repository)."""
|
||
symbols = Symbols()
|
||
sets = read_all()
|
||
errs = validate(sets, symbols)
|
||
if errs:
|
||
raise SourceError("\n".join(errs))
|
||
tr = Translations()
|
||
strings = all_strings(sets)
|
||
out: dict[str, bytes] = {}
|
||
built: dict[str, dict] = {}
|
||
|
||
def put(stem: str, pkg: Package) -> dict:
|
||
data = pkg.build()
|
||
name = f"{stem}.{hashlib.sha256(data).hexdigest()[:12]}.obz" if hash_names else f"{stem}.obz"
|
||
out[name] = data
|
||
return file_entry(name, data, pkg)
|
||
|
||
for ps in sets:
|
||
files: dict[str, Any] = {"en": put(ps.key, Package(ps, symbols, tr, None, None))}
|
||
texts = [t for b in ps.boards.values() for c in b.cells for t in texts_of(c)] + [b.name for b in ps.boards.values()]
|
||
texts += [b.description for b in ps.boards.values() if b.description]
|
||
for code in LANGUAGES:
|
||
if tr.gendered(code, texts):
|
||
files[code] = {g: put(f"{code}/{ps.key}.{g}", Package(ps, symbols, tr, code, g)) for g in ("f", "m")}
|
||
else:
|
||
files[code] = put(f"{code}/{ps.key}", Package(ps, symbols, tr, code, None))
|
||
built[ps.key] = files
|
||
out["index.json"] = (json.dumps(catalogue(sets, built, tr, strings), ensure_ascii=False, indent=1) + "\n").encode()
|
||
out["families.json"] = (json.dumps(families(sets, built), ensure_ascii=False, indent=1) + "\n").encode()
|
||
out["LICENSE.md"] = (HERE / "LICENSE.md").read_bytes()
|
||
reports = {
|
||
REPORTS / "symbols.md": report_symbols(sets, symbols).encode(),
|
||
REPORTS / "translation-status.md": report_translations(strings, tr).encode(),
|
||
REPORTS / "sets.md": report_sets(sets, built).encode(),
|
||
}
|
||
return out, reports
|
||
|
||
|
||
def differences(want: dict, have_dir: Path | None, have: dict | None = None) -> list[str]:
|
||
"""Files of `want` that are missing or different in `have_dir` (or the
|
||
in-memory `have`), and .obz files there that `want` does not have."""
|
||
diffs = []
|
||
for name, data in want.items():
|
||
if have is not None:
|
||
got = have.get(name)
|
||
else:
|
||
path = Path(name) if isinstance(name, Path) else have_dir / name
|
||
got = path.read_bytes() if path.exists() else None
|
||
if got != data:
|
||
diffs.append(str(name))
|
||
if have is None and have_dir is not None and have_dir.exists():
|
||
for old in have_dir.rglob("*.obz"):
|
||
if str(old.relative_to(have_dir)) not in want:
|
||
diffs.append(f"{old.relative_to(have_dir)} (not built any more)")
|
||
return diffs
|
||
|
||
|
||
def write(files: dict, base: Path | None) -> None:
|
||
for name, data in files.items():
|
||
path = Path(name) if base is None else base / name
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_bytes(data)
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
ap.add_argument("--out", type=Path,
|
||
help="write the packages, index.json, families.json and LICENSE.md into this directory")
|
||
ap.add_argument("--hash-names", action="store_true", help="name packages <name>.<sha256-12>.obz")
|
||
ap.add_argument("--check", action="store_true",
|
||
help="fail if the sources are invalid, two builds differ, reports/ is stale, or (with --out) "
|
||
"the directory differs from a fresh build; writes nothing")
|
||
ap.add_argument("--reports", action="store_true", help="rewrite reports/ (committed review aids)")
|
||
ap.add_argument("--strings", action="store_true", help="write work/strings.json for translators")
|
||
args = ap.parse_args()
|
||
if not (args.out or args.check or args.reports or args.strings):
|
||
ap.print_help()
|
||
return 2
|
||
try:
|
||
if args.strings:
|
||
sets = read_all()
|
||
errs = validate(sets, Symbols())
|
||
if errs:
|
||
raise SourceError("\n".join(errs))
|
||
WORK.mkdir(exist_ok=True)
|
||
strings = all_strings(sets)
|
||
(WORK / "strings.json").write_text(json.dumps(strings, ensure_ascii=False, indent=1) + "\n")
|
||
print(f"{len(strings)} strings -> {WORK / 'strings.json'}")
|
||
if not (args.out or args.check or args.reports):
|
||
return 0
|
||
out, reports = build_outputs(args.hash_names)
|
||
except SourceError as e:
|
||
print(f"error:\n{e}", file=sys.stderr)
|
||
return 1
|
||
if args.check:
|
||
again, _ = build_outputs(args.hash_names)
|
||
problems = [f"not reproducible: {d}" for d in differences(out, None, again)]
|
||
problems += [f"stale (run build.py --reports): {d}" for d in differences(reports, None)]
|
||
if args.out:
|
||
problems += [f"stale in {args.out}: {d}" for d in differences(out, args.out)]
|
||
if problems:
|
||
print("\n".join(problems), file=sys.stderr)
|
||
return 1
|
||
print(f"ok: {sum(1 for n in out if n.endswith('.obz'))} packages reproducible"
|
||
+ (f", {args.out} up to date" if args.out else "") + ", reports up to date")
|
||
return 0
|
||
if args.reports:
|
||
write(reports, None)
|
||
print(f"reports written: {', '.join(str(p.relative_to(AAC_RS)) for p in reports)}")
|
||
if args.out:
|
||
if args.out.exists():
|
||
for old in args.out.rglob("*.obz"):
|
||
if str(old.relative_to(args.out)) not in out:
|
||
old.unlink()
|
||
write(out, args.out)
|
||
total = sum(len(d) for d in out.values())
|
||
print(f"{sum(1 for n in out if n.endswith('.obz'))} packages, index.json, families.json and LICENSE.md -> {args.out} "
|
||
f"({total / 1e6:.1f} MB)")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|