open-standards/aac/aac-rs/tools/symbols/build.py

#!/usr/bin/env python3.14
"""Build the AAC symbol sprites and the concept -> symbol index.

Reproducible and offline once the pinned sources are cached: every archive is
pinned by URL and SHA-256, downloaded once into tools/symbols/.cache/, and
verified on every run. The app build never touches the network for symbols;
it only reads what this script writes:

  crates/aac-web/public/symbols/<set>.<sha256-12>.svg   one sprite per set
  crates/aac-web/public/symbols/LICENSES.md             attribution
  crates/aac-core/src/symbols/index.json                sets + concept map
  crates/aac-core/src/symbols/normalize_cases.json      parity fixture
  tools/symbols/mapping-review.md                       what maps to what
  tools/symbols/people-review.md                        who is drawn in the chosen skin tone
  tools/symbols/review/<set>.png                        contact sheets

Concept keys come from the labels of every default board layout
(crates/aac-web/src/defaults.rs) and every word of the built-in core
vocabulary (crates/aac-core/src/vocabulary.rs), normalised with
normalize_concept() -- the same function as aac_core::symbols. Action cells
and navigation homographs get cell concepts ("action:clear", "nav:back"), see
cell_concept().

Matching: each set's own index proposes candidates (Mulberry: symbol name;
OpenMoji: CLDR annotation, with tags only as proposals; Bliss: BCI-AV
glosses), then overrides.toml decides anything ambiguous. A wrong picture is
worse than none, so a concept stays unmapped in a set unless the match is
confident.

People: skin is written as the CSS custom property --aac-skin (OpenMoji's
skin shadow as --aac-skin-shade), falling back to Unicode's non-realistic
neutral, so a client draws every person in the skin tone the person chooses
(see aac_core::symbols::people). OpenMoji's own skin-tone variants say which
of its elements are skin; Mulberry's authors mark skin in some symbols, and
people.toml records a checked decision for every other symbol that uses its
skin colour.

Requires Python >= 3.11 (tomllib), npm (svgo and its dependencies are pinned
by package-lock.json and installed with `npm ci`), rsvg-convert for the
contact sheets, and optionally the `brotli` CLI for size reporting.

Usage: python3.14 tools/symbols/build.py [--review-pages DIR]
"""

from __future__ import annotations

import argparse
import csv
import gzip
import hashlib
import html
import json
import os
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
import textwrap
import tomllib
import unicodedata
import urllib.request
import xml.etree.ElementTree as ET
import zipfile
from dataclasses import dataclass, field
from pathlib import Path

HERE = Path(__file__).resolve().parent
ROOT = HERE.parents[1]  # aac/aac-rs
CACHE = HERE / ".cache"
PUBLIC = ROOT / "crates/aac-web/public/symbols"
CORE_SYMBOLS = ROOT / "crates/aac-core/src/symbols"
DEFAULTS_RS = ROOT / "crates/aac-web/src/defaults.rs"
VOCAB_RS = ROOT / "crates/aac-core/src/vocabulary.rs"
OVERRIDES = HERE / "overrides.toml"
REVIEW_MD = HERE / "mapping-review.md"
REVIEW_DIR = HERE / "review"
# svgo 4.1.0 and every dependency are pinned by package-lock.json; `npm ci`
# installs exactly that tree and the build runs it without resolving anything.
SVGO = [str(HERE / "node_modules/.bin/svgo")]
SVGO_CONFIG = HERE / "svgo.config.mjs"

RETRIEVED = "2026-09-29"
CC_BY_SA = {"spdx": "CC-BY-SA-4.0", "url": "https://creativecommons.org/licenses/by-sa/4.0/"}

# Order matters: it is SymbolSet::ALL, the runtime fallback order.
SET_ORDER = ("mulberry", "openmoji", "bliss")


@dataclass(frozen=True)
class Source:
    file: str
    url: str
    sha256: str


MULBERRY_COMMIT = "9cbab9f400c5de44e2bc58839cca07294aadb086"
OPENMOJI_TAG = "17.0.0"
BLISS_RELEASE = "2026-07-25"

SOURCES: dict[str, list[Source]] = {
    "mulberry": [
        Source(
            f"mulberry-symbols-{MULBERRY_COMMIT[:12]}.tar.gz",
            f"https://codeload.github.com/mulberrysymbols/mulberry-symbols/tar.gz/{MULBERRY_COMMIT}",
            "5f5814600c0f9d0d52b3548bbd961ac281da74f74a90e656684619806850976b",
        ),
    ],
    "openmoji": [
        Source(
            f"openmoji-{OPENMOJI_TAG}-svg-color.zip",
            f"https://github.com/hfg-gmuend/openmoji/releases/download/{OPENMOJI_TAG}/openmoji-svg-color.zip",
            "59b0cd9f6fe033818fc02585cea42bea9fba5d68a4d3a3639bfe5d5cc3805689",
        ),
        Source(
            f"openmoji-{OPENMOJI_TAG}.json",
            f"https://raw.githubusercontent.com/hfg-gmuend/openmoji/{OPENMOJI_TAG}/data/openmoji.json",
            "ab181dd523021dfbe2b98a13d96ab384b7b3014d17815261bc267fb39d9a211b",
        ),
    ],
    # blissymbolics.net serves plain http only; the pinned SHA-256 is what
    # makes the download trustworthy.
    "bliss": [
        Source(
            f"bliss-{BLISS_RELEASE}-svg-id.zip",
            f"http://www.blissymbolics.net/BCI-AV_{BLISS_RELEASE}/bliss_svg_id.zip",
            "b90da15d3a503c2ce7d5f11c54809e37607d6bc6a616528f3c6033009382c00d",
        ),
        Source(
            f"bliss-{BLISS_RELEASE}-id-to-gloss.txt",
            f"http://www.blissymbolics.net/BCI-AV_{BLISS_RELEASE}/BCI-AV_SKOG_{BLISS_RELEASE}_id_to_gloss_map.txt",
            "3c5d63a76e26bede82d93d6bcb1e11259ce7a1abef646bda54d6fa74f5853784",
        ),
    ],
}

# What index.json and LICENSES.md say about each set. Licence terms were read
# from the primary sources on RETRIEVED; the URLs are recorded in LICENSES.md.
SET_INFO: dict[str, dict] = {
    "mulberry": {
        "name": "Mulberry Symbols",
        "license": CC_BY_SA,
        "attribution": (
            "Mulberry Symbols (https://mulberrysymbols.org) © 2018-2026 Steve Lee, "
            "licensed under CC BY-SA 4.0. Symbols designed by Claire Barge; "
            "set originated by Garry Paxton."
        ),
        "conditions": (
            "While you may charge for your product or added value, you must not "
            "charge for the symbols themselves. They are to remain free of cost and "
            "freely available to all. Derived symbols must be shared under CC BY-SA 4.0."
        ),
        "source": {
            "url": f"https://github.com/mulberrysymbols/mulberry-symbols/tree/{MULBERRY_COMMIT}",
            "version": f"3.6.1 (commit {MULBERRY_COMMIT})",
        },
        "retrieved": RETRIEVED,
    },
    "openmoji": {
        "name": "OpenMoji",
        "license": CC_BY_SA,
        "attribution": (
            "All emojis designed by OpenMoji – the open-source emoji and icon project "
            "(https://openmoji.org). License: CC BY-SA 4.0."
        ),
        "conditions": (
            "Give appropriate credit and indicate changes (these symbols are optimised "
            "and recombined into a sprite); share adaptations under CC BY-SA 4.0; do not "
            "suggest the OpenMoji project endorses this app."
        ),
        "source": {
            "url": f"https://github.com/hfg-gmuend/openmoji/releases/tag/{OPENMOJI_TAG}",
            "version": OPENMOJI_TAG,
        },
        "retrieved": RETRIEVED,
    },
    "bliss": {
        "name": "Blissymbolics (BCI-AV)",
        "license": CC_BY_SA,
        "attribution": (
            "The Blissymbolics Communication International Authorized Vocabulary (BCI-AV) "
            "is available at www.blissymbolics.org. The BCI-AV and other BCI Blissymbol "
            "products are copyright Blissymbolics Communication International (BCI) and "
            "made available under the Creative Commons BY-SA 4.0 licence."
        ),
        "conditions": (
            "Redistribute under CC BY-SA 4.0, citing the author and owner (BCI), the "
            "origin (Blissymbolics) and the licence. Bliss-words that are not in the "
            "Authorized Vocabulary must not be published without marking them as "
            "personally created."
        ),
        "source": {
            "url": "https://www.blissymbolics.org/index.php/bci-av-2026",
            "version": f"BCI-AV {BLISS_RELEASE}",
        },
        "retrieved": RETRIEVED,
    },
}

# svgo coordinate precision per set (decimal places, relative to each set's
# own grid). Tuned so the default set stays well under the size budget.
PRECISION = {"mulberry": 0, "openmoji": 1, "bliss": 0}

# ─── Concept normalisation ──────────────────────────────────────────────────

# Unicode White_Space -- exactly what Rust's char::is_whitespace() tests.
_WHITE_SPACE = (
    "\t\n\x0b\x0c\r \x85\xa0        "
    "    

  　"
)
_WS_RUN = re.compile(f"[{re.escape(_WHITE_SPACE)}]+")
_TRAILING_PUNCT = "?!.,;"


def normalize_concept(label: str) -> str:
    """Lowercase, trim, collapse internal whitespace, strip trailing ?!.,;

    Must stay identical to aac_core::symbols::normalize_concept (the parity
    fixture normalize_cases.json is checked by a Rust test).
    """
    s = " ".join(t for t in _WS_RUN.split(label.lower()) if t)
    return s.rstrip(_TRAILING_PUNCT).rstrip(_WHITE_SPACE)


# Edge cases added to the parity fixture on top of every real label.
NORMALIZE_EDGE_CASES = [
    "  all   done  ", "All Done!", "why not?", "why not ?", "stop!!", "\tI want\n",
    "what's that?", "hi, ", "OK.", "?", "...", "", "¿qué?", "ÉCOLE", "ΟΔΟΣ",
    "Straße", "　こんにちは　", "a;b;", "one , two",
]

# ─── Concept extraction ─────────────────────────────────────────────────────

_CELL_RE = re.compile(
    r'\b(cell|fitz_cell|medical_cell|nav_cell|action_cell)\(\s*\d+\s*,\s*\d+\s*,\s*"((?:[^"\\]|\\.)*)"'
    r"(?:\s*,\s*CellAction::(\w+))?"
)
_LAYOUT_FN_RE = re.compile(r"^fn create_(\w+)_boards\(")
_VOCAB_RE = re.compile(r'\(\s*"((?:[^"\\]|\\.)*)"\s*,\s*WordCategory::(\w+)')


# Cells whose picture comes from what they do rather than from their label.
# Message-bar action cells are keyed by their action ("action:clear"), and a
# navigation cell whose label is a homograph by its navigation sense
# ("nav:back" goes back; plain "back" is the body part). Mirrors
# aac_core::symbols::lookup_for_cell.
ACTION_CONCEPTS = {"SpeakAll", "Clear", "Backspace", "Predict"}
NAV_SENSES = {"back"}


def cell_concept(kind: str, label: str, action: str | None) -> str | None:
    if kind == "action_cell" and action in ACTION_CONCEPTS:
        return f"action:{action.lower()}"
    if kind == "nav_cell" and (key := normalize_concept(label)) in NAV_SENSES:
        return f"nav:{key}"
    return None


def _rust_unescape(s: str) -> str:
    return re.sub(r"\\(.)", lambda m: {"n": "\n", "t": "\t"}.get(m.group(1), m.group(1)), s)


@dataclass
class Concept:
    key: str
    labels: set[str] = field(default_factory=set)
    layouts: set[str] = field(default_factory=set)  # default board layouts using it
    kinds: set[str] = field(default_factory=set)  # cell helper kinds (nav_cell, ...)
    vocab: set[str] = field(default_factory=set)  # WordCategory names

    @property
    def on_boards(self) -> bool:
        return bool(self.layouts)


def extract_board_labels() -> list[tuple[str, str, str, str | None]]:
    """(layout, cell kind, label, action_cell's CellAction variant) for every
    cell of every default layout."""
    out, layout = [], None
    for line in DEFAULTS_RS.read_text().splitlines():
        if m := _LAYOUT_FN_RE.match(line):
            layout = m.group(1)
        for m in _CELL_RE.finditer(line):
            if layout is None:
                raise SystemExit(f"cell outside a create_*_boards fn: {line.strip()}")
            if m.group(1) == "action_cell" and not m.group(3):
                raise SystemExit(f"action_cell without a CellAction:: literal: {line.strip()}")
            out.append((layout, m.group(1), _rust_unescape(m.group(2)), m.group(3)))
    if not out:
        raise SystemExit("no board labels found in defaults.rs -- did its helpers change?")
    return out


def extract_vocabulary() -> list[tuple[str, str]]:
    out = [(_rust_unescape(m.group(1)), m.group(2)) for m in _VOCAB_RE.finditer(VOCAB_RS.read_text())]
    if len(out) < 100:
        raise SystemExit("core vocabulary not found in vocabulary.rs -- did its table change?")
    return out


def collect_concepts() -> tuple[dict[str, Concept], list[str]]:
    concepts: dict[str, Concept] = {}
    raw_labels: list[str] = []

    def get(label: str, key: str | None = None) -> Concept | None:
        raw_labels.append(label)
        key = key or normalize_concept(label)
        if not key:
            return None
        c = concepts.setdefault(key, Concept(key))
        c.labels.add(label)
        return c

    for layout, kind, label, action in extract_board_labels():
        if c := get(label, cell_concept(kind, label, action)):
            c.layouts.add(layout)
            c.kinds.add(kind)
    for word, category in extract_vocabulary():
        if c := get(word):
            c.vocab.add(category)
    return concepts, raw_labels


def slugify(concept: str) -> str:
    s = unicodedata.normalize("NFKD", concept).encode("ascii", "ignore").decode()
    s = re.sub(r"[^a-z0-9]+", "-", s.lower().replace("'", "")).strip("-")
    if not s:
        raise SystemExit(f"concept {concept!r} has no ASCII slug")
    return s


# ─── Sources ────────────────────────────────────────────────────────────────


def sha256_file(path: Path) -> str:
    h = hashlib.sha256()
    with path.open("rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def fetch(src: Source) -> Path:
    CACHE.mkdir(parents=True, exist_ok=True)
    dest = CACHE / src.file
    if not dest.exists():
        print(f"  downloading {src.url}", file=sys.stderr)
        tmp = dest.with_suffix(dest.suffix + ".part")
        with urllib.request.urlopen(src.url, timeout=120) as r, tmp.open("wb") as f:
            shutil.copyfileobj(r, f)
        tmp.rename(dest)
    got = sha256_file(dest)
    if got != src.sha256:
        raise SystemExit(f"{dest.name}: SHA-256 {got} != pinned {src.sha256}; delete it to refetch")
    return dest


def unpack(archive: Path) -> Path:
    """Extract once into .cache/<archive>.d (keyed by the verified file name)."""
    out = CACHE / (archive.name + ".d")
    if not (out / ".ok").exists():
        shutil.rmtree(out, ignore_errors=True)
        out.mkdir()
        if archive.name.endswith(".zip"):
            with zipfile.ZipFile(archive) as z:
                z.extractall(out)
        else:
            with tarfile.open(archive) as t:
                t.extractall(out, filter="data")
        (out / ".ok").touch()
    return out


# ─── Set indexes ────────────────────────────────────────────────────────────


@dataclass
class Sym:
    id: str  # native identifier used in overrides.toml
    name: str  # human-readable name for the review
    path: Path  # source SVG
    rel: str  # path inside the source archive, for the review
    # OpenMoji: the source SVG of each skin-tone variant, by modifier (1F3FB..1F3FF)
    tones: dict[str, Path] = field(default_factory=dict)
    # OpenMoji: it draws people or a part of one (group "people-body")
    person: bool = False


@dataclass
class Match:
    sym: Sym
    how: str  # exact | synonym | override
    reason: str = ""


class SymbolIndex:
    set_id: str

    def __init__(self) -> None:
        self.symbols: dict[str, Sym] = {}

    def auto(self, c: Concept) -> Match | None:
        raise NotImplementedError

    def proposals(self, c: Concept) -> list[Sym]:
        """Weak candidates listed in the review but never auto-accepted."""
        return []


def _wants_verb(c: Concept) -> bool:
    return bool(c.vocab) and c.vocab <= {"Verb"}


class Mulberry(SymbolIndex):
    set_id = "mulberry"
    # Letters (the "I" symbol is the letter, not the pronoun), flags and maps
    # never stand for a word; rated symbols are anatomical.
    EXCLUDED_CATEGORIES = {"Alphabet", "Country Flags", "Country Maps"}

    def __init__(self) -> None:
        super().__init__()
        (tar,) = [fetch(s) for s in SOURCES["mulberry"]]
        base = unpack(tar) / f"mulberry-symbols-{MULBERRY_COMMIT}"
        self.by_key: dict[str, list[tuple[Sym, bool, int]]] = {}
        with (base / "scripts/data/symbol-info-en.csv").open(newline="") as f:
            for row in csv.DictReader(f):
                name = row["symbol"]
                path = base / "EN" / f"{name}.svg"
                if not path.exists():
                    continue
                sym = Sym(name, name, path, f"EN/{name}.svg")
                self.symbols[name] = sym
                if row["category"] in self.EXCLUDED_CATEGORIES or row["rated"] != "0":
                    continue
                verb = name.endswith("_,_to") or row["grammar"].startswith("Verb")
                words = [w for w in name.removesuffix("_,_to").replace("_-_", "_").split("_") if w]
                number = 0
                while len(words) > 1 and re.fullmatch(r"\d+[a-z]?", words[-1]):
                    number = int(re.match(r"\d+", words.pop()).group())
                key = " ".join(words).lower()
                self.by_key.setdefault(key, []).append((sym, verb, number))

    def auto(self, c: Concept) -> Match | None:
        cands = self.by_key.get(c.key)
        if not cands:
            return None
        want_verb = _wants_verb(c)
        best = min(cands, key=lambda t: (t[1] != want_verb, t[2], t[0].id))
        return Match(best[0], "exact")


class OpenMoji(SymbolIndex):
    set_id = "openmoji"

    def __init__(self) -> None:
        super().__init__()
        zip_path, meta = [fetch(s) for s in SOURCES["openmoji"]]
        base = unpack(zip_path)
        self.by_annotation: dict[str, list[Sym]] = {}
        self.by_tag: dict[str, list[Sym]] = {}
        entries = json.loads(meta.read_text())
        for e in entries:
            path = base / f"{e['hexcode']}.svg"
            if not path.exists():
                continue
            sym = Sym(e["hexcode"], f"{e['emoji']} {e['annotation']}", path, f"color/svg/{e['hexcode']}.svg",
                      person=e["group"] == "people-body")
            self.symbols[e["hexcode"]] = sym
            # Skin-tone variants, flags, components and brand logos (the "apple"
            # annotation is the Apple logo) never stand for a word.
            if e["skintone"] or e["group"] in ("flags", "component") or e["subgroups"] == "brand":
                continue
            self.by_annotation.setdefault(e["annotation"].lower(), []).append(sym)
            tags = {t.strip().lower() for t in (e["tags"] + "," + e["openmoji_tags"]).split(",")}
            for t in tags - {""}:
                self.by_tag.setdefault(t, []).append(sym)

        # Each person's single-tone variants, as OpenMoji lists them (in a ZWJ
        # sequence the modifier follows the first person, not the end).
        for e in entries:
            base_sym = self.symbols.get(e["skintone_base_hexcode"])
            if base_sym and isinstance(e["skintone"], int) and 1 <= e["skintone"] <= 5:
                variant = base / f"{e['hexcode']}.svg"
                if variant.exists():
                    base_sym.tones[f"1F3F{'BCDEF'[e['skintone'] - 1]}"] = variant

    def auto(self, c: Concept) -> Match | None:
        cands = self.by_annotation.get(c.key)
        return Match(cands[0], "exact") if cands else None

    def proposals(self, c: Concept) -> list[Sym]:
        return self.by_tag.get(c.key, [])[:6]


class Bliss(SymbolIndex):
    set_id = "bliss"
    GENDERED = re.compile(r"(feminine|masculine)")
    LETTER = re.compile(r"_\((uppercase|lowercase)\)")

    def __init__(self) -> None:
        super().__init__()
        zip_path, glosses = [fetch(s) for s in SOURCES["bliss"]]
        base = unpack(zip_path) / "bliss_svg_id"
        # key -> [(sym, is_verb, rank)]; rank 0 = first gloss, no qualifier
        self.by_key: dict[str, list[tuple[Sym, bool, int]]] = {}
        for line in glosses.read_text(encoding="utf-8", errors="replace").splitlines():
            if "\t" not in line:
                continue
            bid, gloss = (p.strip() for p in line.split("\t", 1))
            path = base / f"{bid}.svg"
            if not path.exists():
                continue
            sym = Sym(bid, gloss, path, f"bliss_svg_id/{bid}.svg")
            self.symbols[bid] = sym
            qualifier = ""
            if m := re.search(r"-\(([^()]*)\)$", gloss):
                qualifier, gloss = m.group(1), gloss[: m.start()]
            if self.GENDERED.search(qualifier) or "OLD" in gloss or self.LETTER.search(gloss):
                continue
            for pos, token in enumerate(gloss.split(",")):
                disambiguated = bool(re.search(r"_\([^)]*\)$", token))
                token = re.sub(r"_\([^)]*\)$", "", token)
                key = token.replace("_", " ").strip().lower()
                if key:
                    rank = (0 if pos == 0 else 2) + (1 if disambiguated else 0)
                    self.by_key.setdefault(key, []).append((sym, qualifier == "to", rank))

    def auto(self, c: Concept) -> Match | None:
        cands = self.by_key.get(c.key)
        if not cands:
            return None
        want_verb = _wants_verb(c)
        best = min(cands, key=lambda t: (t[1] != want_verb, t[2], int(t[0].id)))
        return Match(best[0], "exact" if best[2] == 0 else "synonym")


# ─── Mapping ────────────────────────────────────────────────────────────────


def load_overrides(indexes: dict[str, SymbolIndex], concepts: dict[str, Concept]) -> dict:
    data = tomllib.loads(OVERRIDES.read_text()) if OVERRIDES.exists() else {}
    for set_id, entries in data.items():
        if set_id not in indexes:
            raise SystemExit(f"overrides.toml: unknown set [{set_id}]")
        for key, entry in entries.items():
            where = f"overrides.toml [{set_id}] {key!r}"
            if key not in concepts:
                raise SystemExit(f"{where}: not a concept (stale override?)")
            if not entry.get("reason"):
                raise SystemExit(f"{where}: needs a reason")
            if entry.get("none"):
                continue
            if entry.get("symbol") not in indexes[set_id].symbols:
                raise SystemExit(f"{where}: no symbol {entry.get('symbol')!r} in {set_id}")
    return data


def map_concepts(indexes, concepts, overrides) -> dict[str, dict[str, Match]]:
    mapping: dict[str, dict[str, Match]] = {s: {} for s in SET_ORDER}
    for set_id, index in indexes.items():
        ov = overrides.get(set_id, {})
        for key, c in concepts.items():
            if key in ov:
                if not ov[key].get("none"):
                    mapping[set_id][key] = Match(index.symbols[ov[key]["symbol"]], "override", ov[key]["reason"])
            elif m := index.auto(c):
                mapping[set_id][key] = m
    return mapping


# ─── SVG sanitising ─────────────────────────────────────────────────────────

SVG_NS = "http://www.w3.org/2000/svg"
XLINK_NS = "http://www.w3.org/1999/xlink"

ALLOWED_ELEMENTS = {
    "g", "path", "circle", "ellipse", "rect", "line", "polyline", "polygon", "defs",
    "linearGradient", "radialGradient", "stop", "clipPath", "mask", "pattern", "use",
    "text", "tspan",
}
DROPPED_ELEMENTS = {"title", "desc", "metadata", "style", "script", "sodipodi:namedview"}
PRESENTATION = {
    "fill", "fill-opacity", "fill-rule", "stroke", "stroke-width", "stroke-linecap",
    "stroke-linejoin", "stroke-miterlimit", "stroke-dasharray", "stroke-dashoffset",
    "stroke-opacity", "opacity", "clip-path", "clip-rule", "mask", "stop-color",
    "stop-opacity", "font-family", "font-size", "font-weight", "font-style",
    "text-anchor", "display", "visibility", "color", "paint-order",
}
GEOMETRY = {
    "d", "x", "y", "x1", "y1", "x2", "y2", "cx", "cy", "r", "rx", "ry", "fx", "fy", "fr",
    "width", "height", "points", "transform", "offset", "gradientUnits",
    "gradientTransform", "spreadMethod", "patternUnits", "patternContentUnits",
    "patternTransform", "viewBox", "preserveAspectRatio", "clipPathUnits", "maskUnits",
    "maskContentUnits", "dx", "dy", "rotate", "textLength", "lengthAdjust", "pathLength",
}
_URL_REF = re.compile(r"url\(\s*['\"]?#([^)'\"]+)['\"]?\s*\)")


def _local(name: str) -> tuple[str | None, str]:
    if name.startswith("{"):
        ns, local = name[1:].split("}", 1)
        return ns, local
    return None, name


def sanitize(slug: str, svg_text: str) -> tuple[str, str]:
    """Return (viewBox, inner markup) of one optimised SVG, made safe to share
    a document with every other symbol: allow-listed elements and attributes
    only, no styles or classes, local references only, ids prefixed by slug."""
    root = ET.fromstring(svg_text)
    if _local(root.tag)[1] != "svg":
        raise SystemExit(f"{slug}: root is not <svg>")
    view_box = root.get("viewBox")
    if not view_box:
        raise SystemExit(f"{slug}: no viewBox")

    ids = {el.get("id") for el in root.iter() if el.get("id")}

    def fix_ref(value: str) -> str | None:
        refs = _URL_REF.findall(value)
        if "url(" in value and (not refs or any(r not in ids for r in refs)):
            return None
        return _URL_REF.sub(lambda m: f"url(#{slug}-{m.group(1)})", value)

    def clean_attrs(el: ET.Element) -> dict[str, str]:
        attrs: dict[str, str] = {}
        style = el.get("style", "")
        for k, v in el.attrib.items():
            ns, name = _local(k)
            if ns == XLINK_NS and name == "href":
                ns = None
            if ns is not None or name.startswith("on") or name in ("style", "class"):
                continue
            if "javascript:" in v.lower().replace(" ", ""):
                continue
            if name == "id":
                attrs["id"] = f"{slug}-{v}"
            elif name == "href":
                if v.startswith("#") and v[1:] in ids:
                    attrs["href"] = f"#{slug}-{v[1:]}"
            elif name in PRESENTATION or name in GEOMETRY:
                if (fixed := fix_ref(v)) is not None:
                    attrs[name] = fixed
        for decl in style.split(";"):
            if ":" in decl:
                prop, val = (p.strip() for p in decl.split(":", 1))
                if prop in PRESENTATION and prop not in attrs and "javascript:" not in val:
                    if (fixed := fix_ref(val)) is not None:
                        attrs[prop] = fixed
        return attrs

    def emit(el: ET.Element) -> str:
        ns, name = _local(el.tag)
        if name in DROPPED_ELEMENTS:
            return ""
        if ns not in (None, SVG_NS) or name not in ALLOWED_ELEMENTS:
            raise SystemExit(f"{slug}: element <{name}> is not allowed in a sprite")
        attrs = clean_attrs(el)
        if name == "use" and "href" not in attrs:
            return ""
        inner = html.escape(el.text or "", quote=False) if name in ("text", "tspan") else ""
        for child in el:
            inner += emit(child)
            if name in ("text", "tspan"):
                inner += html.escape(child.tail or "", quote=False)
        a = "".join(f' {k}="{html.escape(v, quote=True)}"' for k, v in attrs.items())
        return f"<{name}{a}>{inner}</{name}>" if inner else f"<{name}{a}/>"

    body = "".join(emit(child) for child in root)
    # Presentation attributes on the root (Bliss sets fill-rule and
    # stroke-linecap there) are kept on a wrapper group.
    root_attrs = {k: v for k, v in clean_attrs(root).items() if k in PRESENTATION}
    if root_attrs:
        a = "".join(f' {k}="{html.escape(v, quote=True)}"' for k, v in root_attrs.items())
        body = f"<g{a}>{body}</g>"
    return view_box, body


# ─── People: skin tone ──────────────────────────────────────────────────────
#
# A person is drawn in Unicode's generic, non-realistic colour unless the
# person using the board chooses one of the five skin tones of the emoji
# modifiers (aac_core::symbols::people keeps the same table). Skin, and the
# OpenMoji hair its toned variants change with it, become CSS custom
# properties whose fallback is the neutral colour.

PEOPLE = HERE / "people.toml"
PEOPLE_MD = HERE / "people-review.md"
# The neutral a person is drawn in: UTS #51 §2.4's own example of a generic,
# non-realistic skin tone. OpenMoji's lemon #fcea2b would melt into
# Mulberry's lemon hair, clothes and objects (#fbee34, #fbed24).
NEUTRAL = {"skin": "#ffcc22", "shade": "#f1b31c"}
NEUTRAL_SKIN = NEUTRAL["skin"]
# OpenMoji's own art: its unmodified people, and each modifier's colours for
# skin and the skin's shadow (people.rs keeps the same table). Hair is never
# tied to skin (UTS #51: no hair colour goes with a skin tone).
OPENMOJI_BASE = {"skin": "#fcea2b", "shade": "#f1b31c"}
TONES = {
    "1F3FB": {"skin": "#fadcbc", "shade": "#debb90"},
    "1F3FC": {"skin": "#debb90", "shade": "#c19a65"},
    "1F3FD": {"skin": "#c19a65", "shade": "#a57939"},
    "1F3FE": {"skin": "#a57939", "shade": "#6a462f"},
    "1F3FF": {"skin": "#6a462f", "shade": "#352318"},
}
PROPERTY = {"skin": "--aac-skin", "shade": "--aac-skin-shade"}
STYLE = {role: f"fill:var({PROPERTY[role]},{NEUTRAL[role]})" for role in PROPERTY}
SKIN_STYLE = STYLE["skin"]
MULBERRY_SKIN = "#ffeec8"
# Mulberry's authors mark skin with class="aac-skin-fill" in some symbols. The
# class does not survive svgo and the sanitiser, so before optimising, marked
# skin is painted this colour, which no symbol uses; tone_people() finds it.
SKIN_MARK = "#fd01fd"
# OpenMoji's toned elements are marked the same way, one colour per role, so
# a face and a badge sharing its yellow are told apart (see openmoji_roles).
MARKS = {"skin": SKIN_MARK, "shade": "#fd02fd"}
_DRAWN = {"path", "circle", "ellipse", "rect", "polygon", "polyline", "line"}
# Marked skin is a drawn element, or a group whose fill its children inherit.
_MARKABLE_TAG = re.compile(r"<(?:g|path|circle|ellipse|rect|polygon|polyline|line)\b[^>]*>")
# A drawn element's start tag, in document order (the order _fills() reads).
_DRAWN_ELEMENT = re.compile(r"(<(?:path|circle|ellipse|rect|polygon|polyline|line)\b)[^>]*>")


def mark_skin(text: str) -> str:
    """Paint Mulberry's author-marked skin (aac-skin-fill) SKIN_MARK."""
    if SKIN_MARK in text.lower():
        raise SystemExit(f"a Mulberry symbol already uses {SKIN_MARK}; pick another SKIN_MARK")

    def paint(m: re.Match) -> str:
        el = m.group(0)
        if not re.search(r'class="[^"]*\baac-skin-fill\b', el):
            return el
        painted, n = re.subn(r'fill="[^"]*"', f'fill="{SKIN_MARK}"', el, count=1)
        if not n:
            raise SystemExit(f"marked skin without a fill attribute: {el[:80]}")
        return painted

    return _MARKABLE_TAG.sub(paint, text)


def _hex(colour: str | None) -> str | None:
    """#abc or #AABBCC -> #aabbcc; anything else (none, url(), a name) -> None."""
    c = (colour or "").strip().lower()
    if re.fullmatch(r"#[0-9a-f]{3}", c):
        return "#" + "".join(ch * 2 for ch in c[1:])
    return c if re.fullmatch(r"#[0-9a-f]{6}", c) else None


def _fills(path: Path) -> list[str | None]:
    """Each drawn element's own fill, in document order."""
    out = []
    for el in ET.parse(path).getroot().iter():
        if _local(el.tag)[1] not in _DRAWN:
            continue
        fill = el.get("fill")
        for decl in el.get("style", "").split(";"):
            if ":" in decl and decl.split(":", 1)[0].strip() == "fill":
                fill = decl.split(":", 1)[1]
        out.append(_hex(fill))
    return out


def _layer_roles(path: Path) -> list[str | None]:
    """Each drawn element's OpenMoji layer, in document order: "skin" under
    <g id="skin"> (or a second person's "skin-a"/"skin-b"), "shade" under
    <g id="skin-shadow">, else None. OpenMoji's own tone generator recolours
    by these layers."""
    out: list[str | None] = []

    def walk(el: ET.Element, role: str | None) -> None:
        gid = el.get("id") or ""
        if gid.startswith("skin-shadow"):
            role = "shade"
        elif gid == "skin" or re.fullmatch(r"skin-[a-z]", gid):
            role = "skin"
        if _local(el.tag)[1] in _DRAWN:
            out.append(role)
        for child in el:
            walk(child, role)

    walk(ET.parse(path).getroot(), None)
    return out


def openmoji_roles(sym: Sym) -> tuple[list[str | None], str]:
    """The role of each drawn element of an OpenMoji emoji ("skin", "shade" or
    None), from its skin layers. Where Unicode gives the emoji skin tones,
    OpenMoji's own toned variants must draw exactly those elements in the
    table's colours, or the build stops. Returns ([], why) for no skin."""
    roles = _layer_roles(sym.path)
    if "skin" not in roles:
        return [], ("no skin drawn" if sym.person else "not a person")
    if set(sym.tones) != set(TONES):
        return roles, "Unicode defines no skin tones for it; its skin layer takes the chosen tone"
    for t, path in sym.tones.items():
        toned = _fills(path)
        if len(toned) != len(roles):
            raise SystemExit(f"OpenMoji {sym.id}: its {t} variant is drawn differently")
        for i, (role, colour) in enumerate(zip(roles, toned)):
            if role and colour != TONES[t][role]:
                raise SystemExit(f"OpenMoji {sym.id}: element {i} of its {role} layer is {colour} "
                                 f"in the {t} variant, not {TONES[t][role]}")
    return roles, ""


def mark_roles(text: str, roles: list[str | None]) -> str:
    """Paint each drawn element whose role is known its role's mark."""
    it = iter(roles)

    def paint(m: re.Match) -> str:
        el = m.group(0)
        role = next(it)
        if role is None:
            return el
        el = re.sub(r'\sfill="[^"]*"', "", el)
        el = re.sub(r'(style="[^"]*?)fill:[^;"]*;?', r"\1", el)
        return el.replace(m.group(1), f'{m.group(1)} fill="{MARKS[role]}"', 1)

    out = _DRAWN_ELEMENT.sub(paint, text)
    if next(it, "end") != "end":
        raise SystemExit("mark_roles: fewer drawn elements than roles")
    return out


def load_people(mulberry: SymbolIndex) -> dict[str, dict]:
    """people.toml: {Mulberry symbol: {"tone": bool, "reason": str}}."""
    data = tomllib.loads(PEOPLE.read_text()) if PEOPLE.exists() else {}
    unknown = set(data) - {"mulberry"}
    if unknown:
        raise SystemExit(f"people.toml: unknown section(s) {sorted(unknown)} (only [mulberry] is listed)")
    entries = data.get("mulberry", {})
    for name, e in entries.items():
        where = f"people.toml [mulberry] {name!r}"
        if name not in mulberry.symbols:
            raise SystemExit(f"{where}: no such Mulberry symbol")
        if not isinstance(e.get("tone"), bool):
            raise SystemExit(f"{where}: needs tone = true or false")
        if not e["tone"] and not e.get("reason"):
            raise SystemExit(f"{where}: tone = false needs a reason")
    return entries


def tone_people(set_id: str, sym: Sym, body: str, people: dict[str, dict]) -> tuple[str, dict]:
    """Write skin (and OpenMoji's skin shadow) as custom properties; return
    the body and a record for people-review.md."""
    fills = {_hex(c) for c in re.findall(r'fill="([^"]+)"', body)}
    why = ""
    if set_id == "mulberry" and SKIN_MARK in fills:
        if sym.id in people:
            raise SystemExit(f"people.toml [mulberry] {sym.id!r}: its authors mark its skin (aac-skin-fill); "
                             "it needs no decision")
        roles = {SKIN_MARK: "skin"}
        why = "skin marked by Mulberry's authors"
    elif set_id == "mulberry":
        if MULBERRY_SKIN not in fills:
            if sym.id in people:
                raise SystemExit(f"people.toml [mulberry] {sym.id!r}: it does not use {MULBERRY_SKIN} (stale entry?)")
            return body, {}
        decision = people.get(sym.id)
        if decision is None:
            raise SystemExit(
                f"people.toml [mulberry]: no decision for {sym.id!r}, which uses the skin colour "
                f"{MULBERRY_SKIN}. Check it on a contact sheet with that colour highlighted, then add "
                f"{sym.id!r} = {{ tone = true }} or {{ tone = false, reason = \"...\" }}.")
        if not decision["tone"]:
            return body, {"toned": False, "reason": decision["reason"]}
        roles = {MULBERRY_SKIN: "skin"}
        why = decision.get("reason", "reviewed: every area in the skin colour is skin")
    elif set_id == "openmoji":
        element_roles, why = openmoji_roles(sym)
        if not element_roles:
            # A silhouette or a body part drawn without skin stays as drawn.
            return body, ({"toned": False, "reason": why} if sym.person else {})
        roles = {MARKS[r]: r for r in set(element_roles) if r}
    else:
        return body, {}

    # The neutral colour stays as the presentation attribute: where the style
    # is not applied (a renderer without custom properties, a page whose CSP
    # blocks style attributes) a person is drawn neutral, never black.
    def swap(m: re.Match) -> str:
        role = roles.get(_hex(m.group(1)))
        return f'fill="{NEUTRAL[role]}" style="{STYLE[role]}"' if role else m.group(0)

    toned = re.sub(r'fill="([^"]+)"', swap, body)
    if SKIN_STYLE not in toned or any(mark in toned.lower() for mark in MARKS.values()):
        raise SystemExit(f"{set_id} {sym.id}: skin not found, or not all of it toned, after optimising")
    parts = sorted(set(roles.values()), key=["skin", "shade"].index)
    return toned, {"toned": True, "parts": parts, "reason": why}


def resolve_people(text: str) -> str:
    """Without the custom-property styles, for renderers that lack CSS
    variables (rsvg-convert, the contact sheets): people are drawn neutral."""
    for style in STYLE.values():
        text = text.replace(f' style="{style}"', "")
    return text


def optimise(set_id: str, files: dict[str, Path],
             roles: dict[str, list[str | None]] | None = None) -> dict[str, str]:
    """slug -> optimised SVG text, through the pinned svgo. `roles`: the
    OpenMoji elements to mark before optimising (openmoji_roles)."""
    if not Path(SVGO[0]).exists():
        raise SystemExit(f"svgo is not installed: run `npm ci` in {HERE}")
    with tempfile.TemporaryDirectory() as tmp:
        src, out = Path(tmp, "in"), Path(tmp, "out")
        src.mkdir()
        for slug, path in files.items():
            if set_id == "mulberry":
                (src / f"{slug}.svg").write_text(mark_skin(path.read_text()))
            elif roles and roles.get(slug):
                (src / f"{slug}.svg").write_text(mark_roles(path.read_text(), roles[slug]))
            else:
                shutil.copyfile(path, src / f"{slug}.svg")
        env = dict(os.environ, AAC_SVGO_PRECISION=str(PRECISION[set_id]))
        subprocess.run([*SVGO, "--config", str(SVGO_CONFIG), "-q", "-f", str(src), "-o", str(out)],
                       check=True, env=env)
        return {slug: (out / f"{slug}.svg").read_text() for slug in files}


# ─── Outputs ────────────────────────────────────────────────────────────────


def assign_slugs(set_id: str, matches: dict[str, Match]) -> dict[str, str]:
    """concept -> slug, the <symbol id>. Concepts that share a source symbol
    share its <symbol>, named after the first such concept. Slugs carry the
    set as a prefix ("mulberry-home") so sprites can be inlined together, or
    into a page, without id clashes."""
    by_sym: dict[str, str] = {}
    out: dict[str, str] = {}
    for key in sorted(matches):
        sym_id = matches[key].sym.id
        out[key] = by_sym.setdefault(sym_id, f"{set_id}-{slugify(key)}")
    slugs = {}
    for key, slug in out.items():
        if slugs.setdefault(slug, matches[key].sym.id) != matches[key].sym.id:
            raise SystemExit(f"slug collision: {slug!r}")
    return out


def build_sprite(set_id: str, matches: dict[str, Match], slugs: dict[str, str],
                 people: dict[str, dict]) -> tuple[str, str, dict[str, dict]]:
    """Return (file name, sprite text, people records by slug) and write the
    sprite, replacing the old hash."""
    files = {slugs[k]: matches[k].sym.path for k in matches}
    syms = {slugs[k]: matches[k].sym for k in matches}
    roles = {slug: openmoji_roles(sym)[0] for slug, sym in syms.items()} if set_id == "openmoji" else None
    optimised = optimise(set_id, files, roles)
    records: dict[str, dict] = {}
    info = SET_INFO[set_id]
    parts = [
        '<svg xmlns="http://www.w3.org/2000/svg">',
        f"<!-- {info['name']} {info['source']['version']}. {info['attribution']} "
        f"Adapted: optimised and combined into one sprite. See LICENSES.md. -->",
    ]
    for slug in sorted(files):
        view_box, body = sanitize(slug, optimised[slug])
        body, record = tone_people(set_id, syms[slug], body, people)
        if record:
            records[slug] = {**record, "symbol": syms[slug].id}
        parts.append(f'<symbol id="{slug}" viewBox="{view_box}">{body}</symbol>')
    parts.append("</svg>\n")
    text = "\n".join(parts)
    digest = hashlib.sha256(text.encode()).hexdigest()[:12]
    name = f"{set_id}.{digest}.svg"
    PUBLIC.mkdir(parents=True, exist_ok=True)
    for old in PUBLIC.glob(f"{set_id}.*.svg"):
        if re.fullmatch(rf"{set_id}\.[0-9a-f]{{12}}\.svg", old.name) and old.name != name:
            old.unlink()
    (PUBLIC / name).write_text(text)
    return name, text, records


def check_sprite(text: str) -> None:
    """Belt and braces over sanitize(): nothing executable or external."""
    lowered = text.lower()
    for bad in ("<script", "<style", "<foreignobject", "javascript:", " class=", "xlink:",
                "<image", "@import"):
        if bad in lowered:
            raise SystemExit(f"sprite contains {bad!r}")
    if re.search(r"\son[a-z]+=", lowered) or re.search(r'href="(?!#)', lowered):
        raise SystemExit("sprite contains an event attribute or external href")
    ids = re.findall(r'\sid="([^"]+)"', text)
    if len(ids) != len(set(ids)):
        raise SystemExit("sprite has duplicate ids")
    for style in re.findall(r'\sstyle="([^"]*)"', text):
        if style not in STYLE.values():
            raise SystemExit(f"sprite has a style other than the skin properties: {style!r}")
    ET.fromstring(text)


def sizes(text: str) -> dict[str, int | None]:
    raw = text.encode()
    out: dict[str, int | None] = {"raw": len(raw), "gzip": len(gzip.compress(raw, 9)), "brotli": None}
    if shutil.which("brotli"):
        r = subprocess.run(["brotli", "-q", "11", "-c"], input=raw, capture_output=True, check=True)
        out["brotli"] = len(r.stdout)
    return out


def contact_sheet(set_id: str, sprite: str, matches: dict[str, Match], slugs: dict[str, str],
                  out: Path, cols: int = 16, tile: int = 110) -> None:
    """Render every mapped concept of a set, labelled, via rsvg-convert."""
    keys = sorted(matches)
    label_h = 30
    rows = -(-len(keys) // cols)
    w, h = cols * tile, rows * (tile + label_h) + 40
    defs = resolve_people(sprite.split("\n", 2)[2].rsplit("</svg>", 1)[0])  # the <symbol>s
    tiles = []
    for i, key in enumerate(keys):
        x, y = (i % cols) * tile, 40 + (i // cols) * (tile + label_h)
        pad = 10
        tiles.append(
            f'<rect x="{x + 2}" y="{y + 2}" width="{tile - 4}" height="{tile + label_h - 4}" '
            f'fill="#fff" stroke="#ccc"/>'
            f'<use href="#{slugs[key]}" x="{x + pad}" y="{y + pad}" width="{tile - 2 * pad}" '
            f'height="{tile - 2 * pad}"/>'
            f'<text x="{x + tile / 2}" y="{y + tile + 6}" font-size="13" font-family="Helvetica, Arial, sans-serif" '
            f'text-anchor="middle" fill="#000">{html.escape(key)}</text>'
            f'<text x="{x + tile / 2}" y="{y + tile + 20}" font-size="8" font-family="Helvetica, Arial, sans-serif" '
            f'text-anchor="middle" fill="#777">{html.escape(matches[key].sym.name[:26])}</text>'
        )
    # The sheet is itself an adaptation of the set, so it carries the
    # attribution, licence and change notice (CC BY-SA 4.0 s3(a)(1)).
    info = SET_INFO[set_id]
    title = f"{info['name']} {info['source']['version']} — {len(keys)} concepts"
    notice = (f"{info['attribution']} Adapted (optimised, recombined, rasterised). "
              f"CC BY-SA 4.0 {info['license']['url']}")
    notice_lines = textwrap.wrap(notice, width=max(40, (w - 20) // 7))
    head = 40 + 15 * len(notice_lines)
    h += head - 40
    notice_svg = "".join(
        f'<text x="10" y="{48 + 15 * n}" font-size="11" font-family="Helvetica, Arial, sans-serif" '
        f'fill="#333">{html.escape(line)}</text>'
        for n, line in enumerate(notice_lines)
    )
    svg = (
        f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" viewBox="0 0 {w} {h}">'
        f'<rect width="{w}" height="{h}" fill="#f4f4f4"/><defs>{defs}</defs>'
        f'<text x="10" y="26" font-size="20" font-family="Helvetica, Arial, sans-serif">{html.escape(title)}</text>'
        + notice_svg
        + f'<g transform="translate(0 {head - 40})">' + "".join(tiles) + "</g></svg>"
    )
    out.parent.mkdir(parents=True, exist_ok=True)
    with tempfile.NamedTemporaryFile("w", suffix=".svg", delete=False) as f:
        f.write(svg)
    try:
        subprocess.run(["rsvg-convert", "-o", str(out), f.name], check=True)
    finally:
        os.unlink(f.name)


def write_review(concepts, mapping, indexes, set_files, set_sizes) -> None:
    board_keys = [k for k in sorted(concepts) if concepts[k].on_boards]
    lines = [
        "# Symbol mapping review",
        "",
        "Generated by `tools/symbols/build.py` — do not edit; change `overrides.toml` and rebuild.",
        "",
        "Symbol names and BCI-AV glosses quoted below come from Mulberry Symbols, OpenMoji and",
        "Blissymbolics (BCI-AV) and are shared under CC BY-SA 4.0",
        "(https://creativecommons.org/licenses/by-sa/4.0/); attribution for each set is in",
        "`crates/aac-web/public/symbols/LICENSES.md`.",
        "",
        "Each cell is `native id` — source file, then how it was chosen: **exact** (the set's own",
        "name/annotation/first gloss), **synonym** (a later BCI-AV gloss), **override** (hand-curated,",
        "with the reason in `overrides.toml`). A blank cell is intentionally unmapped: the runtime falls",
        "back to the next set.",
        "",
        "## Coverage",
        "",
        "| set | sprite | all concepts | default-board concepts | raw | gzip | brotli |",
        "|---|---|---|---|---|---|---|",
    ]
    for s in SET_ORDER:
        m = mapping[s]
        on_boards = sum(1 for k in board_keys if k in m)
        z = set_sizes[s]
        lines.append(
            f"| {s} | `{set_files[s]}` | {len(m)} / {len(concepts)} | {on_boards} / {len(board_keys)} | "
            f"{z['raw']:,} | {z['gzip']:,} | {z['brotli'] if z['brotli'] is None else format(z['brotli'], ',')} |"
        )
    any_set = sum(1 for k in concepts if any(k in mapping[s] for s in SET_ORDER))
    lines += ["", f"Concepts with a symbol in at least one set: {any_set} / {len(concepts)}.", ""]
    lines += ["## Concepts", "", "| concept | boards | " + " | ".join(SET_ORDER) + " |",
              "|---|---|" + "---|" * len(SET_ORDER)]
    for key in sorted(concepts):
        c = concepts[key]
        row = [f"`{key}`", ", ".join(sorted(c.layouts)) or "—"]
        for s in SET_ORDER:
            m = mapping[s].get(key)
            row.append(f"`{m.sym.id}` {html.escape(m.sym.name) if m.sym.name != m.sym.id else ''} — "
                       f"{m.sym.rel} ({m.how})" if m else "")
        lines.append("| " + " | ".join(r.replace("|", "\\|") for r in row) + " |")
    for s in SET_ORDER:
        missing = [k for k in sorted(concepts) if k not in mapping[s]]
        lines += ["", f"## Unmapped in {s} ({len(missing)})", ""]
        for k in missing:
            props = indexes[s].proposals(concepts[k])
            hint = f" — proposals (not accepted): {', '.join(f'`{p.id}` {p.name}' for p in props)}" if props else ""
            lines.append(f"- `{k}`{' (on default boards)' if concepts[k].on_boards else ''}{hint}")
    REVIEW_MD.write_text("\n".join(lines) + "\n")


def write_people_review(people_by_set: dict[str, dict[str, dict]], slugs_by_set: dict[str, dict[str, str]]) -> None:
    lines = [
        "# People in symbols",
        "",
        "Generated by `tools/symbols/build.py` — do not edit; change `people.toml` and rebuild.",
        "",
        f"A person is drawn in Unicode's generic, non-realistic colour (`{NEUTRAL_SKIN}`) unless the",
        "person using the board chooses one of the five skin tones of the emoji modifiers",
        "(`aac_core::symbols::people`). Every symbol below draws a person or part of one. A symbol",
        "*in the chosen tone* has its skin — and in OpenMoji the skin's shadow, where its own toned",
        "variants change it — written as the CSS custom properties `--aac-skin` and",
        "`--aac-skin-shade`; one *as drawn* keeps the set's colours, for the reason given. Hair",
        "keeps its colour whatever the tone.",
        "",
        "Symbol names and emoji annotations quoted below come from Mulberry Symbols and OpenMoji",
        "and are shared under CC BY-SA 4.0.",
    ]
    for s in ("mulberry", "openmoji"):
        records = people_by_set.get(s, {})
        concepts_of: dict[str, list[str]] = {}
        for concept, slug in slugs_by_set[s].items():
            concepts_of.setdefault(slug, []).append(concept)
        toned = sum(r["toned"] for r in records.values())
        lines += ["", f"## {SET_INFO[s]['name']}", "",
                  f"{toned} in the chosen tone, {len(records) - toned} as drawn.", "",
                  "| concept(s) | symbol | drawn | why |", "|---|---|---|---|"]
        for slug in sorted(records):
            r = records[slug]
            names = ", ".join(f"`{c}`" for c in sorted(concepts_of.get(slug, [])))
            drawn = f"chosen tone ({', '.join(r['parts'])})" if r["toned"] else "as drawn"
            why = r.get("reason", "").replace("|", "\\|")
            lines.append(f"| {names} | `{r['symbol']}` | {drawn} | {why} |")
    PEOPLE_MD.write_text("\n".join(lines) + "\n")


def write_licenses(set_files: dict[str, str]) -> None:
    urls = {
        "mulberry": [
            f"https://github.com/mulberrysymbols/mulberry-symbols/blob/{MULBERRY_COMMIT}/LICENSE.txt",
            "https://mulberrysymbols.org (Licence and Attribution sections)",
        ],
        "openmoji": [
            f"https://github.com/hfg-gmuend/openmoji/blob/{OPENMOJI_TAG}/LICENSE.txt",
            "https://openmoji.org/faq/ (Licensing and attribution)",
        ],
        "bliss": [
            "https://www.blissymbolics.org/index.php/licensing",
            "https://www.blissymbolics.org/index.php/bci-av-2026",
        ],
    }
    lines = [
        "# Symbol licences",
        "",
        "The symbol sprites in this directory are adapted from the sets below: each source SVG was",
        "optimised with svgo (styles inlined, precision reduced), stripped of anything but drawing",
        "markup, and combined into one sprite per set. In Mulberry and OpenMoji, people's skin (and",
        "in OpenMoji the skin's shadow, as its own skin-tone variants change it) is recoloured:",
        "drawn in Unicode's neutral emoji yellow unless the person using the board chooses a skin",
        "tone (`tools/symbols/people-review.md` lists every symbol). The adapted",
        "sprites are distributed under the same licence as their source, CC BY-SA 4.0",
        "(https://creativecommons.org/licenses/by-sa/4.0/).",
        "They are rebuilt by `tools/symbols/build.py` in the aac-rs workspace, which also writes",
        "three more adaptations of the same sets under the same licence and attribution: the contact",
        "sheets `tools/symbols/review/*.png` (rasterised, each titled with its set's attribution),",
        "`tools/symbols/mapping-review.md` (which quotes the sets' symbol names and BCI-AV glosses)",
        "and `tools/symbols/people-review.md` (which quotes symbol names and emoji annotations).",
        "",
        "Mulberry's condition means the symbols can never sit behind a paid tier or a paid hosted",
        "library: a product that includes them may charge only for its own added value.",
        "",
    ]
    for s in SET_ORDER:
        info = SET_INFO[s]
        lines += [
            f"## {info['name']}",
            "",
            f"- Sprite: `{set_files[s]}`",
            f"- Source: {info['source']['url']} (version {info['source']['version']})",
            f"- Licence: {info['license']['spdx']} — {info['license']['url']}",
            f"- Attribution: {info['attribution']}",
            f"- Conditions: {info['conditions']}",
            f"- Licence verified {info['retrieved']} at: " + "; ".join(urls[s]),
            "",
        ]
    # The optional non-commercial sets are not in this file: a deployment that
    # ships their packs appends their section (build_nc.py's LICENSES-nc.md),
    # so a build without them never names them.
    (PUBLIC / "LICENSES.md").write_text("\n".join(lines))


def nc_license_lines() -> list[str]:
    """The section for the optional non-commercial sets that build_nc.py
    records in index-nc.json, written by build_nc.py to LICENSES-nc.md beside
    the packs (their packs are not in this repository; a free deployment with
    no adverts ships them, and appends this to the app's LICENSES.md)."""
    path = CORE_SYMBOLS / "index-nc.json"
    if not path.exists():
        return []
    doc = json.loads(path.read_text())
    lines = [
        "# Optional sets for non-commercial use",
        "",
        "These sets may be used only for non-commercial purposes. The app offers them as optional",
        "sets where it is free and shows no adverts, downloads one only when a person chooses it,",
        "and they are never offered in a paid product or plan. Their packs are not part of the",
        "open-standards repository, which carries only free-licensed content.",
        "",
    ]
    for s, info in doc["sets"].items():
        lines += [
            f"## {info['name']}",
            "",
            f"- Pack: `{info['file']}` ({info['size']:,} bytes, SHA-256 {info['sha256']})",
            f"- Source: {info['source']['url']} ({info['source']['version']})",
            *([f"- {info['name']}'s own licence: {info['source_license']['name']} — {info['source_license']['url']}",
               f"- Our adapted pack is shared under: {info['license']['spdx']} — {info['license']['url']}"]
              if info.get("source_license") else
              [f"- Licence: {info['license']['spdx']} — {info['license']['url']}"]),
            f"- Author: {info['author']}; owner: {info['owner']}",
            f"- Attribution: {info['attribution']}",
            f"- Conditions: {info['conditions']}",
            f"- Changes: {info['adapted']}",
            f"- Licence verified {info['retrieved']} at: {info['terms_url']}",
            "",
        ]
    return lines


def write_index(mapping, slugs_by_set, set_files) -> None:
    sets = {s: {**SET_INFO[s], "file": set_files[s]} for s in SET_ORDER}
    concepts: dict[str, dict[str, str]] = {}
    for s in SET_ORDER:
        for key, slug in slugs_by_set[s].items():
            concepts.setdefault(key, {})[s] = slug
    doc = {"sets": sets, "concepts": {k: concepts[k] for k in sorted(concepts)}}
    CORE_SYMBOLS.mkdir(parents=True, exist_ok=True)
    (CORE_SYMBOLS / "index.json").write_text(json.dumps(doc, ensure_ascii=False, indent=1) + "\n")


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
    ap.add_argument("--review-pages", type=Path,
                    help="also render contact sheets in pages of 48 into DIR (for close review)")
    args = ap.parse_args()

    concepts, raw_labels = collect_concepts()
    # Parity fixture for the Rust normaliser: every real label plus edge cases.
    cases = sorted(set(raw_labels) | set(NORMALIZE_EDGE_CASES))
    CORE_SYMBOLS.mkdir(parents=True, exist_ok=True)
    (CORE_SYMBOLS / "normalize_cases.json").write_text(
        "[\n" + ",\n".join(json.dumps([c, normalize_concept(c)], ensure_ascii=False) for c in cases) + "\n]\n")

    indexes: dict[str, SymbolIndex] = {"mulberry": Mulberry(), "openmoji": OpenMoji(), "bliss": Bliss()}
    overrides = load_overrides(indexes, concepts)
    mapping = map_concepts(indexes, concepts, overrides)
    people = load_people(indexes["mulberry"])

    set_files, set_sizes, slugs_by_set, people_by_set = {}, {}, {}, {}
    for s in SET_ORDER:
        slugs = assign_slugs(s, mapping[s])
        name, text, records = build_sprite(s, mapping[s], slugs, people)
        check_sprite(text)
        set_files[s], set_sizes[s], slugs_by_set[s] = name, sizes(text), slugs
        people_by_set[s] = records
    # A decision for a symbol no longer drawn is stale: drop it.
    decided = {r["symbol"] for r in people_by_set["mulberry"].values()}
    if stale := sorted(set(people) - decided):
        raise SystemExit(f"people.toml [mulberry]: no mapped symbol uses these any more, remove them: {stale}")
    # Contact sheets: review aids, rendered after every sprite is checked.
    for s in SET_ORDER:
        sprite = (PUBLIC / set_files[s]).read_text()
        contact_sheet(s, sprite, mapping[s], slugs_by_set[s], REVIEW_DIR / f"{s}.png")
        if args.review_pages:
            keys = sorted(mapping[s])
            for p in range(0, len(keys), 48):
                page = {k: mapping[s][k] for k in keys[p:p + 48]}
                contact_sheet(s, sprite, page, slugs_by_set[s], args.review_pages / f"{s}-{p // 48 + 1:02}.png",
                              cols=8, tile=150)

    write_index(mapping, slugs_by_set, set_files)
    write_licenses(set_files)
    write_review(concepts, mapping, indexes, set_files, set_sizes)
    write_people_review(people_by_set, slugs_by_set)

    board_keys = [k for k in concepts if concepts[k].on_boards]
    for s in SET_ORDER:
        z = set_sizes[s]
        print(f"{s:9} {set_files[s]:28} concepts {len(mapping[s]):3}/{len(concepts)}  "
              f"boards {sum(k in mapping[s] for k in board_keys):3}/{len(board_keys)}  "
              f"raw {z['raw']:>9,}  gzip {z['gzip']:>8,}  brotli {z['brotli'] or 0:>8,}")


if __name__ == "__main__":
    main()