1351 lines
66 KiB
Python
1351 lines
66 KiB
Python
#!/usr/bin/env python3.14
|
|
"""Build the optional, non-commercial symbol sets: ARASAAC and Sclera.
|
|
|
|
ARASAAC (CC BY-NC-SA 4.0, Government of Aragón, pictograms by Sergio Palao)
|
|
and Sclera (CC BY-NC 2.0 BE, Sclera vzw) may be used only for non-commercial
|
|
purposes. They are offered as optional sets in a free app with no adverts
|
|
(and never in a paid product or plan: see README.md, "Non-commercial sets").
|
|
|
|
This repository carries only free-licensed content -- it is mirrored to
|
|
Codeberg, whose terms do not accept NC licences -- so the pictograms never
|
|
enter it. What is committed here is ours or a fact about theirs:
|
|
|
|
tools/symbols/build_nc.py this tool
|
|
tools/symbols/overrides-nc.toml mapping decisions, by pictogram number
|
|
tools/symbols/nc-sources.lock.json every source file's URL, date and SHA-256
|
|
crates/aac-core/src/symbols/index-nc.json the sets, their licences and
|
|
concept -> symbol id, and each pack's
|
|
file name, size and SHA-256
|
|
crates/aac-web/public/symbols/LICENSES.md (rewritten by build.py; it
|
|
never names the NC sets: see
|
|
LICENSES-nc.md below)
|
|
|
|
The pictograms themselves are written to --out DIR, outside the repository:
|
|
|
|
DIR/<set>.<sha256-12>.pack one pack per set (format below)
|
|
DIR/LICENSE.md what a copy of the packs must carry
|
|
DIR/LICENSES-nc.md their section of the app's symbols/LICENSES.md,
|
|
appended by a deployment that ships them
|
|
DIR/mapping-review-nc.md what maps to what, and why
|
|
DIR/people-review-nc.md which pictograms take the chosen skin tone
|
|
DIR/review/<set>-NN.png contact sheets, 48 a page
|
|
|
|
and a deployment that offers the sets (a free app, no adverts) ships the packs
|
|
beside the app's symbols. The app downloads a pack only when a person chooses
|
|
its set, checks its size and SHA-256, and keeps it on the device.
|
|
|
|
Sources (downloaded once into $AAC_SYMBOLS_NC_CACHE, default .cache-nc/, and
|
|
checked against nc-sources.lock.json on every run):
|
|
|
|
ARASAAC the API's catalogue of every pictogram per language
|
|
(https://api.arasaac.org/v1/pictograms/all/<lang>), and for each
|
|
mapped pictogram its 2500 px render: the published colours
|
|
(static.arasaac.org) and, for pictograms with skin, the API's own
|
|
skin=white and skin=black renders. The two skin renders differ
|
|
exactly where ARASAAC draws skin, which gives each pictogram a skin
|
|
mask, so the app draws its people in the skin tone the person
|
|
chooses -- the same model as the other sets (Unicode's neutral by
|
|
default, or one of the five emoji tones). Hair keeps ARASAAC's
|
|
colour: no hair colour belongs to a skin tone.
|
|
Sclera the four language downloads of sclera.be (en, nl, fr, de), whose
|
|
identical images are joined across languages by their bytes.
|
|
|
|
Concepts are those of build.py (every default board label and core vocabulary
|
|
word) plus the page sets' words (tools/pagesets/data/lexicon.en.yaml) and the
|
|
symbols the phrase sets ask for (tools/pagesets/phrases/sources/*.yaml,
|
|
`symbol` and `want_symbol`). A concept is matched by meaning, not by English
|
|
spelling alone: a candidate's keywords in each of the languages both sides
|
|
have are compared with the app's audited glosses of the concept
|
|
(crates/aac-core/src/translations_data.json), and a match is accepted only
|
|
when the languages agree (rules in accept()). overrides-nc.toml decides the
|
|
rest. A wrong picture is worse than none: a concept stays unmapped unless the
|
|
match is confident, and the app then falls back to an open set.
|
|
|
|
Pack format (little-endian):
|
|
b"AACSYMP1" magic and version
|
|
u32 length of the header
|
|
header UTF-8 JSON: {"format", "version", "set", "license" (what the
|
|
pack is shared under), "source_license" (the set's own, where
|
|
it differs, else null), "attribution", "conditions", "source"
|
|
({"url", "version"}), "adapted", "symbols": {id: {"w", "h", "image":
|
|
[offset, length, type], "skin": [offset, length, type] | null}}}
|
|
data the images; offsets are from the start of data
|
|
`image` is the pictogram with its skin cut out; `skin` is a grey image of
|
|
where skin is drawn (white = all skin), so a client paints the chosen tone
|
|
through it as a luminance mask. A client checks the whole file's SHA-256.
|
|
|
|
Requires Python >= 3.11, PyYAML, Pillow and NumPy, and cwebp (libwebp) for
|
|
the images; rsvg-convert is not needed. Usage:
|
|
|
|
python3.14 tools/symbols/build_nc.py --out DIR # everything
|
|
python3.14 tools/symbols/build_nc.py --out DIR --refresh # re-download sources
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import html
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import tomllib
|
|
import unicodedata
|
|
import urllib.request
|
|
import zipfile
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
sys.path.insert(0, str(HERE))
|
|
import build as B # noqa: E402 (the open sets' builder: concepts, normalisation, licences)
|
|
|
|
import yaml # noqa: E402
|
|
|
|
ROOT = B.ROOT
|
|
NC_CACHE = Path(os.environ.get("AAC_SYMBOLS_NC_CACHE", HERE / ".cache-nc"))
|
|
LOCK = HERE / "nc-sources.lock.json"
|
|
OVERRIDES_NC = HERE / "overrides-nc.toml"
|
|
INDEX_NC = B.CORE_SYMBOLS / "index-nc.json"
|
|
LEXICON = ROOT / "tools/pagesets/data/lexicon.en.yaml"
|
|
PHRASES = ROOT / "tools/pagesets/phrases/sources"
|
|
TRANSLATIONS = ROOT / "crates/aac-core/src/translations_data.json"
|
|
|
|
NC_SETS = ("arasaac", "sclera")
|
|
RETRIEVED = "2026-10-10"
|
|
USER_AGENT = "aac-open-standard-symbol-builder/1 (+https://aac.transaction.science)"
|
|
|
|
# The app's languages that ARASAAC also keys its pictograms in. Each is a
|
|
# second opinion on a candidate's sense.
|
|
ARASAAC_LANGS = ("es", "fr", "de", "it", "pt", "nl", "pl", "ro", "ru", "ar", "he", "zh", "cs",
|
|
"sv", "tr", "uk", "hu", "el", "da", "nb", "ko", "fa", "hi")
|
|
SCLERA_LANGS = ("en", "nl", "fr", "de")
|
|
|
|
# Pixel size of a pictogram in a pack. A tablet cell is 100-200 CSS px at 2x;
|
|
# a 3 cm printed symbol is 354 px at 300 dpi.
|
|
ARASAAC_SIZE = 320
|
|
SCLERA_SIZE = 283 # Sclera's own size
|
|
|
|
LICENSE = {
|
|
"arasaac": {"spdx": "CC-BY-NC-SA-4.0", "name": "CC BY-NC-SA 4.0",
|
|
"url": "https://creativecommons.org/licenses/by-nc-sa/4.0/"},
|
|
"sclera": {"spdx": "CC-BY-NC-2.0-BE", "name": "CC BY-NC 2.0 BE",
|
|
"url": "https://creativecommons.org/licenses/by-nc/2.0/be/"},
|
|
}
|
|
|
|
# What index-nc.json, LICENSES.md, the app's settings, print credits and OBF
|
|
# exports say about each set. Wording read at the sources on RETRIEVED:
|
|
# arasaac.org/terms-of-use ("How to cite the ARASAAC license", first form) and
|
|
# sclera.be/en/picto/copyright.
|
|
SET_INFO: dict[str, dict] = {
|
|
"arasaac": {
|
|
"name": "ARASAAC",
|
|
"license": LICENSE["arasaac"],
|
|
"attribution": (
|
|
"The pictographic symbols used are the property of the Government of Aragón "
|
|
"and have been created by Sergio Palao for ARASAAC (https://arasaac.org), that "
|
|
"distributes them under Creative Commons License BY-NC-SA."
|
|
),
|
|
"conditions": (
|
|
"Non-commercial use only: ARASAAC authorises use for non-profit purposes and "
|
|
"excludes use within any product or publication for commercial purposes. "
|
|
"Offered only in a free app with no adverts, never in a paid product or plan. "
|
|
"Works derived from the pictograms must be distributed under CC BY-NC-SA 4.0, "
|
|
"citing the author (Sergio Palao), the owner (Government of Aragón), the origin "
|
|
"(ARASAAC) and the licence."
|
|
),
|
|
"author": "Sergio Palao",
|
|
"owner": "Government of Aragón (Gobierno de Aragón)",
|
|
"source": {"url": "https://arasaac.org", "version": ""}, # version: set from the lock
|
|
# ARASAAC's own licence, as its terms of use name it: CC BY-NC-SA with
|
|
# no version. The adapted pictures in the pack are shared under 4.0
|
|
# (`license`), as share-alike allows (a later version of the licence).
|
|
"source_license": {"name": "CC BY-NC-SA (per arasaac.org/terms-of-use)",
|
|
"url": "https://arasaac.org/terms-of-use"},
|
|
"terms_url": "https://arasaac.org/terms-of-use",
|
|
"noncommercial": True,
|
|
"share_alike": True,
|
|
"retrieved": RETRIEVED,
|
|
},
|
|
"sclera": {
|
|
"name": "Sclera",
|
|
"license": LICENSE["sclera"],
|
|
"attribution": (
|
|
"Sclera pictograms by Sclera vzw (www.sclera.be), licensed under Creative "
|
|
"Commons BY-NC 2.0 Belgium."
|
|
),
|
|
"conditions": (
|
|
"Non-commercial use only (commercial use needs a signed agreement with Sclera "
|
|
"vzw). Offered only in a free app with no adverts, never in a paid product or "
|
|
"plan. Credit www.sclera.be as the source, and carry the licence's address "
|
|
"with every copy."
|
|
),
|
|
"author": "Sclera vzw",
|
|
"owner": "Sclera vzw",
|
|
"source": {"url": "https://www.sclera.be/en/picto/downloads", "version": ""},
|
|
"source_license": None, # the pack carries Sclera's own licence unchanged
|
|
"terms_url": "https://www.sclera.be/en/picto/copyright",
|
|
"noncommercial": True,
|
|
"share_alike": False,
|
|
"retrieved": RETRIEVED,
|
|
},
|
|
}
|
|
|
|
ADAPTED = {
|
|
"arasaac": "Adapted: resized from ARASAAC's 2500 px renders, re-encoded and packed into one "
|
|
"file; the skin separated from each picture and recoloured in the skin tone chosen.",
|
|
"sclera": "Adapted: re-encoded and packed into one file.",
|
|
}
|
|
|
|
|
|
# ─── Sources and the lock ───────────────────────────────────────────────────
|
|
|
|
|
|
def load_lock() -> dict:
|
|
return json.loads(LOCK.read_text()) if LOCK.exists() else {"files": {}}
|
|
|
|
|
|
def save_lock(lock: dict) -> None:
|
|
lock["files"] = dict(sorted(lock["files"].items()))
|
|
LOCK.write_text(json.dumps(lock, indent=1, ensure_ascii=False) + "\n")
|
|
|
|
|
|
class Fetcher:
|
|
"""Downloads into NC_CACHE once, then checks every file against the lock.
|
|
With refresh, files not in the lock are fetched and recorded; a file in
|
|
the lock that no longer matches stops the build."""
|
|
|
|
def __init__(self, lock: dict, refresh: bool) -> None:
|
|
import threading
|
|
self.lock, self.refresh = lock, refresh
|
|
self.fetched = 0
|
|
self.used: set[str] = set()
|
|
self._mutex = threading.Lock()
|
|
|
|
def prefetch(self, items: list[tuple[str, str]], workers: int = 6) -> None:
|
|
"""Fetch many files at once (a few at a time: a public service)."""
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
todo = [(n, u) for n, u in items if not (NC_CACHE / n).exists()]
|
|
if todo:
|
|
print(f" downloading {len(todo)} files", file=sys.stderr)
|
|
with ThreadPoolExecutor(max_workers=workers) as pool:
|
|
list(pool.map(lambda nu: self.get(*nu), todo))
|
|
|
|
def get(self, name: str, url: str) -> Path:
|
|
with self._mutex:
|
|
self.used.add(name)
|
|
path = NC_CACHE / name
|
|
entry = self.lock["files"].get(name)
|
|
if not path.exists():
|
|
if entry is None and not self.refresh:
|
|
raise SystemExit(f"{name} is not in nc-sources.lock.json: run with --refresh to fetch it")
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
import threading
|
|
tmp = path.with_name(f"{path.name}.{threading.get_ident()}.part")
|
|
for attempt in range(4):
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
|
with urllib.request.urlopen(req, timeout=300) as r, tmp.open("wb") as f:
|
|
shutil.copyfileobj(r, f)
|
|
break
|
|
except Exception as e: # noqa: BLE001 -- network: retry, then stop
|
|
if attempt == 3:
|
|
raise SystemExit(f"{url}: {e}")
|
|
time.sleep(2 ** attempt)
|
|
tmp.rename(path)
|
|
with self._mutex:
|
|
self.fetched += 1
|
|
time.sleep(0.15) # be gentle with a public service
|
|
got = B.sha256_file(path)
|
|
if entry is None:
|
|
with self._mutex:
|
|
self.lock["files"][name] = {"url": url, "sha256": got, "retrieved": time.strftime("%Y-%m-%d")}
|
|
elif entry["sha256"] != got:
|
|
raise SystemExit(f"{name}: SHA-256 {got} != locked {entry['sha256']}; delete it to refetch")
|
|
return path
|
|
|
|
|
|
# ─── Concepts ───────────────────────────────────────────────────────────────
|
|
|
|
# The app's word classes (vocabulary.rs WordCategory, lexicon pos) -> ARASAAC
|
|
# keyword types (1 people and pronouns, 2 nouns, 3 verbs, 4 descriptions, 5
|
|
# social expressions, 6 miscellaneous: articles, prepositions, questions...).
|
|
POS_TYPES = {
|
|
"Verb": {3}, "verb": {3},
|
|
"CoreNoun": {2}, "FringeNoun": {2}, "noun": {2},
|
|
"Adjective": {4}, "adjective": {4}, "adverb": {4},
|
|
"Pronoun": {1}, "pronoun": {1},
|
|
"Social": {5, 4}, "Interjection": {5, 4}, "phrase": {5}, "interjection": {5, 4},
|
|
"Question": {6, 1}, "question": {6, 1},
|
|
"Preposition": {6}, "preposition": {6}, "Conjunction": {6}, "conjunction": {6},
|
|
"Determiner": {6, 1, 4}, "determiner": {6, 1, 4},
|
|
"Negation": {4, 6},
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class NcConcept:
|
|
key: str
|
|
on_boards: bool = False
|
|
layouts: set[str] = field(default_factory=set)
|
|
pos: set[str] = field(default_factory=set)
|
|
sense: str = ""
|
|
sources: set[str] = field(default_factory=set)
|
|
glosses: dict[str, list[str]] = field(default_factory=dict) # lang -> normalised variants
|
|
|
|
@property
|
|
def types(self) -> set[int]:
|
|
out: set[int] = set()
|
|
for p in self.pos:
|
|
out |= POS_TYPES.get(p, set())
|
|
return out
|
|
|
|
|
|
def norm_word(text: str) -> str:
|
|
"""Casefold, NFKC, strip the punctuation that frames a word or phrase."""
|
|
s = unicodedata.normalize("NFKC", text).casefold().strip()
|
|
s = s.strip("¿¡?!.,;:\"'«»“”„()[]")
|
|
return " ".join(s.split())
|
|
|
|
|
|
def gloss_variants(text: str) -> list[str]:
|
|
out = []
|
|
for part in re.split(r"\s*[/;,]\s*", text):
|
|
n = norm_word(part)
|
|
if n and not n.startswith("-"):
|
|
out.append(n)
|
|
return out
|
|
|
|
|
|
def collect_nc_concepts() -> dict[str, NcConcept]:
|
|
base, _ = B.collect_concepts()
|
|
out: dict[str, NcConcept] = {}
|
|
|
|
def get(key: str, source: str) -> NcConcept:
|
|
c = out.setdefault(key, NcConcept(key))
|
|
c.sources.add(source)
|
|
return c
|
|
|
|
for key, c in base.items():
|
|
n = get(key, "boards" if c.on_boards else "vocabulary")
|
|
n.on_boards = c.on_boards
|
|
n.layouts |= c.layouts
|
|
n.pos |= c.vocab
|
|
if c.vocab:
|
|
n.sources.add("vocabulary")
|
|
lexicon = yaml.safe_load(LEXICON.read_text())["words"]
|
|
for word, e in lexicon.items():
|
|
n = get(B.normalize_concept(str(word)), "pagesets")
|
|
if e and e.get("pos"):
|
|
n.pos.add(e["pos"])
|
|
if e and e.get("sense"):
|
|
n.sense = e["sense"]
|
|
|
|
def walk(o):
|
|
if isinstance(o, dict):
|
|
for k, v in o.items():
|
|
if k in ("symbol", "want_symbol") and isinstance(v, str):
|
|
get(B.normalize_concept(v), f"phrases:{k}")
|
|
walk(v)
|
|
elif isinstance(o, list):
|
|
for v in o:
|
|
walk(v)
|
|
|
|
for f in sorted(PHRASES.glob("*.yaml")):
|
|
walk(yaml.safe_load(f.read_text()))
|
|
|
|
glosses = json.loads(TRANSLATIONS.read_text())
|
|
for raw, by_lang in glosses.items():
|
|
c = out.get(B.normalize_concept(raw))
|
|
if c is None:
|
|
continue
|
|
for lang, text in by_lang.items():
|
|
v = gloss_variants(text)
|
|
if v:
|
|
c.glosses.setdefault(lang, [])
|
|
c.glosses[lang] += [x for x in v if x not in c.glosses[lang]]
|
|
return out
|
|
|
|
|
|
# ─── ARASAAC ────────────────────────────────────────────────────────────────
|
|
|
|
ARASAAC_API = "https://api.arasaac.org/v1"
|
|
ARASAAC_STATIC = "https://static.arasaac.org/pictograms"
|
|
|
|
|
|
@dataclass
|
|
class Candidate:
|
|
id: str
|
|
en: int = 0 # 2: an English keyword is the concept; 1: its plural is
|
|
votes: list[str] = field(default_factory=list) # languages whose keyword is the app's gloss
|
|
seen: list[str] = field(default_factory=list) # languages both sides have words in
|
|
type_ok: bool = False
|
|
preferred: bool = False # the set's own pick for AAC boards
|
|
child: bool = False # draws a child, for a concept that is not about children
|
|
name: str = ""
|
|
note: str = ""
|
|
|
|
def rank(self) -> tuple:
|
|
return (self.en > 0, len(self.votes), self.type_ok, self.preferred, self.en, -int(re.sub(r"\D", "", self.id) or 0))
|
|
|
|
|
|
@dataclass
|
|
class Choice:
|
|
concept: str
|
|
sym: str
|
|
how: str # confirmed | meaning | english | override
|
|
reason: str
|
|
cand: Candidate | None = None
|
|
|
|
|
|
class Arasaac:
|
|
set_id = "arasaac"
|
|
|
|
def __init__(self, fetcher: Fetcher) -> None:
|
|
self.meta: dict[str, dict] = {}
|
|
self.kw: dict[str, dict[str, set[str]]] = {}
|
|
self.kw_type: dict[tuple[str, str], int | None] = {}
|
|
self.plural_en: dict[str, set[str]] = {}
|
|
self.names: dict[str, str] = {}
|
|
for lang in ("en",) + ARASAAC_LANGS:
|
|
path = fetcher.get(f"arasaac/meta/all_{lang}.json", f"{ARASAAC_API}/pictograms/all/{lang}")
|
|
data = json.loads(path.read_text())
|
|
idx = self.kw.setdefault(lang, {})
|
|
for p in data:
|
|
pid = str(p["_id"])
|
|
if lang == "en":
|
|
self.meta[pid] = p
|
|
for k in p.get("keywords", []):
|
|
word = norm_word(k.get("keyword") or "")
|
|
if not word:
|
|
continue
|
|
idx.setdefault(word, set()).add(pid)
|
|
if lang == "en":
|
|
self.kw_type.setdefault((pid, word), k.get("type"))
|
|
self.names.setdefault(pid, k.get("keyword") or "")
|
|
if k.get("plural"):
|
|
self.plural_en.setdefault(norm_word(k["plural"]), set()).add(pid)
|
|
self.symbols = set(self.meta)
|
|
|
|
def usable(self, pid: str) -> bool:
|
|
"""Letters never stand for a word ("I" is also the letter); sexual
|
|
and violent pictograms are never offered for a board word."""
|
|
p = self.meta[pid]
|
|
return not p.get("sex") and not p.get("violence") and "letter" not in p.get("categories", ())
|
|
|
|
def candidates(self, c: NcConcept) -> list[Candidate]:
|
|
ids: set[str] = set(self.kw["en"].get(c.key, ())) | set(self.plural_en.get(c.key, ()))
|
|
for lang, variants in c.glosses.items():
|
|
for v in variants:
|
|
ids |= self.kw.get(lang, {}).get(v, set())
|
|
out = []
|
|
for pid in ids:
|
|
if pid not in self.meta or not self.usable(pid):
|
|
continue
|
|
cand = Candidate(pid, name=self.names.get(pid, ""))
|
|
if pid in self.kw["en"].get(c.key, ()):
|
|
cand.en = 2
|
|
elif pid in self.plural_en.get(c.key, ()):
|
|
cand.en = 1
|
|
for lang in ARASAAC_LANGS:
|
|
variants = c.glosses.get(lang)
|
|
if not variants:
|
|
continue
|
|
if not self._has_words(pid, lang):
|
|
continue
|
|
cand.seen.append(lang)
|
|
if any(pid in self.kw[lang].get(v, ()) for v in variants):
|
|
cand.votes.append(lang)
|
|
t = self.kw_type.get((pid, c.key))
|
|
cand.type_ok = bool(c.types) and t in c.types
|
|
cand.preferred = bool(self.meta[pid].get("aac"))
|
|
cand.child = "child" in self.meta[pid].get("tags", ()) and c.key not in CHILD_CONCEPTS
|
|
out.append(cand)
|
|
out.sort(key=Candidate.rank, reverse=True)
|
|
return out
|
|
|
|
_words_of: dict[str, dict[str, bool]] | None = None
|
|
|
|
def _has_words(self, pid: str, lang: str) -> bool:
|
|
if self._words_of is None:
|
|
self._words_of = {}
|
|
for l, idx in self.kw.items():
|
|
for ids in idx.values():
|
|
for i in ids:
|
|
self._words_of.setdefault(i, {})[l] = True
|
|
return self._words_of.get(pid, {}).get(lang, False)
|
|
|
|
|
|
# ─── Sclera ─────────────────────────────────────────────────────────────────
|
|
|
|
SCLERA_DOWNLOAD = "https://www.sclera.be/resources/download"
|
|
|
|
|
|
class Sclera:
|
|
"""Sclera's four language downloads, joined by identical images. Only
|
|
neutral pictograms are candidates: Sclera also draws "allowed" (green
|
|
ground) and "not allowed" (red cross) versions of the same picture, whose
|
|
file names often lack the qualifier ("eat" is a crossed-out plate)."""
|
|
|
|
set_id = "sclera"
|
|
|
|
def __init__(self, fetcher: Fetcher) -> None:
|
|
self.zips = {lang: zipfile.ZipFile(fetcher.get(f"sclera/picto_{lang}.zip", f"{SCLERA_DOWNLOAD}/picto_{lang}.zip"))
|
|
for lang in SCLERA_LANGS}
|
|
self.where: dict[str, tuple[str, str]] = {} # id (sha256-12 of the image) -> (lang, member)
|
|
self.kw: dict[str, dict[str, set[str]]] = {lang: {} for lang in SCLERA_LANGS}
|
|
self.names: dict[str, dict[str, list[str]]] = {}
|
|
for lang, z in self.zips.items():
|
|
for member in z.namelist():
|
|
if member.endswith("/") or not member.lower().endswith(".png"):
|
|
continue
|
|
data = z.read(member)
|
|
sid = hashlib.sha256(data).hexdigest()[:12]
|
|
self.where.setdefault(sid, (lang, member))
|
|
stem = member.rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
|
stem = re.sub(r"[_ ]\d+$", "", stem.strip()) # "drink_1" -> "drink"
|
|
word = norm_word(stem)
|
|
if word:
|
|
self.kw[lang].setdefault(word, set()).add(sid)
|
|
self.names.setdefault(sid, {}).setdefault(lang, []).append(stem.strip())
|
|
self.symbols = set(self.where)
|
|
self._kind: dict[str, str] = {}
|
|
|
|
def image(self, sid: str) -> bytes:
|
|
lang, member = self.where[sid]
|
|
return self.zips[lang].read(member)
|
|
|
|
def kind(self, sid: str) -> str:
|
|
"""neutral | allowed | not-allowed, from the picture itself: a green
|
|
ground means allowed, red diagonal strokes mean not allowed."""
|
|
if sid not in self._kind:
|
|
import numpy as np
|
|
from PIL import Image
|
|
a = np.asarray(Image.open(io.BytesIO(self.image(sid))).convert("RGB")).astype(int)
|
|
r, g, b = a[..., 0], a[..., 1], a[..., 2]
|
|
green = (g > 90) & (g > r + 50) & (g > b + 50)
|
|
red = (r > 100) & (r > g + 60) & (r > b + 60)
|
|
h, w = r.shape
|
|
corners = np.concatenate([green[:8, :8].ravel(), green[:8, -8:].ravel(), green[-8:, :8].ravel(), green[-8:, -8:].ravel()])
|
|
if corners.mean() > 0.5:
|
|
self._kind[sid] = "allowed"
|
|
else:
|
|
# The cross runs corner to corner: sample both diagonals.
|
|
n = min(h, w)
|
|
d1 = np.array([red[i * h // n, i * w // n] for i in range(n)])
|
|
d2 = np.array([red[i * h // n, w - 1 - i * w // n] for i in range(n)])
|
|
self._kind[sid] = "not-allowed" if d1.mean() > 0.3 and d2.mean() > 0.3 else "neutral"
|
|
return self._kind[sid]
|
|
|
|
def candidates(self, c: NcConcept) -> list[Candidate]:
|
|
ids = set(self.kw["en"].get(c.key, ()))
|
|
for lang in SCLERA_LANGS[1:]:
|
|
for v in c.glosses.get(lang, ()):
|
|
ids |= self.kw[lang].get(v, set())
|
|
out = []
|
|
for sid in ids:
|
|
if self.kind(sid) != "neutral":
|
|
continue
|
|
names = self.names.get(sid, {})
|
|
cand = Candidate(sid, name=" / ".join(f"{l}:{n[0]}" for l, n in names.items()))
|
|
cand.en = 2 if sid in self.kw["en"].get(c.key, ()) else 0
|
|
for lang in SCLERA_LANGS[1:]:
|
|
if lang not in names or lang not in c.glosses:
|
|
continue
|
|
cand.seen.append(lang)
|
|
if any(sid in self.kw[lang].get(v, ()) for v in c.glosses[lang]):
|
|
cand.votes.append(lang)
|
|
out.append(cand)
|
|
out.sort(key=Candidate.rank, reverse=True)
|
|
return out
|
|
|
|
|
|
# ─── Mapping ────────────────────────────────────────────────────────────────
|
|
|
|
|
|
# Concepts about children may show a child; for every other concept a
|
|
# picture of a child is passed over when an equally good one is not (a child
|
|
# figure infantilises an adult who uses the board, SPEC §4.3).
|
|
CHILD_CONCEPTS = {"baby", "boy", "girl", "child", "children", "kid", "kids", "toy", "toys", "diaper", "school",
|
|
"class", "teacher", "playground"}
|
|
|
|
|
|
def accept(set_id: str, c: NcConcept, cands: list[Candidate]) -> Choice | None:
|
|
"""The confident choice among a concept's candidates, or None.
|
|
|
|
Sense first, then the set's own preference:
|
|
- confirmed: an English keyword is the concept, and enough other
|
|
languages agree -- at least two (one for Sclera, which has three) and
|
|
at least a quarter of those compared;
|
|
- meaning: no English keyword qualifies, but at least four languages
|
|
(two for Sclera) and more than half of those compared agree -- the same
|
|
picture under other words;
|
|
- english: no language to compare (a word with no glosses yet), and one
|
|
English candidate stands out: the only one, or the only one ARASAAC
|
|
made for AAC boards, of the concept's word class.
|
|
Among candidates that qualify, those within half the best agreement are
|
|
equals, and the pick is: the concept's word class, ARASAAC's own AAC
|
|
pictogram (its core-vocabulary drawings: a plain, neutral figure), no
|
|
child figure unless the concept is about children, then agreement.
|
|
"""
|
|
if not cands:
|
|
return None
|
|
need = 2 if set_id == "arasaac" else 1
|
|
|
|
def pick(qualified: list[Candidate]) -> Candidate:
|
|
top = max(len(x.votes) for x in qualified)
|
|
equals = [x for x in qualified if 2 * len(x.votes) >= top]
|
|
return max(equals, key=lambda x: (x.type_ok or not c.types, x.preferred, not x.child, len(x.votes),
|
|
x.en, -int(re.sub(r"\D", "", x.id) or 0)))
|
|
|
|
confirmed = [x for x in cands if x.en and len(x.votes) >= max(need, -(-len(x.seen) // 4))]
|
|
if confirmed:
|
|
best = pick(confirmed)
|
|
return Choice(c.key, best.id, "confirmed",
|
|
f"English keyword, and {len(best.votes)} of {len(best.seen)} languages agree", best)
|
|
meaning_need = 4 if set_id == "arasaac" else 2
|
|
meaning = [x for x in cands if len(x.votes) >= meaning_need and 2 * len(x.votes) > len(x.seen)]
|
|
if meaning:
|
|
best = pick(meaning)
|
|
return Choice(c.key, best.id, "meaning", f"{len(best.votes)} of {len(best.seen)} languages agree", best)
|
|
english = [x for x in cands if x.en == 2 and (x.type_ok or not c.types)]
|
|
if english and not any(x.seen for x in english):
|
|
preferred = [x for x in english if x.preferred]
|
|
one = english[0] if len(english) == 1 else preferred[0] if len(preferred) == 1 else None
|
|
if one:
|
|
return Choice(c.key, one.id, "english", "English keyword; no other language to compare", one)
|
|
return None
|
|
|
|
|
|
# overrides-nc.toml tables beside each set's mapping decisions: pictograms
|
|
# reviewed for drawn text ("<set>-text": what is drawn, and why it may stay),
|
|
# and ARASAAC people drawn without skin ("arasaac-people": why no pictogram
|
|
# that takes the tone was used). Keyed by pictogram number or id.
|
|
REVIEW_TABLES = {"arasaac-text": "arasaac", "sclera-text": "sclera", "arasaac-people": "arasaac"}
|
|
|
|
|
|
def load_overrides_nc(indexes: dict, concepts: dict[str, NcConcept]) -> dict:
|
|
data = tomllib.loads(OVERRIDES_NC.read_text()) if OVERRIDES_NC.exists() else {}
|
|
for table, set_id in REVIEW_TABLES.items():
|
|
for sym, entry in data.get(table, {}).items():
|
|
if sym not in indexes[set_id].symbols:
|
|
raise SystemExit(f"overrides-nc.toml [{table}] {sym!r}: no such pictogram in {set_id}")
|
|
if not entry.get("reason"):
|
|
raise SystemExit(f"overrides-nc.toml [{table}] {sym!r}: needs a reason")
|
|
for set_id, entries in data.items():
|
|
if set_id in REVIEW_TABLES:
|
|
continue
|
|
if set_id not in indexes:
|
|
raise SystemExit(f"overrides-nc.toml: unknown set [{set_id}]")
|
|
for key, entry in entries.items():
|
|
where = f"overrides-nc.toml [{set_id}] {key!r}"
|
|
if key not in concepts:
|
|
raise SystemExit(f"{where}: not a concept (stale override?)")
|
|
if not entry.get("reason"):
|
|
raise SystemExit(f"{where}: needs a reason")
|
|
if entry.get("none"):
|
|
continue
|
|
if str(entry.get("symbol")) not in indexes[set_id].symbols:
|
|
raise SystemExit(f"{where}: no pictogram {entry.get('symbol')!r} in {set_id}")
|
|
return data
|
|
|
|
|
|
def map_nc(indexes: dict, concepts: dict[str, NcConcept], overrides: dict) -> dict[str, dict[str, Choice]]:
|
|
out: dict[str, dict[str, Choice]] = {s: {} for s in indexes}
|
|
for set_id, index in indexes.items():
|
|
ov = overrides.get(set_id, {})
|
|
for key, c in concepts.items():
|
|
if key in ov:
|
|
if not ov[key].get("none"):
|
|
out[set_id][key] = Choice(key, str(ov[key]["symbol"]), "override", ov[key]["reason"])
|
|
continue
|
|
if ":" in key: # action and navigation cells: only by override
|
|
continue
|
|
if choice := accept(set_id, c, index.candidates(c)):
|
|
out[set_id][key] = choice
|
|
return out
|
|
|
|
|
|
# ─── One picture per board ──────────────────────────────────────────────────
|
|
|
|
_BOARD_RE = re.compile(r'build_board\(\s*\w+\s*,\s*"((?:[^"\\]|\\.)*)"')
|
|
|
|
|
|
def default_board_concepts() -> dict[str, list[str]]:
|
|
"""The concepts of every default board (defaults.rs), board by board, as
|
|
the app draws them (aac_core::symbols::lookup_for_cell)."""
|
|
text = B.DEFAULTS_RS.read_text()
|
|
marks = sorted([(m.start(), "fn", m.group(1)) for m in re.finditer(r"^fn create_(\w+)_boards\(", text, re.M)]
|
|
+ [(m.start(), "board", B._rust_unescape(m.group(1))) for m in _BOARD_RE.finditer(text)])
|
|
out: dict[str, list[str]] = {}
|
|
for m in B._CELL_RE.finditer(text):
|
|
layout = board = None
|
|
for pos, kind, name in marks:
|
|
if pos > m.start():
|
|
break
|
|
layout, board = (name, None) if kind == "fn" else (layout, name)
|
|
if layout is None or board is None:
|
|
raise SystemExit(f"defaults.rs: a cell outside a build_board(..) call at offset {m.start()}")
|
|
kind, label, action = m.group(1), B._rust_unescape(m.group(2)), m.group(3)
|
|
if key := B.cell_concept(kind, label, action) or B.normalize_concept(label):
|
|
out.setdefault(f"defaults.rs {layout}: {board}", []).append(key)
|
|
if len(out) < 20:
|
|
raise SystemExit("defaults.rs: too few boards found -- did build_board(..) change?")
|
|
return out
|
|
|
|
|
|
def pageset_board_concepts() -> dict[str, list[str]]:
|
|
"""The concepts of every page set's boards, built by the page-set
|
|
generators (tools/pagesets/catalog.py) into a scratch directory: a cell's
|
|
chosen symbol (ext_aurasense_symbol), else its English words."""
|
|
sys.path.insert(0, str(ROOT / "tools/pagesets"))
|
|
import catalog # noqa: PLC0415 (tools/pagesets; its own build.py is not ours)
|
|
# The page-set builder also refreshes the tables in its design documents
|
|
# (tools/pagesets/DESIGN-*.md). This run only reads its boards, so those
|
|
# tracked files are put back exactly as they were.
|
|
docs = {p: p.read_bytes() for p in (ROOT / "tools/pagesets").glob("DESIGN-*.md")}
|
|
out: dict[str, list[str]] = {}
|
|
try:
|
|
with tempfile.TemporaryDirectory() as work:
|
|
for fam, target in catalog.run_generators(Path(work)):
|
|
for pkg in fam["packages"]:
|
|
english = pkg.get("lang", "*") in ("*", "en")
|
|
with zipfile.ZipFile(target / pkg["file"]) as z:
|
|
for name in z.namelist():
|
|
if name.endswith(".obf"):
|
|
board = json.loads(z.read(name))
|
|
out[f"page set {pkg['key']}: {board.get('name') or board.get('id')}"] = _board_keys(board, english)
|
|
finally:
|
|
for path, content in docs.items():
|
|
if path.read_bytes() != content:
|
|
path.write_bytes(content)
|
|
if not out:
|
|
raise SystemExit("the page-set generators made no boards")
|
|
return out
|
|
|
|
|
|
def _board_keys(board: dict, english: bool) -> list[str]:
|
|
"""The concepts a page-set board's cells draw, as the app looks them up."""
|
|
keys = []
|
|
for btn in board.get("buttons", []):
|
|
chosen = btn.get("ext_aurasense_symbol")
|
|
label = btn.get("label") or ""
|
|
if chosen is not None:
|
|
key = B.normalize_concept(chosen) if chosen else ""
|
|
elif not english or not label:
|
|
key = ""
|
|
elif btn.get("load_board") and (k := B.normalize_concept(label)) in B.NAV_SENSES:
|
|
key = f"nav:{k}"
|
|
else:
|
|
key = B.normalize_concept(label)
|
|
if key:
|
|
keys.append(key)
|
|
return keys
|
|
|
|
|
|
def check_one_picture_per_board(mapping: dict[str, dict[str, Choice]], boards: dict[str, list[str]]) -> int:
|
|
"""No two concepts on one board may draw the same pictogram: two cells
|
|
would look the same. Stops the build, naming every such pair. Returns how
|
|
many boards were checked."""
|
|
problems = []
|
|
for s, m in mapping.items():
|
|
seen: set[tuple] = set()
|
|
for board, keys in boards.items():
|
|
by_sym: dict[str, list[str]] = {}
|
|
for key in dict.fromkeys(keys):
|
|
if key in m:
|
|
by_sym.setdefault(m[key].sym, []).append(key)
|
|
for sym, ks in by_sym.items():
|
|
if len(ks) > 1 and (s, sym, tuple(ks)) not in seen:
|
|
seen.add((s, sym, tuple(ks)))
|
|
problems.append(f" {s} {sym}: {', '.join(repr(k) for k in ks)} on {board}")
|
|
if problems:
|
|
raise SystemExit("one pictogram for two concepts on one board (decide in overrides-nc.toml):\n" + "\n".join(problems))
|
|
return len(boards)
|
|
|
|
|
|
# ─── Drawn text ─────────────────────────────────────────────────────────────
|
|
|
|
OCR_SWIFT = HERE / "ocr_vision.swift"
|
|
|
|
|
|
def ocr_tool() -> Path | None:
|
|
"""The drawn-text reader: Apple Vision's text recognition (macOS), built
|
|
from ocr_vision.swift into the cache once. None where it cannot run."""
|
|
if sys.platform != "darwin" or shutil.which("swiftc") is None:
|
|
return None
|
|
tool = NC_CACHE / "tools" / f"ocr-{hashlib.sha256(OCR_SWIFT.read_bytes()).hexdigest()[:12]}"
|
|
if not tool.exists():
|
|
tool.parent.mkdir(parents=True, exist_ok=True)
|
|
res = subprocess.run(["swiftc", "-O", "-o", str(tool), str(OCR_SWIFT)], capture_output=True, text=True)
|
|
if res.returncode != 0:
|
|
print(f" drawn-text check: swiftc failed: {res.stderr[-400:]}", file=sys.stderr)
|
|
return None
|
|
return tool
|
|
|
|
|
|
def drawn_text(images: dict[str, "np.ndarray"]) -> dict[str, list[str]] | None:
|
|
"""sym -> the words read in its picture (RGB 0..1 on white, any size).
|
|
None when no reader runs here."""
|
|
from PIL import Image
|
|
import numpy as np
|
|
tool = ocr_tool()
|
|
if tool is None:
|
|
return None
|
|
out: dict[str, list[str]] = {}
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
paths = {}
|
|
for sym, rgb in images.items():
|
|
path = Path(tmp) / f"{sym}.png"
|
|
im = Image.fromarray(np.clip(np.rint(rgb * 255), 0, 255).astype(np.uint8), "RGB")
|
|
im.resize((max(1, im.width * 3), max(1, im.height * 3)), Image.Resampling.LANCZOS).save(path)
|
|
paths[str(path)] = sym
|
|
names = list(paths)
|
|
for i in range(0, len(names), 200):
|
|
res = subprocess.run([str(tool), *names[i:i + 200]], capture_output=True, text=True)
|
|
if res.returncode != 0:
|
|
print(f" drawn-text check failed to run: {res.stderr[-400:]}", file=sys.stderr)
|
|
return None
|
|
for line in res.stdout.splitlines():
|
|
row = json.loads(line)
|
|
words = [t["text"].strip() for t in row["texts"]
|
|
if t["conf"] >= 0.3 and re.search(r"[^\W\d_]", t["text"])]
|
|
if words:
|
|
out[paths[row["path"]]] = words
|
|
return out
|
|
|
|
|
|
# ─── Images ─────────────────────────────────────────────────────────────────
|
|
|
|
# The tones a pack's people are drawn in: aac_core::symbols::people (and
|
|
# build.py's NEUTRAL and TONES) -- Unicode's neutral, then the five emoji
|
|
# modifiers' skin colours.
|
|
NEUTRAL_SKIN = B.NEUTRAL_SKIN
|
|
TONE_SKINS = [NEUTRAL_SKIN] + [t["skin"] for t in B.TONES.values()]
|
|
SKIN_STYLE = f"fill:var(--aac-skin,{NEUTRAL_SKIN})"
|
|
|
|
|
|
def _rgb(hex_: str) -> tuple[float, float, float]:
|
|
return tuple(int(hex_[i:i + 2], 16) / 255 for i in (1, 3, 5)) # type: ignore[return-value]
|
|
|
|
|
|
def _load_rgba(path_or_bytes) -> "np.ndarray":
|
|
import numpy as np
|
|
from PIL import Image
|
|
src = io.BytesIO(path_or_bytes) if isinstance(path_or_bytes, (bytes, bytearray)) else path_or_bytes
|
|
return np.asarray(Image.open(src).convert("RGBA"), dtype=np.float32) / 255.0
|
|
|
|
|
|
def _box(plane: "np.ndarray", size: int) -> "np.ndarray":
|
|
"""Area-average one float plane down to size x size."""
|
|
import numpy as np
|
|
from PIL import Image
|
|
return np.asarray(Image.fromarray(plane.astype(np.float32), mode="F").resize((size, size), Image.Resampling.BOX),
|
|
dtype=np.float32)
|
|
|
|
|
|
@dataclass
|
|
class Layers:
|
|
"""A pictogram as two layers: `base` (RGBA, its skin cut out) and `skin`
|
|
(coverage 0..1 of skin, or None). Drawing the base and then the chosen
|
|
skin colour at `skin` coverage over it gives the pictogram in that tone."""
|
|
base: "np.ndarray"
|
|
skin: "np.ndarray | None"
|
|
|
|
def compose(self, skin_hex: str, ground=(1.0, 1.0, 1.0)) -> "np.ndarray":
|
|
import numpy as np
|
|
rgb, a = self.base[..., :3], self.base[..., 3:4]
|
|
out_pm = rgb * a
|
|
out_a = a
|
|
if self.skin is not None:
|
|
g = self.skin[..., None]
|
|
s = np.array(_rgb(skin_hex), dtype=np.float32)
|
|
out_pm = s * g + out_pm * (1 - g)
|
|
out_a = g + a * (1 - g)
|
|
return out_pm + np.array(ground, dtype=np.float32) * (1 - out_a)
|
|
|
|
|
|
# The skin colours of ARASAAC's API renders (skin=white, skin=black), read
|
|
# from its renders; every pictogram's re-draw error checks them again.
|
|
ARASAAC_SKIN_WHITE = (245, 229, 223)
|
|
ARASAAC_SKIN_BLACK = (167, 92, 22)
|
|
|
|
|
|
def arasaac_layers(default: Path | None, white: Path | None, black: Path | None, size: int) -> tuple[Layers, float]:
|
|
"""Separate a pictogram's skin from the rest, at 2500 px, then area-
|
|
average both down to `size`. A pictogram without skin is its published
|
|
render (`default`); one with skin comes from the API's skin=white and
|
|
skin=black renders alone, which agree everywhere but the skin (the
|
|
published render can differ slightly in its other colours). Returns the
|
|
layers and the largest error of re-drawing ARASAAC's own skin=white
|
|
render from them (premultiplied, 0..1)."""
|
|
import numpy as np
|
|
if white is None or black is None:
|
|
d = _load_rgba(default)
|
|
pm = d[..., :3] * d[..., 3:4]
|
|
cov = _box(d[..., 3], size)
|
|
rgb = np.stack([_box(pm[..., i], size) for i in range(3)], axis=-1)
|
|
rgb = np.where(cov[..., None] > 1e-6, rgb / np.maximum(cov[..., None], 1e-6), 0)
|
|
return Layers(np.concatenate([np.clip(rgb, 0, 1), cov[..., None]], axis=-1), None), 0.0
|
|
w, k = _load_rgba(white), _load_rgba(black)
|
|
if w.shape != k.shape:
|
|
raise SystemExit(f"{white.name}: renders differ in size")
|
|
d = w # outside the skin both renders are the pictogram's own colours
|
|
# Skin is where the two skin renders differ along ARASAAC's own skin
|
|
# colours. Each such pixel is unmixed linearly: its colour is
|
|
# a*skin + (1-a)*other, `a` read from how far it moved from one skin
|
|
# colour towards the other (an outline's anti-aliased edge is part skin).
|
|
# The renders are palette-quantised separately, so a flat area elsewhere
|
|
# can shift a few levels between them: such a shift is not along the
|
|
# skin colours, or is too small to be skin, and is left as drawn.
|
|
s_w = np.array(ARASAAC_SKIN_WHITE, dtype=np.float32) / 255
|
|
ds = s_w - np.array(ARASAAC_SKIN_BLACK, dtype=np.float32) / 255
|
|
diff = w[..., :3] - k[..., :3]
|
|
frac = np.clip((diff * ds).sum(axis=-1) / float((ds * ds).sum()), 0, 1)
|
|
resid = np.abs(diff - frac[..., None] * ds).max(axis=-1)
|
|
m = (frac > 0.1) & (resid < 0.08) & ((w[..., 3] > 0) | (k[..., 3] > 0))
|
|
if not m.any():
|
|
return arasaac_layers(white, None, None, size)
|
|
frac = np.where(m, frac, 0.0)
|
|
alpha = np.where(m, (w[..., 3] + k[..., 3]) / 2, d[..., 3])
|
|
skin_cov_full = alpha * frac
|
|
# The other part's premultiplied colour: alpha * (c_w - a*s_w), and
|
|
# outside the skin the published pixel.
|
|
other_pm = np.where(m[..., None], alpha[..., None] * np.clip(w[..., :3] - frac[..., None] * s_w, 0, 1),
|
|
d[..., :3] * d[..., 3:4])
|
|
other_a = alpha * (1 - frac)
|
|
g = np.clip(_box(skin_cov_full, size), 0, 1)
|
|
c_o = _box(other_a, size)
|
|
o_pm = np.stack([_box(other_pm[..., i], size) for i in range(3)], axis=-1)
|
|
beta = np.where(1 - g > 1e-4, c_o / np.maximum(1 - g, 1e-4), 0.0)
|
|
rgb = np.where(c_o[..., None] > 1e-6, o_pm / np.maximum(c_o[..., None], 1e-6), 0)
|
|
layers = Layers(np.concatenate([np.clip(rgb, 0, 1), np.clip(beta, 0, 1)[..., None]], axis=-1), g)
|
|
# Check: in ARASAAC's own "white" skin colour the layers re-draw its
|
|
# skin=white render, area-averaged the same way, over white.
|
|
ref_pm = np.where(m[..., None], w[..., :3] * w[..., 3:4], d[..., :3] * d[..., 3:4])
|
|
ref_a = np.where(m, w[..., 3], d[..., 3])
|
|
ref = np.stack([_box(ref_pm[..., i], size) for i in range(3)], axis=-1) + (1 - _box(ref_a, size))[..., None]
|
|
got = layers.compose("#%02x%02x%02x" % tuple(int(round(float(x) * 255)) for x in s_w)) # ARASAAC's "white"
|
|
return layers, float(np.abs(got - ref).max())
|
|
|
|
|
|
def sclera_layers(png: bytes) -> Layers:
|
|
import numpy as np
|
|
rgba = _load_rgba(png)
|
|
rgba[..., 3] = 1.0 # Sclera draws on its own black ground
|
|
return Layers(rgba, None)
|
|
|
|
|
|
CWEBP = "cwebp"
|
|
|
|
|
|
def cwebp_version() -> str:
|
|
r = subprocess.run([CWEBP, "-version"], capture_output=True, text=True, check=True)
|
|
return r.stdout.strip().splitlines()[0]
|
|
|
|
|
|
def encode_webp(rgba: "np.ndarray", lossless: bool, grey: bool = False, near: int | None = None) -> bytes:
|
|
"""cwebp, from an 8-bit PNG. Deterministic for a given libwebp. `near`:
|
|
near-lossless at that level (100 is lossless), which rounds pixel values
|
|
slightly where that saves space -- a fifth of a pictogram's size at 60."""
|
|
import numpy as np
|
|
from PIL import Image
|
|
a8 = np.clip(np.rint(rgba * 255), 0, 255).astype(np.uint8)
|
|
img = Image.fromarray(a8, mode="L") if grey else Image.fromarray(a8, mode="RGBA" if a8.shape[-1] == 4 else "RGB")
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
src, out = Path(tmp, "in.png"), Path(tmp, "out.webp")
|
|
img.save(src, optimize=False)
|
|
args = [CWEBP, "-quiet", "-mt", "-metadata", "none"]
|
|
if near is not None:
|
|
args += ["-near_lossless", str(near), "-z", "9", "-exact"]
|
|
elif lossless:
|
|
args += ["-lossless", "-z", "9", "-exact"]
|
|
else:
|
|
args += ["-q", "92", "-m", "6", "-alpha_q", "100", "-sharp_yuv"]
|
|
subprocess.run(args + [str(src), "-o", str(out)], check=True)
|
|
return out.read_bytes()
|
|
|
|
|
|
# ─── Packs ──────────────────────────────────────────────────────────────────
|
|
|
|
PACK_MAGIC = b"AACSYMP1"
|
|
|
|
|
|
def write_pack(set_id: str, info: dict, items: dict[str, tuple[int, int, bytes, bytes | None]], out_dir: Path) -> tuple[str, int, str]:
|
|
"""items: slug -> (w, h, image webp, skin webp or None). Returns (file
|
|
name, size, sha256). Replaces the set's older packs in out_dir."""
|
|
data = bytearray()
|
|
symbols = {}
|
|
for slug in sorted(items):
|
|
w, h, image, skin = items[slug]
|
|
entry = {"w": w, "h": h, "image": [len(data), len(image), "image/webp"], "skin": None}
|
|
data += image
|
|
if skin is not None:
|
|
entry["skin"] = [len(data), len(skin), "image/webp"]
|
|
data += skin
|
|
symbols[slug] = entry
|
|
if not info["source"].get("version"):
|
|
raise SystemExit(f"{set_id}: the pack's source version is empty")
|
|
header = {
|
|
"format": "aac-symbol-pack", "version": 1, "set": set_id,
|
|
"license": info["license"], "source_license": info.get("source_license"),
|
|
"attribution": info["attribution"], "conditions": info["conditions"],
|
|
"source": info["source"], "adapted": ADAPTED[set_id], "symbols": symbols,
|
|
}
|
|
head = json.dumps(header, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode()
|
|
blob = PACK_MAGIC + struct.pack("<I", len(head)) + head + bytes(data)
|
|
digest = hashlib.sha256(blob).hexdigest()
|
|
name = f"{set_id}.{digest[:12]}.pack"
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
for old in out_dir.glob(f"{set_id}.*.pack"):
|
|
if old.name != name:
|
|
old.unlink()
|
|
(out_dir / name).write_bytes(blob)
|
|
return name, len(blob), digest
|
|
|
|
|
|
def read_pack(path: Path) -> tuple[dict, bytes]:
|
|
blob = path.read_bytes()
|
|
if blob[:8] != PACK_MAGIC:
|
|
raise SystemExit(f"{path}: not a symbol pack")
|
|
(n,) = struct.unpack("<I", blob[8:12])
|
|
return json.loads(blob[12:12 + n]), blob[12 + n:]
|
|
|
|
|
|
# ─── Review ─────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _font(px: int):
|
|
from PIL import ImageFont
|
|
for f in ("/System/Library/Fonts/Supplemental/Arial.ttf", "/System/Library/Fonts/Helvetica.ttc",
|
|
"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"):
|
|
if Path(f).exists():
|
|
return ImageFont.truetype(f, px)
|
|
return ImageFont.load_default()
|
|
|
|
|
|
def contact_sheets(set_id: str, rows: list[tuple[str, str, Layers]], out_dir: Path, skin_hex: str,
|
|
suffix: str = "", per_page: int = 48, cols: int = 8, tile: int = 150) -> list[Path]:
|
|
"""rows: (concept, symbol label, layers). Each page carries the set's
|
|
attribution, as the sheets are themselves adaptations of the set."""
|
|
import numpy as np
|
|
from PIL import Image, ImageDraw
|
|
info = SET_INFO[set_id]
|
|
paths = []
|
|
title_f, label_f, small_f = _font(20), _font(13), _font(10)
|
|
for p in range(0, len(rows), per_page):
|
|
page = rows[p:p + per_page]
|
|
nrows = -(-len(page) // cols)
|
|
w, h = cols * tile, 70 + nrows * (tile + 34)
|
|
sheet = Image.new("RGB", (w, h), (244, 244, 244))
|
|
d = ImageDraw.Draw(sheet)
|
|
d.text((10, 8), f"{info['name']} — {len(rows)} concepts — page {p // per_page + 1} — skin {skin_hex}", fill=(0, 0, 0), font=title_f)
|
|
d.text((10, 36), f"{info['attribution']} {info['license']['name']} {info['license']['url']}", fill=(60, 60, 60), font=small_f)
|
|
d.text((10, 50), ADAPTED[set_id] + " Review aid, not for distribution.", fill=(60, 60, 60), font=small_f)
|
|
for i, (concept, label, layers) in enumerate(page):
|
|
x, y = (i % cols) * tile, 70 + (i // cols) * (tile + 34)
|
|
d.rectangle([x + 2, y + 2, x + tile - 3, y + tile + 31], fill=(255, 255, 255), outline=(200, 200, 200))
|
|
img = np.clip(np.rint(layers.compose(skin_hex) * 255), 0, 255).astype(np.uint8)
|
|
im = Image.fromarray(img, "RGB").resize((tile - 20, tile - 20), Image.Resampling.LANCZOS)
|
|
sheet.paste(im, (x + 10, y + 8))
|
|
d.text((x + tile / 2, y + tile - 6), concept[:22], fill=(0, 0, 0), font=label_f, anchor="mt")
|
|
d.text((x + tile / 2, y + tile + 12), label[:28], fill=(110, 110, 110), font=small_f, anchor="mt")
|
|
path = out_dir / "review" / f"{set_id}{suffix}-{p // per_page + 1:02}.png"
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
sheet.save(path)
|
|
paths.append(path)
|
|
return paths
|
|
|
|
|
|
# ─── Outputs ────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def arasaac_sources(ar: Arasaac, pid: str) -> list[tuple[str, str]]:
|
|
"""(cache name, URL) of a pictogram's 2500 px renders: for a pictogram
|
|
with skin the API's skin=white and skin=black renders, else the published
|
|
one (static.arasaac.org)."""
|
|
if ar.meta[pid].get("skin"):
|
|
q = "resolution=2500&download=false"
|
|
return [(f"arasaac/png/{pid}_2500_skin-{tone}.png", f"{ARASAAC_API}/pictograms/{pid}?skin={tone}&{q}")
|
|
for tone in ("white", "black")]
|
|
return [(f"arasaac/png/{pid}_2500.png", f"{ARASAAC_STATIC}/{pid}/{pid}_2500.png")]
|
|
|
|
|
|
def arasaac_files(fetcher: Fetcher, ar: Arasaac, pid: str) -> tuple[Path | None, Path | None, Path | None]:
|
|
paths = [fetcher.get(n, u) for n, u in arasaac_sources(ar, pid)]
|
|
return (None, paths[0], paths[1]) if len(paths) == 2 else (paths[0], None, None)
|
|
|
|
|
|
def slugs_for(set_id: str, choices: dict[str, Choice]) -> dict[str, str]:
|
|
"""concept -> <symbol id>: concepts sharing a pictogram share its id,
|
|
named after the first such concept (as build.py's assign_slugs)."""
|
|
by_sym: dict[str, str] = {}
|
|
out = {}
|
|
for key in sorted(choices):
|
|
out[key] = by_sym.setdefault(choices[key].sym, f"{set_id}-{B.slugify(key)}")
|
|
seen: dict[str, str] = {}
|
|
for key, slug in out.items():
|
|
if seen.setdefault(slug, choices[key].sym) != choices[key].sym:
|
|
raise SystemExit(f"slug collision: {slug!r}")
|
|
return out
|
|
|
|
|
|
def write_mapping_review(path: Path, concepts: dict[str, NcConcept], mapping, indexes, packs, sizes_) -> None:
|
|
lines = [
|
|
"# Non-commercial symbol sets: mapping review",
|
|
"",
|
|
"Generated by `tools/symbols/build_nc.py` (open-standards, aac/aac-rs) — do not edit; change",
|
|
"`overrides-nc.toml` and rebuild. Quotes ARASAAC keywords (CC BY-NC-SA 4.0, Government of Aragón,",
|
|
"pictograms by Sergio Palao, https://arasaac.org) and Sclera file names (CC BY-NC 2.0 BE, Sclera vzw,",
|
|
"www.sclera.be); for non-commercial use only. Not for the open-standards repository.",
|
|
"",
|
|
"**confirmed**: an English keyword is the concept and enough other languages agree; **meaning**: most",
|
|
"languages compared agree although no English keyword does; **english**: no language to compare, one",
|
|
"English candidate; **override**: decided in `overrides-nc.toml`. A blank cell is unmapped: the app",
|
|
"falls back to an open set.",
|
|
"",
|
|
"## Coverage",
|
|
"",
|
|
"| set | pack | all concepts | default-board concepts | bytes |",
|
|
"|---|---|---|---|---|",
|
|
]
|
|
board = [k for k, c in concepts.items() if c.on_boards]
|
|
for s, m in mapping.items():
|
|
lines.append(f"| {s} | `{packs[s][0]}` | {len(m)} / {len(concepts)} | {sum(k in m for k in board)} / {len(board)} | {packs[s][1]:,} |")
|
|
lines += ["", "## Concepts", "", "| concept | sources | " + " | ".join(mapping) + " |", "|---|---|" + "---|" * len(mapping)]
|
|
for key in sorted(concepts):
|
|
c = concepts[key]
|
|
row = [f"`{key}`", ", ".join(sorted(c.sources))]
|
|
for s, m in mapping.items():
|
|
ch = m.get(key)
|
|
if ch is None:
|
|
row.append("")
|
|
continue
|
|
name = indexes[s].names.get(ch.sym, "")
|
|
if isinstance(name, dict):
|
|
name = " / ".join(f"{l}:{v[0]}" for l, v in name.items())
|
|
row.append(f"`{ch.sym}` {html.escape(str(name))} — {ch.how}: {html.escape(ch.reason)}")
|
|
lines.append("| " + " | ".join(r.replace("|", "\\|") for r in row) + " |")
|
|
for s, m in mapping.items():
|
|
missing = [k for k in sorted(concepts) if k not in m]
|
|
lines += ["", f"## Unmapped in {s} ({len(missing)})", ""]
|
|
for k in missing:
|
|
cands = [x for x in indexes[s].candidates(concepts[k])[:3]] if ":" not in k else []
|
|
hint = "; ".join(f"`{x.id}` {html.escape(str(x.name))[:40]} ({len(x.votes)}/{len(x.seen)})" for x in cands)
|
|
lines.append(f"- `{k}`{' (on default boards)' if concepts[k].on_boards else ''}{' — candidates: ' + hint if hint else ''}")
|
|
path.write_text("\n".join(lines) + "\n")
|
|
|
|
|
|
def write_index_nc(mapping, slugs, packs, versions) -> None:
|
|
sets = {}
|
|
for s in NC_SETS:
|
|
info = dict(SET_INFO[s])
|
|
info["source"] = {**info["source"], "version": versions[s]}
|
|
file, size, digest = packs[s]
|
|
sets[s] = {**info, "file": file, "format": "pack", "size": size, "sha256": digest, "adapted": ADAPTED[s],
|
|
"tones": s == "arasaac"}
|
|
concepts: dict[str, dict[str, str]] = {}
|
|
for s in NC_SETS:
|
|
for key, slug in slugs[s].items():
|
|
concepts.setdefault(key, {})[s] = slug
|
|
doc = {
|
|
"about": ("Optional non-commercial symbol sets, written by tools/symbols/build_nc.py. The packs "
|
|
"themselves are not in this repository: a free deployment with no adverts ships them "
|
|
"beside the app's symbols. Never offer these sets in a paid product or plan."),
|
|
"sets": sets,
|
|
"concepts": {k: concepts[k] for k in sorted(concepts)},
|
|
}
|
|
INDEX_NC.write_text(json.dumps(doc, ensure_ascii=False, indent=1) + "\n")
|
|
|
|
|
|
def write_licences_nc(out: Path) -> None:
|
|
"""DIR/LICENSES-nc.md: the optional sets' section of the app's
|
|
symbols/LICENSES.md, which a deployment that ships the packs appends to
|
|
it (a build without them never names them)."""
|
|
(out / "LICENSES-nc.md").write_text("\n" + "\n".join(B.nc_license_lines()))
|
|
|
|
|
|
def write_pack_licence(out: Path) -> None:
|
|
"""DIR/LICENSE.md: what a copy of the packs must carry wherever it goes
|
|
(attribution, licence and its address, conditions, changes), from
|
|
index-nc.json."""
|
|
doc = json.loads(INDEX_NC.read_text())
|
|
lines = [
|
|
"# Optional symbol packs: licences",
|
|
"",
|
|
"These packs hold pictograms licensed for NON-COMMERCIAL use only. They may be offered",
|
|
"where an app is free of charge and shows no adverts, and never in anything paid for",
|
|
"(a paid product, plan, workspace or hosted library). They are built by",
|
|
"`aac/aac-rs/tools/symbols/build_nc.py` (open-standards) from the sources pinned in",
|
|
"`nc-sources.lock.json`; the app checks each pack against the size and SHA-256 in",
|
|
"`crates/aac-core/src/symbols/index-nc.json`. They are not covered by the licence of",
|
|
"the code beside them.",
|
|
"",
|
|
]
|
|
for s, info in doc["sets"].items():
|
|
lines += [
|
|
f"## {info['name']} — `{info['file']}`",
|
|
"",
|
|
*([f"- {info['name']}'s own licence: {info['source_license']['name']}, {info['source_license']['url']}",
|
|
f"- These adapted pictograms are shared under: {info['license']['name']} ({info['license']['spdx']}), "
|
|
f"{info['license']['url']}"]
|
|
if info.get("source_license") else
|
|
[f"- Licence: {info['license']['name']} ({info['license']['spdx']}), {info['license']['url']}"]),
|
|
f"- Author: {info['author']}. Owner: {info['owner']}.",
|
|
f"- Attribution: {info['attribution']}",
|
|
f"- Conditions: {info['conditions']}",
|
|
f"- Changes: {info['adapted']}",
|
|
f"- Source: {info['source']['url']} ({info['source']['version']})",
|
|
f"- Terms of use: {info['terms_url']} (read {info['retrieved']})",
|
|
f"- Size and SHA-256: {info['size']:,} bytes, {info['sha256']}",
|
|
"",
|
|
]
|
|
if info.get("share_alike"):
|
|
lines += [f"Works derived from these pictograms must be shared under {info['license']['name']}, "
|
|
"with the same attribution.", ""]
|
|
lines += ["`mapping-review-nc.md` and `people-review-nc.md` record which pictogram each concept uses",
|
|
"and how people are drawn; they quote the sets' keywords and file names under the same",
|
|
"licences.", ""]
|
|
(out / "LICENSE.md").write_text("\n".join(lines))
|
|
|
|
|
|
def main() -> None:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
ap.add_argument("--out", type=Path, required=True, help="where the packs and review files go (outside the repository)")
|
|
ap.add_argument("--refresh", action="store_true", help="download sources missing from the lock and record them")
|
|
ap.add_argument("--no-sheets", action="store_true", help="skip the contact sheets")
|
|
ap.add_argument("--skip-drawn-text-check", action="store_true",
|
|
help="only where the drawn-text reader cannot run (not macOS), for packs that are not being "
|
|
"rebuilt for a release; without it the build stops there")
|
|
args = ap.parse_args()
|
|
out = args.out.resolve()
|
|
if ROOT in out.parents or out == ROOT:
|
|
raise SystemExit("--out must be outside the repository: these pictograms are not free-licensed")
|
|
if shutil.which(CWEBP) is None:
|
|
raise SystemExit("cwebp (libwebp) is needed")
|
|
if not args.skip_drawn_text_check and ocr_tool() is None:
|
|
raise SystemExit("the drawn-text check cannot run here (it needs macOS with swiftc and Apple Vision). Build the "
|
|
"packs on a Mac, or pass --skip-drawn-text-check when these packs are not being rebuilt for "
|
|
"a release (their pictures were read where they were built).")
|
|
|
|
lock = load_lock()
|
|
fetcher = Fetcher(lock, args.refresh)
|
|
try:
|
|
concepts = collect_nc_concepts()
|
|
indexes = {"arasaac": Arasaac(fetcher), "sclera": Sclera(fetcher)}
|
|
overrides = load_overrides_nc(indexes, concepts)
|
|
mapping = map_nc(indexes, concepts, overrides)
|
|
boards = check_one_picture_per_board(mapping, default_board_concepts() | pageset_board_concepts())
|
|
slugs = {s: slugs_for(s, mapping[s]) for s in NC_SETS}
|
|
fetcher.prefetch([nu for pid in sorted({c.sym for c in mapping["arasaac"].values()})
|
|
for nu in arasaac_sources(indexes["arasaac"], pid)])
|
|
|
|
meta_dates = sorted({v["retrieved"] for k, v in lock["files"].items() if k.startswith("arasaac/meta/")})
|
|
versions = {
|
|
"arasaac": f"API catalogue of {meta_dates[-1] if meta_dates else RETRIEVED} ({len(indexes['arasaac'].meta)} pictograms); "
|
|
f"2500 px renders; libwebp {cwebp_version()}",
|
|
"sclera": f"sclera.be downloads picto_en/nl/fr/de.zip (last modified 2019-01-20); libwebp {cwebp_version()}",
|
|
}
|
|
packs, layers_by, people, max_err = {}, {}, {}, 0.0
|
|
for s in NC_SETS:
|
|
items: dict[str, tuple[int, int, bytes, bytes | None]] = {}
|
|
layers_by[s] = {}
|
|
people[s] = {}
|
|
sym_of = {slug: mapping[s][k].sym for k, slug in slugs[s].items()}
|
|
for slug in sorted(sym_of):
|
|
sym = sym_of[slug]
|
|
if s == "arasaac":
|
|
d, w, k = arasaac_files(fetcher, indexes[s], sym)
|
|
layers, err = arasaac_layers(d, w, k, ARASAAC_SIZE)
|
|
if err > 0.02:
|
|
raise SystemExit(f"ARASAAC {sym}: its skin could not be separated cleanly (error {err:.3f})")
|
|
max_err = max(max_err, err)
|
|
if indexes[s].meta[sym].get("skin"):
|
|
people[s][slug] = "chosen tone" if layers.skin is not None else "no skin drawn"
|
|
else:
|
|
layers = sclera_layers(indexes[s].image(sym))
|
|
layers_by[s][slug] = layers
|
|
h, w_ = layers.base.shape[:2]
|
|
# The picture near-lossless (pixel values within a few levels);
|
|
# the skin mask exactly, so the tone meets the outline.
|
|
image = (encode_webp(layers.base, lossless=True, near=60) if s == "arasaac"
|
|
else encode_webp(layers.base[..., :3], lossless=True))
|
|
skin = encode_webp(layers.skin, lossless=True, grey=True) if layers.skin is not None else None
|
|
items[slug] = (w_, h, image, skin)
|
|
info = {**SET_INFO[s], "source": {**SET_INFO[s]["source"], "version": versions[s]}}
|
|
packs[s] = write_pack(s, info, items, out)
|
|
# The lock lists exactly the renders this mapping uses.
|
|
for name in [n for n in lock["files"] if n.startswith("arasaac/png/") and n not in fetcher.used]:
|
|
del lock["files"][name]
|
|
finally:
|
|
save_lock(lock)
|
|
|
|
# Words drawn in a picture are read only by those who read that language.
|
|
text_ok = {s: overrides.get(f"{s}-text", {}) for s in NC_SETS}
|
|
found = drawn_text({f"{s}:{mapping[s][k].sym}": layers_by[s][slugs[s][k]].compose(NEUTRAL_SKIN)
|
|
for s in NC_SETS for k in mapping[s]})
|
|
if found is None and not args.skip_drawn_text_check:
|
|
raise SystemExit("the drawn-text check cannot run here (it needs macOS with swiftc and Apple Vision). Build the "
|
|
"packs on a Mac, or pass --skip-drawn-text-check when these packs are not being rebuilt for "
|
|
"a release (their pictures were read where they were built).")
|
|
if found is None:
|
|
print(" drawn-text check skipped (--skip-drawn-text-check): these packs are not for a release", file=sys.stderr)
|
|
else:
|
|
unreviewed = {k: v for k, v in found.items() if k.split(":", 1)[1] not in text_ok[k.split(":", 1)[0]]}
|
|
if unreviewed:
|
|
raise SystemExit("words drawn in chosen pictograms (choose another, or review in overrides-nc.toml [<set>-text]):\n"
|
|
+ "\n".join(f" {k} {[c for c, ch in mapping[k.split(':')[0]].items() if ch.sym == k.split(':', 1)[1]]}: {v}"
|
|
for k, v in sorted(unreviewed.items())))
|
|
write_index_nc(mapping, slugs, packs, versions)
|
|
B.write_licenses({s: json.loads((B.CORE_SYMBOLS / "index.json").read_text())["sets"][s]["file"] for s in B.SET_ORDER})
|
|
write_pack_licence(out)
|
|
write_licences_nc(out)
|
|
write_mapping_review(out / "mapping-review-nc.md", concepts, mapping, indexes, packs, None)
|
|
with (out / "mapping-review-nc.md").open("a") as f:
|
|
f.write(f"\n## One picture per board\n\nChecked on {boards} boards (every default board, every page set's boards): "
|
|
"no two concepts of one board draw the same pictogram.\n\n## Drawn text\n\n")
|
|
if found is None:
|
|
f.write("Not read: built with --skip-drawn-text-check, not for a release.\n")
|
|
else:
|
|
f.write(f"Read in every chosen pictogram with Apple Vision. {len(found)} have words drawn in them, each reviewed:\n\n")
|
|
for k, v in sorted(found.items()):
|
|
set_id, sym = k.split(":", 1)
|
|
f.write(f"- {set_id} `{sym}` {html.escape(str(v), quote=False)}: {html.escape(text_ok[set_id][sym]['reason'], quote=False)}\n")
|
|
f.write("\nThe Spanish ¡ and ¿, which the reader does not report, are checked by eye on the contact sheets.\n")
|
|
(out / "people-review-nc.md").write_text(
|
|
"# ARASAAC people\n\nGenerated by build_nc.py. Every mapped ARASAAC pictogram that ARASAAC marks as drawing skin, and how it is drawn.\n"
|
|
f"Largest error re-drawing ARASAAC's own skin=white render from the separated layers: {max_err:.4f} (0..1).\n\n"
|
|
"| symbol | concept(s) | drawn |\n|---|---|---|\n"
|
|
+ "".join(f"| `{slug}` | {', '.join(f'`{k}`' for k, v in slugs['arasaac'].items() if v == slug)} | {how} |\n"
|
|
for slug, how in sorted(people["arasaac"].items()))
|
|
+ "\n## People drawn without skin\n\nPictograms of people that ARASAAC draws without skin colour, so they never take the "
|
|
"chosen tone, each kept for a reason (overrides-nc.toml [arasaac-people]):\n\n| pictogram | concept(s) | why |\n|---|---|---|\n"
|
|
+ "".join(f"| `{sym}` | {', '.join(f'`{k}`' for k, c in mapping['arasaac'].items() if c.sym == sym) or '(not used)'} | "
|
|
f"{html.escape(e['reason'], quote=False)} |\n" for sym, e in sorted(overrides.get('arasaac-people', {}).items())))
|
|
if not args.no_sheets:
|
|
for s in NC_SETS:
|
|
rows = []
|
|
for key in sorted(mapping[s]):
|
|
slug = slugs[s][key]
|
|
rows.append((key, f"{mapping[s][key].sym} {mapping[s][key].how}", layers_by[s][slug]))
|
|
contact_sheets(s, rows, out, NEUTRAL_SKIN)
|
|
if s == "arasaac":
|
|
toned = [r for r in rows if r[2].skin is not None]
|
|
contact_sheets(s, toned, out, B.TONES["1F3FF"]["skin"], suffix="-tone5")
|
|
board = [k for k, c in concepts.items() if c.on_boards]
|
|
for s in NC_SETS:
|
|
print(f"{s:8} {packs[s][0]:28} concepts {len(mapping[s]):3}/{len(concepts)} boards "
|
|
f"{sum(k in mapping[s] for k in board):3}/{len(board)} symbols {len(set(slugs[s].values())):3} bytes {packs[s][1]:>9,}")
|
|
if fetcher.fetched:
|
|
print(f"downloaded {fetcher.fetched} files into {NC_CACHE}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|