open-standards/aac/aac-rs/tools/pagesets/pslib/samewords.py

"""No board shows the same words on two of its cells, in any language.

A person who sees "encore" twice on one board cannot tell which says "more"
and which says "again"; a partner modelling on it cannot either. So every
board of every package, in every language the package is shown in, is
checked: the words each cell shows there (as the app shows them: a gloss of
its English key, in the sense the cell names; or its words as written) must
differ from every other cell's on that board. Navigation controls (Back,
Home) are the app's controls, drawn and named as such, not words; a cell
with no word in a language, hidden in its place there, shows nothing; nor
does a cell a language says with the same word as another cell of the board
(`ext_aurasense_same_word`: the word is shown once, at the other's place); a
slot a partner fills in (`ext_aurasense_fill_in`: "I enjoy …", each with its
own photo and words) is told apart by what is filled in.

    python3 tools/pagesets/pslib/samewords.py DIR [--json OUT]

checks every package (*.obz) under DIR (catalog.py --out DIR builds them),
prints each clash, and exits 1 if there is one. catalog.py runs the same
check on every build and fails on a clash: it is never a warning.
"""

from __future__ import annotations

import json
import sys
import unicodedata
import zipfile
from pathlib import Path

CONTROLS = ("back", "home")

# Languages whose readers tell two words apart by a capital alone: German
# writes every noun with one ("Essen", food; "essen", eat).
CASED = ("de",)


def normal(text: str, lang: str | None = None) -> str:
    """Words as a person reads them: width and spacing aside, and case too,
    except where the language tells words apart by it (`CASED`)."""
    text = unicodedata.normalize("NFKC", text)
    if (lang or "").split("-")[0] not in CASED:
        text = text.casefold()
    return " ".join(text.split())


def shown(btn: dict, lang: str, package_lang: str) -> str | None:
    """What a button shows in `lang`, as the app shows it; None where it
    shows no words there (a control, an empty cell, a word with none in the
    language on a set made to be glossed)."""
    prov = btn.get("ext_aurasense_provenance") or {}
    if btn.get("load_board") and normal(prov.get("source_label", btn.get("label", ""))) in CONTROLS:
        return None
    label = (btn.get("label") or "").strip()
    if not label or "ext_aurasense_fill_in" in btn:
        return None
    same = btn.get("ext_aurasense_same_word") or {}
    if lang in same or "*" in same:
        return None
    if package_lang != "*":
        # A package in one language: its words as written.
        return label if lang == package_lang.split("-")[0] or lang == package_lang else None
    tr = (btn.get("translations") or {}).get(lang)
    if tr and tr.get("label"):
        return tr["label"]
    if lang == "en":
        return label
    # No word in this language: hidden in place on a glossed set, else its
    # English words (marked English).
    return None if "ext_aurasense_pageset" in btn else label


def board_clashes(board: dict, langs: list[str], package_lang: str) -> list[dict]:
    out = []
    for lang in langs:
        seen: dict[str, list[dict]] = {}
        for btn in board.get("buttons", []):
            words = shown(btn, lang, package_lang)
            if words is None:
                continue
            key = (btn.get("ext_aurasense_provenance") or {}).get("source_label", btn.get("label"))
            sense = btn.get("ext_aurasense_gloss_sense")
            seen.setdefault(normal(words, lang), []).append({"key": key, "sense": sense, "words": words})
        for words, cells in seen.items():
            if len(cells) > 1:
                out.append({"lang": lang, "words": words, "cells": cells})
    return out


def key_of(btn: dict) -> str:
    """The key a button is glossed by: its English word, and the sense it is
    glossed in where it names one ("cold (water)")."""
    prov = btn.get("ext_aurasense_provenance") or {}
    key = prov.get("source_label", btn.get("label", ""))
    sense = btn.get("ext_aurasense_gloss_sense")
    return f"{key} ({sense})" if sense else key


def shown_from(btn: dict) -> int:
    """The step of a set that grows at which the button is first shown (0:
    from the start; aac-core samewords `shown_from`)."""
    return int((btn.get("ext_aurasense_pageset") or {}).get("shown_from", 0))


def stage_gaps(board: dict) -> list[dict]:
    """On a board of a set that grows, every word shown has its words shown
    somewhere, at every step: a word a language says as another key's word
    is left out only where that key is shown too — never for a key that is
    hidden for later (the key shown first keeps the word). Checked as the
    package comes (its hidden words hidden) and at each step of its growth."""
    buttons = board.get("buttons", [])
    steps = sorted({shown_from(b) for b in buttons if "shown_from" in (b.get("ext_aurasense_pageset") or {})})
    views = [("as it comes", [b for b in buttons if not b.get("hidden")])]
    views += [(f"step {t}", [b for b in buttons if shown_from(b) <= t]) for t in steps]
    out = []
    for view, shown in views:
        keys = {key_of(b) for b in shown}
        labels = {normal(b.get("label") or "") for b in shown}
        for b in shown:
            for lang, keeper in sorted((b.get("ext_aurasense_same_word") or {}).items()):
                there = normal(keeper) in labels if lang == "*" else keeper in keys
                if not there:
                    out.append({"view": view, "lang": lang, "key": key_of(b), "keeper": keeper})
    return out


# The families whose sets grow: each of their words says the step it is
# shown from (or the steps could not be checked).
GROWS = ("growing-core", "fixed-paths")


def package_gaps(data: bytes, grows: bool = False) -> list[dict]:
    z = zipfile.ZipFile(__import__("io").BytesIO(data))
    out = []
    for name in sorted(z.namelist()):
        if name.endswith(".obf"):
            board = json.loads(z.read(name))
            out += [{"board": board.get("name", name), **g} for g in stage_gaps(board)]
            if grows:
                for b in board.get("buttons", []):
                    ps = b.get("ext_aurasense_pageset") or {}
                    if ps.get("kind") in ("word", "modifier") and "shown_from" not in ps:
                        out.append({"board": board.get("name", name), "view": "its steps", "lang": "*",
                                    "key": key_of(b), "keeper": "(no shown_from: its step is not known)"})
    return out


def gaps(index: dict, files: dict[str, Path]) -> list[dict]:
    """Every word of every package left out for a key that is not shown with
    it (catalog.py fails on any)."""
    out = []
    for fam in index["families"]:
        grows = fam["key"] in GROWS
        for pkg in fam["packages"]:
            for g in package_gaps(files[pkg["file"]].read_bytes(), grows):
                out.append({"family": fam["key"], "package": pkg["key"], **g})
    return out


def describe_gap(gap: dict) -> str:
    return (f"{gap['family']}/{gap['package']} · {gap['board']} · {gap['view']} · {gap['lang']}: "
            f"“{gap['key']}” is left out for “{gap['keeper']}”, which is not shown")


def package_clashes(data: bytes, langs: list[str], package_lang: str) -> list[dict]:
    z = zipfile.ZipFile(__import__("io").BytesIO(data))
    out = []
    for name in sorted(z.namelist()):
        if not name.endswith(".obf"):
            continue
        board = json.loads(z.read(name))
        for c in board_clashes(board, langs, package_lang):
            out.append({"board": board.get("name", name), **c})
    return out


def check(index: dict, files: dict[str, Path], langs: list[str]) -> list[dict]:
    """Every clash of every package of a catalogue (catalog.py)."""
    out = []
    for fam in index["families"]:
        for pkg in fam["packages"]:
            data = files[pkg["file"]].read_bytes()
            for c in package_clashes(data, langs, pkg.get("lang", "*")):
                out.append({"family": fam["key"], "package": pkg["key"], **c})
    return out


def describe(clash: dict) -> str:
    keys = " / ".join(c["key"] + (f" ({c['sense']})" if c.get("sense") else "") for c in clash["cells"])
    return f"{clash['family']}/{clash['package']} · {clash['board']} · {clash['lang']}: “{clash['words']}” on {keys}"


def main() -> None:
    sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
    from pslib.data import Data  # noqa: E402

    root = Path(sys.argv[1])
    langs = ["en"] + [l["code"] for l in Data().languages if l["code"] != "en"]
    index = json.loads((root / "index.json").read_text(encoding="utf-8"))
    files = {p.name: p for p in root.glob("*.obz")}
    found = check(index, files, langs)
    for c in found:
        print(describe(c))
    left = gaps(index, files)
    for g in left:
        print(describe_gap(g))
    print(f"{len(left)} words left out for a key not shown with them")
    if "--json" in sys.argv:
        Path(sys.argv[sys.argv.index("--json") + 1]).write_text(json.dumps(found, ensure_ascii=False, indent=1), encoding="utf-8")
    print(f"{len(found)} clashes")
    sys.exit(1 if found or left else 0)


if __name__ == "__main__":
    main()