open-standards/aac/aac-rs/third_party/readium-speech/generate.py

#!/usr/bin/env python3
"""Vendor the device-voice facts the app ranks voices with, from Readium Speech.

    python3 third_party/readium-speech/generate.py [--cache DIR] [--check]

Readium Speech (https://github.com/readium/speech, BSD-3-Clause) documents
the voices browsers expose through Web Speech: which are novelty or
very-low-quality voices, how Apple marks a voice's quality in its name in
each interface language, and, per language, every known voice with its
quality, in the project's recommended order. The voice data started as
Hadrien Gardeur's web-speech-recommended-voices, released under CC0.

This script reads those JSON files at one pinned commit (the v0.13.0 release;
its json/ is byte-identical to v0.11.0's), checks them against FILES_DIGEST,
and writes crates/access-core/src/voice/readium.rs: plain Rust tables, so
the app ships the facts inside its own code, fetches nothing at run time,
and sends nothing anywhere. Only what ranking needs is kept:

  - per language the app has (and Mandarin and Cantonese for Chinese): its
    default region, and each voice's names (normalised as
    voice::readium_key does) and quality, in Readium's order. Network
    voices (Edge's "Online", desktop Chrome's Google voices) are left out:
    the app never uses a voice that sends words away, whatever its quality.
  - the novelty voices (names, localised names, Apple identifiers);
  - the very-low-quality ones by kind: Apple's Eloquence voices and the old
    MacinTalk ones (eSpeak is matched by name, so its list is not needed);
  - Apple's localised Enhanced and Premium markers.

--check fails, writing nothing, if the generated file is not what the pinned
data gives (CI and pre-commit use). Python >= 3.11, standard library only.
Downloads about 300 KB once into the cache (default ~/.cache/aac-rs).
"""

from __future__ import annotations

import argparse
import hashlib
import json
import os
import re
import sys
import urllib.request
from pathlib import Path

HERE = Path(__file__).resolve().parent
AAC = HERE.parent.parent
OUT = AAC / "crates" / "access-core" / "src" / "voice" / "readium.rs"

# ─── Pins ────────────────────────────────────────────────────────────────────
# @readium/speech v0.13.0 (npm latest, published 2026-10-02; read 2026-10-04).
# Tag v0.13.0 points at this commit.
VERSION = "0.13.0"
COMMIT = "4a7b73dd37a736e6762497ea7d7bc773c0fe1a30"
RAW = f"https://raw.githubusercontent.com/readium/speech/{COMMIT}"
# sha256 over "<path> <sha256 of its bytes>\n" for FILES, sorted by path.
FILES_DIGEST = "58f761681f5959ca178b80e2c62973830aab6c8cfde49895ca7c917b023a32a6"

# Readium language files the app reads: its languages that Readium documents,
# with Chinese as Mandarin (cmn) and Cantonese (yue).
LANGUAGES = [
    "ar", "bn", "cmn", "cs", "da", "de", "el", "en", "es", "fa", "fi", "fr",
    "he", "hi", "hu", "id", "it", "ja", "kn", "ko", "mr", "ms", "nb", "nl",
    "pl", "pt", "ro", "ru", "sv", "ta", "te", "th", "tr", "uk", "vi", "yue",
]
FILES = sorted(
    ["LICENSE", "json/filters/novelty.json", "json/filters/veryLowQuality.json",
     "json/localizedNames/apple.json"]
    + [f"json/{code}.json" for code in LANGUAGES]
)
QUALITIES = {"veryLow": "VeryLow", "low": "Low", "normal": "Normal", "high": "High", "veryHigh": "VeryHigh"}


def fetch(path: str, cache: Path) -> bytes:
    """A pinned file, from the cache or the commit's raw URL."""
    local = cache / COMMIT / path
    if local.is_file():
        return local.read_bytes()
    with urllib.request.urlopen(f"{RAW}/{path}", timeout=60) as r:  # noqa: S310 (pinned https URL)
        data = r.read()
    local.parent.mkdir(parents=True, exist_ok=True)
    local.write_bytes(data)
    return data


def digest(files: dict[str, bytes]) -> str:
    lines = "".join(f"{p} {hashlib.sha256(files[p]).hexdigest()}\n" for p in sorted(files))
    return hashlib.sha256(lines.encode()).hexdigest()


def key(name: str) -> str:
    """A voice name as voice::readium_key normalises it: lower case, ASCII
    parentheses and what is in them removed, spaces collapsed."""
    s = re.sub(r"\s*\([^)]*\)", "", name.lower())
    return " ".join(s.split())


def is_network(v: dict) -> bool:
    """A voice that sends the words to its provider: Edge's "Online" voices
    and desktop Chrome's Google voices."""
    return "Online" in v["name"] or v.get("browser") == ["ChromeDesktop"]


def rs(s: str) -> str:
    return json.dumps(s, ensure_ascii=False)


def strs(items) -> str:
    return "&[" + ", ".join(rs(i) for i in items) + "]"


def generate(files: dict[str, bytes]) -> str:
    load = lambda p: json.loads(files[p].decode("utf-8"))  # noqa: E731
    out = [
        "// @generated by third_party/readium-speech/generate.py from Readium Speech",
        f"// v{VERSION} (commit {COMMIT}): voice data CC0-1.0, code BSD-3-Clause.",
        "// See third_party/readium-speech/NOTICE.txt. Do not edit by hand.",
        "",
        "USE_LINE",
        "",
        "/// One documented voice: every name it goes by (as `readium_key` gives",
        "/// them) and its quality, lowest first when it comes in several sizes.",
        "pub(super) struct Entry {",
        "    pub keys: &'static [&'static str],",
        "    pub quality: &'static [Quality],",
        "}",
        "",
        "/// A language's documented voices, in Readium's recommended order.",
        "pub(super) struct Language {",
        "    pub code: &'static str,",
        "    pub default_region: &'static str,",
        "    pub voices: &'static [Entry],",
        "}",
        "",
        "/// A novelty voice: its names and Apple identifiers.",
        "pub(super) struct Novelty {",
        "    pub names: &'static [&'static str],",
        "    pub ids: &'static [&'static str],",
        "}",
        "",
        "pub(super) static LANGUAGES: &[Language] = &[",
    ]
    for code in LANGUAGES:
        data = load(f"json/{code}.json")
        region = data["defaultRegion"].split("-")[-1]
        out.append(f"    Language {{ code: {rs(code)}, default_region: {rs(region)}, voices: &[")
        for v in data["voices"]:
            if is_network(v):
                continue
            keys = []
            for n in [v["name"], *v.get("altNames", [])]:
                k = key(n)
                if k and k not in keys:
                    keys.append(k)
            quality = ", ".join(QUALITIES[q] for q in v.get("quality") or [] if q in QUALITIES)
            out.append(f"        Entry {{ keys: {strs(keys)}, quality: &[{quality}] }},")
        out.append("    ] },")
    out.append("];")
    out.append("")

    novelty = load("json/filters/novelty.json")["voices"]
    out.append("pub(super) static NOVELTY: &[Novelty] = &[")
    for v in novelty:
        names = [key(n) for n in [v["name"], *v.get("altNames", [])]]
        out.append(f"    Novelty {{ names: {strs(names)}, ids: {strs(v.get('nativeID', []))} }},")
    out.append("];")
    out.append("")

    low = load("json/filters/veryLowQuality.json")["voices"]
    eloquence = [key(v["name"]) for v in low if v.get("localizedName") == "apple"]
    macintalk = [key(v["name"]) for v in low
                 if v.get("localizedName") != "apple" and not v["name"].lower().startswith("espeak")]
    out.append("/// Apple's Eloquence voices (very low quality).")
    out.append(f"pub(super) static ELOQUENCE: &[&str] = {strs(eloquence)};")
    out.append("")
    out.append("/// Apple's old MacinTalk voices (very low quality).")
    out.append(f"pub(super) static MACINTALK: &[&str] = {strs(macintalk)};")
    out.append("")

    markers = load("json/localizedNames/apple.json")["quality"]
    out.append("/// Apple's quality markers in each interface language: (locale, Enhanced, Premium).")
    out.append("pub(super) static APPLE_MARKERS: &[(&str, &str, &str)] = &[")
    for loc, m in markers.items():
        out.append(f"    ({rs(loc)}, {rs(m['normal'])}, {rs(m['high'])}),")
    out.append("];")
    out.append("")
    text = "\n".join(out)
    used = sorted({q for q in QUALITIES.values() if re.search(rf"\b{q}\b", text.split("pub(super) static LANGUAGES", 1)[1])})
    return text.replace("USE_LINE", f"use super::Quality::{{self, {', '.join(used)}}};", 1)


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--cache", type=Path, default=Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "aac-rs" / "readium-speech")
    ap.add_argument("--check", action="store_true", help="fail if the generated file is out of date")
    args = ap.parse_args()

    files = {p: fetch(p, args.cache) for p in FILES}
    got = digest(files)
    if got != FILES_DIGEST:
        print(f"✗ Readium Speech files digest {got}, pinned {FILES_DIGEST}", file=sys.stderr)
        print("  (a new pin: read the changes, then set FILES_DIGEST, and VERSION/COMMIT if they moved)", file=sys.stderr)
        return 1
    text = generate(files)
    license_text = files["LICENSE"]
    if args.check:
        stale = [p for p, want in ((OUT, text.encode()), (HERE / "LICENSE.txt", license_text))
                 if not p.is_file() or p.read_bytes() != want]
        for p in stale:
            print(f"✗ {p.relative_to(AAC)} is out of date: run third_party/readium-speech/generate.py", file=sys.stderr)
        return 1 if stale else 0
    OUT.parent.mkdir(parents=True, exist_ok=True)
    OUT.write_text(text, encoding="utf-8")
    (HERE / "LICENSE.txt").write_bytes(license_text)
    print(f"✓ {OUT.relative_to(AAC)} ({len(text.encode()):,} bytes) from Readium Speech v{VERSION}")
    return 0


if __name__ == "__main__":
    sys.exit(main())