open-standards/aac/aac-rs/tools/voices/intelligibility.py

#!/usr/bin/env python3
"""Measure whether each voice says the words: the evidence its rating rests on.

Every voice says its language's everyday phrases (phrases.json); a speech
recognizer (Whisper large-v3-turbo, on MLX) writes down what it hears; the
similarity of that to the phrase, 0 to 1 after dropping case, spaces and
punctuation, is the score. The results go into intelligibility.json
(committed), from which build_voices.py rates each voice and withholds any
that says other words (see RATING_RULE there).

  python tools/voices/intelligibility.py                 # every voice in the built packs
  python tools/voices/intelligibility.py de-piper-mls ja # some voices or languages

Needs Node >= 22 and an MLX Python (Apple Silicon) with mlx-whisper; the
model (about 1.6 GB) downloads once. Voices the build withholds are not in the
packs, so their last results are kept: delete one's entry to test it again.
"""

import difflib
import json
import re
import subprocess
import sys
import tempfile
import types
import unicodedata
from datetime import date
from pathlib import Path

HERE = Path(__file__).resolve().parent
RESULTS = HERE / "intelligibility.json"
MODEL = "mlx-community/whisper-large-v3-turbo"

try:
    import mlx_whisper
except ImportError as e:
    if "numba" not in str(e).lower():
        raise
    # Numba only serves word timestamps, unused here, and some environments'
    # Numba refuses their NumPy: stand in for it.
    stub = types.ModuleType("numba")
    stub.jit = stub.njit = lambda *a, **k: (a[0] if len(a) == 1 and callable(a[0]) and not k else (lambda f: f))
    sys.modules["numba"] = stub
    import mlx_whisper


def norm(text):
    return re.sub(r"[\W_]+", "", unicodedata.normalize("NFKC", text).lower())


def main(argv):
    with tempfile.TemporaryDirectory() as tmp:
        subprocess.run(["node", str(HERE / "intelligibility.mjs"), tmp, *argv[1:]], check=True)
        items = json.loads((Path(tmp) / "manifest.json").read_text())
        results = json.loads(RESULTS.read_text()) if RESULTS.exists() else {}
        per = {}
        for it in items:
            heard = mlx_whisper.transcribe(it["file"], path_or_hf_repo=MODEL, language=it["lang"], verbose=None)["text"].strip()
            score = difflib.SequenceMatcher(None, norm(it["text"]), norm(heard)).ratio()
            # Babble makes a recognizer repeat itself at length: keep the start.
            kept = heard if len(heard) <= 200 else heard[:200] + "\u2026"
            per.setdefault(it["pack"], []).append({"text": it["text"], "heard": kept, "score": round(score, 3),
                                                   "seconds": it["seconds"]})
    for pack, phrases in per.items():
        scores = [p["score"] for p in phrases]
        results[pack] = {"mean": round(sum(scores) / len(scores), 3), "min": round(min(scores), 3),
                         "model": MODEL, "measured": date.today().isoformat(), "phrases": phrases}
        print(f"{pack:26} mean {results[pack]['mean']:.2f}  min {results[pack]['min']:.2f}")
    RESULTS.write_text(json.dumps(dict(sorted(results.items())), ensure_ascii=False, indent=1) + "\n")
    print(f"wrote {RESULTS}")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))