open-standards/aac/aac-rs/tools/speech/check_pronunciation.py

#!/usr/bin/env python3
"""Check crates/aac-core/src/speech_lexicon.json against real speech engines.

For every lexicon entry, say the word on its own as written and in its spoken
form (`say`) with an on-device macOS voice, transcribe both with Whisper, and
report what was heard. A good entry turns a letter name ("Capital I",
"Y mayúscula", "E accento grave", "μικρό ήτα") into the word. The words that
tools/speech/review.json records as left alone or unresolved are said as
written, to show they are not read as letters. A lone capital letter,
lower-cased, must no longer be announced as a capital.

Two engines:
- macOS voices (what Safari and Chrome use on a Mac), heard through Whisper.
  For a non-Latin script the audio is also transcribed into English, which
  gives a rough romanisation ("min" versus "man" for Arabic من) where the
  native transcript cannot show vowels.
- eSpeak NG (what Chrome and Firefox use on Linux through speech-dispatcher),
  whose phonemes are printed directly (`espeak-ng --ipa`), with no ASR.

Whisper is weak on a lone syllable, so a transcript that neither names a
letter nor is clearly the word is "inconclusive", not a pass: those entries
need a native speaker's ear (the lexicon's status stays "model-reviewed").

macOS only. Needs mlx-whisper (pip install mlx-whisper), which downloads the
Whisper model below on first run, and optionally espeak-ng (brew install
espeak-ng).

    python3 tools/speech/check_pronunciation.py [--lang pl --lang cs] [--out results.json]
"""
import argparse
import json
import pathlib
import re
import shutil
import subprocess
import tempfile
import unicodedata

import mlx_whisper

ROOT = pathlib.Path(__file__).resolve().parents[2]
LEXICON = ROOT / "crates/aac-core/src/speech_lexicon.json"
REVIEW = pathlib.Path(__file__).resolve().parent / "review.json"
MODEL = "mlx-community/whisper-large-v3-turbo"

# One on-device macOS voice per site language, where macOS has one (checked
# against `say -v '?'` on macOS 27). No macOS voice: ur fa ml mr gu pa tl sw
# am ha yo zu.
VOICES = {
    "en": "Samantha", "es": "Mónica", "fr": "Thomas", "de": "Anna", "pt": "Luciana",
    "it": "Alice", "nl": "Xander", "zh": "Tingting", "ja": "Kyoko", "ko": "Yuna",
    "ar": "Majed", "he": "Carmit", "hi": "Lekha", "bn": "Piya", "ta": "Vani",
    "te": "Geeta", "kn": "Soumya", "th": "Kanya", "vi": "Linh", "id": "Damayanti",
    "ms": "Amira", "ru": "Milena", "pl": "Zosia", "uk": "Lesya", "ro": "Ioana",
    "cs": "Zuzana", "sv": "Alva", "da": "Sara", "nb": "Nora", "fi": "Satu",
    "tr": "Yelda", "el": "Melina", "hu": "Tünde",
}
# eSpeak NG voice per site language; it has none for tl ha yo zu.
ESPEAK = {
    "en": "en-us", "es": "es", "fr": "fr-fr", "de": "de", "pt": "pt-br", "it": "it",
    "nl": "nl", "zh": "cmn", "ja": "ja", "ko": "ko", "ar": "ar", "he": "he", "ur": "ur",
    "fa": "fa", "hi": "hi", "bn": "bn", "ta": "ta", "te": "te", "ml": "ml", "mr": "mr",
    "kn": "kn", "gu": "gu", "pa": "pa", "th": "th", "vi": "vi", "id": "id", "ms": "ms",
    "ru": "ru", "pl": "pl", "uk": "uk", "ro": "ro", "cs": "cs", "sv": "sv", "da": "da",
    "nb": "nb", "fi": "fi", "sw": "sw", "am": "am", "tr": "tr", "el": "el", "hu": "hu",
}
# Capital letters to probe outside the Latin script. Greek is left out: its
# voices name a lone letter whatever its case, so lower-casing cannot help.
CAPITALS = {"ru": "АОЯ", "uk": "АОЯ"}
ALPHABETS = ("LATIN", "CYRILLIC", "GREEK", "ARABIC", "HEBREW")
# Whisper's code where it differs from the site's.
WHISPER_LANG = {"nb": "no"}
LATIN = {"en", "es", "fr", "de", "pt", "it", "nl", "vi", "id", "ms", "tl", "pl", "ro",
         "cs", "sv", "da", "nb", "fi", "sw", "ha", "yo", "zu", "tr", "hu"}
# Transcripts that mean the engine named a letter, a capital or an accent.
NAMED = re.compile(
    r"capital|mayúscula|majuscule|maiúscul|maiuscol|huruf besar|majuscul|großbuchstabe|hoofdletter|"
    r"stort bogstav|stor bokstav|versal|iso kirjain|büyük|заглавн|велика літера|wielk|duże|"
    r"velk|nagy|kis hosszú|kis rövid|ékezetes|accent|acento|accento|acentuad|agudo|"
    r"μικρό|κεφαλαίο|τόνο|όμ[ιη]κρ|όμητρ|ήτα|\bA hoa\b|واو",
    re.I,
)


def installed_voices() -> set[str]:
    """Voice names `say -v` accepts: 'Majed (Arabic (World))' is also 'Majed'."""
    out = subprocess.run(["say", "-v", "?"], capture_output=True, text=True, check=True).stdout
    names = set()
    for line in out.splitlines():
        name = re.split(r"\s{2,}", line.strip())[0]
        if name:
            names.add(name)
            names.add(name.split(" (")[0])
    return names


def transcribe(wav: pathlib.Path, lang: str) -> str:
    r = mlx_whisper.transcribe(str(wav), path_or_hf_repo=MODEL, language=lang, temperature=0.0,
                               condition_on_previous_text=False)
    return r["text"].strip()[:120]


def hear(voice: str, text: str, lang: str, work: pathlib.Path, name: str) -> tuple[str, str | None]:
    """What Whisper hears when `voice` says `text`, and for a non-Latin script a
    rough romanisation (the same audio transcribed as English)."""
    txt, aiff, wav = work / f"{name}.txt", work / f"{name}.aiff", work / f"{name}.wav"
    txt.write_text(text)  # from a file, so a word such as "-e/-a" is not an option
    subprocess.run(["say", "-v", voice, "-o", str(aiff), "-f", str(txt)], check=True)
    subprocess.run(["afconvert", "-f", "WAVE", "-d", "LEI16@16000", str(aiff), str(wav)], check=True)
    heard = transcribe(wav, WHISPER_LANG.get(lang, lang))
    latin = None if lang in LATIN else transcribe(wav, "en")
    return heard, latin


def phonemes(lang: str, text: str, as_letters: bool = False) -> str | None:
    """eSpeak NG's phonemes for `text`, or for its letter names."""
    voice = ESPEAK.get(lang)
    if not voice or not shutil.which("espeak-ng"):
        return None
    cmd = ["espeak-ng", "-v", voice, "-q", "--ipa"]
    if as_letters:
        text = f'<speak><say-as interpret-as="characters">{text}</say-as></speak>'
        cmd.append("-m")
    out = subprocess.run(cmd, input=text, capture_output=True, text=True).stdout
    return " ".join(out.split())


def espeak_verdict(lang: str, written: str, spoken: str | None) -> str | None:
    """For a one-letter word, whether eSpeak says the letter's name."""
    base = [c for c in written if not unicodedata.category(c).startswith("M")]
    # Only alphabets name their letters; a kana, hanzi, Hangul or Indic
    # syllable said alone is simply read.
    if len(base) != 1 or not unicodedata.name(base[0], "").startswith(ALPHABETS):
        return None
    letter, word = phonemes(lang, written, as_letters=True), phonemes(lang, written)
    if not letter or not word:
        return None
    if word != letter:
        return "reads the written word as a word"
    if spoken is None:
        return "says the letter's name (see review.json: the name may be the word)"
    if phonemes(lang, spoken) != letter:
        return "says the letter's name for the written word, not for the spoken form"
    return "the spoken form sounds the same as the letter's name (fine where the name is the word)"


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--lang", action="append", help="only these languages (repeatable)")
    ap.add_argument("--out", type=pathlib.Path)
    args = ap.parse_args()
    lexicon = json.loads(LEXICON.read_text())["languages"]
    review = json.loads(REVIEW.read_text())["languages"]
    have = installed_voices()
    rows = []
    with tempfile.TemporaryDirectory() as tmp:
        work = pathlib.Path(tmp)
        for lang, entry in lexicon.items():
            if args.lang and lang not in args.lang:
                continue
            notes = review.get(lang, {})
            cases = [("lexicon", w, e["say"]) for w, e in entry["words"].items()]
            # The pronoun "I" is written as a capital; so is a word that starts a message.
            cases += [("lexicon", w.upper(), e["say"]) for w, e in entry["words"].items()
                      if len(w) == 1 and w.upper() != w]
            cases += [("left_alone", w, None) for w in notes.get("left_alone", {})]
            cases += [("unresolved", w, None) for w in notes.get("unresolved", {})]
            voice = VOICES.get(lang)
            if voice in have:
                # A capital letter that is not in the lexicon must not be announced.
                capitals = CAPITALS.get(lang, "AIOEY" if lang in LATIN else "")
                probe = next((c for c in capitals if c.lower() not in entry["words"]), None)
                if probe:
                    cases.append(("capital", probe, probe.lower()))
            else:
                voice = None
            for i, (kind, written, spoken) in enumerate(cases):
                row = {"lang": lang, "kind": kind, "written": written, "spoken_form": spoken,
                       "espeak_written": phonemes(lang, written),
                       "espeak_spoken": phonemes(lang, spoken) if spoken else None,
                       "espeak_letter_name": espeak_verdict(lang, written, spoken)}
                if voice:
                    row["voice"] = voice
                    row["heard_written"], row["romanised_written"] = hear(voice, written, lang, work, f"{lang}{i}w")
                    if spoken:
                        row["heard_spoken"], row["romanised_spoken"] = hear(voice, spoken, lang, work, f"{lang}{i}s")
                    named_w = bool(NAMED.search(row["heard_written"]))
                    named_s = spoken is not None and bool(NAMED.search(row["heard_spoken"]))
                    if named_s:
                        row["result"] = "FAIL: the spoken form is still read as a letter"
                    elif spoken is None and named_w and kind == "left_alone":
                        row["result"] = "FAIL: read as a letter; needs a spoken form"
                    elif named_w and spoken is not None:
                        row["result"] = "fixed: the written word is read as a letter, the spoken form is not"
                    elif named_w:
                        row["result"] = "read as a letter (known, see review.json)"
                    else:
                        row["result"] = "inconclusive: not read as a letter; needs a native ear"
                else:
                    row["result"] = "no macOS voice; eSpeak phonemes only"
                rows.append(row)
                heard = row.get("heard_written", "")
                if row.get("romanised_written"):
                    heard += f" ~ {row['romanised_written']}"
                line = f"{lang} {kind[:4]} {written!r:>8} → {heard!r:30}"
                if spoken:
                    heard_s = row.get("heard_spoken", "")
                    if row.get("romanised_spoken"):
                        heard_s += f" ~ {row['romanised_spoken']}"
                    line += f" | {spoken!r:>6} → {heard_s!r:30}"
                line += f" | eSpeak [{row['espeak_written'] or '-'}]"
                if spoken:
                    line += f" → [{row['espeak_spoken'] or '-'}]"
                if row["espeak_letter_name"]:
                    line += f" ({row['espeak_letter_name']})"
                print(f"{line}  {row['result'].split(':')[0]}", flush=True)
    if args.out:
        args.out.write_text(json.dumps(rows, ensure_ascii=False, indent=1) + "\n")
    fails = [r for r in rows if r["result"].startswith("FAIL")]
    print(f"{len(rows)} checks, {len(fails)} failures")
    raise SystemExit(1 if fails else 0)


if __name__ == "__main__":
    main()