open-standards/aac/aac-rs/tools/finder/build_ja_kana.py

#!/usr/bin/env python3
"""The kana word list for reading Japanese typed without spaces (aac-core
finder.rs `words_in_listed`, aac-web finder.rs).

Japanese is written without spaces, and a word typed in kana (はなや,
めがね) must be read as the word it is, not as the one-kana words of the
boards inside it (はな + や, め + が + ね). The finder reads a kana run by the
words of this list and of the boards, the fewest words first. The list is
the kana readings of JMdict's common entries — an entry with a priority tag
(news1, ichi1, spec1, spec2, gai1) on any of its spellings — each reading
that is itself marked common, or the only way the entry is written; readings
JMdict marks out-dated, search-only or rare are left out. Hiragana and
katakana, one word a line, sorted. It is fetched only by a device whose
person writes Japanese, as the pinyin index is for Chinese.

Each word that inflects carries its classes after a tab, from JMdict's parts
of speech, so a verb or adjective typed in an inflected form is read as one
word and led to its dictionary form (aac-core ja_inflect.rs). The classes are
each entry's own, the entries apart by "|": かえる is 変える (v1), 帰る (v5) and
蛙 (none), "v1|v5|", so a spelling whose entries differ is known to be more
than one word, and a form of one of them is never taken for another:
  v1   Ichidan verbs (v1, v1-s, vz)
  v5   Godan verbs (v5*)
  vk   くる
  vs   する itself (vs-i, vs-s)
  vsn  nouns that take する (vs)
  adj  i-adjectives (adj-i, adj-ix)
  exp  set phrases and interjections (exp, int): a word that may look
       inflected but is the phrase it is (すみません is not すむ's)
  adv  adverbs (adv, adv-to): どうした, もうした — a verb's form after one
       at the end of a sentence is the verb
  vi   a verb whose senses are all intransitive (vi, and no vt): after an
       object with を it is not the verb meant (ふくをきて is not 来る's)

Writes:
  data/kana/ja.<sha256-12>.txt   the words, one a line (word, or word<TAB>classes,
                                 one entry's classes apart from the next's by "|")
  data/kana/index.json           file, sha256, bytes, entries, source
  data/kana/LICENSE.md           its licence (JMdict's, CC BY-SA 4.0)

Usage:
    python3 tools/finder/build_ja_kana.py JMdict_e.gz

JMdict is the property of the Electronic Dictionary Research and Development
Group (EDRDG), used under its licence (CC BY-SA 4.0,
https://www.edrdg.org/edrdg/licence.html); the list this writes is shared
under the same licence.
"""

import gzip
import hashlib
import json
import re
import sys
import xml.etree.ElementTree as ET
from pathlib import Path

OUT = Path(__file__).resolve().parents[2] / "data" / "kana"
PRIORITY = {"news1", "ichi1", "spec1", "spec2", "gai1"}
KANA = re.compile(r"^[ぁ-ゖァ-ヺー]+$")
SKIP_READING = {"out-dated or obsolete kana usage", "search-only kana form", "rarely used kana form"}

LICENSE = """# Kana word list — licence

Generated by `aac/aac-rs/tools/finder/build_ja_kana.py`; do not edit by hand.

`{file}` lists the kana readings of the common entries of **JMdict**
(`JMdict_e`), the Japanese–multilingual dictionary of the Electronic
Dictionary Research and Development Group (EDRDG), one word a line, so that
Japanese typed without spaces is read as its words (the finder of the AAC
reference app).

JMdict is the property of the EDRDG and is used in conformance with the
Group's licence: Creative Commons Attribution-ShareAlike 4.0 International
(CC BY-SA 4.0), https://creativecommons.org/licenses/by-sa/4.0/ ;
https://www.edrdg.org/edrdg/licence.html ;
https://www.edrdg.org/jmdict/j_jmdict.html .

The list is a work made from JMdict and is shared under the same licence,
CC BY-SA 4.0. Changes: only the readings in kana of the entries JMdict marks
common (priority tags news1, ichi1, spec1, spec2, gai1) were kept — each
reading itself marked common, or the only way its entry is written — without
readings marked out-dated, search-only or rare, each with the classes of
each of its entries' parts of speech (verb and adjective classes, whether it
is a set phrase or interjection, an adverb, or a verb that is only
intransitive, only);
nothing else of JMdict is in it.
"""


def pos_codes(jmdict: str) -> dict[str, str]:
    """JMdict's part-of-speech descriptions -> their entity names (the parser
    gives the descriptions)."""
    with gzip.open(jmdict, "rt", encoding="utf-8") as f:
        head = f.read(300_000)
    return {desc: name for name, desc in re.findall(r'<!ENTITY ([\w-]+) "([^"]*)">', head)}


def word_class(code: str) -> str | None:
    if code in ("v1", "v1-s", "vz"):
        return "v1"
    if code.startswith("v5"):
        return "v5"
    if code == "vk":
        return "vk"
    if code in ("vs-i", "vs-s"):
        return "vs"
    if code == "vs":
        return "vsn"
    if code in ("adj-i", "adj-ix"):
        return "adj"
    if code in ("exp", "int"):
        return "exp"
    if code in ("adv", "adv-to"):
        return "adv"
    return None


ORDER = ["v1", "v5", "vk", "vs", "vsn", "adj", "exp", "adv", "vi"]
VERBS = {"v1", "v5", "vk", "vs"}


def entry_classes(el: ET.Element, codes: dict[str, str]) -> set[str]:
    """An entry's classes, from all its senses' parts of speech, with "vi"
    for a verb none of whose senses is transitive and some intransitive."""
    pos = {codes.get(p.text or "", "") for s in el.findall("sense") for p in s.findall("pos")}
    found = {c for p in pos for c in [word_class(p)] if c}
    if found & VERBS and "vi" in pos and "vt" not in pos:
        found.add("vi")
    return found


def main(jmdict: str) -> None:
    codes = pos_codes(jmdict)
    words: set[str] = set()
    # Each spelling's entries, by their classes (a set of class sets).
    classes: dict[str, set[frozenset[str]]] = {}
    with gzip.open(jmdict, "rb") as f:
        for _, el in ET.iterparse(f, events=("end",)):
            if el.tag != "entry":
                continue
            kebs = [{p.text for p in k.findall("ke_pri")} for k in el.findall("k_ele")]
            rebs = [
                (r.findtext("reb"), {p.text for p in r.findall("re_pri")}, {i.text for i in r.findall("re_inf")})
                for r in el.findall("r_ele")
            ]
            common = any(p & PRIORITY for _, p, _ in rebs) or any(k & PRIORITY for k in kebs)
            if common:
                found = entry_classes(el, codes)
                for reb, pri, inf in rebs:
                    if reb and KANA.match(reb) and not inf & SKIP_READING and (pri & PRIORITY or not kebs):
                        words.add(reb)
                        classes.setdefault(reb, set()).add(frozenset(found))
            el.clear()
    lines = []
    for w in sorted(words):
        entries = sorted(",".join(c for c in ORDER if c in found) for found in classes.get(w, ()))
        if any(entries):
            lines.append(f"{w}\t{'|'.join(entries)}\n")
        else:
            lines.append(f"{w}\n")
    text = "".join(lines).encode("utf-8")
    sha = hashlib.sha256(text).hexdigest()
    OUT.mkdir(parents=True, exist_ok=True)
    for old in OUT.glob("ja.*.txt"):
        old.unlink()
    name = f"ja.{sha[:12]}.txt"
    (OUT / name).write_bytes(text)
    index = {
        "format": "aac-kana-words",
        "builder": "aac/aac-rs/tools/finder/build_ja_kana.py",
        "file": name,
        "sha256": sha,
        "bytes": len(text),
        "entries": len(words),
        "source": "JMdict_e (EDRDG), https://www.edrdg.org/jmdict/j_jmdict.html: kana readings of common entries, with each entry's inflecting classes from its parts of speech",
        "licence": "CC-BY-SA-4.0",
    }
    (OUT / "index.json").write_text(json.dumps(index, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
    (OUT / "LICENSE.md").write_text(LICENSE.format(file=name), encoding="utf-8")
    print(f"{len(words)} kana words, {len(text)} bytes -> {OUT / name}")


if __name__ == "__main__":
    if len(sys.argv) != 2:
        sys.exit(__doc__)
    main(sys.argv[1])