2344443d9f
The catalogue can now list more than one model. Each language names the smallest model that passed its test; a model may cover a smaller one (it serves that one's languages too), and asks for a device with at least min_memory_gb of memory. A device keeps one model: in a language the model on it does not serve, Settings says the language uses a larger model, with its download size and memory, and offers it in place of the one there. Nothing downloads by itself. build_ai.py builds any set of pinned models (--source ID=FILE, --models). The evaluation kit gains run.mjs LANGUAGES=1 (test languages before they are published) and ITEMS=, merge_languages.py (add a language's exchanges and examples as not yet tested), and judge.py (a local Gemma 4 31B as one more reader of the answers). Co-Authored-By: Claude Opus 5.5 <[email protected]>
86 lines
3.3 KiB
Python
86 lines
3.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Merge new languages' test material into the eval set and languages.json.
|
|
|
|
python3 tools/ai/eval/merge_languages.py DIR [code...]
|
|
|
|
DIR holds one <code>.json per language: {lang, name, items: [20 x [partner,
|
|
target, words]], examples: {sentence: [...], reply: [...]}, notes}. Each one's
|
|
items go into items.json, and its examples into ../languages.json as a
|
|
language not yet tested ("not-yet", note "Not tested yet."): it is offered
|
|
only once run.mjs and the reviewers have passed it. A language already ready
|
|
keeps its status, note and examples. Checks the shape before writing anything.
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
ITEMS = HERE / "items.json"
|
|
LANGS = HERE.parent / "languages.json"
|
|
EN_ITEMS = 20
|
|
|
|
|
|
def load(path):
|
|
return json.loads(Path(path).read_text(encoding="utf-8"))
|
|
|
|
|
|
def check(code, d):
|
|
problems = []
|
|
if d.get("lang") != code:
|
|
problems.append(f"lang {d.get('lang')!r}")
|
|
if not str(d.get("name", "")).strip():
|
|
problems.append("no name")
|
|
items = d.get("items", [])
|
|
if len(items) != EN_ITEMS or any(len(i) != 3 or not all(isinstance(x, str) and x.strip() for x in i) for i in items):
|
|
problems.append(f"items: need {EN_ITEMS} rows of [partner, target, words]")
|
|
ex = d.get("examples", {})
|
|
sent, rep = ex.get("sentence", []), ex.get("reply", [])
|
|
if len(sent) != 2 or any(len(s.get("out", [])) != 3 or not s.get("words") for s in sent):
|
|
problems.append("sentence examples: 2, with words and 3 sentences each")
|
|
if not any(s.get("partner") for s in sent) or not any(s.get("partner") is None for s in sent):
|
|
problems.append("sentence examples: one with a partner line, one without")
|
|
if len(rep) != 2 or any(len(r.get("out", [])) != 4 or not r.get("partner") for r in rep):
|
|
problems.append("reply examples: 2, with a partner line and 4 replies each")
|
|
return problems
|
|
|
|
|
|
def main(argv):
|
|
if len(argv) < 2:
|
|
print(__doc__, file=sys.stderr)
|
|
return 2
|
|
src = Path(argv[1])
|
|
wanted = argv[2:] or sorted(p.stem for p in src.glob("*.json"))
|
|
items, langs = load(ITEMS), load(LANGS)
|
|
bad = False
|
|
for code in wanted:
|
|
d = load(src / f"{code}.json")
|
|
problems = check(code, d)
|
|
if problems:
|
|
print(f"✗ {code}: {'; '.join(problems)}", file=sys.stderr)
|
|
bad = True
|
|
continue
|
|
items[code] = d["items"]
|
|
entry = langs["languages"].get(code)
|
|
if entry and entry.get("status") == "ready":
|
|
continue
|
|
langs["languages"][code] = {
|
|
"status": "not-yet",
|
|
"name": d["name"],
|
|
"note": "Not tested yet.",
|
|
"examples": {
|
|
"sentence": [{"partner": s.get("partner"), "words": s["words"], "out": s["out"]} for s in d["examples"]["sentence"]],
|
|
"reply": [{"partner": r["partner"], "out": r["out"]} for r in d["examples"]["reply"]],
|
|
},
|
|
}
|
|
if bad:
|
|
return 1
|
|
langs["languages"] = dict(sorted(langs["languages"].items()))
|
|
ITEMS.write_text(json.dumps(items, indent=1, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
LANGS.write_text(json.dumps(langs, indent=1, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
print(f"✓ merged {', '.join(wanted)}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv))
|