d7d11fbd9b
tools/ae/build_ae.py publishes the abbreviation-expansion model made in the model library (gemma4-web/abbrev): Qwen3-0.6B-Base (Apache-2.0) fine-tuned to turn first letters into sentences, INT4 block 16 for WebGPU, every node on WebGPU. Each file is checked against its pinned size and SHA-256, named by its contents and kept at most 64 MiB: the one weights file that holds a single 74 MiB tensor (the word embeddings) is stored as two parts. The content-named catalogue (aac-ae-index/1, committed as data/ae/index.json) lists every part, the credits (the model's licence, and the CC BY 4.0 conversations it learned from), and each language's status and evidence; check_packs.py matches a pack directory to it before a site ships it. A language is ready only when we measured it on the app's own worker with the reply meant among the five sentences for at least 50 in 100 of at least 200 replies (tools/ae/eval/run.mjs). English, through the app's worker in Chrome on WebGPU: 52.8% among five and 32.9% first on 252 Turk Dialogues replies with what the partner said (56.3% punctuation aside), 85.5% on the held-out conversations; forgiving slips to a neighbouring key doubles the hits on replies typed with slips (33.5% -> 66.5% held out) for a loss of 0.5-1.6 points on clean typing. The 50-item check gives 26 first and 36 among five (the model library: 26/37 Chrome, 27/37 WebKit). Every other language says why not yet. Co-Authored-By: Claude Opus 5.5 <[email protected]>
148 lines
2.7 KiB
JSON
148 lines
2.7 KiB
JSON
{
|
|
"date": "2026-10-01",
|
|
"runtime": {
|
|
"model": "ae-en-qwen3-0.6b",
|
|
"load_ms": 1173,
|
|
"adapter": "apple metal-3",
|
|
"ua": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/154.0.0.0 Safari/537.36"
|
|
},
|
|
"items": "ae-en/check_full.json",
|
|
"sets": {
|
|
"turk_t2:ctx": {
|
|
"n": 252,
|
|
"top1": 32.9,
|
|
"top5": 52.8,
|
|
"top5_words": 56.3,
|
|
"ms_median": 161,
|
|
"ms_p90": 224
|
|
},
|
|
"turk_t2:ctx+near-keys": {
|
|
"n": 252,
|
|
"top1": 31.7,
|
|
"top5": 51.2,
|
|
"top5_words": 54.4,
|
|
"ms_median": 163,
|
|
"ms_p90": 241
|
|
},
|
|
"turk_t2:noctx": {
|
|
"n": 252,
|
|
"top1": 21.8,
|
|
"top5": 33.3,
|
|
"top5_words": 34.5,
|
|
"ms_median": 172,
|
|
"ms_p90": 234
|
|
},
|
|
"turk_t2:noctx+near-keys": {
|
|
"n": 252,
|
|
"top1": 20.6,
|
|
"top5": 32.9,
|
|
"top5_words": 34.1,
|
|
"ms_median": 179,
|
|
"ms_p90": 247
|
|
},
|
|
"turk_t2:noisy": {
|
|
"n": 252,
|
|
"top1": 23.4,
|
|
"top5": 37.3,
|
|
"top5_words": 39.7,
|
|
"ms_median": 172,
|
|
"ms_p90": 238
|
|
},
|
|
"heldout:ctx": {
|
|
"n": 200,
|
|
"top1": 68,
|
|
"top5": 85.5,
|
|
"top5_words": 85.5,
|
|
"ms_median": 185,
|
|
"ms_p90": 264
|
|
},
|
|
"heldout:ctx+near-keys": {
|
|
"n": 200,
|
|
"top1": 67,
|
|
"top5": 85,
|
|
"top5_words": 85,
|
|
"ms_median": 190,
|
|
"ms_p90": 271
|
|
},
|
|
"heldout:noisy": {
|
|
"n": 200,
|
|
"top1": 47,
|
|
"top5": 66.5,
|
|
"top5_words": 69.5,
|
|
"ms_median": 196,
|
|
"ms_p90": 281
|
|
},
|
|
"base20:ctx": {
|
|
"n": 20,
|
|
"top1": 65,
|
|
"top5": 85,
|
|
"top5_words": 85,
|
|
"ms_median": 172,
|
|
"ms_p90": 213
|
|
},
|
|
"base20:ctx+near-keys": {
|
|
"n": 20,
|
|
"top1": 65,
|
|
"top5": 85,
|
|
"top5_words": 85,
|
|
"ms_median": 188,
|
|
"ms_p90": 240
|
|
}
|
|
},
|
|
"full_results": "model library: gemma4-web/abbrev/evals/app_worker_full_2026-10-01.json (outside git: it holds the test sentences)",
|
|
"check50": {
|
|
"items": "ae-en/check.json (25 turk_t2:ctx + 25 heldout:ctx)",
|
|
"n": 50,
|
|
"top1": 26,
|
|
"top5": 36,
|
|
"lab_chrome": {
|
|
"top1": 26,
|
|
"top5": 37
|
|
},
|
|
"lab_webkit": {
|
|
"top1": 27,
|
|
"top5": 37
|
|
},
|
|
"lab_mlx": {
|
|
"top1": 29,
|
|
"top5": 36
|
|
}
|
|
},
|
|
"slips": {
|
|
"about": "the items typed with slips (sigma 0.3 on the 3 x 10 QWERTY grid), asked with slips not forgiven and forgiven (near_keys)",
|
|
"sets": {
|
|
"turk_t2:noisy": {
|
|
"n": 252,
|
|
"top1": 17.5,
|
|
"top5": 27,
|
|
"top5_words": 29,
|
|
"ms_median": 174,
|
|
"ms_p90": 263
|
|
},
|
|
"turk_t2:noisy+near-keys": {
|
|
"n": 252,
|
|
"top1": 23.4,
|
|
"top5": 37.3,
|
|
"top5_words": 39.7,
|
|
"ms_median": 178,
|
|
"ms_p90": 249
|
|
},
|
|
"heldout:noisy": {
|
|
"n": 200,
|
|
"top1": 26.5,
|
|
"top5": 33.5,
|
|
"top5_words": 34,
|
|
"ms_median": 215,
|
|
"ms_p90": 338
|
|
},
|
|
"heldout:noisy+near-keys": {
|
|
"n": 200,
|
|
"top1": 47,
|
|
"top5": 66.5,
|
|
"top5_words": 69.5,
|
|
"ms_median": 206,
|
|
"ms_p90": 301
|
|
}
|
|
}
|
|
}
|
|
}
|