e86ed1f6a4
DESIGN-core.md and DESIGN-motor.md explain every choice with the metrics
build.py writes into them: the location rule and why points live in atoms
(6x8 to 7x9 makes 8x10 centres provably unable to keep every size full), the
designed first boards, the sources of each word, the Fitzgerald palette, the
Back cell decision, the social repairs, the Word forms plan (UniMorph), the
access-method cost models and their measured trade-offs, the patent position,
target sizes on a 10.2-inch tablet and a 6.1-inch phone, each language's
status and the native work it needs, how the app should offer the sets after
Track B merges, and the decisions left to the founder.
Changes found while writing them down:
- Fixed Paths pages free places at the least weight: a group key never holds a
single word, and the heaviest words stay on the page.
- A page is closed by Back in the place of the key that opened it, to the board
that key is on; a key or control never reads like a word on the same board
(build check), so the body part "back" is left out.
- Endings count towards reach only on words that take them ("a" + s is not
"as"); "be", "would", "could", "as" and other top-100 words added.
- Key pictures must exist in the symbol index, and no two keys share one.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
353 lines
14 KiB
Python
353 lines
14 KiB
Python
"""Metrics that show whether the designs work (DESIGN-core.md, DESIGN-motor.md).
|
|
|
|
Paths are measured on the packages actually written, so a metric is always
|
|
about the files that ship.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
|
|
from . import costs
|
|
from .boards import Package, motor_filter
|
|
from .data import Data
|
|
from .grid import Size, centre, rect
|
|
from .layout import CoreLayout, MotorLayout, _home_words, _pages_at, motor_design
|
|
from .solve import hungarian
|
|
|
|
TOP_N = (25, 50, 100, 200, 400)
|
|
|
|
# Which words an ending applies to, for the reach metrics (an ending added to
|
|
# "a" is not the word "as").
|
|
ENDING_TAKES = {
|
|
"plural": ("core_noun", "fringe_noun"),
|
|
"possessive": ("core_noun", "fringe_noun"),
|
|
"past": ("verb",),
|
|
"present-progressive": ("verb",),
|
|
}
|
|
# Forms of be, have, do and the past forms: words of their own, not stems.
|
|
NOT_INFLECTED = {"is", "am", "are", "was", "were", "be", "been", "has", "had", "did", "does", "got", "went", "said", "can", "will"}
|
|
|
|
|
|
def apply_modifier(word: str, modifier: str) -> str:
|
|
"""aac-web actions.rs apply_modifier: English-regular endings only."""
|
|
if modifier == "plural":
|
|
if word.endswith(("s", "x", "z", "sh", "ch")):
|
|
return word + "es"
|
|
if word.endswith("y") and not word.endswith(("ay", "ey", "oy", "uy")):
|
|
return word[:-1] + "ies"
|
|
return word + "s"
|
|
if modifier == "past":
|
|
return word + "d" if word.endswith("e") else word + "ed"
|
|
if modifier == "present-progressive":
|
|
return word[:-1] + "ing" if word.endswith("e") else word + "ing"
|
|
if modifier == "future":
|
|
return f"will {word}"
|
|
if modifier == "possessive":
|
|
return f"{word}'s"
|
|
if modifier == "negation":
|
|
return f"not {word}"
|
|
if modifier == "question":
|
|
return f"{word}?"
|
|
return word
|
|
|
|
|
|
# ── paths through a package ──────────────────────────────────────────────
|
|
|
|
|
|
def paths(pkg: Package) -> dict[str, list[tuple[str, tuple[int, int], Size]]]:
|
|
"""Every word's shortest visible path from the root: [(board id, cell, size)]."""
|
|
boards = {b.id: b for b in pkg.boards}
|
|
best: dict[str, list] = {}
|
|
frontier = [(pkg.root, [])]
|
|
seen = {pkg.root}
|
|
while frontier:
|
|
nxt = []
|
|
for bid, path in frontier:
|
|
b = boards[bid]
|
|
size = Size(b.rows, b.cols)
|
|
for btn in b.buttons:
|
|
if btn.hidden:
|
|
continue
|
|
step = path + [(bid, (btn.row, btn.col), size)]
|
|
if btn.action == "speak":
|
|
if btn.key not in best or len(step) < len(best[btn.key]):
|
|
best[btn.key] = step
|
|
elif btn.action == "modifier":
|
|
best.setdefault("\0mod:" + btn.modifier, step)
|
|
elif btn.action == "navigate" and btn.control is None and btn.target not in seen:
|
|
seen.add(btn.target)
|
|
nxt.append((btn.target, step))
|
|
frontier = nxt
|
|
return best
|
|
|
|
|
|
def path_costs(path) -> dict[str, float]:
|
|
return {
|
|
"hits": len(path),
|
|
"scan": sum(costs.scan_steps(c, s) for _, c, s in path),
|
|
"touch": sum(costs.touch_cost(c, s) for _, c, s in path),
|
|
"gaze": sum(costs.gaze_cost(c, s) for _, c, s in path),
|
|
}
|
|
|
|
|
|
def reach(data: Data, pkg: Package) -> dict[str, dict]:
|
|
"""Words the package can say: directly, or through the app's endings
|
|
(base word, then the Word forms page and the ending)."""
|
|
ps = paths(pkg)
|
|
out = {}
|
|
mods = {k[5:]: v for k, v in ps.items() if k.startswith("\0mod:")}
|
|
for w, p in ps.items():
|
|
if w.startswith("\0"):
|
|
continue
|
|
out[w.lower()] = {"how": "direct", **path_costs(p)}
|
|
for w, p in ps.items():
|
|
if w.startswith("\0") or " " in w:
|
|
continue
|
|
e = data.lexicon.get(w)
|
|
for m, mp in mods.items():
|
|
if not e or e.pos not in ENDING_TAKES.get(m, ()) or (e.pos == "verb" and e.sense and "modal" in e.sense):
|
|
continue
|
|
if e.pos == "verb" and w.lower() in NOT_INFLECTED:
|
|
continue
|
|
form = apply_modifier(w.lower(), m)
|
|
if form in out:
|
|
continue
|
|
c = path_costs(p + mp)
|
|
# the ending is chosen from the Word forms page: open it, then the ending
|
|
out[form] = {"how": f"ending:{m}", **c}
|
|
return out
|
|
|
|
|
|
def spoken_reference(data: Data):
|
|
ranking = data.spoken_ranking
|
|
total = sum(n for _, n in ranking)
|
|
return ranking, total
|
|
|
|
|
|
def coverage(data: Data, r: dict[str, dict]) -> dict:
|
|
ranking, total = spoken_reference(data)
|
|
out = {}
|
|
for n in TOP_N:
|
|
top = [w for w, _ in ranking[:n]]
|
|
direct = [w for w in top if w in r and r[w]["how"] == "direct"]
|
|
forms = [w for w in top if w in r]
|
|
one = [w for w in top if w in r and r[w]["how"] == "direct" and r[w]["hits"] == 1]
|
|
out[f"top{n}"] = {"one_hit": len(one), "direct": len(direct), "with_endings": len(forms)}
|
|
out["missing_top100"] = [w for w, _ in ranking[:100] if w not in r]
|
|
said = sum(c for w, c in ranking if w in r and r[w]["how"] == "direct")
|
|
out["token_share_direct"] = round(said / total, 4)
|
|
out["token_share_with_endings"] = round(sum(c for w, c in ranking if w in r) / total, 4)
|
|
return out
|
|
|
|
|
|
def weighted(data: Data, r: dict[str, dict], weight=None) -> dict:
|
|
"""Frequency-weighted mean of each path cost over the words said directly."""
|
|
ranking, _ = spoken_reference(data)
|
|
counts = dict(ranking)
|
|
tot = {"hits": 0.0, "scan": 0.0, "touch": 0.0, "gaze": 0.0}
|
|
wsum = 0.0
|
|
for w, c in r.items():
|
|
if c["how"] != "direct":
|
|
continue
|
|
f = weight(w) if weight else counts.get(w, 0)
|
|
if not f:
|
|
continue
|
|
wsum += f
|
|
for k in tot:
|
|
tot[k] += f * c[k]
|
|
return {k: round(v / wsum, 3) for k, v in tot.items()} if wsum else {}
|
|
|
|
|
|
def hits_histogram(r: dict[str, dict]) -> dict[str, int]:
|
|
h: dict[str, int] = {}
|
|
for c in r.values():
|
|
if c["how"] == "direct":
|
|
h[str(c["hits"])] = h.get(str(c["hits"]), 0) + 1
|
|
return dict(sorted(h.items()))
|
|
|
|
|
|
def list_coverage(data: Data, r: dict[str, dict]) -> dict:
|
|
uc = [("all done" if w == "finished" else w).lower() for w in data.universal_core]
|
|
bj = [w.lower() for w in data.banajee]
|
|
|
|
def hits(ws):
|
|
return {w: r[w]["hits"] if w in r and r[w]["how"] == "direct" else None for w in ws}
|
|
|
|
u, b = hits(uc), hits(bj)
|
|
return {
|
|
"universal_core": {"reachable": sum(1 for v in u.values() if v), "of": len(uc), "one_hit": sum(1 for v in u.values() if v == 1)},
|
|
"banajee_2003": {"reachable": sum(1 for v in b.values() if v), "of": len(bj), "one_hit": sum(1 for v in b.values() if v == 1)},
|
|
}
|
|
|
|
|
|
# ── stability ────────────────────────────────────────────────────────────
|
|
|
|
|
|
def stability(sizes_of: dict[str, dict[str, tuple[int, int]]], sizes: list[Size]) -> dict:
|
|
"""For items present at two sizes A < B: how far the centre of their button
|
|
moves (in board widths/heights, and in cells of the smaller board A), and
|
|
whether all of an item's buttons share a point."""
|
|
by_name = {s.name: s for s in sizes}
|
|
worst = {"du": 0.0, "dv": 0.0, "cells": 0.0, "item": None, "from": None, "to": None}
|
|
total, n = 0.0, 0
|
|
worst_step = 0.0
|
|
common_ok = True
|
|
order = [s.name for s in sizes]
|
|
for key, at in sizes_of.items():
|
|
names = [s for s in order if s in at]
|
|
rects = [rect(at[s], by_name[s]) for s in names]
|
|
v0 = max(r[0] for r in rects)
|
|
v1 = min(r[1] for r in rects)
|
|
u0 = max(r[2] for r in rects)
|
|
u1 = min(r[3] for r in rects)
|
|
if not (v0 < v1 and u0 < u1):
|
|
common_ok = False
|
|
for i, a in enumerate(names):
|
|
for b in names[i + 1:]:
|
|
sa, sb = by_name[a], by_name[b]
|
|
ca, cb = centre(at[a], sa), centre(at[b], sb)
|
|
dv, du = abs(ca[0] - cb[0]), abs(ca[1] - cb[1])
|
|
cells = max(dv * sa.rows, du * sa.cols)
|
|
total += math.hypot(du, dv)
|
|
n += 1
|
|
if cells > worst["cells"]:
|
|
worst = {"du": round(du, 3), "dv": round(dv, 3), "cells": round(cells, 3), "item": key, "from": a, "to": b}
|
|
if order.index(b) == order.index(a) + 1:
|
|
worst_step = max(worst_step, math.hypot(du, dv))
|
|
return {
|
|
"pairs": n,
|
|
"mean_shift": round(total / n, 4) if n else 0.0,
|
|
"worst": worst,
|
|
"worst_shift_one_step": round(worst_step, 4),
|
|
"within_one_cell_of_smaller": worst["cells"] < 1.0,
|
|
"every_item_keeps_a_common_point": common_ok,
|
|
}
|
|
|
|
|
|
# ── the sets ─────────────────────────────────────────────────────────────
|
|
|
|
|
|
def core_metrics(data: Data, lay: CoreLayout, pkgs: list[Package]) -> dict:
|
|
out = {"sizes": {}}
|
|
for pkg in pkgs:
|
|
if pkg.variant != "full":
|
|
continue
|
|
size = Size.parse(pkg.size)
|
|
r = reach(data, pkg)
|
|
root = next(b for b in pkg.boards if b.id == pkg.root)
|
|
out["sizes"][pkg.size] = {
|
|
"cells": size.cells,
|
|
"core_words": sum(1 for b in root.buttons if b.action == "speak"),
|
|
"pages": sum(1 for b in root.buttons if b.action == "navigate"),
|
|
"words_reachable": sum(1 for c in r.values() if c["how"] == "direct"),
|
|
"hits": hits_histogram(r),
|
|
"weighted": weighted(data, r),
|
|
"coverage": coverage(data, r),
|
|
"lists": list_coverage(data, r),
|
|
"targets": costs.target_sizes(size),
|
|
}
|
|
core_at = {p.key: p.at for p in lay.core}
|
|
out["stability_core"] = stability(core_at, lay.sizes)
|
|
out["stability_pages"] = {
|
|
name: stability({p.key: p.at for p in page.items}, page.sizes) for name, page in lay.pages.items()
|
|
}
|
|
return out
|
|
|
|
|
|
def motor_metrics(data: Data, lay: MotorLayout, pkgs: list[Package]) -> dict:
|
|
d = data.fixed_paths
|
|
out = {"sizes": {}}
|
|
_, _, _, _, _, w_aac = motor_design(data)
|
|
_, _, _, _, _, w_spoken = motor_design(data, aac_boost=0)
|
|
for pkg in pkgs:
|
|
if pkg.variant != "full":
|
|
continue
|
|
ms = next(m for m in lay.sizes if m.label == pkg.size)
|
|
r = reach(data, pkg)
|
|
lookup = {k.lower(): k for k in data.lexicon}
|
|
out["sizes"][pkg.size] = {
|
|
"grid": ms.size.name,
|
|
"home_words": sum(1 for k in ms.home if k not in ms.pages),
|
|
"pages": len(ms.pages),
|
|
"groups": len(ms.groups),
|
|
"words": sum(1 for c in r.values() if c["how"] == "direct"),
|
|
"hits": hits_histogram(r),
|
|
"mean_hits_unweighted": round(sum(c["hits"] for c in r.values() if c["how"] == "direct") / max(1, sum(1 for c in r.values() if c["how"] == "direct")), 3),
|
|
"weighted_spoken": weighted(data, r, lambda w: w_spoken.get(lookup.get(w, w), 0.0)),
|
|
"weighted_design": weighted(data, r, lambda w: w_aac.get(lookup.get(w, w), 0.0)),
|
|
"coverage": coverage(data, r),
|
|
"lists": list_coverage(data, r),
|
|
"targets": costs.target_sizes(ms.size),
|
|
"tradeoffs": tradeoffs(data, lay, ms),
|
|
}
|
|
out["stability_home"] = stability(
|
|
{k: {m.size.name: m.home[k] for m in lay.sizes if k in m.home} for k in lay.points}, [m.size for m in lay.sizes]
|
|
)
|
|
out["filters"] = {f: sorted(motor_filter(data, f)) for f in d.get("filters", {})}
|
|
return out
|
|
|
|
|
|
def tradeoffs(data: Data, lay: MotorLayout, ms) -> dict:
|
|
"""The chosen layout against layouts that serve one access method only.
|
|
|
|
For each method: put the Home items where that method alone says (the
|
|
cheapest assignment for this size, with no tie to the other sizes), lay
|
|
out the pages the same way, and measure every layout under every method:
|
|
the design-weighted mean cost of saying a word."""
|
|
d, labels, sizes, page_of, group_of, weights = motor_design(data)
|
|
k = labels.index(ms.label)
|
|
homes = _home_words(d, sizes, weights)
|
|
items = list(d["pages"]) + homes[k]
|
|
pw = {p: sum(w for x, w in weights.items() if page_of[x] == p and x not in homes[k]) for p in d["pages"]}
|
|
|
|
def layout_for(table) -> tuple[dict, dict, dict]:
|
|
cells = list(table)
|
|
m = [[(pw[it] if it in pw else weights[it]) * table[c] for c in cells] for it in items]
|
|
_, assign = hungarian(m)
|
|
home = {it: cells[assign[i]] for i, it in enumerate(items)}
|
|
pages, groups = _pages_at(d, ms.size, home, page_of, group_of, weights, table)
|
|
return home, pages, groups
|
|
|
|
def evaluate(home, pages, groups) -> dict:
|
|
tot = {"hits": 0.0, "scan": 0.0, "touch": 0.0, "gaze": 0.0}
|
|
wsum = 0.0
|
|
size = ms.size
|
|
for w, wt in weights.items():
|
|
if w in home:
|
|
path = [home[w]]
|
|
else:
|
|
p = page_of[w]
|
|
if w in pages[p]["words"]:
|
|
path = [home[p], pages[p]["words"][w]]
|
|
else:
|
|
g = group_of[w]
|
|
path = [home[p], pages[p]["groups"][g], groups[g]["words"][w]]
|
|
wsum += wt
|
|
tot["hits"] += wt * len(path)
|
|
tot["scan"] += wt * sum(costs.scan_steps(c, size) for c in path)
|
|
tot["touch"] += wt * sum(costs.touch_cost(c, size) for c in path)
|
|
tot["gaze"] += wt * sum(costs.gaze_cost(c, size) for c in path)
|
|
return {k2: round(v / wsum, 3) for k2, v in tot.items()}
|
|
|
|
out = {"chosen": evaluate(ms.home, ms.pages, ms.groups)}
|
|
out["blend_alone"] = evaluate(*layout_for(costs.blended(ms.size, d["blend"][ms.label])))
|
|
for method in ("touch", "scan", "gaze"):
|
|
out[f"{method}_only"] = evaluate(*layout_for(costs.blended(ms.size, {method: 1.0})))
|
|
return out
|
|
|
|
|
|
def endings_report(data: Data, words: set[str]) -> list[dict]:
|
|
"""Where the app's English-regular endings get a word of the sets wrong."""
|
|
names = {"past": "past", "ing": "present-progressive", "plural": "plural"}
|
|
out = []
|
|
for w in sorted(words):
|
|
e = data.lexicon.get(w)
|
|
if not e:
|
|
continue
|
|
for k, m in names.items():
|
|
if k in e.forms:
|
|
app = apply_modifier(w, m)
|
|
if app != e.forms[k]:
|
|
out.append({"word": w, "ending": m, "app_says": app, "correct": e.forms[k]})
|
|
return out
|