open-standards/aac/aac-rs/tools/asr/eval/run.mjs

// Measure the listener on FLEURS clips, on this app's own runtime.
//
//   node tools/asr/eval/run.mjs VENDOR PACKS CLIPS OUT.json [code ...]
//
// VENDOR: a site's vendor/onnxruntime-web-<version>-<rev>/ (third_party/
// onnxruntime-web/vendor.sh writes it); PACKS: what tools/asr/build_asr.py
// wrote (aac-asr-packs/); CLIPS: what fetch_fleurs.py wrote. With no codes,
// every language with a prompt slot in tools/asr/languages.json.
//
// Each clip goes through the app's own worker (asr-worker.js), in installed
// Chrome on its WebGPU, exactly as the app runs it: the worker reads the
// runtime and the model from the model store, the VAD decides when speech
// starts and ends, and one utterance is heard per "press" (it ends 1 s
// after the speaker stops). The clip is fed as the microphone would: its
// samples, then a second of silence, in 20 ms blocks (faster than real time:
// what is heard does not depend on the clock). Each language uses the
// prompt slot the app uses for it. Writes each clip's reference and what
// was heard; tools/asr/eval/score.py counts the errors.
//
// Needs Chrome with WebGPU on a machine with a graphics chip, and node with
// Playwright (aac-aurasense-web's node_modules: NODE_PATH or run from there).

import { createServer } from 'node:http';
import { readFileSync, existsSync, statSync, writeFileSync } from 'node:fs';
import { extname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { createRequire } from 'node:module';

const [vendor, packs, clips, out, ...codes] = process.argv.slice(2);
if (!out) {
  console.error('usage: node run.mjs VENDOR PACKS CLIPS OUT.json [code ...]');
  process.exit(2);
}
const here = fileURLToPath(new URL('.', import.meta.url));
const languages = JSON.parse(readFileSync(join(here, '..', 'languages.json'), 'utf8')).languages;
const wanted = codes.length ? codes : Object.keys(languages).filter((c) => languages[c].slot !== undefined);
const require = createRequire(join(process.cwd(), 'package.json'));
const { chromium } = require('playwright');

// The page: the model store filled, the worker started from it, and clips fed
// to it through its own messages.
const PAGE = `<!doctype html><meta charset="utf-8"><script type="module">
const CACHE = 'aac-models-v1';
const abs = (u) => new URL(u, location.href).href;
function wav(buf) {
  const dv = new DataView(buf); let p = 12, fmt;
  while (p + 8 <= dv.byteLength) {
    const id = String.fromCharCode(dv.getUint8(p), dv.getUint8(p + 1), dv.getUint8(p + 2), dv.getUint8(p + 3));
    const size = dv.getUint32(p + 4, true);
    if (id === 'fmt ') fmt = { format: dv.getUint16(p + 8, true), rate: dv.getUint32(p + 12, true), bits: dv.getUint16(p + 22, true) };
    if (id === 'data') {
      if (fmt.rate !== 16000) throw new Error('16 kHz only');
      if (fmt.bits === 32) return new Float32Array(buf.slice(p + 8, p + 8 + size));
      const s = new Int16Array(buf.slice(p + 8, p + 8 + size)); return Float32Array.from(s, (v) => v / 32768);
    }
    p += 8 + size + (size & 1);
  }
  throw new Error('no data');
}
let worker, nextId = 1; const calls = new Map(); let onEvent = () => {};
const call = (msg) => new Promise((resolve, reject) => { const id = nextId++; calls.set(id, { resolve, reject }); worker.postMessage({ ...msg, id }); });
window.load = async () => {
  const cache = await caches.open(CACHE);
  const idx = await (await fetch('/asr/index.json')).json();
  const cat = await (await fetch('/asr/' + idx.current)).json();
  const [id, m] = Object.entries(cat.models)[0];
  const g = (x) => ({ url: abs('/asr/' + x.file.url), data: x.data.map((d) => ({ name: d.name, url: abs('/asr/' + d.url), bytes: d.bytes })) });
  const files = ['/vendor/asr-worker.js', '/vendor/ort.jspi.bundle.min.mjs', '/vendor/ort-wasm-simd-threaded.jspi.wasm',
    ...[m.encoder.file, ...m.encoder.data, m.step.file, ...m.step.data, m.vad.file, m.vocab].map((p) => '/asr/' + p.url)];
  for (const f of files) if (!(await cache.match(abs(f)))) await cache.put(abs(f), await fetch(f));
  const src = await (await cache.match(abs('/vendor/asr-worker.js'))).blob();
  worker = new Worker(URL.createObjectURL(src), { type: 'module' });
  worker.onmessage = (e) => {
    const msg = e.data;
    if (msg.event) { onEvent(msg); return; }
    const c = calls.get(msg.id); calls.delete(msg.id);
    msg.ok ? c.resolve(msg) : c.reject(new Error(msg.error));
  };
  await call({ type: 'init', bundle: abs('/vendor/ort.jspi.bundle.min.mjs'), wasm: abs('/vendor/ort-wasm-simd-threaded.jspi.wasm') });
  const t0 = performance.now();
  await call({ type: 'load', key: id, model: { vad: g(m.vad), encoder: g(m.encoder), step: g(m.step), vocab: abs('/asr/' + m.vocab.url) } });
  return { model: id, load_ms: Math.round(performance.now() - t0), ua: navigator.userAgent };
};
window.hear = async (url, slot) => {
  const samples = wav(await (await fetch(url)).arrayBuffer());
  const all = new Float32Array(samples.length + 16000); all.set(samples);
  let final = null; const events = [];
  onEvent = (e) => { events.push(e.event); if (e.event === 'final') final = e.text; };
  const t0 = performance.now();
  await call({ type: 'start', slot });
  for (let i = 0; i < all.length; i += 320) worker.postMessage({ type: 'audio', samples: all.slice(i, i + 320) });
  const r = await call({ type: 'finish' });
  return { text: final ?? r.text, ended_by: final !== null ? 'silence' : 'end of clip', quiet: events.filter((e) => e === 'quiet').length,
    seconds: samples.length / 16000, ms: Math.round(performance.now() - t0) };
};
window.ready = true;
</script>`;

const roots = { '/vendor/': resolve(vendor), '/asr/': resolve(packs), '/clips/': resolve(clips) };
const server = createServer((req, res) => {
  const path = decodeURIComponent(new URL(req.url, 'http://x').pathname);
  if (path === '/' || path === '/eval.html') {
    res.writeHead(200, { 'Content-Type': 'text/html' });
    return res.end(PAGE);
  }
  for (const [prefix, root] of Object.entries(roots)) {
    if (!path.startsWith(prefix)) continue;
    const file = join(root, path.slice(prefix.length));
    if (!file.startsWith(root) || !existsSync(file) || !statSync(file).isFile()) break;
    const type = { '.js': 'text/javascript', '.mjs': 'text/javascript', '.json': 'application/json', '.wasm': 'application/wasm' }[extname(file)] || 'application/octet-stream';
    res.writeHead(200, { 'Content-Type': type });
    return res.end(readFileSync(file));
  }
  res.writeHead(404);
  res.end();
});
await new Promise((r) => server.listen(0, '127.0.0.1', r));
const base = `http://127.0.0.1:${server.address().port}`;

const browser = await chromium.launch({ channel: 'chrome', args: ['--enable-unsafe-webgpu'] });
const page = await browser.newPage();
page.on('pageerror', (e) => console.error('page error:', e.message));
await page.goto(`${base}/eval.html`);
await page.waitForFunction(() => window.ready);
const result = { date: new Date().toISOString().slice(0, 10), runtime: await page.evaluate(() => window.load()), languages: {} };
console.log(`loaded ${result.runtime.model} in ${result.runtime.load_ms} ms`);
for (const code of wanted) {
  const refs = JSON.parse(readFileSync(join(clips, code, 'refs.json'), 'utf8'));
  const slot = languages[code].slot;
  const rows = [];
  for (const clip of refs.clips) {
    const heard = await page.evaluate(([u, s]) => window.hear(u, s), [`/clips/${code}/${clip.file}`, slot]);
    rows.push({ file: clip.file, id: clip.id, ref: clip.raw, ...heard });
  }
  result.languages[code] = { source: refs.source, slot, clips: rows };
  console.log(code, rows.length, 'clips');
  writeFileSync(out, JSON.stringify(result, null, 1) + '\n');
}
await browser.close();
server.close();