// Measure the listener on FLEURS clips, on this app's own runtime. // // node tools/asr/eval/run.mjs VENDOR PACKS CLIPS OUT.json [code ...] // // VENDOR: a site's vendor/onnxruntime-web--/ (third_party/ // onnxruntime-web/vendor.sh writes it); PACKS: what tools/asr/build_asr.py // wrote (aac-asr-packs/); CLIPS: what fetch_fleurs.py wrote. With no codes, // every language with a prompt slot in tools/asr/languages.json. // // Each clip goes through the app's own worker (asr-worker.js), in installed // Chrome on its WebGPU, exactly as the app runs it: the worker reads the // runtime and the model from the model store, the VAD decides when speech // starts and ends, and one utterance is heard per "press" (it ends 1 s // after the speaker stops). The clip is fed as the microphone would: its // samples, then a second of silence, in 20 ms blocks (faster than real time: // what is heard does not depend on the clock). Each language uses the // prompt slot the app uses for it. Writes each clip's reference and what // was heard; tools/asr/eval/score.py counts the errors. // // Needs Chrome with WebGPU on a machine with a graphics chip, and node with // Playwright (aac-aurasense-web's node_modules: NODE_PATH or run from there). import { createServer } from 'node:http'; import { readFileSync, existsSync, statSync, writeFileSync } from 'node:fs'; import { extname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { createRequire } from 'node:module'; const [vendor, packs, clips, out, ...codes] = process.argv.slice(2); if (!out) { console.error('usage: node run.mjs VENDOR PACKS CLIPS OUT.json [code ...]'); process.exit(2); } const here = fileURLToPath(new URL('.', import.meta.url)); const languages = JSON.parse(readFileSync(join(here, '..', 'languages.json'), 'utf8')).languages; const wanted = codes.length ? codes : Object.keys(languages).filter((c) => languages[c].slot !== undefined); const require = createRequire(join(process.cwd(), 'package.json')); const { chromium } = require('playwright'); // The page: the model store filled, the worker started from it, and clips fed // to it through its own messages. const PAGE = ``; const roots = { '/vendor/': resolve(vendor), '/asr/': resolve(packs), '/clips/': resolve(clips) }; const server = createServer((req, res) => { const path = decodeURIComponent(new URL(req.url, 'http://x').pathname); if (path === '/' || path === '/eval.html') { res.writeHead(200, { 'Content-Type': 'text/html' }); return res.end(PAGE); } for (const [prefix, root] of Object.entries(roots)) { if (!path.startsWith(prefix)) continue; const file = join(root, path.slice(prefix.length)); if (!file.startsWith(root) || !existsSync(file) || !statSync(file).isFile()) break; const type = { '.js': 'text/javascript', '.mjs': 'text/javascript', '.json': 'application/json', '.wasm': 'application/wasm' }[extname(file)] || 'application/octet-stream'; res.writeHead(200, { 'Content-Type': type }); return res.end(readFileSync(file)); } res.writeHead(404); res.end(); }); await new Promise((r) => server.listen(0, '127.0.0.1', r)); const base = `http://127.0.0.1:${server.address().port}`; const browser = await chromium.launch({ channel: 'chrome', args: ['--enable-unsafe-webgpu'] }); const page = await browser.newPage(); page.on('pageerror', (e) => console.error('page error:', e.message)); await page.goto(`${base}/eval.html`); await page.waitForFunction(() => window.ready); const result = { date: new Date().toISOString().slice(0, 10), runtime: await page.evaluate(() => window.load()), languages: {} }; console.log(`loaded ${result.runtime.model} in ${result.runtime.load_ms} ms`); for (const code of wanted) { const refs = JSON.parse(readFileSync(join(clips, code, 'refs.json'), 'utf8')); const slot = languages[code].slot; const rows = []; for (const clip of refs.clips) { const heard = await page.evaluate(([u, s]) => window.hear(u, s), [`/clips/${code}/${clip.file}`, slot]); rows.push({ file: clip.file, id: clip.id, ref: clip.raw, ...heard }); } result.languages[code] = { source: refs.source, slot, clips: rows }; console.log(code, rows.length, 'clips'); writeFileSync(out, JSON.stringify(result, null, 1) + '\n'); } await browser.close(); server.close();