// WAI Studio in Chromium, WebKit and Firefox, against a `wai dev` stack on loopback that // serves it at /_wai/ (e2e/run.sh builds everything this needs): // // node e2e/studio.e2e.mjs --wai BIN --relay BIN --room BIN --web DIR --studio DIST // --wai-dir DIR --out DIR [--browsers chromium,webkit,firefox] // // For each browser, the local host (/_wai/) and the hosted host standing alone // (/_wai/hosted.html, the core in the tab and nothing else): // - each panel the host shows loads, and the Run panel only on the local host; // - Inspect reads a vector of http-conformance (O5, with its receipt index and groups): // its verdict here, its content hash, its receipts delivered-verified, and, on the local // host, the local wai's own report; // - Verify checks http-conformance's receipts as its cases expect them: O9c held whole, // delivered-verified (receipts_walk_two), and O9a's index naming a group whose signature // was changed, signature_verify_failed (receipts_group_bad_signature); // - Encode, decode, package: FLAC decoded in the tab, a picture encoded as PNG and read back, // a payload wrapped and inspected; // - Rooms: the room shown, joined as a viewer through the room client against the stack's // room service and relay, and left; the pre-call test against the stack's relay. A viewer // follows the room's clock, whose events track opens with the room's first participant, // so a publisher is in the room first: the room page itself, in Chromium with its fake // camera and microphone, for the whole run; // - Run (local): the stack, every service on loopback, the room service checked, a log; // - Quantum: Quantum Studio mounted, its first example linted by the engine; // - the command palette: every operation by its id, one this host runs opening its panel, // one it does not saying where it runs; // - axe finds no WCAG 2.2 A or AA violation on any panel (WebKit and Firefox too); // - no console error and no failed request; nothing scrolls sideways at a phone's width. // Exit 0 iff every check passes. import { spawn } from 'node:child_process'; import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { createRequire } from 'node:module'; import { join, resolve } from 'node:path'; import { createInterface } from 'node:readline'; const args = Object.fromEntries(process.argv.slice(2).reduce((a, x, i, all) => (x.startsWith('--') ? [...a, [x.slice(2), all[i + 1]]] : a), [])); for (const k of ['wai', 'relay', 'room', 'web', 'studio', 'wai-dir', 'out']) if (!args[k]) { console.error(`--${k} is needed`); process.exit(2); } const BROWSERS = (args.browsers || 'chromium,webkit,firefox').split(','); const OUT = resolve(args.out); const WAI = resolve(args['wai-dir']); const HTTP = join(WAI, 'http-conformance'); const SHOTS = join(OUT, 'screenshots'); rmSync(OUT, { recursive: true, force: true }); mkdirSync(SHOTS, { recursive: true }); const pw = (await import(process.env.WAI_E2E_DRIVER ?? 'playwright')).default; const AXE = createRequire(import.meta.url).resolve('axe-core/axe.min.js'); const TAGS = ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa', 'wcag22a', 'wcag22aa']; const LONG = 90_000; let fails = 0; const check = (ok, what) => { console.log(`${ok ? 'PASS' : 'FAIL'} ${what}`); if (!ok) fails += 1; return ok; }; // http-conformance's vectors (objects.json). const O5 = 'e43e038c10c0494d9647444fd4daf54b19fc3a946a3c7980431bc6e0a236093b'; const O9A = '31abc1da2cdf927191f3f447f68508ead30d078690cb7bea2435ae3d1df5f4f3'; const O9C = 'a8ab2cd0cc277ef7109aacdee6bb55ff713aa0d4dc4c2e3dfe40aab759a90133'; const O10 = '1eb764beb4a8a739054341df8b5b7d065e490032b187a93787be15f1478080a8'; const objects = (...names) => names.map((n) => join(HTTP, 'objects', n)); const rcpt = (dir) => readdirSync(join(HTTP, dir)).map((n) => join(HTTP, dir, n)); const cases = JSON.parse(readFileSync(join(HTTP, 'cases.json'), 'utf8')); const expect = (name) => cases.find((c) => c.name === name)?.expect.checks[0]; // ---- the stack ---- const proj = join(OUT, 'project'); mkdirSync(proj, { recursive: true }); const dev = spawn(resolve(args.wai), ['dev', '--port', '0', '--dir', join(OUT, 'dev'), '--web', resolve(args.web), '--relay', resolve(args.relay), '--room', resolve(args.room)], { cwd: proj, env: { ...process.env, WAI_STUDIO: resolve(args.studio) }, stdio: ['ignore', 'pipe', 'inherit'], }); const origin = await new Promise((ok, no) => { const t = setTimeout(() => no(new Error('wai dev did not come up in 60 s')), 60_000); createInterface({ input: dev.stdout }).on('line', (l) => { process.stdout.write(` wai dev | ${l}\n`); const m = l.match(/^wai dev: ready on (http:\/\/127\.0\.0\.1:\d+)/); if (m) { clearTimeout(t); ok(m[1]); } }); dev.on('exit', (c) => no(new Error(`wai dev exited (${c})`))); }); const stop = () => { try { dev.kill('SIGTERM'); } catch { /* gone */ } }; process.on('exit', stop); // The room's first participant: the room page, publishing a fake camera and microphone. async function publisher() { const b = await pw.chromium.launch({ headless: true, args: ['--use-fake-device-for-media-stream', '--use-fake-ui-for-media-stream', '--autoplay-policy=no-user-gesture-required'] }); const c = await b.newContext({ permissions: ['camera', 'microphone'] }); const p = await c.newPage(); await p.goto(`${origin}/room/?service=/room-api`, { waitUntil: 'domcontentloaded' }); await p.waitForFunction(() => window.roomTest?.state?.phase === 'idle', null, { timeout: LONG }); await p.click('#join'); const ok = await p.waitForFunction(() => window.roomTest.state.phase === 'publishing', null, { timeout: LONG }).then(() => true, () => false); check(ok, `the room's first participant publishes from the room page (${await p.evaluate(() => JSON.stringify({ phase: window.roomTest.state.phase, errors: window.roomTest.state.errors.slice(0, 1) }))})`); return b; } async function axe(page, where, frame = null) { const target = frame || page; await target.addScriptTag({ path: AXE }); const r = await target.evaluate((tags) => window.axe.run(document, { runOnly: { type: 'tag', values: tags }, iframes: false }), TAGS); const v = r.violations.map((x) => `${x.id} (${x.impact}): ${x.nodes.slice(0, 3).map((n) => n.target.join(' ')).join(', ')}`); check(v.length === 0, `${where}: axe finds no WCAG 2.2 A/AA violation${v.length ? ': ' + v.join(' | ') : ''}`); } async function run(browserName, host) { const tag = `${browserName} · ${host}`; const browser = await pw[browserName].launch({ headless: true }); const ctx = await browser.newContext({ viewport: { width: 1360, height: 980 }, acceptDownloads: true }); const page = await ctx.newPage(); const seen = { errors: [], failed: [] }; page.on('console', (m) => { if (m.type() === 'error') seen.errors.push(m.text()); }); page.on('pageerror', (e) => seen.errors.push(String(e))); page.on('requestfailed', (r) => { if (!/ERR_ABORTED|NS_BINDING_ABORTED|cancelled/i.test(r.failure()?.errorText ?? '')) seen.failed.push(`${r.url()} (${r.failure()?.errorText})`); }); page.on('response', (r) => { if (r.status() >= 400) seen.failed.push(`${r.url()} (${r.status()})`); }); const url = `${origin}/_wai/${host === 'hosted' ? 'hosted.html' : ''}`; await page.goto(url, { waitUntil: 'domcontentloaded' }); await page.waitForFunction(() => window.waiStudio && /Ready\./.test(document.querySelector('#ws-status')?.textContent ?? ''), null, { timeout: LONG }); const tabs = await page.$$eval('[role=tab]', (ts) => ts.map((t) => t.id.replace('ws-tab-', ''))); const want = host === 'local' ? ['inspect', 'verify', 'media', 'rooms', 'run', 'quantum'] : ['inspect', 'verify', 'media', 'rooms', 'quantum']; check(JSON.stringify(tabs) === JSON.stringify(want), `${tag}: the panels this host shows (${tabs.join(', ')})`); check((await page.$$('h1')).length === 1 && (await page.textContent('h1')) === 'WAI Studio', `${tag}: one h1, WAI Studio`); check((await page.getAttribute('#ws-host', 'data-host')) === host, `${tag}: the host is chosen at start (${await page.textContent('#ws-host')})`); const tab = async (id) => { await page.click(`#ws-tab-${id}`); await page.waitForSelector(`#ws-panel-${id}:not([hidden])`); }; // ---- Inspect ---- await tab('inspect'); await axe(page, `${tag}: Inspect, empty`); await page.setInputFiles('#ws-inspect-file', [...objects(`${O5}.wai`, `${O5}.wai.receipts`), ...rcpt('objects/rcpt')]); await page.waitForSelector('#ws-verdict', { timeout: LONG }); await page.waitForFunction(() => document.querySelector('#ws-inspect-receipts')?.childElementCount > 0, null, { timeout: LONG }); const ins = await page.evaluate(() => window.waiStudio.ctx.last); check(ins.inspect?.data?.verdict?.code === 'decodable' && ins.inspect.data.cid === 'e43e038c10c0494d9647444fd4daf54b19fc3a946a3c7980431bc6e0a236093b', `${tag}: Inspect reads http-conformance O5: decodable here, its content hash its name (${ins.inspect?.data?.verdict?.code})`); check(ins.inspectReceipts?.data?.ok === true && ins.inspectReceipts.data.indexes[0].lines[0].code === 'delivered-verified', `${tag}: Inspect checks O5's receipts, delivered-verified`); if (host === 'local') { await page.waitForFunction(() => /wai inspect/.test(document.querySelector('#ws-inspect-local')?.textContent ?? ''), null, { timeout: LONG }); const local = await page.evaluate(() => window.waiStudio.ctx.last.inspectLocal); check(local?.ok === true && /wai\.text\.zstd/.test(local.stdout), `${tag}: the local wai's report beside it (${local?.argv?.join(' ')})`); } await page.click('#ws-registry > summary'); await page.waitForSelector('#ws-registry-out table', { timeout: LONG }); await axe(page, `${tag}: Inspect, an envelope read`); await page.screenshot({ path: join(SHOTS, `${browserName}-${host}-inspect.png`), fullPage: true }); // ---- Verify ---- await tab('verify'); await page.setInputFiles('#ws-verify-files', [...objects(`${O9C}.wai`, `${O9C}.wai.receipts`), ...rcpt('objects/rcpt')]); await page.waitForSelector('#ws-verify-summary', { timeout: LONG }); const good = (await page.evaluate(() => window.waiStudio.ctx.last.verify)).data; const walk = expect('receipts_walk_two'); check(good.ok && good.indexes[0].lines[0].code === walk.delivery[0].evidence, `${tag}: Verify: O9c held whole is ${walk.delivery[0].evidence}, as receipts_walk_two expects`); // The corpus keeps this index under a name of its own; served beside O9a it is O9a's. const badIndex = { name: `${O9A}.wai.receipts`, mimeType: 'application/octet-stream', buffer: readFileSync(join(HTTP, 'cases/receipts/O9a-bad-signature.wai.receipts')) }; await page.setInputFiles('#ws-verify-files', [...objects(`${O9A}.wai`), ...rcpt('objects/rcpt'), ...rcpt('cases/rcpt')].map((p) => ({ name: p.split('/').pop(), mimeType: 'application/octet-stream', buffer: readFileSync(p) })).concat([badIndex])); await page.waitForFunction(() => window.waiStudio.ctx.last.verify?.data?.ok === false, null, { timeout: LONG }); const bad = (await page.evaluate(() => window.waiStudio.ctx.last.verify)).data; const sig = expect('receipts_group_bad_signature'); check(bad.indexes[0].lines[0].code === sig.delivery[0].error, `${tag}: Verify: a group whose signature was changed is ${sig.delivery[0].error}, as receipts_group_bad_signature expects`); check(await page.isVisible(`#ws-verify-out li[data-code="${sig.delivery[0].error}"]`), `${tag}: Verify shows the refusal with its reason code`); await axe(page, `${tag}: Verify, a refusal shown`); // ---- Encode, decode, package ---- await tab('media'); await page.setInputFiles('#ws-decode-file', objects(`${O10}.wai`)); await page.waitForSelector('#ws-decode-verdict', { timeout: LONG }); const dec = await page.evaluate(() => window.waiStudio.ctx.last.decode); check(dec?.ok && dec.data.capability === 'wai.audio.flac' && dec.data.kind === 'audio', `${tag}: FLAC (http-conformance O10) decoded in the tab`); await page.selectOption('#ws-encode-codec', 'png'); await page.setInputFiles('#ws-encode-file', join(WAI, 'pack-conformance/stills/plain.png')); await page.click('#ws-encode-run'); await page.waitForFunction(() => window.waiStudio.ctx.last.encode, null, { timeout: LONG }); const enc = await page.evaluate(() => window.waiStudio.ctx.last.encode); check(enc?.ok === true && enc.data.capability === 'wai.image.png', `${tag}: a picture encoded as PNG in the tab (${enc?.error?.message ?? enc?.data?.envelope_bytes + ' B'})`); await page.waitForSelector('#ws-wrap-cap option[value="wai.text.zstd"]', { state: 'attached', timeout: LONG }); await page.selectOption('#ws-wrap-cap', 'wai.text.zstd'); await page.setInputFiles('#ws-wrap-file', { name: 'note.txt', mimeType: 'text/plain', buffer: Buffer.from('carried byte for byte') }); await page.click('#ws-wrap-run'); await page.waitForFunction(() => window.waiStudio.ctx.last.wrap, null, { timeout: LONG }); const wr = await page.evaluate(() => window.waiStudio.ctx.last.wrap); check(wr?.ok === true && wr.data.intent === 'replicate', `${tag}: a payload wrapped as wai.text.zstd`); await axe(page, `${tag}: Encode, decode, package`); await page.click('#ws-wrap-out button:has-text("Inspect it")'); await page.waitForFunction(() => window.waiStudio.ctx.last.inspect?.data?.capability === 'wai.text.zstd', null, { timeout: LONG }); check(true, `${tag}: the wrapped envelope opens in Inspect`); // ---- Rooms ---- await tab('rooms'); await page.click('#ws-room-show'); await page.waitForFunction(() => window.waiStudio.ctx.last.room, null, { timeout: LONG }); check(/^[0-9a-f]{64}$/.test((await page.evaluate(() => window.waiStudio.ctx.last.room)).room), `${tag}: the room shown from its service (room.show)`); await page.click('#ws-room-join'); const joined = await page.waitForFunction(() => window.waiStudio.ctx.last.roomState?.phase === 'watching', null, { timeout: LONG }).then(() => true, () => false); const rs = await page.evaluate(() => window.waiStudio.ctx.last.roomState); check(joined && rs.errors === 0, `${tag}: joined the wai dev room as a viewer (${JSON.stringify(rs)})`); if (joined) await axe(page, `${tag}: the room client, as a viewer`, page.frame({ url: /\/room\// })); await axe(page, `${tag}: Rooms, in the room`); await page.click('#ws-room-leave'); check(await page.isHidden('#ws-room-frame'), `${tag}: left the room`); await page.click('#ws-relay-test'); await page.waitForFunction(() => window.waiStudio.ctx.last.relayTest, null, { timeout: LONG }); const rt = await page.evaluate(() => window.waiStudio.ctx.last.relayTest); check(rt.ok === true && rt.protocol === 'moqt-22', `${tag}: the pre-call test: the relay's binding opened and its SETUP answered (${JSON.stringify(rt)})`); // ---- Run ---- if (host === 'local') { await tab('run'); await page.waitForSelector('#ws-run-stack table', { timeout: LONG }); const st = await page.evaluate(() => window.waiStudio.ctx.state); check(st.services.length === 3 && st.services.every((s) => /^127\.0\.0\.1:/.test(s.listen)) && /checked/.test(await page.textContent('#ws-run-loopback')), `${tag}: Run: the stack, every service on loopback (${st.services.map((s) => s.listen).join(', ')})`); await page.click('#ws-run-check'); await page.waitForFunction(() => window.waiStudio.ctx.last.health, null, { timeout: LONG }); check((await page.evaluate(() => window.waiStudio.ctx.last.health)).relay === true, `${tag}: Run: the room service answers, its clock on the relay (service.check)`); await page.waitForFunction(() => (document.querySelector('#ws-run-log')?.textContent ?? '').length > 0, null, { timeout: LONG }); check(true, `${tag}: Run: the relay's log, followed`); await page.click('#ws-run-sessions-show'); await page.waitForFunction(() => window.waiStudio.ctx.last.sessions, null, { timeout: LONG }); check(Array.isArray((await page.evaluate(() => window.waiStudio.ctx.last.sessions)).sessions), `${tag}: Run: the relay's session records (session.show)`); await axe(page, `${tag}: Run`); } else { check(!(await page.$('#ws-tab-run')), `${tag}: no Run panel on the hosted host`); } // ---- Quantum ---- await tab('quantum'); const lint = await page.waitForFunction(() => /qubits/.test(document.querySelector('#qs-lint')?.textContent ?? ''), null, { timeout: LONG }).then(() => true, () => false); check(lint, `${tag}: Quantum Studio mounted, its first example linted by the engine (${await page.textContent('#qs-lint').catch(() => 'nothing')})`); await axe(page, `${tag}: Quantum`); // ---- the command palette ---- await page.click('#ws-palette-open'); await page.waitForSelector('#ws-palette[open]', { timeout: 10_000 }); const total = await page.$$eval('#ws-palette-list > li', (l) => l.length); const ids = await page.evaluate(() => window.waiStudio.actions.length); check(total === ids && ids > 0, `${tag}: the palette lists every operation of the catalogue (${total})`); await page.fill('#ws-palette-q', 'room.grant'); const grant = await page.textContent('#ws-palette-list > li[data-id="room.grant"] .ws-op-where'); check(/Runs elsewhere/.test(grant) && (host === 'local' ? /\/room-api\/grant/.test(grant) : /POST \/grant/.test(grant)), `${tag}: an operation this host does not run says where it runs (${grant})`); await page.fill('#ws-palette-q', 'dev.start'); const devStart = await page.textContent('#ws-palette-list > li[data-id="dev.start"] .ws-op-where'); check(host === 'local' ? /^Runs here/.test(devStart) : /Studio served by wai dev/.test(devStart), `${tag}: dev.start (${devStart})`); await axe(page, `${tag}: the command palette`); await page.keyboard.press('Escape'); check(!(await page.$('#ws-palette[open]')), `${tag}: Escape closes the palette, from its search field`); await page.click('#ws-palette-open'); await page.fill('#ws-palette-q', 'envelope.inspect'); await page.click('#ws-palette-list > li[data-id="envelope.inspect"] button'); check(await page.getAttribute('#ws-tab-inspect', 'aria-selected') === 'true' && await page.evaluate(() => document.activeElement?.id) === 'ws-inspect-file', `${tag}: envelope.inspect from the palette opens Inspect at its control`); // ---- a phone's width ---- await page.setViewportSize({ width: 390, height: 844 }); for (const id of tabs) { await tab(id); const wide = await page.evaluate(() => document.documentElement.scrollWidth - window.innerWidth); check(wide <= 1, `${tag}: ${id} at a phone's width, nothing scrolls sideways (${wide} px)`); } await page.screenshot({ path: join(SHOTS, `${browserName}-${host}-phone.png`), fullPage: true }); check(seen.errors.length === 0, `${tag}: no console error${seen.errors.length ? ': ' + seen.errors.slice(0, 5).join(' | ') : ''}`); check(seen.failed.length === 0, `${tag}: no failed request${seen.failed.length ? ': ' + seen.failed.slice(0, 5).join(' | ') : ''}`); await browser.close(); } let host1 = null; try { host1 = await publisher(); for (const b of BROWSERS) { for (const host of ['local', 'hosted']) { try { await run(b, host); } catch (e) { check(false, `${b} · ${host}: ${e.stack || e}`); } } } } finally { await host1?.close().catch(() => {}); stop(); } writeFileSync(join(OUT, 'result.txt'), `${fails === 0 ? 'every check passed' : `${fails} check(s) failed`}\n`); console.log(`\n${fails === 0 ? 'every check passed' : `${fails} check(s) failed`}`); process.exit(fails === 0 ? 0 : 1);