open-standards/wai/wai-studio/e2e/studio.e2e.mjs

// WAI Studio in Chromium, WebKit and Firefox, against a `wai dev` stack on loopback that
// serves it at /_wai/ (e2e/run.sh builds everything this needs):
//
//   node e2e/studio.e2e.mjs --wai BIN --relay BIN --room BIN --web DIR --studio DIST
//        --wai-dir DIR --out DIR [--browsers chromium,webkit,firefox]
//
// For each browser, the local host (/_wai/) and the hosted host standing alone
// (/_wai/hosted.html, the core in the tab and nothing else):
//   - each panel the host shows loads, and the Run panel only on the local host;
//   - Inspect reads a vector of http-conformance (O5, with its receipt index and groups):
//     its verdict here, its content hash, its receipts delivered-verified, and, on the local
//     host, the local wai's own report;
//   - Verify checks http-conformance's receipts as its cases expect them: O9c held whole,
//     delivered-verified (receipts_walk_two), and O9a's index naming a group whose signature
//     was changed, signature_verify_failed (receipts_group_bad_signature);
//   - Encode, decode, package: FLAC decoded in the tab, a picture encoded as PNG and read back,
//     a payload wrapped and inspected;
//   - Rooms: the room shown, joined as a viewer through the room client against the stack's
//     room service and relay, and left; the pre-call test against the stack's relay. A viewer
//     follows the room's clock, whose events track opens with the room's first participant,
//     so a publisher is in the room first: the room page itself, in Chromium with its fake
//     camera and microphone, for the whole run;
//   - Run (local): the stack, every service on loopback, the room service checked, a log;
//   - Quantum: Quantum Studio mounted, its first example linted by the engine;
//   - the command palette: every operation by its id, one this host runs opening its panel,
//     one it does not saying where it runs;
//   - axe finds no WCAG 2.2 A or AA violation on any panel (WebKit and Firefox too);
//   - no console error and no failed request; nothing scrolls sideways at a phone's width.
// Exit 0 iff every check passes.
import { spawn } from 'node:child_process';
import { mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
import { createRequire } from 'node:module';
import { join, resolve } from 'node:path';
import { createInterface } from 'node:readline';

const args = Object.fromEntries(process.argv.slice(2).reduce((a, x, i, all) => (x.startsWith('--') ? [...a, [x.slice(2), all[i + 1]]] : a), []));
for (const k of ['wai', 'relay', 'room', 'web', 'studio', 'wai-dir', 'out']) if (!args[k]) { console.error(`--${k} is needed`); process.exit(2); }
const BROWSERS = (args.browsers || 'chromium,webkit,firefox').split(',');
const OUT = resolve(args.out);
const WAI = resolve(args['wai-dir']);
const HTTP = join(WAI, 'http-conformance');
const SHOTS = join(OUT, 'screenshots');
rmSync(OUT, { recursive: true, force: true });
mkdirSync(SHOTS, { recursive: true });
const pw = (await import(process.env.WAI_E2E_DRIVER ?? 'playwright')).default;
const AXE = createRequire(import.meta.url).resolve('axe-core/axe.min.js');
const TAGS = ['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa', 'wcag22a', 'wcag22aa'];
const LONG = 90_000;

let fails = 0;
const check = (ok, what) => { console.log(`${ok ? 'PASS' : 'FAIL'}  ${what}`); if (!ok) fails += 1; return ok; };

// http-conformance's vectors (objects.json).
const O5 = 'e43e038c10c0494d9647444fd4daf54b19fc3a946a3c7980431bc6e0a236093b';
const O9A = '31abc1da2cdf927191f3f447f68508ead30d078690cb7bea2435ae3d1df5f4f3';
const O9C = 'a8ab2cd0cc277ef7109aacdee6bb55ff713aa0d4dc4c2e3dfe40aab759a90133';
const O10 = '1eb764beb4a8a739054341df8b5b7d065e490032b187a93787be15f1478080a8';
const objects = (...names) => names.map((n) => join(HTTP, 'objects', n));
const rcpt = (dir) => readdirSync(join(HTTP, dir)).map((n) => join(HTTP, dir, n));
const cases = JSON.parse(readFileSync(join(HTTP, 'cases.json'), 'utf8'));
const expect = (name) => cases.find((c) => c.name === name)?.expect.checks[0];

// ---- the stack ----
const proj = join(OUT, 'project');
mkdirSync(proj, { recursive: true });
const dev = spawn(resolve(args.wai), ['dev', '--port', '0', '--dir', join(OUT, 'dev'), '--web', resolve(args.web), '--relay', resolve(args.relay), '--room', resolve(args.room)], {
  cwd: proj,
  env: { ...process.env, WAI_STUDIO: resolve(args.studio) },
  stdio: ['ignore', 'pipe', 'inherit'],
});
const origin = await new Promise((ok, no) => {
  const t = setTimeout(() => no(new Error('wai dev did not come up in 60 s')), 60_000);
  createInterface({ input: dev.stdout }).on('line', (l) => {
    process.stdout.write(`  wai dev | ${l}\n`);
    const m = l.match(/^wai dev: ready on (http:\/\/127\.0\.0\.1:\d+)/);
    if (m) { clearTimeout(t); ok(m[1]); }
  });
  dev.on('exit', (c) => no(new Error(`wai dev exited (${c})`)));
});
const stop = () => { try { dev.kill('SIGTERM'); } catch { /* gone */ } };
process.on('exit', stop);

// The room's first participant: the room page, publishing a fake camera and microphone.
async function publisher() {
  const b = await pw.chromium.launch({ headless: true, args: ['--use-fake-device-for-media-stream', '--use-fake-ui-for-media-stream', '--autoplay-policy=no-user-gesture-required'] });
  const c = await b.newContext({ permissions: ['camera', 'microphone'] });
  const p = await c.newPage();
  await p.goto(`${origin}/room/?service=/room-api`, { waitUntil: 'domcontentloaded' });
  await p.waitForFunction(() => window.roomTest?.state?.phase === 'idle', null, { timeout: LONG });
  await p.click('#join');
  const ok = await p.waitForFunction(() => window.roomTest.state.phase === 'publishing', null, { timeout: LONG }).then(() => true, () => false);
  check(ok, `the room's first participant publishes from the room page (${await p.evaluate(() => JSON.stringify({ phase: window.roomTest.state.phase, errors: window.roomTest.state.errors.slice(0, 1) }))})`);
  return b;
}

async function axe(page, where, frame = null) {
  const target = frame || page;
  await target.addScriptTag({ path: AXE });
  const r = await target.evaluate((tags) => window.axe.run(document, { runOnly: { type: 'tag', values: tags }, iframes: false }), TAGS);
  const v = r.violations.map((x) => `${x.id} (${x.impact}): ${x.nodes.slice(0, 3).map((n) => n.target.join(' ')).join(', ')}`);
  check(v.length === 0, `${where}: axe finds no WCAG 2.2 A/AA violation${v.length ? ': ' + v.join(' | ') : ''}`);
}

async function run(browserName, host) {
  const tag = `${browserName} · ${host}`;
  const browser = await pw[browserName].launch({ headless: true });
  const ctx = await browser.newContext({ viewport: { width: 1360, height: 980 }, acceptDownloads: true });
  const page = await ctx.newPage();
  const seen = { errors: [], failed: [] };
  page.on('console', (m) => { if (m.type() === 'error') seen.errors.push(m.text()); });
  page.on('pageerror', (e) => seen.errors.push(String(e)));
  page.on('requestfailed', (r) => { if (!/ERR_ABORTED|NS_BINDING_ABORTED|cancelled/i.test(r.failure()?.errorText ?? '')) seen.failed.push(`${r.url()} (${r.failure()?.errorText})`); });
  page.on('response', (r) => { if (r.status() >= 400) seen.failed.push(`${r.url()} (${r.status()})`); });
  const url = `${origin}/_wai/${host === 'hosted' ? 'hosted.html' : ''}`;
  await page.goto(url, { waitUntil: 'domcontentloaded' });
  await page.waitForFunction(() => window.waiStudio && /Ready\./.test(document.querySelector('#ws-status')?.textContent ?? ''), null, { timeout: LONG });
  const tabs = await page.$$eval('[role=tab]', (ts) => ts.map((t) => t.id.replace('ws-tab-', '')));
  const want = host === 'local' ? ['inspect', 'verify', 'media', 'rooms', 'run', 'quantum'] : ['inspect', 'verify', 'media', 'rooms', 'quantum'];
  check(JSON.stringify(tabs) === JSON.stringify(want), `${tag}: the panels this host shows (${tabs.join(', ')})`);
  check((await page.$$('h1')).length === 1 && (await page.textContent('h1')) === 'WAI Studio', `${tag}: one h1, WAI Studio`);
  check((await page.getAttribute('#ws-host', 'data-host')) === host, `${tag}: the host is chosen at start (${await page.textContent('#ws-host')})`);
  const tab = async (id) => { await page.click(`#ws-tab-${id}`); await page.waitForSelector(`#ws-panel-${id}:not([hidden])`); };

  // ---- Inspect ----
  await tab('inspect');
  await axe(page, `${tag}: Inspect, empty`);
  await page.setInputFiles('#ws-inspect-file', [...objects(`${O5}.wai`, `${O5}.wai.receipts`), ...rcpt('objects/rcpt')]);
  await page.waitForSelector('#ws-verdict', { timeout: LONG });
  await page.waitForFunction(() => document.querySelector('#ws-inspect-receipts')?.childElementCount > 0, null, { timeout: LONG });
  const ins = await page.evaluate(() => window.waiStudio.ctx.last);
  check(ins.inspect?.data?.verdict?.code === 'decodable' && ins.inspect.data.cid === 'e43e038c10c0494d9647444fd4daf54b19fc3a946a3c7980431bc6e0a236093b',
    `${tag}: Inspect reads http-conformance O5: decodable here, its content hash its name (${ins.inspect?.data?.verdict?.code})`);
  check(ins.inspectReceipts?.data?.ok === true && ins.inspectReceipts.data.indexes[0].lines[0].code === 'delivered-verified', `${tag}: Inspect checks O5's receipts, delivered-verified`);
  if (host === 'local') {
    await page.waitForFunction(() => /wai inspect/.test(document.querySelector('#ws-inspect-local')?.textContent ?? ''), null, { timeout: LONG });
    const local = await page.evaluate(() => window.waiStudio.ctx.last.inspectLocal);
    check(local?.ok === true && /wai\.text\.zstd/.test(local.stdout), `${tag}: the local wai's report beside it (${local?.argv?.join(' ')})`);
  }
  await page.click('#ws-registry > summary');
  await page.waitForSelector('#ws-registry-out table', { timeout: LONG });
  await axe(page, `${tag}: Inspect, an envelope read`);
  await page.screenshot({ path: join(SHOTS, `${browserName}-${host}-inspect.png`), fullPage: true });

  // ---- Verify ----
  await tab('verify');
  await page.setInputFiles('#ws-verify-files', [...objects(`${O9C}.wai`, `${O9C}.wai.receipts`), ...rcpt('objects/rcpt')]);
  await page.waitForSelector('#ws-verify-summary', { timeout: LONG });
  const good = (await page.evaluate(() => window.waiStudio.ctx.last.verify)).data;
  const walk = expect('receipts_walk_two');
  check(good.ok && good.indexes[0].lines[0].code === walk.delivery[0].evidence, `${tag}: Verify: O9c held whole is ${walk.delivery[0].evidence}, as receipts_walk_two expects`);
  // The corpus keeps this index under a name of its own; served beside O9a it is O9a's.
  const badIndex = { name: `${O9A}.wai.receipts`, mimeType: 'application/octet-stream', buffer: readFileSync(join(HTTP, 'cases/receipts/O9a-bad-signature.wai.receipts')) };
  await page.setInputFiles('#ws-verify-files', [...objects(`${O9A}.wai`), ...rcpt('objects/rcpt'), ...rcpt('cases/rcpt')].map((p) => ({ name: p.split('/').pop(), mimeType: 'application/octet-stream', buffer: readFileSync(p) })).concat([badIndex]));
  await page.waitForFunction(() => window.waiStudio.ctx.last.verify?.data?.ok === false, null, { timeout: LONG });
  const bad = (await page.evaluate(() => window.waiStudio.ctx.last.verify)).data;
  const sig = expect('receipts_group_bad_signature');
  check(bad.indexes[0].lines[0].code === sig.delivery[0].error, `${tag}: Verify: a group whose signature was changed is ${sig.delivery[0].error}, as receipts_group_bad_signature expects`);
  check(await page.isVisible(`#ws-verify-out li[data-code="${sig.delivery[0].error}"]`), `${tag}: Verify shows the refusal with its reason code`);
  await axe(page, `${tag}: Verify, a refusal shown`);

  // ---- Encode, decode, package ----
  await tab('media');
  await page.setInputFiles('#ws-decode-file', objects(`${O10}.wai`));
  await page.waitForSelector('#ws-decode-verdict', { timeout: LONG });
  const dec = await page.evaluate(() => window.waiStudio.ctx.last.decode);
  check(dec?.ok && dec.data.capability === 'wai.audio.flac' && dec.data.kind === 'audio', `${tag}: FLAC (http-conformance O10) decoded in the tab`);
  await page.selectOption('#ws-encode-codec', 'png');
  await page.setInputFiles('#ws-encode-file', join(WAI, 'pack-conformance/stills/plain.png'));
  await page.click('#ws-encode-run');
  await page.waitForFunction(() => window.waiStudio.ctx.last.encode, null, { timeout: LONG });
  const enc = await page.evaluate(() => window.waiStudio.ctx.last.encode);
  check(enc?.ok === true && enc.data.capability === 'wai.image.png', `${tag}: a picture encoded as PNG in the tab (${enc?.error?.message ?? enc?.data?.envelope_bytes + ' B'})`);
  await page.waitForSelector('#ws-wrap-cap option[value="wai.text.zstd"]', { state: 'attached', timeout: LONG });
  await page.selectOption('#ws-wrap-cap', 'wai.text.zstd');
  await page.setInputFiles('#ws-wrap-file', { name: 'note.txt', mimeType: 'text/plain', buffer: Buffer.from('carried byte for byte') });
  await page.click('#ws-wrap-run');
  await page.waitForFunction(() => window.waiStudio.ctx.last.wrap, null, { timeout: LONG });
  const wr = await page.evaluate(() => window.waiStudio.ctx.last.wrap);
  check(wr?.ok === true && wr.data.intent === 'replicate', `${tag}: a payload wrapped as wai.text.zstd`);
  await axe(page, `${tag}: Encode, decode, package`);
  await page.click('#ws-wrap-out button:has-text("Inspect it")');
  await page.waitForFunction(() => window.waiStudio.ctx.last.inspect?.data?.capability === 'wai.text.zstd', null, { timeout: LONG });
  check(true, `${tag}: the wrapped envelope opens in Inspect`);

  // ---- Rooms ----
  await tab('rooms');
  await page.click('#ws-room-show');
  await page.waitForFunction(() => window.waiStudio.ctx.last.room, null, { timeout: LONG });
  check(/^[0-9a-f]{64}$/.test((await page.evaluate(() => window.waiStudio.ctx.last.room)).room), `${tag}: the room shown from its service (room.show)`);
  await page.click('#ws-room-join');
  const joined = await page.waitForFunction(() => window.waiStudio.ctx.last.roomState?.phase === 'watching', null, { timeout: LONG }).then(() => true, () => false);
  const rs = await page.evaluate(() => window.waiStudio.ctx.last.roomState);
  check(joined && rs.errors === 0, `${tag}: joined the wai dev room as a viewer (${JSON.stringify(rs)})`);
  if (joined) await axe(page, `${tag}: the room client, as a viewer`, page.frame({ url: /\/room\// }));
  await axe(page, `${tag}: Rooms, in the room`);
  await page.click('#ws-room-leave');
  check(await page.isHidden('#ws-room-frame'), `${tag}: left the room`);
  await page.click('#ws-relay-test');
  await page.waitForFunction(() => window.waiStudio.ctx.last.relayTest, null, { timeout: LONG });
  const rt = await page.evaluate(() => window.waiStudio.ctx.last.relayTest);
  check(rt.ok === true && rt.protocol === 'moqt-22', `${tag}: the pre-call test: the relay's binding opened and its SETUP answered (${JSON.stringify(rt)})`);

  // ---- Run ----
  if (host === 'local') {
    await tab('run');
    await page.waitForSelector('#ws-run-stack table', { timeout: LONG });
    const st = await page.evaluate(() => window.waiStudio.ctx.state);
    check(st.services.length === 3 && st.services.every((s) => /^127\.0\.0\.1:/.test(s.listen)) && /checked/.test(await page.textContent('#ws-run-loopback')),
      `${tag}: Run: the stack, every service on loopback (${st.services.map((s) => s.listen).join(', ')})`);
    await page.click('#ws-run-check');
    await page.waitForFunction(() => window.waiStudio.ctx.last.health, null, { timeout: LONG });
    check((await page.evaluate(() => window.waiStudio.ctx.last.health)).relay === true, `${tag}: Run: the room service answers, its clock on the relay (service.check)`);
    await page.waitForFunction(() => (document.querySelector('#ws-run-log')?.textContent ?? '').length > 0, null, { timeout: LONG });
    check(true, `${tag}: Run: the relay's log, followed`);
    await page.click('#ws-run-sessions-show');
    await page.waitForFunction(() => window.waiStudio.ctx.last.sessions, null, { timeout: LONG });
    check(Array.isArray((await page.evaluate(() => window.waiStudio.ctx.last.sessions)).sessions), `${tag}: Run: the relay's session records (session.show)`);
    await axe(page, `${tag}: Run`);
  } else {
    check(!(await page.$('#ws-tab-run')), `${tag}: no Run panel on the hosted host`);
  }

  // ---- Quantum ----
  await tab('quantum');
  const lint = await page.waitForFunction(() => /qubits/.test(document.querySelector('#qs-lint')?.textContent ?? ''), null, { timeout: LONG }).then(() => true, () => false);
  check(lint, `${tag}: Quantum Studio mounted, its first example linted by the engine (${await page.textContent('#qs-lint').catch(() => 'nothing')})`);
  await axe(page, `${tag}: Quantum`);

  // ---- the command palette ----
  await page.click('#ws-palette-open');
  await page.waitForSelector('#ws-palette[open]', { timeout: 10_000 });
  const total = await page.$$eval('#ws-palette-list > li', (l) => l.length);
  const ids = await page.evaluate(() => window.waiStudio.actions.length);
  check(total === ids && ids > 0, `${tag}: the palette lists every operation of the catalogue (${total})`);
  await page.fill('#ws-palette-q', 'room.grant');
  const grant = await page.textContent('#ws-palette-list > li[data-id="room.grant"] .ws-op-where');
  check(/Runs elsewhere/.test(grant) && (host === 'local' ? /\/room-api\/grant/.test(grant) : /POST \/grant/.test(grant)), `${tag}: an operation this host does not run says where it runs (${grant})`);
  await page.fill('#ws-palette-q', 'dev.start');
  const devStart = await page.textContent('#ws-palette-list > li[data-id="dev.start"] .ws-op-where');
  check(host === 'local' ? /^Runs here/.test(devStart) : /Studio served by wai dev/.test(devStart), `${tag}: dev.start (${devStart})`);
  await axe(page, `${tag}: the command palette`);
  await page.keyboard.press('Escape');
  check(!(await page.$('#ws-palette[open]')), `${tag}: Escape closes the palette, from its search field`);
  await page.click('#ws-palette-open');
  await page.fill('#ws-palette-q', 'envelope.inspect');
  await page.click('#ws-palette-list > li[data-id="envelope.inspect"] button');
  check(await page.getAttribute('#ws-tab-inspect', 'aria-selected') === 'true' && await page.evaluate(() => document.activeElement?.id) === 'ws-inspect-file', `${tag}: envelope.inspect from the palette opens Inspect at its control`);

  // ---- a phone's width ----
  await page.setViewportSize({ width: 390, height: 844 });
  for (const id of tabs) {
    await tab(id);
    const wide = await page.evaluate(() => document.documentElement.scrollWidth - window.innerWidth);
    check(wide <= 1, `${tag}: ${id} at a phone's width, nothing scrolls sideways (${wide} px)`);
  }
  await page.screenshot({ path: join(SHOTS, `${browserName}-${host}-phone.png`), fullPage: true });

  check(seen.errors.length === 0, `${tag}: no console error${seen.errors.length ? ': ' + seen.errors.slice(0, 5).join(' | ') : ''}`);
  check(seen.failed.length === 0, `${tag}: no failed request${seen.failed.length ? ': ' + seen.failed.slice(0, 5).join(' | ') : ''}`);
  await browser.close();
}

let host1 = null;
try {
  host1 = await publisher();
  for (const b of BROWSERS) {
    for (const host of ['local', 'hosted']) {
      try {
        await run(b, host);
      } catch (e) {
        check(false, `${b} · ${host}: ${e.stack || e}`);
      }
    }
  }
} finally {
  await host1?.close().catch(() => {});
  stop();
}
writeFileSync(join(OUT, 'result.txt'), `${fails === 0 ? 'every check passed' : `${fails} check(s) failed`}\n`);
console.log(`\n${fails === 0 ? 'every check passed' : `${fails} check(s) failed`}`);
process.exit(fails === 0 ? 0 : 1);