open-standards/wai/wai-web/reach/test/run-modern.mjs

// Every rung forced, on a modern Node: the reach conformance at every profile, and the
// existing wai-web verify scripts (demo/verify_*.mjs) on every full build.
//
//   node run-modern.mjs --dist <dist> --shipped <package-dir> [--out <dir>] [--demo-commit <commit>]
//
//   1. The shipped package, chosen by the loader itself (it must choose `shipped` here),
//      runs the workloads; its outputs are the reference.
//   2. Every rung and profile, forced, runs the same workloads, and each output must
//      equal the reference byte for byte: L1; L2 at full, verify_decode, verify and
//      receipt; J at the same four profiles (with WebAssembly deleted). Then the engine
//      is stripped to ES5 (no WebAssembly, BigInt, Map, Symbol, ...), and the loader must
//      choose J on its own and give the same outputs, at full and receipt.
//   3. The verify scripts of demo/ (integer payloads, int_synth, the staged corpus with
//      its guards and fuzz, the menu, the writers, the staged measurement) run against
//      shipped, L1, L2 and J, each loaded through the loader by a generated adapter
//      module; the measurement script's written outputs must equal the shipped build's
//      file for file.
//
//      The scripts are this tree's by default. With --demo-commit, they and the corpora
//      they read are that commit's: its wai/, exported with `git archive` into
//      <out>/demo-tree. A package is held to the scripts of the commit it was built
//      from, not to checks written after it: a check a later commit adds describes code
//      the package does not hold. CI's leg on the package of record passes the commit
//      reach/manifest.json names (input.commit). A script this tree runs that the commit
//      does not have is reported as SKIP, not as a pass; if the commit has none of them,
//      the run fails.
//
// Exit 0 only if every run passes. --out keeps the reference, the logs and a JSON
// summary there (default: a temporary folder, removed).
import { spawnSync } from 'node:child_process';
import { mkdtempSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync, existsSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';

const here = dirname(fileURLToPath(import.meta.url));
const web = resolve(here, '../..');
const wai = resolve(web, '..');
const args = Object.fromEntries(process.argv.slice(2).reduce((acc, a, i, all) => (a.startsWith('--') ? [...acc, [a.slice(2), all[i + 1]]] : acc), []));
if (!args.dist || !args.shipped) { console.log('usage: node run-modern.mjs --dist <dist> --shipped <package-dir> [--out <dir>] [--demo-commit <commit>]'); process.exit(2); }
const dist = resolve(args.dist), shipped = resolve(args.shipped);
const keep = !!args.out;
const out = keep ? resolve(args.out) : mkdtempSync(join(tmpdir(), 'wai-reach-'));
mkdirSync(out, { recursive: true });
const ref = join(out, 'reference.json');
const runs = [];

// Where step 3's scripts and corpora come from: this tree, or --demo-commit's wai/.
let demoWai = wai, demoFrom = 'this tree';
if ('demo-commit' in args) {
  if (!args['demo-commit'] || args['demo-commit'].startsWith('--')) { console.log('FAIL  --demo-commit names no commit'); process.exit(2); }
  // From the top of the repository: run in a subfolder, `git archive` keeps only that
  // subfolder's part of the tree it is given.
  const top = spawnSync('git', ['-C', wai, 'rev-parse', '--show-toplevel'], { encoding: 'utf8' }).stdout.trim();
  const git = (...a) => spawnSync('git', ['-C', top || wai, ...a], { encoding: 'utf8', maxBuffer: 1 << 26 });
  const rev = git('rev-parse', '--verify', '--quiet', `${args['demo-commit']}^{commit}`);
  if (rev.status !== 0) {
    console.log(`FAIL  --demo-commit ${args['demo-commit']} is not a commit in this clone (a shallow clone must fetch it first)`);
    process.exit(1);
  }
  const commit = rev.stdout.trim();
  const root = join(out, 'demo-tree'), tar = join(out, 'demo-tree.tar');
  rmSync(root, { recursive: true, force: true });
  mkdirSync(root, { recursive: true });
  const a = git('archive', '--format=tar', '--prefix=wai/', '-o', tar, `${commit}:wai`);
  const x = a.status === 0 ? spawnSync('tar', ['-xf', tar, '-C', root], { encoding: 'utf8' }) : a;
  rmSync(tar, { force: true });
  if (x.status !== 0) { console.log(`FAIL  could not export wai/ at ${commit}: ${(x.stderr || '').trim()}`); process.exit(1); }
  demoWai = join(root, 'wai');
  demoFrom = `commit ${commit}`;
}
const demoWeb = join(demoWai, 'wai-web');
console.log(`the demo verify scripts and their corpora: ${demoFrom === 'this tree' ? demoFrom : `wai/ at ${demoFrom}`}`);

function conformance(label, extra) {
  const r = spawnSync(process.execPath, [join(here, 'conformance.js'), '--dist', dist, '--wai', wai, '--shipped', shipped, ...extra], { encoding: 'utf8', maxBuffer: 1 << 26 });
  const text = (r.stdout || '') + (r.stderr || '');
  writeFileSync(join(out, `conformance.${label.replace(/[^\w.-]+/g, '_')}.log`), text);
  const lines = text.split('\n').filter((l) => /^(PASS|FAIL) /.test(l));
  const summary = (text.match(/^SUMMARY (.*)$/m) || [])[1];
  const ok = r.status === 0;
  runs.push({ kind: 'workloads', label, ok, passed: lines.filter((l) => l.startsWith('PASS')).length, failed: lines.filter((l) => l.startsWith('FAIL')), summary: summary ? JSON.parse(summary) : null });
  console.log(`${ok ? 'PASS' : 'FAIL'}  ${label}: ${lines.length} checks${ok ? '' : '\n      ' + lines.filter((l) => l.startsWith('FAIL')).join('\n      ')}`);
  return ok;
}

// 1. the reference
conformance('shipped/full auto (reference)', ['--expect-rung', 'shipped', '--write-reference', ref]);
if (!existsSync(ref)) { console.log('FAIL  no reference was written'); process.exit(1); }

// 2. every rung and profile, forced
conformance('shipped/full forced', ['--rung', 'shipped', '--reference', ref]);
conformance('L1/full forced', ['--rung', 'L1', '--reference', ref]);
for (const p of ['full', 'verify_decode', 'verify', 'receipt']) conformance(`L2/${p} forced`, ['--rung', 'L2', '--profile', p, '--reference', ref]);
for (const p of ['full', 'verify_decode', 'verify', 'receipt']) conformance(`J/${p} forced, no WebAssembly`, ['--rung', 'J', '--profile', p, '--reference', ref]);
for (const p of ['full', 'receipt']) conformance(`J/${p} chosen on an ES5-stripped engine`, ['--profile', p, '--strip', 'es5', '--expect-rung', 'J', '--reference', ref]);

// 3. the demo verify scripts through an adapter per rung
const glue = await import(pathToFileURL(join(shipped, 'wai_web.js')).href);
const names = Object.keys(glue).filter((n) => n !== 'default' && n !== 'initSync').sort();
const adapters = join(out, 'adapters');
mkdirSync(adapters, { recursive: true });
function adapter(rung) {
  const file = join(adapters, `wai_web.${rung}.mjs`);
  writeFileSync(file, `// Generated by run-modern.mjs: the ${rung} rung of wai-web, loaded through wai-load.js, as an ES module.
import { createRequire } from 'node:module';
const require = createRequire(import.meta.url);
const H = require(${JSON.stringify(join(here, 'node-host.js'))});
${rung === 'J' ? `// J must not use WebAssembly: every entry point throws. RuntimeError stays, and counts a
// wasm2js trap (an Error('abort')) as one, so the scripts' "nothing traps" checks still hold.
const trap = class RuntimeError extends Error { static [Symbol.hasInstance](e) { return e instanceof Error && e.message === 'abort'; } };
const refuse = () => { throw new Error('the J rung called WebAssembly'); };
globalThis.WebAssembly = { RuntimeError: trap, validate: refuse, compile: refuse, instantiate: refuse, Module: refuse, Instance: refuse, Memory: refuse, Table: refuse };
` : ''}const WaiLoad = H.loadLoader(${JSON.stringify(dist)});
const st = await new Promise((res, rej) => WaiLoad.load({
  base: ${JSON.stringify(dist + '/')}, profile: 'full', force: ${JSON.stringify(rung)}, host: H.host,
  shipped: { glue: ${JSON.stringify(join(shipped, 'wai_web.js'))}, wasm: ${JSON.stringify(join(shipped, 'wai_web_bg.wasm'))} },
}, (e, s) => (e ? rej(e) : res(s))));
if (st.rung !== ${JSON.stringify(rung)}) throw new Error('the loader gave ' + st.rung);
export const { ${names.join(', ')} } = st.api;
`);
  return file;
}
const SCRIPTS = [
  ['verify_int_payload_wasm', (p) => [p, join(demoWai, 'wai-rs/tests/vectors')]],
  ['verify_int_synth_wasm', (p) => [p, join(demoWai, 'neural-conformance')]],
  ['verify_staged_wasm', (p) => [p, join(demoWai, 'staged-conformance')]],
  ['verify_menu', (p) => [p, join(demoWai, 'registry/capabilities.json')]],
  ['verify_pack', (p) => [p, join(demoWai, 'registry/capabilities.json')]],
  ['verify_measure_wasm', (p, rung) => [p, join(demoWai, 'staged-measure-conformance'), '--out', join(out, `measure.${rung}`)]],
];
const absent = SCRIPTS.map(([s]) => s).filter((s) => !existsSync(join(demoWeb, 'demo', `${s}.mjs`)));
if (absent.length === SCRIPTS.length) {
  runs.push({ kind: 'demo', label: 'the demo verify scripts', ok: false, failed: [`${demoFrom} has none of them`] });
  console.log(`FAIL  ${demoFrom} has none of the demo verify scripts`);
}
for (const rung of ['shipped', 'L1', 'L2', 'J']) {
  const a = adapter(rung);
  for (const [script, argv] of SCRIPTS) {
    if (absent.includes(script)) {
      runs.push({ kind: 'demo', label: `${rung} ${script}`, ok: true, skipped: `${demoFrom} has no demo/${script}.mjs` });
      console.log(`SKIP  ${rung} demo/${script}.mjs: ${demoFrom} has no such script`);
      continue;
    }
    const r = spawnSync(process.execPath, [join(demoWeb, 'demo', `${script}.mjs`), ...argv(a, rung)], { encoding: 'utf8', cwd: demoWeb, maxBuffer: 1 << 26 });
    const text = (r.stdout || '') + (r.stderr || '');
    writeFileSync(join(out, `${script}.${rung}.log`), text);
    const ok = r.status === 0;
    runs.push({ kind: 'demo', label: `${rung} ${script}`, ok, failed: ok ? [] : text.split('\n').filter((l) => /FAIL|Error/.test(l)).slice(0, 8) });
    console.log(`${ok ? 'PASS' : 'FAIL'}  ${rung} demo/${script}.mjs${ok ? '' : '\n      ' + text.trim().split('\n').slice(-6).join('\n      ')}`);
  }
  if (rung !== 'shipped' && !absent.includes('verify_measure_wasm')) {
    const A = join(out, 'measure.shipped'), B = join(out, `measure.${rung}`);
    const files = existsSync(A) ? readdirSync(A).sort() : [];
    const differ = files.filter((f) => !existsSync(join(B, f)) || !readFileSync(join(A, f)).equals(readFileSync(join(B, f))));
    const ok = files.length > 0 && differ.length === 0 && readdirSync(B).length === files.length;
    runs.push({ kind: 'demo', label: `${rung} measurement outputs equal the shipped build's`, ok, failed: differ });
    console.log(`${ok ? 'PASS' : 'FAIL'}  ${rung}: the measurement script's ${files.length} written files equal the shipped build's${ok ? '' : ': ' + differ.join(', ')}`);
  }
}

const failed = runs.filter((r) => !r.ok);
const skipped = runs.filter((r) => r.skipped);
writeFileSync(join(out, 'summary.json'), JSON.stringify({ engine: `node ${process.version}`, demo_scripts: demoFrom, runs }, null, 1) + '\n');
console.log(`\n${runs.length - failed.length - skipped.length} of ${runs.length - skipped.length} runs passed${skipped.length ? ` (${skipped.length} skipped)` : ''} on node ${process.version}; demo scripts from ${demoFrom}${keep ? `; logs in ${out}` : ''}`);
if (!keep) rmSync(out, { recursive: true, force: true });
process.exit(failed.length ? 1 : 0);