5189e5f50b
- wai/wai-vector/wasm: the sink's exports (wv_alloc, wv_free, wv_open, wv_close, wv_render, wv_out_ptr, wv_out_len, wv_sha256, wv_cost, wv_transcript, wv_projection), no_std with its own allocator, built for wasm32v1-none so the module is WebAssembly 1.0 only; check-mvp.mjs refuses any post-1.0 feature, and fails on the wasm32-unknown-unknown build. - build-pkg.sh: the package a page serves. - browser/: a harness that renders every listed frame of every vector in Chromium, WebKit and Firefox, hashes it in the page twice, re-hashes sampled frames in Node, recomputes the arithmetic vectors and authoring forms, and compares each a11y payload's transcript and projections with its text record. A missing engine fails the run. - wai-vector.yml: the browser job, in a Playwright container pinned by digest, on the linux runner. Co-Authored-By: Claude Opus 5.5 <[email protected]>
439 lines
21 KiB
JavaScript
439 lines
21 KiB
JavaScript
// The sink of wai.graphics.vector in three browser engines, against
|
|
// wai/vector-conformance.
|
|
//
|
|
// node run.mjs --engines chromium,webkit,firefox [--pkg <dir>] [--recompute <module.wasm>]
|
|
// <vector-conformance dir>
|
|
//
|
|
// It serves page.html, the two modules and the corpus from a local server, launches
|
|
// each engine named, and in each one:
|
|
//
|
|
// 1. Through the package (<pkg dir>/wai_vector.wasm, which wasm/build-pkg.sh builds),
|
|
// every payload of the corpus:
|
|
// - it loads to the expected file's code: accepted, or the first refusal;
|
|
// - an accepted payload's SHA-256 is the expected `instr`, and each listed
|
|
// size's cost is the expected one;
|
|
// - in the groups that render, frames come through the page's hook,
|
|
// window.__wai_overlay.frame(size, rate, g), which returns the renderer's own
|
|
// buffer: every listed frame at sizes up to 1280 x 720, and at larger sizes the
|
|
// sampled frames. The page hashes each frame twice, with the module's SHA-256
|
|
// and with the platform's digest, and both must be the expected digest;
|
|
// - the sampled frames (the first, the last, the middle one, and the frames on
|
|
// each side of every key tick and of every item's window edges) are also sent
|
|
// here as bytes, and Node hashes them itself: the run does not rest on a hash
|
|
// the page computed;
|
|
// - a frame outside the range, a frame number above 2^32, an unlisted size and
|
|
// an unlisted rate are each refused with their code;
|
|
// - where the expected file states a text record, the sink's transcript and its
|
|
// projection at every frame of every listed rate, written out as the record's
|
|
// lines, are the committed <name>.a11y.jsonl byte for byte; and a projection
|
|
// outside the range or at an unlisted rate is refused with its code.
|
|
// 2. Through the recomputation module (the build verify.mjs runs under Node), every
|
|
// vector's expected file and every payload's authoring form, byte for byte.
|
|
//
|
|
// A missing engine is a failure, never a skip: the run passes only when chromium,
|
|
// webkit and firefox each ran and passed. --engines says which to launch; any of the
|
|
// three left out, or that cannot launch, fails the run.
|
|
//
|
|
// Nothing is read from a canvas, and nothing waits for a length of time.
|
|
//
|
|
// --lying-page is for testing this script. The page then changes one byte of every
|
|
// frame and reports the expected digest as both of its hashes; Node's own hash of the
|
|
// bytes sent must report every frame it was sent. A run with it never passes.
|
|
//
|
|
// Exit status 0: all three engines pass; 1: a failure; 2: usage.
|
|
import { createHash } from 'node:crypto';
|
|
import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
|
|
import { createServer } from 'node:http';
|
|
import { dirname, join, resolve, sep } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
const REQUIRED = ['chromium', 'webkit', 'firefox'];
|
|
// The groups whose vectors are arithmetic inputs, not payloads or outline sets.
|
|
const INPUT_GROUPS = ['arith', 'flatten'];
|
|
// Every listed frame is rendered at sizes up to this many pixels; sampled frames above.
|
|
const FULL = 1280 * 720;
|
|
|
|
function usage(why) {
|
|
if (why) console.error(`run.mjs: ${why}`);
|
|
console.error('usage: node run.mjs --engines chromium,webkit,firefox [--pkg <dir>] [--recompute <module.wasm>] <vector-conformance dir>');
|
|
process.exit(2);
|
|
}
|
|
|
|
let engines = null, pkgDir = join(HERE, '..', 'wasm', 'dist', 'pkg-vector');
|
|
let recomputePath = join(HERE, '..', 'wasm', 'target', 'wasm32-unknown-unknown', 'release', 'wai_vector_wasm.wasm');
|
|
let root = null, lying = false;
|
|
for (let a = process.argv.slice(2), i = 0; i < a.length; i++) {
|
|
if (a[i] === '--engines' && i + 1 < a.length) engines = [...new Set(a[++i].split(',').filter(Boolean))];
|
|
else if (a[i] === '--pkg' && i + 1 < a.length) pkgDir = a[++i];
|
|
else if (a[i] === '--recompute' && i + 1 < a.length) recomputePath = a[++i];
|
|
else if (a[i] === '--lying-page') lying = true;
|
|
else if (!a[i].startsWith('--') && root === null) root = a[i];
|
|
else usage(`unexpected argument ${a[i]}`);
|
|
}
|
|
if (engines === null || engines.length === 0) usage('--engines is required');
|
|
if (root === null) usage('the vector-conformance directory is required');
|
|
for (const e of engines) if (!REQUIRED.includes(e)) usage(`no engine named ${e}`);
|
|
root = resolve(root);
|
|
const pkgPath = resolve(pkgDir, 'wai_vector.wasm');
|
|
recomputePath = resolve(recomputePath);
|
|
const pagePath = join(HERE, 'page.html');
|
|
|
|
const sha256 = (b) => createHash('sha256').update(b).digest('hex');
|
|
const firstLine = (s) => String(s).split('\n')[0];
|
|
const failuresOf = (n) => `${n} failure${n === 1 ? '' : 's'}`;
|
|
|
|
for (const [what, p] of [['the package', pkgPath], ['the recomputation module', recomputePath], ['the corpus', join(root, 'vectors')]]) {
|
|
if (!existsSync(p)) {
|
|
console.error(`run.mjs: ${what} is not at ${p}`);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
console.log(`the package: ${pkgPath}, ${statSync(pkgPath).size} bytes, SHA-256 ${sha256(readFileSync(pkgPath))}`);
|
|
console.log(`the recomputation module: ${recomputePath}, ${statSync(recomputePath).size} bytes, SHA-256 ${sha256(readFileSync(recomputePath))}`);
|
|
|
|
// ---- the server ---------------------------------------------------------------------
|
|
|
|
// The frames the page sent: id -> what Node read, hashed here.
|
|
const received = new Map();
|
|
|
|
function fileFor(path) {
|
|
if (path === '/' || path === '/page.html') return [pagePath, 'text/html; charset=utf-8'];
|
|
if (path === '/pkg/wai_vector.wasm') return [pkgPath, 'application/wasm'];
|
|
if (path === '/recompute.wasm') return [recomputePath, 'application/wasm'];
|
|
if (path.startsWith('/corpus/')) {
|
|
const f = resolve(root, path.slice('/corpus/'.length));
|
|
if (f.startsWith(root + sep)) return [f, 'application/octet-stream'];
|
|
}
|
|
return null;
|
|
}
|
|
|
|
const server = createServer((req, res) => {
|
|
let path;
|
|
try {
|
|
path = decodeURIComponent(new URL(req.url, 'http://localhost').pathname);
|
|
} catch {
|
|
res.writeHead(400).end();
|
|
return;
|
|
}
|
|
if (req.method === 'POST' && path.startsWith('/frame/')) {
|
|
const h = createHash('sha256');
|
|
let bytes = 0;
|
|
req.on('data', (d) => { h.update(d); bytes += d.length; });
|
|
req.on('end', () => {
|
|
received.set(path.slice('/frame/'.length), { sha256: h.digest('hex'), bytes });
|
|
res.writeHead(204).end();
|
|
});
|
|
return;
|
|
}
|
|
const f = req.method === 'GET' ? fileFor(path) : null;
|
|
if (f === null || !existsSync(f[0]) || !statSync(f[0]).isFile()) {
|
|
res.writeHead(404).end();
|
|
return;
|
|
}
|
|
const body = readFileSync(f[0]);
|
|
res.writeHead(200, { 'content-type': f[1], 'content-length': body.length, 'cache-control': 'no-store' }).end(body);
|
|
});
|
|
await new Promise((ok) => server.listen(0, '127.0.0.1', ok));
|
|
const origin = `http://127.0.0.1:${server.address().port}`;
|
|
|
|
// ---- the corpus ---------------------------------------------------------------------
|
|
|
|
// Every vector, in the order verify.mjs reads them: an index file is not a vector, and
|
|
// a .json file is one only in the input groups (elsewhere it is an authoring form).
|
|
const vectors = [];
|
|
for (const group of readdirSync(join(root, 'vectors')).sort()) {
|
|
for (const f of readdirSync(join(root, 'vectors', group)).sort()) {
|
|
const dot = f.lastIndexOf('.');
|
|
const name = f.slice(0, dot), ext = f.slice(dot + 1);
|
|
if (name === 'index' || (ext === 'json' && !INPUT_GROUPS.includes(group))) continue;
|
|
vectors.push({ group, name, ext, id: `${group}/${name}` });
|
|
}
|
|
}
|
|
const setNames = readdirSync(join(root, 'sets')).filter((n) => n.endsWith('.wos1')).sort();
|
|
const expectedBytes = (v) => readFileSync(join(root, 'expected', v.group, `${v.name}.json`));
|
|
const key = (f) => `${f.size[0]}x${f.size[1]}@${f.rate[0]}/${f.rate[1]}#${f.g}`;
|
|
|
|
// The sampled frames of a list: the first, the last, the middle one, and the frame at
|
|
// or after each tick with the frame before it.
|
|
function sampled(gs, ticks, ticksPerFrame) {
|
|
const first = gs[0], last = gs[gs.length - 1];
|
|
const s = new Set([first, last, gs[gs.length >> 1]]);
|
|
for (const t of ticks) {
|
|
const g = Math.ceil(t / ticksPerFrame);
|
|
for (const c of [g - 1, g]) if (c >= first && c <= last) s.add(c);
|
|
}
|
|
return s;
|
|
}
|
|
|
|
// The first pair [n, 1] that `listed` does not hold.
|
|
function unlisted(listed) {
|
|
for (let n = 1; ; n++) if (!listed.some((p) => p[0] === n && p[1] === 1)) return [n, 1];
|
|
}
|
|
|
|
// What to ask of a payload, from its expected file: the sizes whose cost is asked, the
|
|
// frames to render (with each one's expected digest, and an id when it is also sent),
|
|
// the requests that must be refused (with each one's code), and, where the expected
|
|
// file states a text record, every projection the record holds and the projections
|
|
// that must be refused.
|
|
let nextId = 0;
|
|
function planOf(v, expected) {
|
|
const plan = { sizes: [], frames: [], requests: [], text: null };
|
|
if (expected.code !== 'accepted') return plan;
|
|
plan.sizes = expected.sizes.map((s) => [s[0], s[1]]);
|
|
if (expected.frames) {
|
|
const form = JSON.parse(readFileSync(join(root, 'vectors', v.group, `${v.name}.json`), 'utf8'));
|
|
const ticks = [...form.keys.map((k) => k[0]), ...form.items.flatMap((i) => [i.t_on, i.t_off])];
|
|
for (const s of expected.frames) {
|
|
if (s.frames.length === 0) continue;
|
|
const rate = expected.rates.find((r) => r[0] === s.num && r[1] === s.den);
|
|
const sample = sampled(s.frames.map((f) => f[0]), ticks, rate[2]);
|
|
const full = s.w * s.h <= FULL;
|
|
for (const [g, digest] of s.frames) {
|
|
if (!full && !sample.has(g)) continue;
|
|
plan.frames.push({ size: [s.w, s.h], rate: [s.num, s.den], g, send: sample.has(g) ? nextId++ : null, digest });
|
|
}
|
|
}
|
|
}
|
|
if (plan.sizes.length > 0 && expected.rates.length > 0) {
|
|
const size = plan.sizes[0];
|
|
for (const [num, den, , first, last] of expected.rates) {
|
|
plan.requests.push({ size, rate: [num, den], g: last + 1, want: 'frame_range' });
|
|
if (first > 0) plan.requests.push({ size, rate: [num, den], g: first - 1, want: 'frame_range' });
|
|
}
|
|
const [num, den, , first] = expected.rates[0];
|
|
// A frame number whose low half alone would be in range.
|
|
plan.requests.push({ size, rate: [num, den], g: 4294967296 + Math.max(first, 0), want: 'frame_range' });
|
|
plan.requests.push({ size: unlisted(plan.sizes), rate: [num, den], g: Math.max(first, 0), want: 'size_unlisted' });
|
|
plan.requests.push({ size, rate: unlisted(expected.rates), g: Math.max(first, 0), want: 'rate_unlisted' });
|
|
}
|
|
if (expected.text_record !== undefined) {
|
|
// The record's order: each listed rate, and within it each frame of the document.
|
|
plan.text = { projections: [], requests: [] };
|
|
for (const [num, den, , first, last] of expected.rates) {
|
|
for (let g = first; g <= last; g++) plan.text.projections.push({ rate: [num, den], g });
|
|
plan.text.requests.push({ rate: [num, den], g: last + 1, want: 'frame_range' });
|
|
}
|
|
if (expected.rates.length > 0) {
|
|
const [num, den, , first] = expected.rates[0];
|
|
plan.text.requests.push({ rate: [num, den], g: 4294967296 + Math.max(first, 0), want: 'frame_range' });
|
|
plan.text.requests.push({ rate: unlisted(expected.rates), g: Math.max(first, 0), want: 'rate_unlisted' });
|
|
}
|
|
}
|
|
return plan;
|
|
}
|
|
|
|
// A text record's line for a projection.
|
|
const recordLine = (q, projection) => `{"den":${q.rate[1]},"g":${q.g},"num":${q.rate[0]},"projection":${projection}}`;
|
|
// Two lines that differ, each shown around the first character at which they do.
|
|
function differing(got, want) {
|
|
if (got === undefined || want === undefined) return `the sink gives ${got === undefined ? 'no such line' : 'a line'}; the committed record has ${want === undefined ? 'no such line' : 'one'}`;
|
|
let at = 0;
|
|
while (at < got.length && at < want.length && got[at] === want[at]) at++;
|
|
const from = Math.max(0, at - 60);
|
|
const part = (s) => `${from > 0 ? '…' : ''}${s.slice(from, at + 60)}${s.length > at + 60 ? '…' : ''}`;
|
|
return `at character ${at} the sink gives ${part(got)}; expected ${part(want)}`;
|
|
}
|
|
|
|
// ---- one engine ---------------------------------------------------------------------
|
|
|
|
async function runEngine(playwright, name) {
|
|
const failures = [];
|
|
const fail = (where, what) => {
|
|
failures.push(`${where}: ${what}`);
|
|
console.error(`FAIL ${name} ${where}: ${what}`);
|
|
};
|
|
const n = { payloads: 0, accepted: 0, frames: 0, sent: 0, sentBytes: 0, records: 0, lines: 0, recomputed: 0, forms: 0 };
|
|
let browser;
|
|
try {
|
|
browser = await playwright[name].launch();
|
|
} catch (e) {
|
|
fail('launch', firstLine(e.message));
|
|
return { failures, version: null };
|
|
}
|
|
const version = browser.version();
|
|
try {
|
|
const page = await browser.newPage();
|
|
let dead = false;
|
|
page.on('pageerror', (e) => fail('the page', `uncaught: ${firstLine(e.message)}`));
|
|
page.on('crash', () => { dead = true; fail('the page', 'crashed'); });
|
|
await page.goto(`${origin}/page.html`);
|
|
await page.waitForFunction(() => window.__wai_ready !== undefined);
|
|
const ready = await page.evaluate(() => window.__wai_ready);
|
|
if (!ready.platformDigest) fail('the page', 'has no platform digest (crypto.subtle)');
|
|
await page.evaluate((names) => window.__wai_harness.sets(names), setNames);
|
|
if (lying) {
|
|
await page.evaluate(() => {
|
|
const h = window.__wai_harness;
|
|
const honest = h.check.bind(h);
|
|
h.check = async (size, rate, g) => {
|
|
const r = await honest(size, rate, g);
|
|
const claim = window.__wai_claims[`${size[0]}x${size[1]}@${rate[0]}/${rate[1]}#${g}`];
|
|
r.bytes[r.bytes.length >> 1] ^= 0x01;
|
|
return { bytes: r.bytes, inModule: claim, platform: claim };
|
|
};
|
|
});
|
|
}
|
|
|
|
// 1. Every payload through the package.
|
|
for (const v of vectors.filter((x) => x.ext === 'wvg1')) {
|
|
if (dead) break;
|
|
const expected = JSON.parse(expectedBytes(v).toString('utf8'));
|
|
const plan = planOf(v, expected);
|
|
const asked = {
|
|
sizes: plan.sizes,
|
|
frames: plan.frames.map((f) => ({ size: f.size, rate: f.rate, g: f.g, send: f.send })),
|
|
requests: plan.requests.map((q) => ({ size: q.size, rate: q.rate, g: q.g })),
|
|
text: plan.text === null ? null : {
|
|
projections: plan.text.projections,
|
|
requests: plan.text.requests.map((q) => ({ rate: q.rate, g: q.g })),
|
|
},
|
|
};
|
|
const claims = lying ? Object.fromEntries(plan.frames.map((f) => [key(f), f.digest])) : null;
|
|
let got;
|
|
try {
|
|
got = await page.evaluate(([path, p, c]) => {
|
|
if (c !== null) window.__wai_claims = c;
|
|
return window.__wai_harness.payload(path, p);
|
|
}, [`vectors/${v.group}/${v.name}.wvg1`, asked, claims]);
|
|
} catch (e) {
|
|
fail(v.id, firstLine(e.message));
|
|
continue;
|
|
}
|
|
n.payloads += 1;
|
|
if (got.code !== expected.code) {
|
|
fail(v.id, `loads as ${got.code}; expected ${expected.code}`);
|
|
continue;
|
|
}
|
|
if (expected.code !== 'accepted') continue;
|
|
n.accepted += 1;
|
|
if (got.instr.inModule !== expected.instr) fail(v.id, `the module's SHA-256 of the payload is ${got.instr.inModule}; expected ${expected.instr}`);
|
|
if (got.instr.platform !== expected.instr) fail(v.id, `the platform's digest of the payload is ${got.instr.platform}; expected ${expected.instr}`);
|
|
expected.sizes.forEach(([w, h, cost], i) => {
|
|
if (got.costs[i] !== cost) fail(v.id, `the cost at ${w}x${h} is ${got.costs[i]}; expected ${cost}`);
|
|
});
|
|
plan.frames.forEach((f, i) => {
|
|
const r = got.frames[i];
|
|
const where = `${v.id} frame ${key(f)}`;
|
|
n.frames += 1;
|
|
if (r === undefined) { fail(where, 'not rendered'); return; }
|
|
if (r.bytes !== 4 * f.size[0] * f.size[1]) fail(where, `${r.bytes} bytes; expected ${4 * f.size[0] * f.size[1]}`);
|
|
if (r.inModule !== f.digest) fail(where, `the module's SHA-256 is ${r.inModule}; expected ${f.digest}`);
|
|
if (r.platform !== f.digest) fail(where, `the platform's digest is ${r.platform}; expected ${f.digest}`);
|
|
if (f.send === null) return;
|
|
const sent = received.get(String(f.send));
|
|
received.delete(String(f.send));
|
|
if (sent === undefined) { fail(where, 'its bytes did not arrive'); return; }
|
|
n.sent += 1;
|
|
n.sentBytes += sent.bytes;
|
|
if (sent.sha256 !== f.digest) fail(where, `Node's SHA-256 of the bytes sent is ${sent.sha256}; expected ${f.digest}`);
|
|
});
|
|
plan.requests.forEach((q, i) => {
|
|
if (got.requests[i] !== q.want) fail(`${v.id} request ${key(q)}`, `${got.requests[i]}; expected ${q.want}`);
|
|
});
|
|
if (plan.text === null) continue;
|
|
|
|
// The text record: the transcript and every projection, as the record's lines,
|
|
// against the committed file's bytes.
|
|
const recordPath = join(root, 'expected', v.group, `${v.name}.a11y.jsonl`);
|
|
if (!existsSync(recordPath)) {
|
|
fail(v.id, `its expected file states a text record, and expected/${v.group}/${v.name}.a11y.jsonl is missing`);
|
|
continue;
|
|
}
|
|
const want = readFileSync(recordPath);
|
|
if (sha256(want) !== expected.text_record) fail(v.id, `the committed text record's SHA-256 is ${sha256(want)}; its expected file states ${expected.text_record}`);
|
|
if (got.text === null) {
|
|
fail(v.id, 'the page returned no transcript and no projection');
|
|
continue;
|
|
}
|
|
const lines = [got.text.transcript, ...plan.text.projections.map((q, i) => recordLine(q, got.text.projections[i]))];
|
|
n.records += 1;
|
|
n.lines += lines.length;
|
|
if (Buffer.compare(Buffer.from(lines.map((l) => `${l}\n`).join(''), 'utf8'), want) !== 0) {
|
|
const wantLines = want.toString('utf8').split('\n');
|
|
if (wantLines[wantLines.length - 1] === '') wantLines.pop();
|
|
const differ = [];
|
|
for (let i = 0; i < Math.max(lines.length, wantLines.length); i++) if (lines[i] !== wantLines[i]) differ.push(i);
|
|
if (differ.length === 0) {
|
|
fail(`${v.id} text record`, 'every line is the committed one, and the bytes differ: the committed file does not end each line with one U+000A');
|
|
} else {
|
|
const i = differ[0];
|
|
const q = plan.text.projections[i - 1];
|
|
const which = i === 0 ? 'the transcript' : q === undefined ? `line ${i + 1}, which the sink was not asked for` : `the projection at ${q.rate.join('/')}#${q.g}`;
|
|
fail(
|
|
`${v.id} text record`,
|
|
`${differ.length} of ${wantLines.length} lines differ (${differ.includes(0) ? 'the transcript and ' : ''}${differ.filter((d) => d > 0).length} projections); `
|
|
+ `the first is ${which}: ${differing(lines[i], wantLines[i])}`,
|
|
);
|
|
}
|
|
}
|
|
plan.text.requests.forEach((q, i) => {
|
|
if (got.text.requests[i] !== q.want) fail(`${v.id} projection request ${q.rate.join('/')}#${q.g}`, `${got.text.requests[i]}; expected ${q.want}`);
|
|
});
|
|
}
|
|
|
|
// 2. Every vector through the recomputation.
|
|
for (const v of vectors) {
|
|
if (dead) break;
|
|
let got;
|
|
try {
|
|
got = await page.evaluate(([g, name, ext]) => window.__wai_harness.recompute(g, name, ext), [v.group, v.name, v.ext]);
|
|
} catch (e) {
|
|
fail(v.id, `recomputation: ${firstLine(e.message)}`);
|
|
continue;
|
|
}
|
|
n.recomputed += 1;
|
|
if (got.status !== 0) fail(`${v.id}.${v.ext}`, `recomputation: ${got.expected}`);
|
|
else if (Buffer.compare(Buffer.from(got.expected, 'utf8'), expectedBytes(v)) !== 0) fail(`${v.id}.${v.ext}`, 'the recomputed expected file differs');
|
|
if (v.ext === 'wvg1') {
|
|
n.forms += 1;
|
|
const form = readFileSync(join(root, 'vectors', v.group, `${v.name}.json`));
|
|
if (Buffer.compare(Buffer.from(got.form, 'utf8'), form) !== 0) fail(`${v.id}.${v.ext}`, 'the authoring form differs');
|
|
}
|
|
}
|
|
} catch (e) {
|
|
fail('the run', firstLine(e.message));
|
|
} finally {
|
|
await browser.close();
|
|
}
|
|
console.log(
|
|
`${name} ${version}: ${n.payloads} payloads through the package, ${n.accepted} accepted; `
|
|
+ `${n.frames} frames rendered and hashed twice in the page, ${n.sent} of them hashed again in Node (${n.sentBytes} bytes); `
|
|
+ `${n.records} text records, ${n.lines} lines; `
|
|
+ `${n.recomputed} vectors and ${n.forms} authoring forms recomputed; ${failuresOf(failures.length)}`,
|
|
);
|
|
return { failures, version, n };
|
|
}
|
|
|
|
// ---- the run ------------------------------------------------------------------------
|
|
|
|
let playwright;
|
|
try {
|
|
playwright = await import('playwright');
|
|
} catch (e) {
|
|
console.error(`run.mjs: playwright is not installed here (npm ci): ${firstLine(e.message)}`);
|
|
process.exit(1);
|
|
}
|
|
if (lying) console.log('--lying-page: the page changes one byte of every frame and reports the expected digest; this run cannot pass');
|
|
|
|
const results = new Map();
|
|
for (const name of engines) results.set(name, await runEngine(playwright, name));
|
|
server.close();
|
|
|
|
let bad = 0;
|
|
for (const name of REQUIRED) {
|
|
const r = results.get(name);
|
|
if (r === undefined) {
|
|
bad += 1;
|
|
console.error(`FAIL ${name}: not run. A missing engine is a failure: chromium, webkit and firefox must each run.`);
|
|
} else if (r.failures.length > 0) {
|
|
bad += 1;
|
|
console.error(`FAIL ${name}: ${failuresOf(r.failures.length)}, the first: ${r.failures[0]}`);
|
|
}
|
|
}
|
|
if (lying && bad === 0) {
|
|
console.error('FAIL --lying-page: no engine reported the changed frames');
|
|
bad += 1;
|
|
}
|
|
if (bad > 0) process.exit(1);
|
|
console.log('vector-conformance: every payload, frame, text record and vector verifies in chromium, webkit and firefox');
|