ae91bc200a
wasm-smith writes valid modules, or ones invalid only as it is told to be, so the differential tested how the two validators accept far more than how they refuse: with check-module.mjs changed to accept an untyped select of references, or to read an uninitialised local, the 10,000 modules still all agreed. The differential now also writes two mutated copies of each module, one or two bytes changed (a byte replaced, inserted or removed anywhere after the preamble, or a code-section byte replaced by a one-byte opcode), most of them outside R2. On the committed seed the checkers agree on all 60,000 verdicts of its 30,000 modules, and it catches a changed section order, an unchecked if without else or an unchecked alignment. examples/spectest.rs writes every module the specification's test scripts define, assert invalid or assert malformed, with the Rust checker's verdicts, for differential.mjs. On the scripts of the specification build of 2026-11-04 (commit 859d3232a55c, test/core, 259 scripts, one of which the text parser of this release does not read) it writes 5,712 modules: the Rust checker refuses every invalid and malformed one by R2 and no valid one, and the two checkers agree on all 11,424 verdicts. With check-module.mjs changed in any of ten ways a validator can go wrong (a select of references, an uninitialised local, a u32's unused bits, br_if's pushed types, duplicate exports, a custom section's name, a final supertype, a field's mutability in subtyping, catch labels, undeclared function references), they disagree. Its scripts are not in this repository, so it is run by hand on a copy of them. Co-Authored-By: Claude Opus 5.5 <[email protected]>
58 lines
2.5 KiB
JavaScript
58 lines
2.5 KiB
JavaScript
// differential.mjs: the differential's second half. examples/differential.rs writes
|
|
// modules from wasm-smith and the Rust checker's verdicts; this runs check-module.mjs on
|
|
// every one at both levels, under the caps the Rust side used, and names each verdict
|
|
// that differs.
|
|
//
|
|
// node differential.mjs DIR
|
|
//
|
|
// DIR/verdicts.txt holds one line a module, `<name> <caps> <integer verdict> | <float
|
|
// verdict>`, the module being DIR/<name>.wasm: `<i>` for wasm-smith's i-th, `<i>-<k>` for a
|
|
// mutated copy of it, `<script>.<n>` for a module of the specification's tests
|
|
// (examples/spectest.rs). Prints each disagreement, then how the verdicts spread over held and
|
|
// the rules. Exit status 0 when every verdict agrees, 1 otherwise, 2 for an input error.
|
|
// No dependency.
|
|
import { readFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import { analyse, verdictOf, parseCaps } from './check-module.mjs';
|
|
|
|
const dir = process.argv[2];
|
|
if (!dir || process.argv.length !== 3) {
|
|
console.error('usage: node differential.mjs DIR');
|
|
process.exit(2);
|
|
}
|
|
let lines;
|
|
try {
|
|
lines = readFileSync(join(dir, 'verdicts.txt'), 'utf8').split('\n').filter(Boolean);
|
|
} catch (e) {
|
|
console.error(`differential: ${e.message}`);
|
|
process.exit(2);
|
|
}
|
|
|
|
const spread = new Map();
|
|
const count = (k) => spread.set(k, (spread.get(k) || 0) + 1);
|
|
let differ = 0, modules = 0;
|
|
for (const line of lines) {
|
|
const m = /^(\S+) (\S+) (.+) \| (.+)$/.exec(line);
|
|
const caps = m && parseCaps(m[2]);
|
|
if (!m || !caps) {
|
|
console.error(`differential: not a verdict line: ${line}`);
|
|
process.exit(2);
|
|
}
|
|
const bytes = new Uint8Array(readFileSync(join(dir, `${m[1]}.wasm`)));
|
|
const analysis = bytes.length > caps.module_bytes ? null : analyse(bytes);
|
|
modules++;
|
|
for (const [level, want] of [['integer', m[3]], ['float', m[4]]]) {
|
|
const got = verdictOf(analysis, bytes.length, level, caps);
|
|
const kind = m[1].includes('.') ? 'spec' : m[1].includes('-') ? 'mutated' : 'wasm-smith';
|
|
count(`${kind} ${level} ${want === 'held' ? 'held' : want.split(' ').slice(0, 2).join(' ')}`);
|
|
if (got !== want) {
|
|
differ++;
|
|
if (differ <= 50) console.log(`${m[1]}.wasm ${level} ${m[2]}: the Rust checker ${want}, check-module.mjs ${got}`);
|
|
}
|
|
}
|
|
}
|
|
for (const k of [...spread.keys()].sort()) console.log(` ${k}: ${spread.get(k)}`);
|
|
console.log(differ
|
|
? `differential: ${differ} verdicts of ${modules * 2} differ, over ${modules} modules`
|
|
: `differential: the two checkers agree on all ${modules * 2} verdicts of ${modules} modules`);
|
|
process.exit(differ ? 1 : 0);
|