/* Reach conformance runner (ECMAScript 5, Node 0.12 onward). * * node conformance.js --dist --wai [options] * * --rung auto|shipped|L1|L2|J auto (default) lets the loader choose; the others force it * --profile full|verify_decode|verify|receipt (default full) * --expect-rung R fail unless the loader chose R * --shipped the shipped package (wai_web.js, wai_web_bg.wasm), for the shipped rung * --reference fail unless every workload's output equals the file's (the shipped build's) * --write-reference write this run's outputs there * --strip es5 delete the post-ES5 builtins first (WebAssembly, BigInt, Map, Symbol, ...), * so a modern engine behaves like the oldest engines J serves * --only a,b run only these workloads * * Under the J rung WebAssembly is deleted before anything loads, so the run proves the * translation does not use it. Prints one line per workload (PASS/FAIL, named * "/ ") and a JSON summary line; exits 0 only when every * workload passes its own check and, with --reference, equals the shipped build's output. */ 'use strict'; var fs = require('fs'), path = require('path'), crypto = require('crypto'); var H = require('./node-host.js'); var a = process.argv.slice(2), opt = { rung: 'auto', profile: 'full' }; for (var i = 0; i < a.length; i++) { var k = a[i]; if (k.slice(0, 2) !== '--' || i + 1 >= a.length) { console.log('usage: see the header of conformance.js'); process.exit(2); } opt[k.slice(2)] = a[++i]; } if (!opt.dist || !opt.wai) { console.log('conformance: --dist and --wai are required'); process.exit(2); } var DIST = path.resolve(opt.dist), WAI = path.resolve(opt.wai); var engine = 'node ' + process.version + ' (V8 ' + process.versions.v8 + ')'; function sha256(u8) { var b = typeof Buffer.from === 'function' && Buffer.from !== Uint8Array.from ? Buffer.from(u8.buffer, u8.byteOffset, u8.byteLength) : new Buffer(u8); return crypto.createHash('sha256').update(b).digest('hex'); } function now() { var t = process.hrtime(); return t[0] * 1e3 + t[1] / 1e6; } var ctx = { bytes: function (rel) { return H.bytes(path.join(WAI, rel)); }, text: function (rel) { return fs.readFileSync(path.join(WAI, rel), 'utf8'); }, json: function (rel) { return JSON.parse(fs.readFileSync(path.join(WAI, rel), 'utf8')); }, dirs: function (rel) { var d = path.join(WAI, rel); return fs.readdirSync(d).filter(function (n) { return fs.statSync(path.join(d, n)).isDirectory(); }).sort(); }, sha256: sha256, now: now }; // Node's own modules are loaded above; what is deleted now is what the code under test sees. var stripped = []; if (opt.strip === 'es5') { require('./es5-strip.js'); stripped = global.__WAI_ES5_STRIPPED || []; } if (opt.rung === 'J' && typeof global.WebAssembly !== 'undefined') { try { delete global.WebAssembly; } catch (e) { /* not configurable */ } if (typeof global.WebAssembly !== 'undefined') global.WebAssembly = undefined; stripped.push('WebAssembly'); } H.runScript(path.join(__dirname, 'workloads.js')); var Wk = global.WaiReachWorkloads; var WaiLoad = H.loadLoader(DIST); var summary = { engine: engine, requested: opt.rung, profile: opt.profile, stripped: stripped }; var loadOpts = { base: DIST + path.sep, profile: opt.profile, host: H.host }; if (opt.rung !== 'auto') loadOpts.force = opt.rung; if (opt.shipped) { loadOpts.shipped = { glue: path.join(path.resolve(opt.shipped), 'wai_web.js'), wasm: path.join(path.resolve(opt.shipped), 'wai_web_bg.wasm') }; } var t0 = now(); WaiLoad.load(loadOpts, function (err, st) { summary.rung = st ? st.rung : null; summary.reason = st ? st.reason : null; summary.features = st ? st.features : null; summary.load_ms = Math.round(now() - t0); var label = (st && st.rung ? st.rung : opt.rung) + '/' + (st && st.profile ? st.profile : opt.profile); var pass = true; if (err) { summary.load_error = String(err && err.message || err); console.log('FAIL ' + label + ' load: ' + summary.load_error); finish(false); return; } console.log(' ' + engine + ': ' + label + ' (' + st.reason + ')'); if (opt['expect-rung'] && st.rung !== opt['expect-rung']) { console.log('FAIL ' + label + ' rung: the loader chose ' + st.rung + ', expected ' + opt['expect-rung']); pass = false; } var names = Wk.forProfile(st.profile); if (opt.only) names = names.filter(function (n) { return opt.only.split(',').indexOf(n) >= 0; }); var ref = opt.reference ? JSON.parse(fs.readFileSync(opt.reference, 'utf8')).outputs : null; var R = Wk.run(st.api, ctx, names), outs = {}; for (var j = 0; j < names.length; j++) { var n = names[j], r = R[n], ok = r.ok && !r.err, why = r.err ? r.err : (r.ok ? '' : 'its own check failed'); if (ok && ref) { if (!Object.prototype.hasOwnProperty.call(ref, n)) { ok = false; why = 'no shipped output to compare with'; } else if (JSON.stringify(ref[n]) !== JSON.stringify(r.out)) { ok = false; why = 'output differs from the shipped build'; } } outs[n] = r.out; pass = pass && ok; console.log((ok ? 'PASS ' : 'FAIL ') + label + ' ' + n + ' ' + r.ms + ' ms' + (why ? ' ' + why : '')); } try { summary.memory_bytes = st.api.wasmMemoryBytes(); } catch (e) { summary.memory_bytes = null; } summary.results = R; if (opt['write-reference']) { fs.writeFileSync(opt['write-reference'], JSON.stringify({ engine: engine, rung: st.rung, profile: st.profile, outputs: outs }, null, 1) + '\n'); } finish(pass); }); function finish(pass) { summary.pass = pass; console.log('SUMMARY ' + JSON.stringify({ engine: summary.engine, rung: summary.rung, profile: summary.profile, pass: pass, load_ms: summary.load_ms, memory_bytes: summary.memory_bytes, reason: summary.reason, stripped: summary.stripped.length, ms: (function () { var o = {}; for (var n in summary.results || {}) o[n] = summary.results[n].ms; return o; })() })); // exitCode, not exit(): a pipe on some platforms is written asynchronously. process.exitCode = pass ? 0 : 1; }