// The reach build's steps that are not binaryen. build.sh calls each one; every step // refuses input it does not recognise rather than produce a quietly different build. // // node cli.mjs sigs // node cli.mjs graph // node cli.mjs exports the module exports exactly the graph's names // node cli.mjs same-sigs the two modules have one i64 boundary // node cli.mjs factory // node cli.mjs glue // node cli.mjs minify // node cli.mjs es5 ... each file parses as ECMAScript 5 // node cli.mjs manifest // // The JavaScript tools (TypeScript, esbuild, acorn) are the versions package-lock.json // pins, installed by build.sh into $WAI_REACH_NODE_MODULES. import fs from 'node:fs'; import path from 'node:path'; import zlib from 'node:zlib'; import crypto from 'node:crypto'; import { createRequire } from 'node:module'; import { readModule, i64Boundary } from './wasm.mjs'; const NM = process.env.WAI_REACH_NODE_MODULES; const req = NM ? createRequire(path.join(NM, 'noop.js')) : null; const tool = (name) => { if (!req) throw new Error('WAI_REACH_NODE_MODULES is not set (build.sh sets it)'); return req(name); }; const read = (p) => fs.readFileSync(p); const readText = (p) => fs.readFileSync(p, 'utf8'); const die = (msg) => { console.error(`reach: ${msg}`); process.exit(1); }; const ROOT_EXPR = "typeof self !== 'undefined' ? self : typeof global !== 'undefined' ? global : this"; // ---- sigs: the i64 boundary table the runtime needs ------------------------------- function sigs(wasmPath, outPath) { const s = i64Boundary(readModule(read(wasmPath))); const js = '/* The i64 boundary of wai_web_bg.wasm: every import and export with an i64 parameter or result (generated by the reach build). */\n' + `(function (root) { root.WaiReachSigs = ${JSON.stringify(s)}; })(${ROOT_EXPR});\n`; fs.writeFileSync(outPath, js); console.log(`sigs: ${Object.keys(s.exports).length} exports and ${Object.keys(s.imports).length} imports cross the i64 boundary`); } function sameSigs(a, b) { const x = JSON.stringify(i64Boundary(readModule(read(a)))); const y = JSON.stringify(i64Boundary(readModule(read(b)))); if (x !== y) throw new Error(`${path.basename(a)} and ${path.basename(b)} have different i64 boundaries`); console.log(`same-sigs: ${path.basename(a)} and ${path.basename(b)} agree`); } // ---- graph: the wasm-metadce graph of one profile --------------------------------- function profileRules(profilesPath, profile) { const P = JSON.parse(readText(profilesPath)); const names = new Set(P.always.names), prefixes = [...P.always.prefixes]; const seen = new Set(); const add = (p) => { if (seen.has(p)) return; seen.add(p); const r = P.profiles[p]; if (!r) throw new Error(`no profile '${p}' in ${profilesPath}`); for (const inc of r.includes ?? []) add(inc); for (const n of r.names ?? []) names.add(n); for (const x of r.prefixes ?? []) prefixes.push(x); }; add(profile); return { names, prefixes }; } function keptExports(wasmPath, profilesPath, profile) { const all = readModule(read(wasmPath)).exports.map((e) => e.name); const { names, prefixes } = profileRules(profilesPath, profile); const frees = new Set(prefixes.filter((x) => !x.startsWith('__')).map((x) => `__wbg_${x}free`)); const keep = all.filter((n) => names.has(n) || frees.has(n) || prefixes.some((x) => n.startsWith(x))); const missing = [...names].filter((n) => !all.includes(n)); if (missing.length) throw new Error(`profile ${profile} names exports the module lacks: ${missing.join(', ')}`); for (const x of prefixes) if (!all.some((n) => n.startsWith(x))) throw new Error(`profile ${profile}: no export starts with ${x}`); return { all, keep }; } function graph(wasmPath, profilesPath, profile, outPath) { const { all, keep } = keptExports(wasmPath, profilesPath, profile); const g = [{ name: 'root', root: true, reaches: keep.map((_, i) => `e${i}`) }, ...keep.map((n, i) => ({ name: `e${i}`, export: n }))]; fs.writeFileSync(outPath, JSON.stringify(g)); console.log(`graph ${profile}: keeps ${keep.length} of ${all.length} exports`); } function checkExports(wasmPath, graphPath) { const want = JSON.parse(readText(graphPath)).filter((n) => n.export).map((n) => n.export).sort(); const got = readModule(read(wasmPath)).exports.map((e) => e.name).sort(); if (JSON.stringify(want) !== JSON.stringify(got)) { const extra = got.filter((n) => !want.includes(n)), lost = want.filter((n) => !got.includes(n)); throw new Error(`${path.basename(wasmPath)}: exports differ from the profile (extra: ${extra.slice(0, 5).join(', ') || 'none'}; lost: ${lost.slice(0, 5).join(', ') || 'none'})`); } console.log(`exports: ${path.basename(wasmPath)} exports exactly its profile's ${got.length}`); } // ---- factory: the wasm2js ES module as a classic script ---------------------------- // wasm2js writes `import * as X from '...'` lines, the translation, then // `var retasmFunc = asmFunc({...});` and one `export var` line per export. The imports // and everything from the instantiation on are dropped; the rest is wrapped in // WaiReachFactory(imports), which returns the module's exports. function factory(inPath, outPath) { const lines = readText(inPath).split('\n'); const body = []; let imports = 0, cut = false; for (const l of lines) { if (cut) { if (/^export var \S+ = retasmFunc\.\S+;$/.test(l) || /^\s*("[^"]+": \S+,?|\}\);)?\s*$/.test(l)) continue; throw new Error(`unexpected line after the instantiation: ${l.slice(0, 80)}`); } if (/^import \* as \S+ from '[^']+';\s*$/.test(l)) { imports++; continue; } if (/^var retasmFunc = asmFunc\(/.test(l)) { cut = true; continue; } body.push(l); } if (!cut || imports === 0) throw new Error('the wasm2js layout changed (no imports or no instantiation found)'); const text = body.join('\n'); if (!/^function asmFunc\(imports\) \{$/m.test(text)) throw new Error('no asmFunc(imports)'); if (/^\s*(import|export)\s/m.test(text)) throw new Error('an import or export statement is left in the translation'); if (/\bWebAssembly\b/.test(text)) throw new Error('the translation refers to WebAssembly'); const out = '/* WAI wasm2js factory: binaryen wasm2js translation of the MVP-lowered wai_web_bg.wasm. ES5 and typed arrays; no WebAssembly. */\n' + '(function (root) {\nfunction WaiReachFactory(imports) {\n' + text + '\nreturn asmFunc(imports);\n}\n' + `root.WaiReachFactory = WaiReachFactory;\n})(${ROOT_EXPR});\n`; fs.writeFileSync(outPath, out); console.log(`factory: ${path.basename(outPath)} (${imports} import lines and the instantiation dropped)`); } // ---- glue: the wasm-bindgen glue, made loadable by the reach rungs ----------------- // The shipped glue is an ES module for engines with WebAssembly, BigInt-to-i64 and ES // modules. The reach glue is the same code with five changes, each counted and checked: // 1. init: the WebAssembly loading at the end is replaced by two hooks the loader // calls, __reachImports() and __reachFinalize(instance); // 2. memory views: the glue rebuilt a cached view only when its buffer was detached, // which a real WebAssembly.Memory does on growth and wasm2js's memory does not; it // now rebuilds whenever wasm.memory.buffer is a different buffer; // 3. Symbol.dispose is read only where Symbol exists; // 4. 64-bit values: with BigInt the helpers do what the glue did; without it they // carry the value as a Number (exact below 2^53); // 5. typed-array copies: `view.slice()` on wasm memory becomes an explicit copy, as // early typed-array engines return a view from slice(). // It is then compiled to ES5 by TypeScript and wrapped as a classic script that // defines the global WaiWeb. function glue(inPath, outPath, notesPath) { let g = readText(inPath); const notes = {}; const count = (re) => (g.match(re) || []).length; if (/\bimport\.meta\b/.test(g.slice(0, g.indexOf('async function __wbg_load(')))) throw new Error('import.meta before the init code'); const cutAt = g.indexOf('async function __wbg_load('); if (cutAt < 0) throw new Error('no __wbg_load: the glue layout changed'); const tail = g.slice(cutAt); if (!tail.includes('__wbg_finalize_init(instance, module)') || !/export \{ initSync, __wbg_init as default \};\s*$/.test(tail)) { throw new Error('the init code changed'); } if (!/^function __wbg_get_imports\(\)/m.test(g.slice(0, cutAt)) || !/^function __wbg_finalize_init\(instance, module\)/m.test(g.slice(0, cutAt))) { throw new Error('no __wbg_get_imports or __wbg_finalize_init before the init code'); } g = g.slice(0, cutAt) + [ '// reach init: the reach loader instantiates the module (a legalised wasm, or the wasm2js', '// translation) and hands the instance here; the glue never fetches or compiles.', 'function __reachImports() { return __wbg_get_imports(); }', 'function __reachFinalize(instance) {', ' if (wasm !== undefined) return wasm;', ' return __wbg_finalize_init(instance, null);', '}', 'export { __reachImports, __reachFinalize };', '', ].join('\n'); if (/\bWebAssembly\b/.test(g)) throw new Error('the glue still refers to WebAssembly outside the init code'); // 2. memory views keyed on buffer identity const caches = count(/^let cached\w+Memory0 = null;$/gm); let n = 0; g = g.replace(/(cached\w+Memory0) === null \|\| \1\.byteLength === 0/g, (m, v) => { n++; return `${v} === null || ${v}.buffer !== wasm.memory.buffer`; }); const dv = 'cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)'; if (g.includes(dv)) { g = g.replace(dv, 'cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer !== wasm.memory.buffer'); n++; } if (n !== caches || caches === 0) throw new Error(`memory views: ${caches} caches, ${n} checks rewritten`); if (/Memory0\.byteLength === 0|\.detached\b/.test(g)) throw new Error('a memory-view check was left unrewritten'); notes.memory_view_checks = n; // 3. Symbol.dispose guarded notes.symbol_dispose_guards = 0; g = g.replace(/^if \(Symbol\.dispose\)/gm, () => { notes.symbol_dispose_guards++; return "if (typeof Symbol !== 'undefined' && Symbol.dispose)"; }); if (/^(?!if \(typeof Symbol !== 'undefined' && Symbol\.dispose\)).*\bSymbol\.dispose\b/m.test(g)) throw new Error('an unguarded Symbol.dispose is left'); // 4. BigInt made optional let b = 0; g = g.replace(/BigInt\.asUintN\(64, (\w+)\)/g, (m, v) => { b++; return `__wai_u64(${v})`; }); g = g.replace(/getDataViewMemory0\(\)\.getBigInt64\(([^,()]+), true\)/g, (m, p) => { b++; return `__wai_get_i64(${p})`; }); g = g.replace(/getDataViewMemory0\(\)\.setBigInt64\(([^,()]+), isLikeNone\(ret\) \? BigInt\(0\) : ret, true\)/g, (m, p) => { b++; return `__wai_set_i64(${p}, isLikeNone(ret) ? 0 : ret)`; }); notes.bigint_sites_made_optional = b; // What may still need BigInt, by design: the u128 value IntPayloadCost.macs, and the // BigInt64Array / BigUint64Array views (built only when a function returns one). const stillBigInt = g.split('\n').filter((l) => /\bBigInt/.test(l) && !/^\s*(\*|\/\/)/.test(l)); const allowed = [/<< BigInt\(64\)\)\);$/, /new BigInt64Array\(wasm\.memory\.buffer\)/, /new BigUint64Array\(wasm\.memory\.buffer\)/, /cachedBig(Int|Uint)64ArrayMemory0/, /getBig(Int|Uint)64ArrayMemory0\(\)/, /^function getArrayBig(I|U)\w*FromWasm0/]; const unexpected = stillBigInt.filter((l) => !allowed.some((re) => re.test(l))); if (unexpected.length) throw new Error(`BigInt left where a Number path was expected: ${unexpected[0].trim().slice(0, 100)}`); notes.bigint_required_by = stillBigInt.length; // 5. typed-array copies that do not rely on TypedArray#slice let c = 0; g = g.replace(/getArray(\w+)FromWasm0\(([^()]*)\)\.slice\(\)/g, (m, t, a) => { c++; return `__wai_ta_copy(getArray${t}FromWasm0(${a}))`; }); if (/FromWasm0\([^()]*\)\.slice\(/.test(g)) throw new Error('a typed-array slice() on wasm memory is left'); notes.typed_array_copies = c; g = [ '// reach helpers (see tools/cli.mjs, glue)', "const __WAI_HAS_BIGINT = typeof BigInt === 'function';", 'function __wai_u64(v) { return __WAI_HAS_BIGINT ? BigInt.asUintN(64, v) : (v < 0 ? v + 18446744073709551616 : v); }', 'function __wai_get_i64(p) {', ' const dv = getDataViewMemory0();', ' return __WAI_HAS_BIGINT ? dv.getBigInt64(p, true) : dv.getInt32(p + 4, true) * 4294967296 + dv.getUint32(p, true);', '}', 'function __wai_set_i64(p, v) {', ' const dv = getDataViewMemory0();', " if (__WAI_HAS_BIGINT) { dv.setBigInt64(p, typeof v === 'bigint' ? v : BigInt(v), true); return; }", ' const lo = v % 4294967296;', ' dv.setUint32(p, lo < 0 ? lo + 4294967296 : lo, true);', ' dv.setInt32(p + 4, Math.floor(v / 4294967296), true);', '}', 'function __wai_ta_copy(v) { const o = new v.constructor(v.length); o.set(v); return o; }', g, ].join('\n'); const ts = tool('typescript'); const r = ts.transpileModule(g, { compilerOptions: { target: ts.ScriptTarget.ES5, module: ts.ModuleKind.CommonJS, allowJs: true, downlevelIteration: true, importHelpers: false, removeComments: true, }, reportDiagnostics: true, }); if (r.diagnostics && r.diagnostics.length) { throw new Error(`TypeScript: ${ts.flattenDiagnosticMessageText(r.diagnostics[0].messageText, ' ')}`); } const out = '/* WAI wasm-bindgen glue, reach build: ES5 classic script. Global: WaiWeb. */\n' + '(function (root) {\nvar exports = {}; var module = { exports: exports };\n' + r.outputText + `\nroot.WaiWeb = module.exports;\n})(${ROOT_EXPR});\n`; fs.writeFileSync(outPath, out); fs.writeFileSync(notesPath, JSON.stringify(notes, null, 2) + '\n'); console.log(`glue: ${JSON.stringify(notes)}`); } // ---- minify: esbuild, ES5 output --------------------------------------------------- function minify(inPath, outPath) { const esbuild = tool('esbuild'); const r = esbuild.transformSync(readText(inPath), { minify: true, target: 'es5', charset: 'ascii', legalComments: 'none', logLevel: 'error' }); fs.writeFileSync(outPath, r.code); } // ---- es5: each file parses as ECMAScript 5 ----------------------------------------- function es5(...files) { const acorn = tool('acorn'); for (const f of files) { try { acorn.parse(readText(f), { ecmaVersion: 5, sourceType: 'script', allowReserved: true }); } catch (e) { throw new Error(`${path.basename(f)} is not ECMAScript 5: ${e.message}`); } } console.log(`es5: ${files.length} files parse as ECMAScript 5`); } // ---- manifest ------------------------------------------------------------------------ const sha256 = (buf) => crypto.createHash('sha256').update(buf).digest('hex'); const sizes = (buf) => ({ sha256: sha256(buf), bytes: buf.length, gzip: zlib.gzipSync(buf, { level: 9 }).length, brotli: zlib.brotliCompressSync(buf, { params: { [zlib.constants.BROTLI_PARAM_QUALITY]: 11, [zlib.constants.BROTLI_PARAM_LGWIN]: 22 } }).length, }); function manifest(dist, inputDir, notesPath, toolsPath, outPath) { const RUNG = [ [/^wai_web_bg\.l1\.wasm$/, 'L1', 'full', 'the shipped binary, legalised at the JS interface'], [/^wai_web_bg\.l2\.(\w+)\.wasm$/, 'L2', null, 'MVP-lowered and legalised'], [/^wai_web\.reach\.js$/, 'L1, L2', 'full', 'runtime + i64 table + glue, for the wasm rungs'], [/^wai_web\.j\.(\w+)\.js$/, 'J', null, 'wasm2js bundle: runtime + i64 table + translation + glue'], [/^wai-load\.js$/, 'all', null, 'the loader'], ]; const files = {}; for (const f of fs.readdirSync(dist).sort()) { const st = fs.statSync(path.join(dist, f)); if (!st.isFile() || f === 'manifest.json') continue; const row = RUNG.find(([re]) => re.test(f)); if (!row) throw new Error(`${f} is not a reach artifact`); const [re, rung, profile, what] = row; files[f] = { rung, profile: profile ?? (f.match(re)[1] ?? null), what, ...sizes(read(path.join(dist, f))) }; } const input = { files: {}, source: process.env.WAI_REACH_SOURCE || null, commit: process.env.WAI_REACH_INPUT_COMMIT || null }; for (const f of ['wai_web_bg.wasm', 'wai_web.js']) input.files[f] = { sha256: sha256(read(path.join(inputDir, f))), bytes: read(path.join(inputDir, f)).length }; const m = { format: 'wai-reach-manifest/1', input, tools: JSON.parse(readText(toolsPath)), glue: JSON.parse(readText(notesPath)), outputs: files, }; fs.writeFileSync(outPath, JSON.stringify(m, null, 2) + '\n'); console.log(`manifest: ${Object.keys(files).length} artifacts`); } // ---- dispatch (last, so every step above is initialised) ---------------------------- const [cmd, ...args] = process.argv.slice(2); const commands = { sigs, graph, exports: checkExports, 'same-sigs': sameSigs, factory, glue, minify, es5, manifest }; if (!commands[cmd]) die(`unknown step '${cmd}'`); try { commands[cmd](...args); } catch (e) { die(`${cmd}: ${e.message}`); }