#!/usr/bin/env bash # The reach builds of wai-web, from one wai-web package (wai_web.js + wai_web_bg.wasm, # the wasm-bindgen web target). # # bash wai/wai-web/reach/build.sh [] # # Writes into (default wai/wai-web/reach/dist, which git ignores): # # wai_web_bg.l1.wasm L1 the shipped binary legalised at the JS interface # wai_web_bg.l2..wasm L2 MVP-lowered and legalised; full, verify_decode, # verify and receipt # wai_web.reach.js ES5 runtime + i64 table + glue, for L1 and L2 # wai_web.j..js J wasm2js translation in one ES5 script per profile # wai-load.js the loader # manifest.json every file's SHA-256 and sizes, the input's SHA-256, # the tool pins and the glue changes # # Reproducibility. The tools are pinned: binaryen by release and by the SHA-256 of each # platform's archive (checked before use), and TypeScript, esbuild, acorn and binaryen's # own WebAssembly build by package-lock.json (npm ci checks their integrity hashes). J's # wasm2js is that WebAssembly build, run on Node: the native wasm2js of the x86_64 Linux # archive gives the translation's locals a different allocation than the macOS archive # does, while the WebAssembly build gives the same output on every host (and equals the # macOS archive's). The same package gives the same bytes on every platform below. When the package is the one reach/manifest.json # names as its input, the build is refused unless every output's SHA-256 equals the # manifest's; WAI_REACH_RECORD=1 writes the new manifest there instead. # # Environment: # WAI_REACH_TOOLS where binaryen and the JS tools are installed (default # wai/wai-web/reach/.tools, which git ignores) # WAI_REACH_RECORD 1: write /manifest.json to reach/manifest.json # WAI_REACH_REQUIRE_RECORD # 1: refuse unless the package is the input reach/manifest.json names, # so the outputs are always checked against the manifest (CI's leg on # the package of record) # WAI_REACH_SOURCE a line saying where the package came from, kept in the manifest # WAI_REACH_INPUT_COMMIT # the open-standards commit the package was built from, kept in the # manifest as input.commit # # Needs bash, curl, tar, node >= 18 and npm. Platforms: macOS arm64 and x86_64, Linux # x86_64 and aarch64. set -euo pipefail if [ $# -lt 1 ] || [ $# -gt 2 ]; then echo "usage: bash build.sh []" >&2 exit 2 fi REACH="$(cd "$(dirname "$0")" && pwd -P)" PKG="$(cd "$1" && pwd -P)" DIST="${2:-$REACH/dist}" mkdir -p "$DIST" DIST="$(cd "$DIST" && pwd -P)" TOOLS="${WAI_REACH_TOOLS:-$REACH/.tools}" mkdir -p "$TOOLS" TOOLS="$(cd "$TOOLS" && pwd -P)" WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT for f in wai_web_bg.wasm wai_web.js; do [ -f "$PKG/$f" ] || { echo "reach: $PKG has no $f" >&2; exit 1; } done for t in curl tar node npm; do command -v "$t" >/dev/null || { echo "reach: needs $t" >&2; exit 1; } done NODE_MAJOR="$(node -p 'process.versions.node.split(".")[0]')" [ "$NODE_MAJOR" -ge 18 ] || { echo "reach: the build needs node >= 18 (this is $(node --version))" >&2; exit 1; } sha256() { if command -v sha256sum >/dev/null; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi; } # ---- binaryen, pinned ------------------------------------------------------------- BINARYEN=version_123 case "$(uname -s)-$(uname -m)" in Darwin-arm64) PLAT=arm64-macos; BIN_SHA=74428be348c1a09863e7b642a1fa948cabf8ec9561052233d8288e941951725b ;; Darwin-x86_64) PLAT=x86_64-macos; BIN_SHA=cc18b14d2b673d9c66bf54f31ff2b0ceb23ba5132455b893965ae2792f9e00dd ;; Linux-x86_64) PLAT=x86_64-linux; BIN_SHA=e959f2170af4c20c552e9de3a0253704d6a9d2766e8fdb88e4d6ac4bae9388fe ;; Linux-aarch64) PLAT=aarch64-linux; BIN_SHA=4b6bd61ba6cd3b18c993b4657d93426c782f9b91b74be0d38018cd8be1319376 ;; *) echo "reach: no pinned binaryen archive for $(uname -s)-$(uname -m)" >&2; exit 1 ;; esac ARCHIVE="binaryen-$BINARYEN-$PLAT.tar.gz" B="$TOOLS/binaryen-$BINARYEN-$PLAT/bin" if [ ! -x "$B/wasm-opt" ] || [ "$(cat "$TOOLS/binaryen-$BINARYEN-$PLAT/.sha256" 2>/dev/null)" != "$BIN_SHA" ]; then rm -rf "$TOOLS/binaryen-$BINARYEN-$PLAT" "$TOOLS/$ARCHIVE" curl -fsSL --retry 3 -o "$TOOLS/$ARCHIVE" \ "https://github.com/WebAssembly/binaryen/releases/download/$BINARYEN/$ARCHIVE" # allow-name: the binaryen project's release download got="$(sha256 "$TOOLS/$ARCHIVE")" if [ "$got" != "$BIN_SHA" ]; then echo "reach: $ARCHIVE has SHA-256 $got, not the pinned $BIN_SHA" >&2 rm -f "$TOOLS/$ARCHIVE" exit 1 fi mkdir -p "$TOOLS/binaryen-$BINARYEN-$PLAT" tar -xzf "$TOOLS/$ARCHIVE" -C "$TOOLS/binaryen-$BINARYEN-$PLAT" --strip-components=1 rm -f "$TOOLS/$ARCHIVE" echo "$BIN_SHA" > "$TOOLS/binaryen-$BINARYEN-$PLAT/.sha256" fi "$B/wasm-opt" --version | grep -q "($BINARYEN)" || { echo "reach: $B/wasm-opt is not binaryen $BINARYEN" >&2; exit 1; } # ---- TypeScript, esbuild, acorn, pinned by package-lock.json ------------------------ LOCK_SHA="$(sha256 "$REACH/package-lock.json")" if [ "$(cat "$TOOLS/js/.lock-sha256" 2>/dev/null)" != "$LOCK_SHA" ] || [ ! -d "$TOOLS/js/node_modules" ]; then rm -rf "$TOOLS/js" mkdir -p "$TOOLS/js" cp "$REACH/package.json" "$REACH/package-lock.json" "$TOOLS/js/" (cd "$TOOLS/js" && npm ci --ignore-scripts --no-audit --no-fund --loglevel=error) echo "$LOCK_SHA" > "$TOOLS/js/.lock-sha256" fi export WAI_REACH_NODE_MODULES="$TOOLS/js/node_modules" CLI=(node "$REACH/tools/cli.mjs") # binaryen's wasm2js compiled to WebAssembly, pinned by package-lock.json: host-independent. W2J=(node "$TOOLS/js/node_modules/binaryen/bin/wasm2js") "${W2J[@]}" --version | grep -q "($BINARYEN)" || { echo "reach: the pinned WebAssembly wasm2js is not binaryen $BINARYEN" >&2; exit 1; } IN="$PKG/wai_web_bg.wasm" FEATURES=(--enable-bulk-memory --enable-nontrapping-float-to-int --enable-sign-ext) LOWER=(--signext-lowering --llvm-nontrapping-fptoint-lowering --llvm-memory-copy-fill-lowering) PROFILES=(verify_decode verify receipt) rm -f "$DIST"/wai_web_bg.l1.wasm "$DIST"/wai_web_bg.l2.*.wasm "$DIST"/wai_web.reach.js "$DIST"/wai_web.j.*.js "$DIST"/wai-load.js "$DIST"/manifest.json # The input must be exactly the shipped build's feature set: MVP + sign-extension, # non-trapping float-to-int and bulk memory. A module using anything more (multi-value, # reference types, SIMD, ...) fails here, before any rung is built from it. if ! "$B/wasm-opt" "$IN" --mvp-features "${FEATURES[@]}" -o "$WORK/shipped.check.wasm" 2> "$WORK/check.err"; then cat "$WORK/check.err" >&2 echo "reach: $IN uses a feature beyond MVP, sign-extension, non-trapping float-to-int and bulk memory" >&2 exit 1 fi # ---- L1: legalised at the JS interface, nothing lowered ------------------------------ "$B/wasm-opt" "$IN" --mvp-features "${FEATURES[@]}" --legalize-js-interface -o "$DIST/wai_web_bg.l1.wasm" # ---- L2: MVP-lowered, then legalised ------------------------------------------------- # The lowering runs with the post-MVP features enabled for reading; reading the result # back with MVP features only is the check that nothing post-MVP is left. "$B/wasm-opt" "$IN" --mvp-features "${FEATURES[@]}" "${LOWER[@]}" -o "$WORK/mvp.full.wasm" "$B/wasm-opt" "$WORK/mvp.full.wasm" --mvp-features --legalize-js-interface -o "$DIST/wai_web_bg.l2.full.wasm" for p in "${PROFILES[@]}"; do "${CLI[@]}" graph "$WORK/mvp.full.wasm" "$REACH/profiles.json" "$p" "$WORK/graph.$p.json" "$B/wasm-metadce" "$WORK/mvp.full.wasm" --mvp-features --graph-file "$WORK/graph.$p.json" -o "$WORK/mvp.$p.dce.wasm" > /dev/null "$B/wasm-opt" "$WORK/mvp.$p.dce.wasm" --mvp-features -Oz -ocimfs=200 -o "$WORK/mvp.$p.wasm" "${CLI[@]}" exports "$WORK/mvp.$p.wasm" "$WORK/graph.$p.json" "$B/wasm-opt" "$WORK/mvp.$p.wasm" --mvp-features --legalize-js-interface -o "$DIST/wai_web_bg.l2.$p.wasm" done # ---- the i64 table, the runtime and the glue ---------------------------------------- "${CLI[@]}" same-sigs "$IN" "$WORK/mvp.full.wasm" "${CLI[@]}" sigs "$IN" "$WORK/sigs.js" "${CLI[@]}" minify "$REACH/src/wai-reach-runtime.js" "$WORK/runtime.min.js" "${CLI[@]}" glue "$PKG/wai_web.js" "$WORK/glue.js" "$WORK/glue-notes.json" "${CLI[@]}" minify "$WORK/glue.js" "$WORK/glue.min.js" cat "$WORK/runtime.min.js" "$WORK/sigs.js" "$WORK/glue.min.js" > "$DIST/wai_web.reach.js" # ---- J: wasm2js on the MVP-lowered modules ------------------------------------------- # The translations run side by side (each is single-threaded), then in order. pids=() for p in full "${PROFILES[@]}"; do "${W2J[@]}" "$WORK/mvp.$p.wasm" -Oz -ocimfs=200 -o "$WORK/j.$p.mjs" & pids+=($!) done for pid in "${pids[@]}"; do wait "$pid" || { echo "reach: wasm2js failed" >&2; exit 1; }; done for p in full "${PROFILES[@]}"; do "${CLI[@]}" factory "$WORK/j.$p.mjs" "$WORK/j.$p.factory.js" "${CLI[@]}" minify "$WORK/j.$p.factory.js" "$WORK/j.$p.factory.min.js" cat "$WORK/runtime.min.js" "$WORK/sigs.js" "$WORK/j.$p.factory.min.js" "$WORK/glue.min.js" > "$DIST/wai_web.j.$p.js" done cp "$REACH/src/wai-load.js" "$DIST/wai-load.js" "${CLI[@]}" es5 "$DIST/wai-load.js" "$DIST/wai_web.reach.js" "$DIST"/wai_web.j.*.js # ---- manifest -------------------------------------------------------------------------- cat > "$WORK/tools.json" < JSON.parse(require("fs").readFileSync(f, "utf8"))); process.exit(JSON.stringify(rec.input.files) === JSON.stringify(out.input.files) ? 0 : 1);' "$REC" "$DIST/manifest.json"; then node -e ' const [rec, out] = process.argv.slice(1).map((f) => JSON.parse(require("fs").readFileSync(f, "utf8"))); const names = [...new Set([...Object.keys(rec.outputs), ...Object.keys(out.outputs)])].sort(); let bad = 0; for (const n of names) { const a = rec.outputs[n], b = out.outputs[n]; if (!a || !b || a.sha256 !== b.sha256 || a.bytes !== b.bytes) { bad++; console.error(`reach: ${n}: ${b ? b.sha256 : "missing"}, recorded ${a ? a.sha256 : "nothing"}`); } } if (bad) { console.error(`reach: ${bad} outputs differ from reach/manifest.json`); process.exit(1); } console.log(`reach: all ${names.length} outputs equal reach/manifest.json`);' "$REC" "$DIST/manifest.json" elif [ "${WAI_REACH_REQUIRE_RECORD:-}" = 1 ]; then echo "reach: WAI_REACH_REQUIRE_RECORD=1, but the package is not the input reach/manifest.json names" >&2 exit 1 else echo "reach: the package is not the recorded input; outputs are in $DIST/manifest.json only" fi