open-standards/wai/wai-web/reach/build.sh

#!/usr/bin/env bash
# The reach builds of wai-web, from one wai-web package (wai_web.js + wai_web_bg.wasm,
# the wasm-bindgen web target).
#
#   bash wai/wai-web/reach/build.sh <package-dir> [<dist-dir>]
#
# Writes into <dist-dir> (default wai/wai-web/reach/dist, which git ignores):
#
#   wai_web_bg.l1.wasm             L1  the shipped binary legalised at the JS interface
#   wai_web_bg.l2.<profile>.wasm   L2  MVP-lowered and legalised; full, verify_decode,
#                                      verify and receipt
#   wai_web.reach.js                   ES5 runtime + i64 table + glue, for L1 and L2
#   wai_web.j.<profile>.js         J   wasm2js translation in one ES5 script per profile
#   wai-load.js                        the loader
#   manifest.json                      every file's SHA-256 and sizes, the input's SHA-256,
#                                      the tool pins and the glue changes
#
# Reproducibility. The tools are pinned: binaryen by release and by the SHA-256 of each
# platform's archive (checked before use), and TypeScript, esbuild, acorn and binaryen's
# own WebAssembly build by package-lock.json (npm ci checks their integrity hashes). J's
# wasm2js is that WebAssembly build, run on Node: the native wasm2js of the x86_64 Linux
# archive gives the translation's locals a different allocation than the macOS archive
# does, while the WebAssembly build gives the same output on every host (and equals the
# macOS archive's). The same package gives the same bytes on every platform below. When the package is the one reach/manifest.json
# names as its input, the build is refused unless every output's SHA-256 equals the
# manifest's; WAI_REACH_RECORD=1 writes the new manifest there instead.
#
# Environment:
#   WAI_REACH_TOOLS   where binaryen and the JS tools are installed (default
#                     wai/wai-web/reach/.tools, which git ignores)
#   WAI_REACH_RECORD  1: write <dist-dir>/manifest.json to reach/manifest.json
#   WAI_REACH_REQUIRE_RECORD
#                     1: refuse unless the package is the input reach/manifest.json names,
#                     so the outputs are always checked against the manifest (CI's leg on
#                     the package of record)
#   WAI_REACH_SOURCE  a line saying where the package came from, kept in the manifest
#   WAI_REACH_INPUT_COMMIT
#                     the open-standards commit the package was built from, kept in the
#                     manifest as input.commit
#
# Needs bash, curl, tar, node >= 18 and npm. Platforms: macOS arm64 and x86_64, Linux
# x86_64 and aarch64.
set -euo pipefail

if [ $# -lt 1 ] || [ $# -gt 2 ]; then
  echo "usage: bash build.sh <package-dir> [<dist-dir>]" >&2
  exit 2
fi
REACH="$(cd "$(dirname "$0")" && pwd -P)"
PKG="$(cd "$1" && pwd -P)"
DIST="${2:-$REACH/dist}"
mkdir -p "$DIST"
DIST="$(cd "$DIST" && pwd -P)"
TOOLS="${WAI_REACH_TOOLS:-$REACH/.tools}"
mkdir -p "$TOOLS"
TOOLS="$(cd "$TOOLS" && pwd -P)"
WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT

for f in wai_web_bg.wasm wai_web.js; do
  [ -f "$PKG/$f" ] || { echo "reach: $PKG has no $f" >&2; exit 1; }
done
for t in curl tar node npm; do
  command -v "$t" >/dev/null || { echo "reach: needs $t" >&2; exit 1; }
done
NODE_MAJOR="$(node -p 'process.versions.node.split(".")[0]')"
[ "$NODE_MAJOR" -ge 18 ] || { echo "reach: the build needs node >= 18 (this is $(node --version))" >&2; exit 1; }
sha256() { if command -v sha256sum >/dev/null; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi; }

# ---- binaryen, pinned -------------------------------------------------------------
BINARYEN=version_123
case "$(uname -s)-$(uname -m)" in
  Darwin-arm64)   PLAT=arm64-macos;   BIN_SHA=74428be348c1a09863e7b642a1fa948cabf8ec9561052233d8288e941951725b ;;
  Darwin-x86_64)  PLAT=x86_64-macos;  BIN_SHA=cc18b14d2b673d9c66bf54f31ff2b0ceb23ba5132455b893965ae2792f9e00dd ;;
  Linux-x86_64)   PLAT=x86_64-linux;  BIN_SHA=e959f2170af4c20c552e9de3a0253704d6a9d2766e8fdb88e4d6ac4bae9388fe ;;
  Linux-aarch64)  PLAT=aarch64-linux; BIN_SHA=4b6bd61ba6cd3b18c993b4657d93426c782f9b91b74be0d38018cd8be1319376 ;;
  *) echo "reach: no pinned binaryen archive for $(uname -s)-$(uname -m)" >&2; exit 1 ;;
esac
ARCHIVE="binaryen-$BINARYEN-$PLAT.tar.gz"
B="$TOOLS/binaryen-$BINARYEN-$PLAT/bin"
if [ ! -x "$B/wasm-opt" ] || [ "$(cat "$TOOLS/binaryen-$BINARYEN-$PLAT/.sha256" 2>/dev/null)" != "$BIN_SHA" ]; then
  rm -rf "$TOOLS/binaryen-$BINARYEN-$PLAT" "$TOOLS/$ARCHIVE"
  curl -fsSL --retry 3 -o "$TOOLS/$ARCHIVE" \
    "https://github.com/WebAssembly/binaryen/releases/download/$BINARYEN/$ARCHIVE" # allow-name: the binaryen project's release download
  got="$(sha256 "$TOOLS/$ARCHIVE")"
  if [ "$got" != "$BIN_SHA" ]; then
    echo "reach: $ARCHIVE has SHA-256 $got, not the pinned $BIN_SHA" >&2
    rm -f "$TOOLS/$ARCHIVE"
    exit 1
  fi
  mkdir -p "$TOOLS/binaryen-$BINARYEN-$PLAT"
  tar -xzf "$TOOLS/$ARCHIVE" -C "$TOOLS/binaryen-$BINARYEN-$PLAT" --strip-components=1
  rm -f "$TOOLS/$ARCHIVE"
  echo "$BIN_SHA" > "$TOOLS/binaryen-$BINARYEN-$PLAT/.sha256"
fi
"$B/wasm-opt" --version | grep -q "($BINARYEN)" || { echo "reach: $B/wasm-opt is not binaryen $BINARYEN" >&2; exit 1; }

# ---- TypeScript, esbuild, acorn, pinned by package-lock.json ------------------------
LOCK_SHA="$(sha256 "$REACH/package-lock.json")"
if [ "$(cat "$TOOLS/js/.lock-sha256" 2>/dev/null)" != "$LOCK_SHA" ] || [ ! -d "$TOOLS/js/node_modules" ]; then
  rm -rf "$TOOLS/js"
  mkdir -p "$TOOLS/js"
  cp "$REACH/package.json" "$REACH/package-lock.json" "$TOOLS/js/"
  (cd "$TOOLS/js" && npm ci --ignore-scripts --no-audit --no-fund --loglevel=error)
  echo "$LOCK_SHA" > "$TOOLS/js/.lock-sha256"
fi
export WAI_REACH_NODE_MODULES="$TOOLS/js/node_modules"
CLI=(node "$REACH/tools/cli.mjs")
# binaryen's wasm2js compiled to WebAssembly, pinned by package-lock.json: host-independent.
W2J=(node "$TOOLS/js/node_modules/binaryen/bin/wasm2js")
"${W2J[@]}" --version | grep -q "($BINARYEN)" || { echo "reach: the pinned WebAssembly wasm2js is not binaryen $BINARYEN" >&2; exit 1; }

IN="$PKG/wai_web_bg.wasm"
FEATURES=(--enable-bulk-memory --enable-nontrapping-float-to-int --enable-sign-ext)
LOWER=(--signext-lowering --llvm-nontrapping-fptoint-lowering --llvm-memory-copy-fill-lowering)
PROFILES=(verify_decode verify receipt)
rm -f "$DIST"/wai_web_bg.l1.wasm "$DIST"/wai_web_bg.l2.*.wasm "$DIST"/wai_web.reach.js "$DIST"/wai_web.j.*.js "$DIST"/wai-load.js "$DIST"/manifest.json

# The input must be exactly the shipped build's feature set: MVP + sign-extension,
# non-trapping float-to-int and bulk memory. A module using anything more (multi-value,
# reference types, SIMD, ...) fails here, before any rung is built from it.
if ! "$B/wasm-opt" "$IN" --mvp-features "${FEATURES[@]}" -o "$WORK/shipped.check.wasm" 2> "$WORK/check.err"; then
  cat "$WORK/check.err" >&2
  echo "reach: $IN uses a feature beyond MVP, sign-extension, non-trapping float-to-int and bulk memory" >&2
  exit 1
fi

# ---- L1: legalised at the JS interface, nothing lowered ------------------------------
"$B/wasm-opt" "$IN" --mvp-features "${FEATURES[@]}" --legalize-js-interface -o "$DIST/wai_web_bg.l1.wasm"

# ---- L2: MVP-lowered, then legalised -------------------------------------------------
# The lowering runs with the post-MVP features enabled for reading; reading the result
# back with MVP features only is the check that nothing post-MVP is left.
"$B/wasm-opt" "$IN" --mvp-features "${FEATURES[@]}" "${LOWER[@]}" -o "$WORK/mvp.full.wasm"
"$B/wasm-opt" "$WORK/mvp.full.wasm" --mvp-features --legalize-js-interface -o "$DIST/wai_web_bg.l2.full.wasm"
for p in "${PROFILES[@]}"; do
  "${CLI[@]}" graph "$WORK/mvp.full.wasm" "$REACH/profiles.json" "$p" "$WORK/graph.$p.json"
  "$B/wasm-metadce" "$WORK/mvp.full.wasm" --mvp-features --graph-file "$WORK/graph.$p.json" -o "$WORK/mvp.$p.dce.wasm" > /dev/null
  "$B/wasm-opt" "$WORK/mvp.$p.dce.wasm" --mvp-features -Oz -ocimfs=200 -o "$WORK/mvp.$p.wasm"
  "${CLI[@]}" exports "$WORK/mvp.$p.wasm" "$WORK/graph.$p.json"
  "$B/wasm-opt" "$WORK/mvp.$p.wasm" --mvp-features --legalize-js-interface -o "$DIST/wai_web_bg.l2.$p.wasm"
done

# ---- the i64 table, the runtime and the glue ----------------------------------------
"${CLI[@]}" same-sigs "$IN" "$WORK/mvp.full.wasm"
"${CLI[@]}" sigs "$IN" "$WORK/sigs.js"
"${CLI[@]}" minify "$REACH/src/wai-reach-runtime.js" "$WORK/runtime.min.js"
"${CLI[@]}" glue "$PKG/wai_web.js" "$WORK/glue.js" "$WORK/glue-notes.json"
"${CLI[@]}" minify "$WORK/glue.js" "$WORK/glue.min.js"
cat "$WORK/runtime.min.js" "$WORK/sigs.js" "$WORK/glue.min.js" > "$DIST/wai_web.reach.js"

# ---- J: wasm2js on the MVP-lowered modules -------------------------------------------
# The translations run side by side (each is single-threaded), then in order.
pids=()
for p in full "${PROFILES[@]}"; do
  "${W2J[@]}" "$WORK/mvp.$p.wasm" -Oz -ocimfs=200 -o "$WORK/j.$p.mjs" & pids+=($!)
done
for pid in "${pids[@]}"; do wait "$pid" || { echo "reach: wasm2js failed" >&2; exit 1; }; done
for p in full "${PROFILES[@]}"; do
  "${CLI[@]}" factory "$WORK/j.$p.mjs" "$WORK/j.$p.factory.js"
  "${CLI[@]}" minify "$WORK/j.$p.factory.js" "$WORK/j.$p.factory.min.js"
  cat "$WORK/runtime.min.js" "$WORK/sigs.js" "$WORK/j.$p.factory.min.js" "$WORK/glue.min.js" > "$DIST/wai_web.j.$p.js"
done

cp "$REACH/src/wai-load.js" "$DIST/wai-load.js"
"${CLI[@]}" es5 "$DIST/wai-load.js" "$DIST/wai_web.reach.js" "$DIST"/wai_web.j.*.js

# ---- manifest --------------------------------------------------------------------------
cat > "$WORK/tools.json" <<EOF
{
  "binaryen": { "release": "$BINARYEN", "archives_sha256": {
    "arm64-macos": "74428be348c1a09863e7b642a1fa948cabf8ec9561052233d8288e941951725b",
    "x86_64-macos": "cc18b14d2b673d9c66bf54f31ff2b0ceb23ba5132455b893965ae2792f9e00dd",
    "x86_64-linux": "e959f2170af4c20c552e9de3a0253704d6a9d2766e8fdb88e4d6ac4bae9388fe",
    "aarch64-linux": "4b6bd61ba6cd3b18c993b4657d93426c782f9b91b74be0d38018cd8be1319376" } },
  "package_lock_sha256": "$LOCK_SHA",
  "l1": "wasm-opt --mvp-features ${FEATURES[*]} --legalize-js-interface",
  "l2": "wasm-opt --mvp-features ${FEATURES[*]} ${LOWER[*]}, then wasm-opt --mvp-features --legalize-js-interface; reduced profiles: wasm-metadce with the profile's exports, wasm-opt -Oz -ocimfs=200, then legalised",
  "j": "wasm2js -Oz -ocimfs=200 on the MVP-lowered module of each profile, run from binaryen's WebAssembly build of the same release (npm binaryen, pinned by package-lock.json)"
}
EOF
"${CLI[@]}" manifest "$DIST" "$PKG" "$WORK/glue-notes.json" "$WORK/tools.json" "$DIST/manifest.json"

# ---- the recorded build --------------------------------------------------------------
REC="$REACH/manifest.json"
if [ "${WAI_REACH_RECORD:-}" = 1 ]; then
  cp "$DIST/manifest.json" "$REC"
  echo "reach: recorded $REC"
elif [ -f "$REC" ] && node -e '
  const [rec, out] = process.argv.slice(1).map((f) => JSON.parse(require("fs").readFileSync(f, "utf8")));
  process.exit(JSON.stringify(rec.input.files) === JSON.stringify(out.input.files) ? 0 : 1);' "$REC" "$DIST/manifest.json"; then
  node -e '
    const [rec, out] = process.argv.slice(1).map((f) => JSON.parse(require("fs").readFileSync(f, "utf8")));
    const names = [...new Set([...Object.keys(rec.outputs), ...Object.keys(out.outputs)])].sort();
    let bad = 0;
    for (const n of names) {
      const a = rec.outputs[n], b = out.outputs[n];
      if (!a || !b || a.sha256 !== b.sha256 || a.bytes !== b.bytes) { bad++; console.error(`reach: ${n}: ${b ? b.sha256 : "missing"}, recorded ${a ? a.sha256 : "nothing"}`); }
    }
    if (bad) { console.error(`reach: ${bad} outputs differ from reach/manifest.json`); process.exit(1); }
    console.log(`reach: all ${names.length} outputs equal reach/manifest.json`);' "$REC" "$DIST/manifest.json"
elif [ "${WAI_REACH_REQUIRE_RECORD:-}" = 1 ]; then
  echo "reach: WAI_REACH_REQUIRE_RECORD=1, but the package is not the input reach/manifest.json names" >&2
  exit 1
else
  echo "reach: the package is not the recorded input; outputs are in $DIST/manifest.json only"
fi