#!/usr/bin/env bash
# ============================================================================
# vendor.sh — put first letters' own runtime files (its worker, and Hugging
# Face's tokenizer library) on the app's own site.
#
#   third_party/hf-tokenizers/vendor.sh <site-dir> [cache-dir]
#
# Writes <site-dir>/vendor/hf-tokenizers-<version>-<revision>/ (and removes
# any other vendor/hf-tokenizers-* there):
#   ae-worker.js            this directory's worker.js, unchanged
#   tokenizers.min.mjs      @huggingface/tokenizers' browser build, from the
#                           npm package, unchanged
#   LICENSE.txt, NOTICE.txt this directory's licence (the package's) and notice
#
# First letters runs on onnxruntime-web, the listener's runtime: it is not
# vendored again here. third_party/onnxruntime-web/vendor.sh puts it on the
# site, and the app stores its JSPI build once for both.
#
# The app never imports these URLs directly. When someone downloads first
# letters, src/abbrev.rs stores these files (and the runtime's) in the model
# store (SHA-256 checked, kept across app updates), and src/bindings/abbrev.js
# runs them from there, in a worker, offline.
#
# The version and revision come from TOKENIZERS_VERSION and
# TOKENIZERS_VENDOR_REV in crates/aac-web/src/abbrev.rs, and each file's size
# and SHA-256 from RUNTIME_FILES there. This script fails loudly on any drift:
# another version asked for than pinned here, an npm tarball that is not the
# pinned one, a vendored file that differs from the Rust pin, or a directory
# whose bytes differ from VENDOR_DIGEST. Changed bytes need a new revision,
# because browsers and the model store treat the directory's URLs as
# immutable.
#
# Idempotent; the download is cached in cache-dir (default ~/.cache/aac-rs).
# Needs node (npm).
# ============================================================================
set -euo pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
AAC="$(cd "$HERE/../.." && pwd)"
SITE="${1:?usage: vendor.sh <site-dir> [cache-dir]}"
CACHE="${2:-${XDG_CACHE_HOME:-$HOME/.cache}/aac-rs}"

AE="$AAC/crates/aac-web/src/abbrev.rs"
TOK_VERSION="$(sed -n 's/^pub const TOKENIZERS_VERSION: &str = "\([^"]*\)";$/\1/p' "$AE")"
TOK_REV="$(sed -n 's/^pub const TOKENIZERS_VENDOR_REV: &str = "\([^"]*\)";$/\1/p' "$AE")"
[ -n "$TOK_VERSION" ] || { echo "✗ TOKENIZERS_VERSION not found in $AE" >&2; exit 1; }
[ -n "$TOK_REV" ] || { echo "✗ TOKENIZERS_VENDOR_REV not found in $AE" >&2; exit 1; }
DIR="hf-tokenizers-$TOK_VERSION-$TOK_REV"

# ─── Pins ────────────────────────────────────────────────────────────────────
# The npm release, by its registry integrity (npm view
# @huggingface/tokenizers@0.2.0 dist.integrity, read 2026-10-01; 0.2.0 is the
# latest release).
PINNED_VERSION="0.2.0"
NPM_INTEGRITY="sha512-LidMHe1FpcSYH4vcSjXooda34pC0M8m1gmDOv7SQi6Iv+ib1zX7J3sHBCC9/3FPCkn2k1b55FneVqMRxrr99pg=="
# The package's LICENSE (Apache-2.0), verbatim.
LICENSE_SHA256="c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4"
# What vendor/$VENDOR_PINNED holds, as a sha256 over its sorted file list and
# each file's sha256. Re-pin only together with a new TOKENIZERS_VENDOR_REV.
VENDOR_PINNED="hf-tokenizers-0.2.0-aac1"
VENDOR_DIGEST="11936747c9537a33132ae6834b78deed15f9669e9ad8c23a0428f0ecf7527102"

[ "$TOK_VERSION" = "$PINNED_VERSION" ] || {
  echo "✗ abbrev.rs asks for @huggingface/tokenizers $TOK_VERSION, but vendor.sh is pinned to $PINNED_VERSION —" >&2
  echo "  check the new release (licence, no network use, the same tokens as the model's tokenizer), then re-pin PINNED_VERSION and NPM_INTEGRITY" >&2
  exit 1
}
[ "$DIR" = "$VENDOR_PINNED" ] || {
  echo "✗ abbrev.rs asks for $DIR, but vendor.sh is pinned to $VENDOR_PINNED — vendor it, check it, and re-pin VENDOR_PINNED and VENDOR_DIGEST" >&2
  exit 1
}

sha256() { shasum -a 256 "$1" | cut -c1-64; }
fsize() { wc -c < "$1" | tr -d ' '; }

echo "▸ vendoring @huggingface/tokenizers $TOK_VERSION and first letters' worker ($DIR)"
mkdir -p "$CACHE"

# Tell (but do not fail) when npm has a newer release to review.
LATEST="$(npm view @huggingface/tokenizers dist-tags.latest 2>/dev/null || true)"
if [ -n "$LATEST" ] && [ "$LATEST" != "$TOK_VERSION" ]; then
  echo "  note: npm's latest @huggingface/tokenizers is $LATEST (pinned: $TOK_VERSION)"
fi

# The package, fetched and checked against its pinned integrity beside the
# cache entry, then moved in whole, so an interrupted run never leaves half a
# package that looks complete.
TOK_CACHE="$CACHE/hf-tokenizers-$TOK_VERSION"
if [ ! -f "$TOK_CACHE/package/package.json" ]; then
  tmp="$(mktemp -d "$CACHE/npm.XXXXXX")"
  (cd "$tmp" && npm pack --silent "@huggingface/tokenizers@$TOK_VERSION" >/dev/null)
  tgz="$(ls "$tmp"/*.tgz)"
  got="sha512-$(openssl dgst -sha512 -binary "$tgz" | base64 | tr -d '\n')"
  [ "$got" = "$NPM_INTEGRITY" ] || {
    rm -rf "$tmp"
    echo "✗ @huggingface/tokenizers@$TOK_VERSION from npm is not the pinned tarball (got $got)" >&2
    exit 1
  }
  (cd "$tmp" && tar xzf "$tgz")
  rm -f "$tgz"
  rm -rf "$TOK_CACHE"
  mv "$tmp" "$TOK_CACHE"
fi
grep -q "\"version\": \"$TOK_VERSION\"" "$TOK_CACHE/package/package.json" || { echo "✗ cached @huggingface/tokenizers is not $TOK_VERSION" >&2; exit 1; }
grep -q '"license": "Apache-2.0"' "$TOK_CACHE/package/package.json" || { echo "✗ @huggingface/tokenizers $TOK_VERSION no longer declares Apache-2.0 — review it" >&2; exit 1; }
[ "$(sha256 "$TOK_CACHE/package/LICENSE")" = "$LICENSE_SHA256" ] || { echo "✗ the package's LICENSE is not the one pinned here — review it" >&2; exit 1; }
[ "$(sha256 "$HERE/LICENSE.txt")" = "$LICENSE_SHA256" ] || { echo "✗ $HERE/LICENSE.txt is not the package's LICENSE" >&2; exit 1; }

rm -rf "$SITE"/vendor/hf-tokenizers-*
OUT="$SITE/vendor/$DIR"
mkdir -p "$OUT"
cp "$HERE/worker.js" "$OUT/ae-worker.js"
cp "$HERE/LICENSE.txt" "$HERE/NOTICE.txt" "$OUT/"
cp "$TOK_CACHE/package/dist/tokenizers.min.mjs" "$OUT/"
chmod 644 "$OUT"/*

# Neither may ever reach another site: no URL in either at all, and the
# library imports nothing.
if grep -nE 'https?://' "$OUT/ae-worker.js"; then
  echo "✗ first letters' worker names another site — review it" >&2; exit 1
fi
if grep -oE 'https?://[^"'"'"'` )]*' "$OUT/tokenizers.min.mjs"; then
  echo "✗ tokenizers.min.mjs names another site — review it" >&2; exit 1
fi
if grep -qE '(^|[;}])import[ ({"'"'"']|import\(' "$OUT/tokenizers.min.mjs"; then
  echo "✗ tokenizers.min.mjs imports another module — review it" >&2; exit 1
fi

# Each file must be exactly what abbrev.rs pins (RUNTIME_FILES): the app
# downloads it by that size and SHA-256.
PINS="$(sed -n 's/^ *RuntimeFile { name: "\([^"]*\)", bytes: \([0-9_]*\), sha256: "\([0-9a-f]*\)" },$/\1 \2 \3/p' "$AE" | awk '{ gsub(/_/, "", $2); print }')"
[ "$(printf '%s\n' "$PINS" | grep -c .)" = 2 ] || { echo "✗ RUNTIME_FILES in $AE should pin 2 files" >&2; exit 1; }
bad=0
while read -r name bytes want; do
  f="$OUT/$name"
  [ -s "$f" ] || { echo "✗ vendored runtime is missing $name" >&2; exit 1; }
  got_bytes="$(fsize "$f")"; got="$(sha256 "$f")"
  if [ "$got_bytes" != "$bytes" ] || [ "$got" != "$want" ]; then
    echo "✗ vendor/$DIR/$name is $got_bytes bytes, sha256 $got;" >&2
    echo "  abbrev.rs RUNTIME_FILES pins $bytes bytes, sha256 $want." >&2
    bad=1
  fi
done <<< "$PINS"
[ "$bad" = 0 ] || { echo "  Changed bytes need a new TOKENIZERS_VENDOR_REV (a new URL), then new pins there and here." >&2; exit 1; }

DIGEST="$(cd "$OUT" && find . -type f | LC_ALL=C sort | while IFS= read -r f; do printf '%s %s\n' "$f" "$(sha256 "$f")"; done | shasum -a 256 | cut -c1-64)"
[ "$DIGEST" = "$VENDOR_DIGEST" ] || {
  echo "✗ vendor/$DIR differs from what was pinned (got $DIGEST)." >&2
  echo "  Its URLs are cached as immutable: bump TOKENIZERS_VENDOR_REV in abbrev.rs," >&2
  echo "  then set VENDOR_PINNED and VENDOR_DIGEST in $HERE/vendor.sh." >&2
  exit 1
}
echo "✓ vendor/$DIR ($(du -sh "$OUT" | cut -f1))"