open-standards/joule-code/joule-code-rs/crates/joule-moq-web/web/binding/test.js

// WAI browser-sink (extensions/browser-sink.md) in a real page, for the end-to-end scenario
// `binding`. Everything the test reads is on `window.bindingTest`; `done` is set last.
//
// Query parameters: corpus=<url> (live-conformance/browser/vectors.json), envelope=<url>
// (its int_motion.wai), digests=<url> (wai-rs/tests/vectors/int_video/sha256.json).

import initWai, * as wai from "../pkg/wai_live_web.js";
import initDecode, * as dec from "../pkg/wai_live_web_decode.js";
import { SideTable, runSequence, buildReport, stateOf, metadataProbe, installFramePatches } from "../room/binding.js";

const q = new URLSearchParams(location.search);
const out = { done: false, errors: [], corpus: null, wasm_route: null, life: null, native: null, metadata: null, report: null };
window.bindingTest = out;
const lines = [];
const say = (m) => { lines.push(m); document.getElementById("out").textContent = lines.join("\n"); };
const hex = (b) => Array.from(new Uint8Array(b), (x) => x.toString(16).padStart(2, "0")).join("");
const unhex = (h) => new Uint8Array((h.match(/../g) || []).map((x) => parseInt(x, 16)));
const sha256 = async (b) => hex(await crypto.subtle.digest("SHA-256", b));
const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);

installFramePatches();

// ------------------------------------------------------------- the corpus through the page

function corpusInPage(v) {
  const fails = [];
  let n = 0;
  for (const c of v.routes) {
    n++;
    const got = JSON.parse(wai.routeReport(JSON.stringify(c.report)));
    if (got.outcome !== c.outcome || (c.outcome === "routed" ? !same(got.table, c.table) : got.reason !== c.reason)) fails.push(`route ${c.name}`);
  }
  for (const c of v.choices || []) {
    n++;
    const got = JSON.parse(wai.chooseReport(JSON.stringify(c.report)));
    if (got.outcome !== c.outcome || (c.outcome === "chosen" ? !same(got.choices, c.choices) : got.reason !== c.reason)) fails.push(`choice ${c.name}`);
  }
  for (const c of v.constraints) {
    n++;
    const got = JSON.parse(wai.evaluateConstraints(JSON.stringify(c.sets), JSON.stringify(c.params)));
    if (got.outcome !== c.outcome || got.reason !== c.reason) fails.push(`constraints ${c.name}`);
  }
  for (const c of v.bindings) {
    n++;
    const got = JSON.parse(wai.readBinding(JSON.stringify(c.value)));
    if (got.outcome !== c.outcome || (got.canonical || null) !== c.canonical || (got.reason || null) !== c.reason) fails.push(`binding ${c.name}`);
  }
  for (const c of v.frame_tiers) {
    n++;
    if (wai.frameTier(c.route, c.route_tier, c.format_null, c.altered) !== c.tier) fails.push(`frame tier ${JSON.stringify(c)}`);
  }
  for (const c of v.frames) {
    n++;
    const got = JSON.parse(wai.frameState(JSON.stringify(c.binding), c.object, c.chain, c.provenance));
    if (!same(got, c.result)) fails.push(`frame ${c.name}`);
  }
  // The side table is this page's own JavaScript (web/room/binding.js), held to the sequences.
  const canonicalOf = (b) => JSON.parse(wai.readBinding(JSON.stringify(b))).canonical;
  for (const c of v.tables) {
    n++;
    let got;
    try { got = runSequence(c.bound, c.ops, canonicalOf); } catch (e) { got = { error: String(e) }; }
    // Compared member by member: the order of frames in an object is not part of the result.
    const norm = (r) => JSON.stringify({ ...r, frames: Object.fromEntries(Object.entries(r.frames || {}).sort()) });
    if (norm(got) !== norm(c.result)) fails.push(`table ${c.name}: ${JSON.stringify(got)}`);
  }
  return { cases: n, fails };
}

// ------------------------------------------- the wasm route: verified, decoded, every frame bound

async function frameRgbDigest(f) {
  const buf = new Uint8Array(f.allocationSize());
  await f.copyTo(buf);
  const rgb = new Uint8Array((buf.length / 4) * 3);
  for (let i = 0, j = 0; i < buf.length; i += 4, j += 3) { rgb[j] = buf[i]; rgb[j + 1] = buf[i + 1]; rgb[j + 2] = buf[i + 2]; }
  return sha256(rgb);
}

async function wasmRoute(v, envelopeFile, digests) {
  const table = new SideTable();
  const want = digests.int_video.frames_sha256;
  const objects = [];
  const kept = []; // bound frames kept for the life-cycle tests
  let t = 0;
  for (const [gi, g] of v.chain.groups.entries()) {
    for (const o of g.objects) {
      const env = typeof o.envelope === "string" ? unhex(o.envelope) : envelopeFile;
      const clip = dec.decodeBound(env, JSON.stringify(g.receipt), JSON.stringify(o.receipt));
      const r = { name: o.name, outcome: o.outcome, decoded: clip.nFrames, binding: clip.binding || null, reason: clip.reason || null, frames: [] };
      for (let i = 0; i < clip.nFrames; i++) {
        const f = new VideoFrame(clip.rgbx(i), { format: "RGBX", codedWidth: clip.width, codedHeight: clip.height, timestamp: t++ * 40000 });
        table.attach(f, clip.binding ? { binding: clip.binding, group: g.receipt.group_id } : null);
        const e = SideTable.of(f);
        const digest = await frameRgbDigest(f);
        // The state of this frame were it on screen: the chain anchors at the first group a
        // verifier holds and links the next (jwp-receipts §3 step 3).
        const s = stateOf(wai, e, gi === 0 ? "anchor" : "linked", "not_applicable");
        r.frames.push({ format: f.format, bound: e ? e.binding : null, digest_ok: digest === want[i], state: s.state, tier: s.tier });
        if (i === 0 && o.name === "int_motion") {
          const ctx = document.getElementById("frame").getContext("2d");
          ctx.drawImage(f, 0, 0);
        }
        if (o.name === "int_motion" && kept.length < 3) kept.push(f); else f.close();
      }
      clip.free();
      objects.push(r);
    }
  }
  // Decoded with no receipts: frames, none of them bound.
  const bare = dec.decodeBound(envelopeFile, "", "");
  const unverified = { decoded: bare.nFrames, binding: bare.binding || null, reason: bare.reason || null };
  bare.free();
  return { objects, unverified, kept };
}

// ------------------------------------------------- the binding through the frame's life (§8)

function postToWorker(worker, frame, binding) {
  return new Promise((resolve, reject) => {
    worker.onmessage = (e) => resolve(e.data);
    worker.onerror = (e) => reject(e.message || e);
    worker.postMessage({ frame, binding }, [frame]);
  });
}

async function encodeChunks(n, codec, width, height, at = 0) {
  const chunks = [];
  let config = null;
  const enc = new VideoEncoder({ output: (c, md) => { const b = new Uint8Array(c.byteLength); c.copyTo(b); chunks.push({ type: c.type, timestamp: c.timestamp, data: b }); if (md && md.decoderConfig) config = md.decoderConfig; }, error: (e) => { throw e; } });
  enc.configure({ codec, width, height, bitrate: 600000, framerate: 30, latencyMode: "realtime" });
  const cv = new OffscreenCanvas(width, height);
  const ctx = cv.getContext("2d");
  for (let i = 0; i < n; i++) {
    ctx.fillStyle = `hsl(${(i * 37) % 360} 70% 50%)`;
    ctx.fillRect(0, 0, width, height);
    ctx.fillStyle = "#fff";
    ctx.fillRect((i * 7) % width, (i * 5) % height, width >> 3, height >> 3);
    const f = new VideoFrame(cv, { timestamp: (at + i) * 33333 });
    enc.encode(f, { keyFrame: i === 0 });
    f.close();
  }
  await enc.flush();
  enc.close();
  return { chunks, config: config || { codec, codedWidth: width, codedHeight: height } };
}

async function lifeCycle(kept) {
  const r = {};
  const [f, g, h] = kept;
  const b = SideTable.of(f).binding;
  // clone(): carried.
  const c = f.clone();
  r.clone = { kept: SideTable.of(c) ? SideTable.of(c).binding === b : false };
  // close(): gone with the frame.
  c.close();
  r.close = { detached: SideTable.of(c) === null };
  // new VideoFrame(frame, init) without the member: carried only when the page sets it.
  const n = new VideoFrame(f, { timestamp: f.timestamp });
  r.construct = { carried: SideTable.of(n) !== null };
  n.close();
  // Transfer: posted with the binding beside it, attached in the worker's realm, posted back.
  const table = new SideTable();
  const worker = new Worker(new URL("./worker.js", import.meta.url), { type: "module" });
  const sent = table.transferOut(g);
  const there = await postToWorker(worker, g, sent);
  table.transferIn(there.frame, there.binding);
  r.transfer = {
    in_worker: there.held === b, cloned_in_worker: there.cloned === b,
    back: SideTable.of(there.frame) ? SideTable.of(there.frame).binding === b : false,
    source_detached: SideTable.of(g) === null,
  };
  there.frame.close();
  // Posted without its binding: unbound on arrival.
  const bare = await postToWorker(worker, h, null);
  r.transfer.without = bare.held === null;
  bare.frame.close();
  worker.terminate();
  // copyTo(): the samples, unbound bytes; with a format option, a conversion.
  const own = new Uint8Array(f.allocationSize());
  await f.copyTo(own);
  let converted = null;
  try {
    const opt = { format: "BGRX" };
    const conv = new Uint8Array(f.allocationSize(opt));
    await f.copyTo(conv, opt);
    converted = (await sha256(conv)) !== (await sha256(own));
  } catch (e) {
    converted = `no conversion: ${e.message || e}`;
  }
  r.copy_to = { samples: own.length, conversion_differs: converted };
  // Re-encode: a new object. Its chunks carry nothing, and the frame decoded from them is
  // unbound, though its timestamp is the bound frame's.
  let reencoded = null;
  try {
    const chunks = [];
    let cfg = null;
    const enc = new VideoEncoder({ output: (ch, md) => { const x = new Uint8Array(ch.byteLength); ch.copyTo(x); chunks.push({ type: ch.type, timestamp: ch.timestamp, data: x }); if (md && md.decoderConfig) cfg = md.decoderConfig; }, error: (e) => { reencoded = `encoder: ${e.message}`; } });
    enc.configure({ codec: "vp8", width: f.codedWidth, height: f.codedHeight, bitrate: 200000 });
    enc.encode(f, { keyFrame: true });
    await enc.flush();
    enc.close();
    const outs = [];
    const t2 = new SideTable();
    let d2 = null;
    d2 = new VideoDecoder({ output: (vf) => { t2.output(d2, vf); outs.push({ ts: vf.timestamp, bound: SideTable.of(vf) !== null }); vf.close(); }, error: () => {} });
    d2.configure(cfg || { codec: "vp8" });
    for (const ch of chunks) { t2.chunk(d2, ch.timestamp, null); d2.decode(new EncodedVideoChunk(ch)); }
    await d2.flush();
    d2.close();
    reencoded = { frames: outs.length, same_timestamp: outs.length > 0 && outs[0].ts === f.timestamp, bound: outs.some((o) => o.bound) };
  } catch (e) {
    reencoded = `re-encode: ${e.message || e}`;
  }
  r.reencode = reencoded;
  f.close();
  return r;
}

// ------------------------------------- the side table over the user agent's own decoder (§9.2)

async function nativeDecode() {
  const { chunks, config } = await encodeChunks(12, "vp09.00.10.08", 320, 180);
  const table = new SideTable();
  const got = [];
  let d = null;
  d = new VideoDecoder({ output: (vf) => { const e = table.output(d, vf); got.push({ ts: vf.timestamp, format: vf.format, binding: e ? e.binding : null }); vf.close(); }, error: (e) => got.push({ error: e.message }) });
  d.configure(config);
  const expect = new Map();
  for (const ch of chunks) {
    const digest = await sha256(ch.data);
    const b = wai.makeBinding(digest, "", wai.frameTier("ua", "none", false, false));
    expect.set(ch.timestamp, b);
    table.chunk(d, ch.timestamp, { binding: b });
    d.decode(new EncodedVideoChunk(ch));
  }
  await d.flush();
  table.reset(d); // flush resolved: anything still pending is struck
  const each = got.filter((g) => !g.error).every((g) => g.binding === expect.get(g.ts));
  // Two chunks under one timestamp: neither frame bound.
  const t2 = new SideTable();
  const got2 = [];
  let d2 = null;
  d2 = new VideoDecoder({ output: (vf) => { got2.push(t2.output(d2, vf)); vf.close(); }, error: () => {} });
  d2.configure(config);
  const k = chunks[0];
  for (const ch of [k, { ...chunks[1], timestamp: k.timestamp }]) {
    t2.chunk(d2, k.timestamp, { binding: wai.makeBinding("11".repeat(32), "", "none") });
    d2.decode(new EncodedVideoChunk({ ...ch }));
  }
  await d2.flush();
  d2.close();
  d.close();
  return {
    chunks: chunks.length, outputs: got.length, every_output_bound_to_its_chunk: each && got.length === chunks.length,
    formats: [...new Set(got.map((g) => g.format))], errors: got.filter((g) => g.error).map((g) => g.error),
    ambiguous: { outputs: got2.length, bound: got2.filter(Boolean).length, refusals: t2.refusals },
  };
}

// --------------------------------------------- a capability report from this user agent (§3, §4)

const OBJECTS = [
  ["vp9", "wai.video.vp9", "vp09.00.10.08"], ["av1", "wai.video.av1", "av01.0.04M.08"], ["avc", "wai.video.avc", "avc1.42001e"],
  ["hevc", "wai.video.hevc", "hev1.1.6.L93.B0"], ["vp8", "wai.video.vp8", "vp8"], ["opus", "wai.audio.opus", "opus"],
  ["flac", "wai.audio.flac", "flac"], ["aac", "wai.audio.aac", "mp4a.40.2"], ["av2", "wai.video.av2", "av02.0.08M.08"],
  ["vvc", "wai.video.vvc", "vvc1.1.L123.CQA"], ["iamf", "wai.audio.iamf", "iamf.001.001.Opus"], ["int_motion", "wai.video.int_motion", null],
].map(([name, capability, codec]) => ({
  name, capability, codec, need: name === "int_motion" ? "decode-equivalence" : "none",
  params: capability.startsWith("wai.audio") ? { "wai:sample_rate": 48000, "wai:channel_count": 2 } : { "wai:frame_width": capability === "wai.video.int_motion" ? 48 : 640, "wai:frame_height": capability === "wai.video.int_motion" ? 48 : 360 },
}));
const SINK = { wasm: ["wai.video.int_motion"], constraints: { "wai.video.int_motion": [{ "wai:frame_width": { maximum: 4096 }, "wai:frame_height": { maximum: 4096 } }] } };

async function liveReport() {
  const r = await buildReport(wai, OBJECTS, SINK);
  return { asked: r.asked, outcome: r.outcome, ua: r.report.ua, table: r.table };
}

// The energy entry beside powerEfficient (§6), for a measurement the host gives the page.
let burstChunks = null;
out.prepareBurst = async () => {
  const { chunks, config } = await encodeChunks(60, "vp09.00.10.08", 640, 360, 1000);
  burstChunks = { chunks, config };
  return chunks.length;
};
// Decode the prepared chunks again and again for `ms`, counting the frames output.
out.burst = async (ms) => {
  const { chunks, config } = burstChunks;
  let decodes = 0;
  const end = performance.now() + ms;
  const t0 = performance.now();
  while (performance.now() < end) {
    const d = new VideoDecoder({ output: (vf) => { decodes++; vf.close(); }, error: () => {} });
    d.configure(config);
    for (const ch of chunks) d.decode(new EncodedVideoChunk(ch));
    await d.flush();
    d.close();
  }
  return { decodes, ms: performance.now() - t0 };
};
out.energyEntry = async (measurement) => {
  const vp9 = OBJECTS.find((o) => o.name === "vp9");
  const r = await buildReport(wai, [vp9], { ...SINK, energy: measurement ? { "wai.video.vp9 ua": measurement } : {} });
  return r.table[0];
};

// --------------------------------------------------------------------------------- run

async function main() {
  await initWai();
  await initDecode();
  const [v, envelopeFile, digests] = await Promise.all([
    fetch(q.get("corpus") || "/vectors/browser.json").then((r) => r.json()),
    fetch(q.get("envelope") || "/vectors/int_motion.wai").then((r) => r.arrayBuffer()).then((b) => new Uint8Array(b)),
    fetch(q.get("digests") || "/vectors/int_video_sha256.json").then((r) => r.json()),
  ]);
  out.corpus = corpusInPage(v);
  say(`corpus: ${out.corpus.cases} cases, ${out.corpus.fails.length} differ`);
  const w = await wasmRoute(v, envelopeFile, digests);
  out.wasm_route = { objects: w.objects, unverified: w.unverified };
  say(`wasm route: ${w.objects.map((o) => `${o.name} ${o.outcome} ${o.decoded} frames`).join("; ")}`);
  out.life = await lifeCycle(w.kept);
  say(`life: ${JSON.stringify(out.life)}`);
  out.native = await nativeDecode();
  say(`native: ${JSON.stringify(out.native)}`);
  out.metadata = metadataProbe();
  say(`metadata: ${JSON.stringify(out.metadata)}`);
  out.report = await liveReport();
  say(`report: asked ${out.report.asked.join(", ")}`);
  for (const e of out.report.table) say(`  ${e.name}: ${e.route}${e.reason ? ` (${e.reason})` : ""} legs ${JSON.stringify(e.legs)} tier ${e.tier} powerEfficient ${e.energy.power_efficient}`);
}

main().catch((e) => { out.errors.push((e && e.stack) || String(e)); say(`error: ${e && e.message}`); }).finally(() => { out.done = true; });