// WAI browser-sink (extensions/browser-sink.md) in a real page, for the end-to-end scenario // `binding`. Everything the test reads is on `window.bindingTest`; `done` is set last. // // Query parameters: corpus= (live-conformance/browser/vectors.json), envelope= // (its int_motion.wai), digests= (wai-rs/tests/vectors/int_video/sha256.json). import initWai, * as wai from "../pkg/wai_live_web.js"; import initDecode, * as dec from "../pkg/wai_live_web_decode.js"; import { SideTable, runSequence, buildReport, stateOf, metadataProbe, installFramePatches } from "../room/binding.js"; const q = new URLSearchParams(location.search); const out = { done: false, errors: [], corpus: null, wasm_route: null, life: null, native: null, metadata: null, report: null }; window.bindingTest = out; const lines = []; const say = (m) => { lines.push(m); document.getElementById("out").textContent = lines.join("\n"); }; const hex = (b) => Array.from(new Uint8Array(b), (x) => x.toString(16).padStart(2, "0")).join(""); const unhex = (h) => new Uint8Array((h.match(/../g) || []).map((x) => parseInt(x, 16))); const sha256 = async (b) => hex(await crypto.subtle.digest("SHA-256", b)); const same = (a, b) => JSON.stringify(a) === JSON.stringify(b); installFramePatches(); // ------------------------------------------------------------- the corpus through the page function corpusInPage(v) { const fails = []; let n = 0; for (const c of v.routes) { n++; const got = JSON.parse(wai.routeReport(JSON.stringify(c.report))); if (got.outcome !== c.outcome || (c.outcome === "routed" ? !same(got.table, c.table) : got.reason !== c.reason)) fails.push(`route ${c.name}`); } for (const c of v.choices || []) { n++; const got = JSON.parse(wai.chooseReport(JSON.stringify(c.report))); if (got.outcome !== c.outcome || (c.outcome === "chosen" ? !same(got.choices, c.choices) : got.reason !== c.reason)) fails.push(`choice ${c.name}`); } for (const c of v.constraints) { n++; const got = JSON.parse(wai.evaluateConstraints(JSON.stringify(c.sets), JSON.stringify(c.params))); if (got.outcome !== c.outcome || got.reason !== c.reason) fails.push(`constraints ${c.name}`); } for (const c of v.bindings) { n++; const got = JSON.parse(wai.readBinding(JSON.stringify(c.value))); if (got.outcome !== c.outcome || (got.canonical || null) !== c.canonical || (got.reason || null) !== c.reason) fails.push(`binding ${c.name}`); } for (const c of v.frame_tiers) { n++; if (wai.frameTier(c.route, c.route_tier, c.format_null, c.altered) !== c.tier) fails.push(`frame tier ${JSON.stringify(c)}`); } for (const c of v.frames) { n++; const got = JSON.parse(wai.frameState(JSON.stringify(c.binding), c.object, c.chain, c.provenance)); if (!same(got, c.result)) fails.push(`frame ${c.name}`); } // The side table is this page's own JavaScript (web/room/binding.js), held to the sequences. const canonicalOf = (b) => JSON.parse(wai.readBinding(JSON.stringify(b))).canonical; for (const c of v.tables) { n++; let got; try { got = runSequence(c.bound, c.ops, canonicalOf); } catch (e) { got = { error: String(e) }; } // Compared member by member: the order of frames in an object is not part of the result. const norm = (r) => JSON.stringify({ ...r, frames: Object.fromEntries(Object.entries(r.frames || {}).sort()) }); if (norm(got) !== norm(c.result)) fails.push(`table ${c.name}: ${JSON.stringify(got)}`); } return { cases: n, fails }; } // ------------------------------------------- the wasm route: verified, decoded, every frame bound async function frameRgbDigest(f) { const buf = new Uint8Array(f.allocationSize()); await f.copyTo(buf); const rgb = new Uint8Array((buf.length / 4) * 3); for (let i = 0, j = 0; i < buf.length; i += 4, j += 3) { rgb[j] = buf[i]; rgb[j + 1] = buf[i + 1]; rgb[j + 2] = buf[i + 2]; } return sha256(rgb); } async function wasmRoute(v, envelopeFile, digests) { const table = new SideTable(); const want = digests.int_video.frames_sha256; const objects = []; const kept = []; // bound frames kept for the life-cycle tests let t = 0; for (const [gi, g] of v.chain.groups.entries()) { for (const o of g.objects) { const env = typeof o.envelope === "string" ? unhex(o.envelope) : envelopeFile; const clip = dec.decodeBound(env, JSON.stringify(g.receipt), JSON.stringify(o.receipt)); const r = { name: o.name, outcome: o.outcome, decoded: clip.nFrames, binding: clip.binding || null, reason: clip.reason || null, frames: [] }; for (let i = 0; i < clip.nFrames; i++) { const f = new VideoFrame(clip.rgbx(i), { format: "RGBX", codedWidth: clip.width, codedHeight: clip.height, timestamp: t++ * 40000 }); table.attach(f, clip.binding ? { binding: clip.binding, group: g.receipt.group_id } : null); const e = SideTable.of(f); const digest = await frameRgbDigest(f); // The state of this frame were it on screen: the chain anchors at the first group a // verifier holds and links the next (jwp-receipts §3 step 3). const s = stateOf(wai, e, gi === 0 ? "anchor" : "linked", "not_applicable"); r.frames.push({ format: f.format, bound: e ? e.binding : null, digest_ok: digest === want[i], state: s.state, tier: s.tier }); if (i === 0 && o.name === "int_motion") { const ctx = document.getElementById("frame").getContext("2d"); ctx.drawImage(f, 0, 0); } if (o.name === "int_motion" && kept.length < 3) kept.push(f); else f.close(); } clip.free(); objects.push(r); } } // Decoded with no receipts: frames, none of them bound. const bare = dec.decodeBound(envelopeFile, "", ""); const unverified = { decoded: bare.nFrames, binding: bare.binding || null, reason: bare.reason || null }; bare.free(); return { objects, unverified, kept }; } // ------------------------------------------------- the binding through the frame's life (§8) function postToWorker(worker, frame, binding) { return new Promise((resolve, reject) => { worker.onmessage = (e) => resolve(e.data); worker.onerror = (e) => reject(e.message || e); worker.postMessage({ frame, binding }, [frame]); }); } async function encodeChunks(n, codec, width, height, at = 0) { const chunks = []; let config = null; const enc = new VideoEncoder({ output: (c, md) => { const b = new Uint8Array(c.byteLength); c.copyTo(b); chunks.push({ type: c.type, timestamp: c.timestamp, data: b }); if (md && md.decoderConfig) config = md.decoderConfig; }, error: (e) => { throw e; } }); enc.configure({ codec, width, height, bitrate: 600000, framerate: 30, latencyMode: "realtime" }); const cv = new OffscreenCanvas(width, height); const ctx = cv.getContext("2d"); for (let i = 0; i < n; i++) { ctx.fillStyle = `hsl(${(i * 37) % 360} 70% 50%)`; ctx.fillRect(0, 0, width, height); ctx.fillStyle = "#fff"; ctx.fillRect((i * 7) % width, (i * 5) % height, width >> 3, height >> 3); const f = new VideoFrame(cv, { timestamp: (at + i) * 33333 }); enc.encode(f, { keyFrame: i === 0 }); f.close(); } await enc.flush(); enc.close(); return { chunks, config: config || { codec, codedWidth: width, codedHeight: height } }; } async function lifeCycle(kept) { const r = {}; const [f, g, h] = kept; const b = SideTable.of(f).binding; // clone(): carried. const c = f.clone(); r.clone = { kept: SideTable.of(c) ? SideTable.of(c).binding === b : false }; // close(): gone with the frame. c.close(); r.close = { detached: SideTable.of(c) === null }; // new VideoFrame(frame, init) without the member: carried only when the page sets it. const n = new VideoFrame(f, { timestamp: f.timestamp }); r.construct = { carried: SideTable.of(n) !== null }; n.close(); // Transfer: posted with the binding beside it, attached in the worker's realm, posted back. const table = new SideTable(); const worker = new Worker(new URL("./worker.js", import.meta.url), { type: "module" }); const sent = table.transferOut(g); const there = await postToWorker(worker, g, sent); table.transferIn(there.frame, there.binding); r.transfer = { in_worker: there.held === b, cloned_in_worker: there.cloned === b, back: SideTable.of(there.frame) ? SideTable.of(there.frame).binding === b : false, source_detached: SideTable.of(g) === null, }; there.frame.close(); // Posted without its binding: unbound on arrival. const bare = await postToWorker(worker, h, null); r.transfer.without = bare.held === null; bare.frame.close(); worker.terminate(); // copyTo(): the samples, unbound bytes; with a format option, a conversion. const own = new Uint8Array(f.allocationSize()); await f.copyTo(own); let converted = null; try { const opt = { format: "BGRX" }; const conv = new Uint8Array(f.allocationSize(opt)); await f.copyTo(conv, opt); converted = (await sha256(conv)) !== (await sha256(own)); } catch (e) { converted = `no conversion: ${e.message || e}`; } r.copy_to = { samples: own.length, conversion_differs: converted }; // Re-encode: a new object. Its chunks carry nothing, and the frame decoded from them is // unbound, though its timestamp is the bound frame's. let reencoded = null; try { const chunks = []; let cfg = null; const enc = new VideoEncoder({ output: (ch, md) => { const x = new Uint8Array(ch.byteLength); ch.copyTo(x); chunks.push({ type: ch.type, timestamp: ch.timestamp, data: x }); if (md && md.decoderConfig) cfg = md.decoderConfig; }, error: (e) => { reencoded = `encoder: ${e.message}`; } }); enc.configure({ codec: "vp8", width: f.codedWidth, height: f.codedHeight, bitrate: 200000 }); enc.encode(f, { keyFrame: true }); await enc.flush(); enc.close(); const outs = []; const t2 = new SideTable(); let d2 = null; d2 = new VideoDecoder({ output: (vf) => { t2.output(d2, vf); outs.push({ ts: vf.timestamp, bound: SideTable.of(vf) !== null }); vf.close(); }, error: () => {} }); d2.configure(cfg || { codec: "vp8" }); for (const ch of chunks) { t2.chunk(d2, ch.timestamp, null); d2.decode(new EncodedVideoChunk(ch)); } await d2.flush(); d2.close(); reencoded = { frames: outs.length, same_timestamp: outs.length > 0 && outs[0].ts === f.timestamp, bound: outs.some((o) => o.bound) }; } catch (e) { reencoded = `re-encode: ${e.message || e}`; } r.reencode = reencoded; f.close(); return r; } // ------------------------------------- the side table over the user agent's own decoder (§9.2) async function nativeDecode() { const { chunks, config } = await encodeChunks(12, "vp09.00.10.08", 320, 180); const table = new SideTable(); const got = []; let d = null; d = new VideoDecoder({ output: (vf) => { const e = table.output(d, vf); got.push({ ts: vf.timestamp, format: vf.format, binding: e ? e.binding : null }); vf.close(); }, error: (e) => got.push({ error: e.message }) }); d.configure(config); const expect = new Map(); for (const ch of chunks) { const digest = await sha256(ch.data); const b = wai.makeBinding(digest, "", wai.frameTier("ua", "none", false, false)); expect.set(ch.timestamp, b); table.chunk(d, ch.timestamp, { binding: b }); d.decode(new EncodedVideoChunk(ch)); } await d.flush(); table.reset(d); // flush resolved: anything still pending is struck const each = got.filter((g) => !g.error).every((g) => g.binding === expect.get(g.ts)); // Two chunks under one timestamp: neither frame bound. const t2 = new SideTable(); const got2 = []; let d2 = null; d2 = new VideoDecoder({ output: (vf) => { got2.push(t2.output(d2, vf)); vf.close(); }, error: () => {} }); d2.configure(config); const k = chunks[0]; for (const ch of [k, { ...chunks[1], timestamp: k.timestamp }]) { t2.chunk(d2, k.timestamp, { binding: wai.makeBinding("11".repeat(32), "", "none") }); d2.decode(new EncodedVideoChunk({ ...ch })); } await d2.flush(); d2.close(); d.close(); return { chunks: chunks.length, outputs: got.length, every_output_bound_to_its_chunk: each && got.length === chunks.length, formats: [...new Set(got.map((g) => g.format))], errors: got.filter((g) => g.error).map((g) => g.error), ambiguous: { outputs: got2.length, bound: got2.filter(Boolean).length, refusals: t2.refusals }, }; } // --------------------------------------------- a capability report from this user agent (§3, §4) const OBJECTS = [ ["vp9", "wai.video.vp9", "vp09.00.10.08"], ["av1", "wai.video.av1", "av01.0.04M.08"], ["avc", "wai.video.avc", "avc1.42001e"], ["hevc", "wai.video.hevc", "hev1.1.6.L93.B0"], ["vp8", "wai.video.vp8", "vp8"], ["opus", "wai.audio.opus", "opus"], ["flac", "wai.audio.flac", "flac"], ["aac", "wai.audio.aac", "mp4a.40.2"], ["av2", "wai.video.av2", "av02.0.08M.08"], ["vvc", "wai.video.vvc", "vvc1.1.L123.CQA"], ["iamf", "wai.audio.iamf", "iamf.001.001.Opus"], ["int_motion", "wai.video.int_motion", null], ].map(([name, capability, codec]) => ({ name, capability, codec, need: name === "int_motion" ? "decode-equivalence" : "none", params: capability.startsWith("wai.audio") ? { "wai:sample_rate": 48000, "wai:channel_count": 2 } : { "wai:frame_width": capability === "wai.video.int_motion" ? 48 : 640, "wai:frame_height": capability === "wai.video.int_motion" ? 48 : 360 }, })); const SINK = { wasm: ["wai.video.int_motion"], constraints: { "wai.video.int_motion": [{ "wai:frame_width": { maximum: 4096 }, "wai:frame_height": { maximum: 4096 } }] } }; async function liveReport() { const r = await buildReport(wai, OBJECTS, SINK); return { asked: r.asked, outcome: r.outcome, ua: r.report.ua, table: r.table }; } // The energy entry beside powerEfficient (§6), for a measurement the host gives the page. let burstChunks = null; out.prepareBurst = async () => { const { chunks, config } = await encodeChunks(60, "vp09.00.10.08", 640, 360, 1000); burstChunks = { chunks, config }; return chunks.length; }; // Decode the prepared chunks again and again for `ms`, counting the frames output. out.burst = async (ms) => { const { chunks, config } = burstChunks; let decodes = 0; const end = performance.now() + ms; const t0 = performance.now(); while (performance.now() < end) { const d = new VideoDecoder({ output: (vf) => { decodes++; vf.close(); }, error: () => {} }); d.configure(config); for (const ch of chunks) d.decode(new EncodedVideoChunk(ch)); await d.flush(); d.close(); } return { decodes, ms: performance.now() - t0 }; }; out.energyEntry = async (measurement) => { const vp9 = OBJECTS.find((o) => o.name === "vp9"); const r = await buildReport(wai, [vp9], { ...SINK, energy: measurement ? { "wai.video.vp9 ua": measurement } : {} }); return r.table[0]; }; // --------------------------------------------------------------------------------- run async function main() { await initWai(); await initDecode(); const [v, envelopeFile, digests] = await Promise.all([ fetch(q.get("corpus") || "/vectors/browser.json").then((r) => r.json()), fetch(q.get("envelope") || "/vectors/int_motion.wai").then((r) => r.arrayBuffer()).then((b) => new Uint8Array(b)), fetch(q.get("digests") || "/vectors/int_video_sha256.json").then((r) => r.json()), ]); out.corpus = corpusInPage(v); say(`corpus: ${out.corpus.cases} cases, ${out.corpus.fails.length} differ`); const w = await wasmRoute(v, envelopeFile, digests); out.wasm_route = { objects: w.objects, unverified: w.unverified }; say(`wasm route: ${w.objects.map((o) => `${o.name} ${o.outcome} ${o.decoded} frames`).join("; ")}`); out.life = await lifeCycle(w.kept); say(`life: ${JSON.stringify(out.life)}`); out.native = await nativeDecode(); say(`native: ${JSON.stringify(out.native)}`); out.metadata = metadataProbe(); say(`metadata: ${JSON.stringify(out.metadata)}`); out.report = await liveReport(); say(`report: asked ${out.report.asked.join(", ")}`); for (const e of out.report.table) say(` ${e.name}: ${e.route}${e.reason ? ` (${e.reason})` : ""} legs ${JSON.stringify(e.legs)} tier ${e.tier} powerEfficient ${e.energy.power_efficient}`); } main().catch((e) => { out.errors.push((e && e.stack) || String(e)); say(`error: ${e && e.message}`); }).finally(() => { out.done = true; });