// The browser sink in a page (WAI extensions/browser-sink.md): the capability report a page // builds and the route of each object (§3, §4), the content binding every decoded frame // carries (§7) in the side table a page keeps until a registry lists the member (§9.2), and // the state of the frame on screen (§10). // // What is normative is computed by wai-live-web's wasm, the same Rust the corpus // live-conformance/browser holds: `routeReport`, `makeBinding`, `frameTier`, `frameState`. // This module asks the user agent what the report needs, keeps the side table over live // decoders and frames, and draws the state behind a debug control. export const ALG = "jwp-object-1"; // ------------------------------------------------------------------ §9.2: the side table // Every frame's binding, by frame object, in this realm. Weak: an entry lives as long as // the frame object does. Shared by every table, so that `clone()` (patched below) carries a // binding whichever table attached it. const frameEntries = new WeakMap(); const STRUCK = Symbol("struck"); // The pending entries of each decoder, the refusals and the count struck, for a page's // decoders; `of(frame)` for any frame. An entry is `{ binding, ... }`, `binding` the // canonical form (or a function of the frame that returns it, for a tier that depends on // the frame's format, §7.2), and anything else the page wants back with it. export class SideTable { constructor(bound = 64) { this.bound = bound; this.pending = new Map(); // decoder -> [[timestamp, entry | STRUCK], ...], in entry order this.refusals = []; this.struck = 0; this.ops = 0; this.bound_frames = 0; this.unbound_frames = 0; } // Step 1: before a decoder is given a chunk. chunk(decoder, timestamp, entry) { const at = this.ops++; let q = this.pending.get(decoder); if (!q) { q = []; this.pending.set(decoder, q); } const hit = q.find((e) => e[0] === timestamp); if (hit) { if (hit[1] !== STRUCK) this.refusals.push({ at, reason: "binding_ambiguous" }); hit[1] = STRUCK; return; } q.push([timestamp, entry || null]); if (q.length > this.bound) { q.shift(); this.struck++; } } // Step 2: a decoder's output. Returns the entry attached, or null. output(decoder, frame) { this.ops++; const q = this.pending.get(decoder) || []; const i = q.findIndex((e) => e[0] === frame.timestamp); let entry = null; if (i >= 0) { const e = q.splice(i, 1)[0][1]; entry = e === STRUCK ? null : e; } if (entry && typeof entry.binding === "function") entry = { ...entry, binding: entry.binding(frame) }; if (entry && entry.binding) { frameEntries.set(frame, entry); this.bound_frames++; } else { frameEntries.delete(frame); this.unbound_frames++; entry = null; } return entry; } // A frame a wasm decoder output with its binding (§7.2): attached directly. attach(frame, entry) { if (entry && entry.binding) frameEntries.set(frame, entry); else frameEntries.delete(frame); } // Step 3: reset(), close() or an error; and flush() once it resolves. reset(decoder) { this.ops++; const q = this.pending.get(decoder); this.pending.delete(decoder); if (q) this.struck += q.length; } static of(frame) { return frameEntries.get(frame) || null; } of(frame) { return SideTable.of(frame); } // Step 4, for a frame object that is not a VideoFrame (the corpus's sequences): the // source's entry attached to the clone. cloneOnto(src, dst) { const e = frameEntries.get(src); if (e) frameEntries.set(dst, e); else frameEntries.delete(dst); } // Step 5: what a frame posted to another realm is posted with; the source keeps none. transferOut(frame) { const e = frameEntries.get(frame) || null; frameEntries.delete(frame); return e; } transferIn(frame, entry) { this.attach(frame, entry); } // Step 6. close(frame) { frameEntries.delete(frame); } } // Step 4 and 6 for VideoFrame itself: clone() attaches the source's binding to the clone, // and close() detaches it. Installed once per realm. export function installFramePatches() { if (typeof VideoFrame === "undefined" || VideoFrame.prototype.clone.__waiBinding) return; const clone = VideoFrame.prototype.clone; const close = VideoFrame.prototype.close; const c = function () { const out = clone.call(this); const e = frameEntries.get(this); if (e) frameEntries.set(out, e); return out; }; c.__waiBinding = true; VideoFrame.prototype.clone = c; VideoFrame.prototype.close = function () { frameEntries.delete(this); return close.call(this); }; } // The corpus's side-table sequences (live-conformance/browser `tables`), run through a // SideTable over stand-in objects: the result in the corpus's form. export function runSequence(bound, ops, canonicalOf) { const t = new SideTable(bound); const decoders = new Map(); const frames = new Map(); const dec = (n) => { if (!decoders.has(n)) decoders.set(n, { name: n }); return decoders.get(n); }; for (const [i, op] of ops.entries()) { t.ops = i; // a refusal names the index of its operation in the sequence switch (op.op) { case "chunk": t.chunk(dec(op.decoder), op.timestamp, op.binding ? { binding: canonicalOf(op.binding) } : null); break; case "output": { const f = { name: op.frame, timestamp: op.timestamp }; t.output(dec(op.decoder), f); frames.set(op.frame, f); break; } case "clone": { const src = frames.get(op.from); if (!src) throw new Error(`clone of ${op.from}, not open`); const f = { name: op.frame }; t.cloneOnto(src, f); frames.set(op.frame, f); break; } case "transfer": { const src = frames.get(op.from); if (!src) throw new Error(`transfer of ${op.from}, not open`); const e = t.transferOut(src); frames.delete(op.from); const f = { name: op.frame }; t.transferIn(f, op.with_binding === true ? e : null); frames.set(op.frame, f); break; } case "close": { const f = frames.get(op.frame); if (f) t.close(f); frames.delete(op.frame); break; } case "reset": case "flush": t.reset(dec(op.decoder)); break; default: throw new Error(`op ${op.op}`); } } const out = {}; for (const [n, f] of frames) { const e = t.of(f); out[n] = e ? e.binding : null; } return { frames: out, refusals: t.refusals, struck: t.struck }; } // ------------------------------------------------------------------ §3, §4: the report // What the user agent answers for one object: isConfigSupported for the configuration the // object's parameters give, then, for a supported one only (http-delivery §12.2), Media // Capabilities' smooth and powerEfficient. No probe runs in this revision (§4.4). export async function askUserAgent(o, log = () => {}) { const p = o.params || {}; const video = o.capability.startsWith("wai.video"); const width = p["wai:frame_width"] || 640, height = p["wai:frame_height"] || 360; const cfg = video ? { codec: o.codec, codedWidth: width, codedHeight: height } : { codec: o.codec, sampleRate: p["wai:sample_rate"] || 48000, numberOfChannels: p["wai:channel_count"] || 1 }; let supported = false; try { const r = await (video ? VideoDecoder : AudioDecoder).isConfigSupported(cfg); supported = !!r.supported; } catch (e) { log(`isConfigSupported(${o.codec}): ${e.message || e}`); } let smooth = null, power_efficient = null; if (supported && typeof navigator !== "undefined" && navigator.mediaCapabilities) { const q = video ? { type: "file", video: { contentType: `video/mp4; codecs="${o.codec}"`, width, height, bitrate: p["wai:bitrate"] || 1000000, framerate: (p["wai:frame_rate_milli"] || 30000) / 1000 } } : { type: "file", audio: { contentType: `audio/mp4; codecs="${o.codec}"`, channels: String(p["wai:channel_count"] || 1), bitrate: p["wai:bitrate"] || 64000, samplerate: p["wai:sample_rate"] || 48000 } }; try { const mc = await navigator.mediaCapabilities.decodingInfo(q); smooth = !!mc.smooth; power_efficient = !!mc.powerEfficient; } catch (e) { log(`decodingInfo(${o.codec}): ${e.message || e}`); } } return { supported, smooth, power_efficient, probe: "not_run" }; } // §4.6: the report in two passes. The objects alone give each entry's `asked`; exactly the // codec strings asked are put to the user agent; then the routes. `ask` is the question // (askUserAgent by default); every codec string it was called with is returned. export async function buildReport(wai, objects, { host = {}, wasm = [], constraints = {}, energy = {}, ask = askUserAgent } = {}) { const report = { browser_sink: 1, ua: {}, host, wasm, constraints, energy, objects }; const plan = JSON.parse(wai.routeReport(JSON.stringify(report))); const asked = []; if (plan.outcome === "routed") { for (const e of plan.table) { if (!e.asked) continue; const o = objects.find((x) => x.name === e.name); if (o.codec in report.ua) continue; asked.push(o.codec); report.ua[o.codec] = await ask(o); } } const routed = JSON.parse(wai.routeReport(JSON.stringify(report))); return { report, asked, outcome: routed.outcome, reason: routed.reason || null, table: routed.table || [] }; } // ------------------------------------------------------------------ §10: the frame on screen // A group's admission to its chain, as a page's verifier reports it, in §10's tokens. export function chainToken(s) { if (s == null) return "held"; if (s === "linked" || s === "already") return "linked"; if (s === "anchor" || s === "history") return s; if (s.startsWith("refused")) return "refused"; return "held"; } export function stateOf(wai, entry, chain, provenance = "not_applicable") { return JSON.parse(wai.frameState(entry ? entry.binding : "null", "verified", chain, provenance)); } // The debug view: one row a tile, as §10 lists what it shows. export function renderStates(el, rows) { if (!el) return; const tbody = el.querySelector("tbody"); if (!tbody) return; tbody.textContent = ""; for (const r of rows) { const tr = document.createElement("tr"); tr.dataset.state = r.state; tr.dataset.principal = r.principal; const pe = r.energy ? r.energy.power_efficient : null; const fig = r.energy && r.energy.figure ? `${r.energy.figure.per_decode_micro} µJ/decode (${r.energy.figure.class}, ${r.energy.figure.coverage})` : "unmetered"; for (const v of [r.name, r.state, r.tier || "", r.digest ? r.digest.slice(0, 12) : "", r.receipt ? r.receipt.slice(0, 12) : "", r.chain, r.provenance, r.route, pe == null ? "not stated" : String(pe), fig]) { const td = document.createElement("td"); td.textContent = v; tr.appendChild(td); } tbody.appendChild(tr); } el.hidden = false; } // §9.1: whether this user agent keeps a `contentBinding` member it does not register, and // a registered one, in `init.metadata`; through the buffer constructor, the frame // constructor and clone(). What a test reports, not something a page relies on. export function metadataProbe() { const binding = { alg: ALG, digest: "00".repeat(32), tier: "none" }; const out = { buffer: null, frame: null, clone: null, registered: null, errors: [] }; if (typeof VideoFrame === "undefined") return { ...out, errors: ["no VideoFrame"] }; let f = null; try { f = new VideoFrame(new Uint8Array(2 * 2 * 4), { format: "RGBX", codedWidth: 2, codedHeight: 2, timestamp: 0, metadata: { contentBinding: binding, rtpTimestamp: 7 } }); const md = f.metadata(); out.buffer = { keys: Object.keys(md), kept: "contentBinding" in md }; out.registered = { rtpTimestamp: md.rtpTimestamp === 7 }; } catch (e) { out.errors.push(`buffer constructor: ${e.message || e}`); } if (f) { try { const g = new VideoFrame(f, { metadata: { contentBinding: binding } }); const md = g.metadata(); out.frame = { keys: Object.keys(md), kept: "contentBinding" in md }; const c = g.clone(); out.clone = { keys: Object.keys(c.metadata()), kept: "contentBinding" in c.metadata() }; c.close(); g.close(); } catch (e) { out.errors.push(`frame constructor: ${e.message || e}`); } f.close(); } out.member_kept = !!((out.buffer && out.buffer.kept) || (out.frame && out.frame.kept)); return out; }