// Platform decode capabilities (extensions/platform-decode.md) and browser-sink §4.7's // choice, in a page. // // For each AV1 floor object of the packaging corpus (WAI1 envelopes whose payload is SPEC // §5's AV1 framing): the query the reference reads from the payload (platformQuery), the // platform's answers to it (WebCodecs isConfigSupported with prefer-hardware and // no-preference; Media Capabilities decodingInfo, asked only once isConfigSupported said // supported for the preference the page configures), and a decode of every temporal unit // through WebCodecs with the configuration the query gives. Then an HEVC Annex B stream the // platform's encoder writes, configured with prefer-hardware (a hint), read by the // reference's query and decoded back. Then browser-sink §4.7's choice (platformChoose) over // a capability report of a learned alternative on the page's wasm decoder and the first // AV1 floor object, from the answers the platform gave for that object's own query: with no // figure for the learned alternative, and with a synthetic one. Every query is read from a // payload; none is written by hand. // // The report is left in window.platformDecode for verify_platform_decode.mjs to read. import init, { parse, platformQuery, platformAnswer, platformChoose } from "./pkg-platform/wai_platform_decode.js"; const lines = []; const say = (l) => lines.push(l); const info = (d) => d ? `supported=${d.supported} smooth=${d.smooth} powerEfficient=${d.powerEfficient}` : "none"; /// SPEC §5's AV1 framing: n, h, w, fps_num, fps_den, then each unit with its length. function av1Units(p) { const dv = new DataView(p.buffer, p.byteOffset, p.byteLength); const n = dv.getUint32(0, true); const units = []; let at = 20; for (let k = 0; k < n; k++) { const len = dv.getUint32(at, true); units.push(p.subarray(at + 4, at + 4 + len)); at += 4 + len; } return units; } /// Whether a temporal unit carries a sequence header OBU (type 1): the corpus floors start /// each closed group with one, before its key frame. function hasSequenceHeader(tu) { let at = 0; while (at < tu.length) { const h = tu[at]; const ext = (h & 4) ? 1 : 0; let size = 0, shift = 0, i = at + 1 + ext; for (;;) { const b = tu[i++]; size += (b & 0x7f) * 2 ** shift; shift += 7; if (!(b & 0x80)) break; } if (((h >> 3) & 15) === 1) return true; at = i + size; } return false; } /// WebCodecs first; Media Capabilities only once isConfigSupported has said supported for the /// preference the page configures (http-delivery §12.2), its own `supported` not read. async function answers(q) { const out = { decodingInfo: null, preferHardware: null, noPreference: null }; out.preferHardware = (await VideoDecoder.isConfigSupported(q.decoderConfigPreferHardware)).supported; out.noPreference = (await VideoDecoder.isConfigSupported(q.decoderConfigNoPreference)).supported; if (q.decodingConfiguration && (out.preferHardware || out.noPreference)) { const d = await navigator.mediaCapabilities.decodingInfo(q.decodingConfiguration); out.decodingInfo = { supported: d.supported, smooth: d.smooth, powerEfficient: d.powerEfficient }; } return out; } /// Decode `chunks` ([data, key]) with the configuration `config`: the frames output. async function decodeAll(config, chunks, frameUs) { let frames = 0, error = null; const dec = new VideoDecoder({ output: (f) => { frames++; f.close(); }, error: (e) => { error = String(e); } }); dec.configure(config); chunks.forEach(([data, key], k) => dec.decode(new EncodedVideoChunk({ type: key ? "key" : "delta", timestamp: k * frameUs, data }))); try { await dec.flush(); } catch (e) { error = error || String(e); } if (dec.state !== "closed") dec.close(); return { frames, error }; } async function av1Objects(paths) { let first = null; for (const path of paths) { const bytes = new Uint8Array(await (await fetch(path)).arrayBuffer()); const env = parse(bytes); if (env.capability !== "wai.video.av1") continue; const payload = env.payload; const q = JSON.parse(platformQuery("wai.video.av1", payload, undefined, undefined)); const a = await answers(q); const units = av1Units(payload); const config = { ...(a.preferHardware ? q.decoderConfigPreferHardware : q.decoderConfigNoPreference) }; const r = await decodeAll(config, units.map((u) => [u, hasSequenceHeader(u)]), Math.round(1e6 * q.framerate[1] / q.framerate[0])); say(`platform av1 object: ${path.split("/").slice(-4).join("/")} codec=${q.codec} size=${q.width}x${q.height} rate=${q.framerate.join("/")} bitrate=${q.bitrate} decodingInfo=[${info(a.decodingInfo)}] isConfigSupported prefer-hardware=${a.preferHardware} no-preference=${a.noPreference} decoded=${r.frames}/${units.length} with ${config.hardwareAcceleration}${r.error ? " error=" + r.error : ""}`); first = first || { q, a }; } return first; } /// The first SPS NAL unit (type 33) of an Annex B stream, as hexadecimal. function firstSps(s) { for (let i = 0; i + 4 < s.length; i++) { if (s[i] === 0 && s[i + 1] === 0 && s[i + 2] === 1 && ((s[i + 3] >> 1) & 63) === 33) { let j = i + 3; while (j + 3 <= s.length && !(s[j] === 0 && s[j + 1] === 0 && (s[j + 2] === 1 || (s[j + 2] === 0 && s[j + 3] === 1)))) j++; return Array.from(s.subarray(i + 3, j), (b) => b.toString(16).padStart(2, "0")).join(""); } } return ""; } async function hevcRoundTrip() { const configured = "hvc1.1.6.L93.B0"; const [w, h, n] = [640, 360, 12]; const enc = { codec: configured, width: w, height: h, bitrate: 1000000, framerate: 30, hevc: { format: "annexb" }, hardwareAcceleration: "prefer-hardware" }; const es = await VideoEncoder.isConfigSupported(enc); if (!es.supported) { say(`platform hevc encoder: not supported (${configured} ${w}x${h}, prefer-hardware requested)`); return; } const chunks = []; let declared = ""; const e = new VideoEncoder({ output: (c, meta) => { const b = new Uint8Array(c.byteLength); c.copyTo(b); chunks.push([b, c.type === "key"]); if (meta && meta.decoderConfig) declared = meta.decoderConfig.codec; }, error: (x) => say(`platform hevc encoder: error ${x}`) }); e.configure(enc); const rgba = new Uint8Array(w * h * 4); for (let k = 0; k < n; k++) { for (let y = 0; y < h; y++) for (let x = 0; x < w; x++) { const o = 4 * (y * w + x); rgba[o] = (x + 4 * k) & 255; rgba[o + 1] = (y * 2) & 255; rgba[o + 2] = (x ^ y) & 255; rgba[o + 3] = 255; } const f = new VideoFrame(rgba, { format: "RGBA", codedWidth: w, codedHeight: h, timestamp: k * 33333 }); e.encode(f, { keyFrame: k === 0 }); f.close(); } await e.flush(); e.close(); const stream = new Uint8Array(chunks.reduce((s, [b]) => s + b.length, 0)); let at = 0; for (const [b] of chunks) { stream.set(b, at); at += b.length; } const q = JSON.parse(platformQuery("wai.video.hevc", stream, 30, 1)); const a = await answers(q); const r = await decodeAll(a.preferHardware ? q.decoderConfigPreferHardware : q.decoderConfigNoPreference, chunks, 33333); say(`platform hevc encoder: configured=${configured} encoder-declares=${declared || "nothing"} derived=${q.codec} size=${q.width}x${q.height} coded=${q.codedWidth}x${q.codedHeight} depth=${q.bitDepth} bitrate=${q.bitrate} decodingInfo=[${info(a.decodingInfo)}] isConfigSupported prefer-hardware=${a.preferHardware} no-preference=${a.noPreference} decoded=${r.frames}/${chunks.length}${r.error ? " error=" + r.error : ""} sps=${firstSps(stream)}`); } /// browser-sink §4.7 over a report of a learned alternative on the page's wasm decoder and the /// AV1 floor object `first`, from the answers the platform gave for that object's own query. function choose(first) { const { q, a } = first; // The answer for the preference the page configures: prefer-hardware where accepted. const supported = a.preferHardware || a.noPreference; const d = a.decodingInfo; const answer = JSON.parse(platformAnswer(supported, d ? d.smooth : undefined, d ? d.powerEfficient : undefined)); const report = (energy) => ({ browser_sink: 1, ua: { [q.codec]: answer }, host: {}, wasm: ["wai.video.int_hyper"], constraints: {}, energy, objects: [ { name: "int_hyper", capability: "wai.video.int_hyper", codec: null, need: "none", params: {}, choice: "clip" }, { name: "av1", capability: "wai.video.av1", codec: q.codec, need: "none", params: {}, choice: "clip" }, ], }); const line = (label, r) => { const c = JSON.parse(platformChoose(JSON.stringify(r))); const t = c.choices ? c.choices[0] : {}; say(`${label}: av1=[${q.codec} ${JSON.stringify(answer)}] order=${JSON.stringify(t.order)} ranks=${JSON.stringify(t.ranks)} chosen=${t.chosen}`); }; line("platform choose", report({})); // Synthetic: no measurement. It exists only to exercise the rule, and is marked so. const figure = { joules_micro: 2460000, decodes: 120, window_us: 6000000, class: "OnChipCounter", relative_ppm: 40000, filtering: "Unknown", coverage: "Whole", fabrics: ["cpu", "gpu"], note: "synthetic: no measurement" }; line("platform choose with a synthetic figure", report({ "wai.video.int_hyper wasm": figure })); } window.platformDecode = { done: false, report: "" }; try { await init(); say(`platform: userAgent=${navigator.userAgent}`); const paths = await (await fetch("objects.json")).json(); const first = await av1Objects(paths); await hevcRoundTrip(); if (first) choose(first); else say("platform choose: no AV1 object to choose with"); } catch (e) { say(`platform: page error: ${e && e.stack || e}`); } window.platformDecode.report = lines.join("\n"); document.getElementById("report").textContent = window.platformDecode.report; window.platformDecode.done = true;