open-standards/wai/wai-web/demo/platform_decode.js

// Platform decode capabilities (extensions/platform-decode.md) and browser-sink §4.7's
// choice, in a page.
//
// For each AV1 floor object of the packaging corpus (WAI1 envelopes whose payload is SPEC
// §5's AV1 framing): the query the reference reads from the payload (platformQuery), the
// platform's answers to it (WebCodecs isConfigSupported with prefer-hardware and
// no-preference; Media Capabilities decodingInfo, asked only once isConfigSupported said
// supported for the preference the page configures), and a decode of every temporal unit
// through WebCodecs with the configuration the query gives. Then an HEVC Annex B stream the
// platform's encoder writes, configured with prefer-hardware (a hint), read by the
// reference's query and decoded back. Then browser-sink §4.7's choice (platformChoose) over
// a capability report of a learned alternative on the page's wasm decoder and the first
// AV1 floor object, from the answers the platform gave for that object's own query: with no
// figure for the learned alternative, and with a synthetic one. Every query is read from a
// payload; none is written by hand.
//
// The report is left in window.platformDecode for verify_platform_decode.mjs to read.
import init, { parse, platformQuery, platformAnswer, platformChoose } from "./pkg-platform/wai_platform_decode.js";

const lines = [];
const say = (l) => lines.push(l);
const info = (d) => d ? `supported=${d.supported} smooth=${d.smooth} powerEfficient=${d.powerEfficient}` : "none";

/// SPEC §5's AV1 framing: n, h, w, fps_num, fps_den, then each unit with its length.
function av1Units(p) {
  const dv = new DataView(p.buffer, p.byteOffset, p.byteLength);
  const n = dv.getUint32(0, true);
  const units = [];
  let at = 20;
  for (let k = 0; k < n; k++) {
    const len = dv.getUint32(at, true);
    units.push(p.subarray(at + 4, at + 4 + len));
    at += 4 + len;
  }
  return units;
}

/// Whether a temporal unit carries a sequence header OBU (type 1): the corpus floors start
/// each closed group with one, before its key frame.
function hasSequenceHeader(tu) {
  let at = 0;
  while (at < tu.length) {
    const h = tu[at];
    const ext = (h & 4) ? 1 : 0;
    let size = 0, shift = 0, i = at + 1 + ext;
    for (;;) { const b = tu[i++]; size += (b & 0x7f) * 2 ** shift; shift += 7; if (!(b & 0x80)) break; }
    if (((h >> 3) & 15) === 1) return true;
    at = i + size;
  }
  return false;
}

/// WebCodecs first; Media Capabilities only once isConfigSupported has said supported for the
/// preference the page configures (http-delivery §12.2), its own `supported` not read.
async function answers(q) {
  const out = { decodingInfo: null, preferHardware: null, noPreference: null };
  out.preferHardware = (await VideoDecoder.isConfigSupported(q.decoderConfigPreferHardware)).supported;
  out.noPreference = (await VideoDecoder.isConfigSupported(q.decoderConfigNoPreference)).supported;
  if (q.decodingConfiguration && (out.preferHardware || out.noPreference)) {
    const d = await navigator.mediaCapabilities.decodingInfo(q.decodingConfiguration);
    out.decodingInfo = { supported: d.supported, smooth: d.smooth, powerEfficient: d.powerEfficient };
  }
  return out;
}

/// Decode `chunks` ([data, key]) with the configuration `config`: the frames output.
async function decodeAll(config, chunks, frameUs) {
  let frames = 0, error = null;
  const dec = new VideoDecoder({ output: (f) => { frames++; f.close(); }, error: (e) => { error = String(e); } });
  dec.configure(config);
  chunks.forEach(([data, key], k) => dec.decode(new EncodedVideoChunk({ type: key ? "key" : "delta", timestamp: k * frameUs, data })));
  try { await dec.flush(); } catch (e) { error = error || String(e); }
  if (dec.state !== "closed") dec.close();
  return { frames, error };
}

async function av1Objects(paths) {
  let first = null;
  for (const path of paths) {
    const bytes = new Uint8Array(await (await fetch(path)).arrayBuffer());
    const env = parse(bytes);
    if (env.capability !== "wai.video.av1") continue;
    const payload = env.payload;
    const q = JSON.parse(platformQuery("wai.video.av1", payload, undefined, undefined));
    const a = await answers(q);
    const units = av1Units(payload);
    const config = { ...(a.preferHardware ? q.decoderConfigPreferHardware : q.decoderConfigNoPreference) };
    const r = await decodeAll(config, units.map((u) => [u, hasSequenceHeader(u)]), Math.round(1e6 * q.framerate[1] / q.framerate[0]));
    say(`platform av1 object: ${path.split("/").slice(-4).join("/")} codec=${q.codec} size=${q.width}x${q.height} rate=${q.framerate.join("/")} bitrate=${q.bitrate} decodingInfo=[${info(a.decodingInfo)}] isConfigSupported prefer-hardware=${a.preferHardware} no-preference=${a.noPreference} decoded=${r.frames}/${units.length} with ${config.hardwareAcceleration}${r.error ? " error=" + r.error : ""}`);
    first = first || { q, a };
  }
  return first;
}

/// The first SPS NAL unit (type 33) of an Annex B stream, as hexadecimal.
function firstSps(s) {
  for (let i = 0; i + 4 < s.length; i++) {
    if (s[i] === 0 && s[i + 1] === 0 && s[i + 2] === 1 && ((s[i + 3] >> 1) & 63) === 33) {
      let j = i + 3;
      while (j + 3 <= s.length && !(s[j] === 0 && s[j + 1] === 0 && (s[j + 2] === 1 || (s[j + 2] === 0 && s[j + 3] === 1)))) j++;
      return Array.from(s.subarray(i + 3, j), (b) => b.toString(16).padStart(2, "0")).join("");
    }
  }
  return "";
}

async function hevcRoundTrip() {
  const configured = "hvc1.1.6.L93.B0";
  const [w, h, n] = [640, 360, 12];
  const enc = { codec: configured, width: w, height: h, bitrate: 1000000, framerate: 30, hevc: { format: "annexb" }, hardwareAcceleration: "prefer-hardware" };
  const es = await VideoEncoder.isConfigSupported(enc);
  if (!es.supported) { say(`platform hevc encoder: not supported (${configured} ${w}x${h}, prefer-hardware requested)`); return; }
  const chunks = [];
  let declared = "";
  const e = new VideoEncoder({ output: (c, meta) => { const b = new Uint8Array(c.byteLength); c.copyTo(b); chunks.push([b, c.type === "key"]); if (meta && meta.decoderConfig) declared = meta.decoderConfig.codec; }, error: (x) => say(`platform hevc encoder: error ${x}`) });
  e.configure(enc);
  const rgba = new Uint8Array(w * h * 4);
  for (let k = 0; k < n; k++) {
    for (let y = 0; y < h; y++) for (let x = 0; x < w; x++) { const o = 4 * (y * w + x); rgba[o] = (x + 4 * k) & 255; rgba[o + 1] = (y * 2) & 255; rgba[o + 2] = (x ^ y) & 255; rgba[o + 3] = 255; }
    const f = new VideoFrame(rgba, { format: "RGBA", codedWidth: w, codedHeight: h, timestamp: k * 33333 });
    e.encode(f, { keyFrame: k === 0 });
    f.close();
  }
  await e.flush();
  e.close();
  const stream = new Uint8Array(chunks.reduce((s, [b]) => s + b.length, 0));
  let at = 0;
  for (const [b] of chunks) { stream.set(b, at); at += b.length; }
  const q = JSON.parse(platformQuery("wai.video.hevc", stream, 30, 1));
  const a = await answers(q);
  const r = await decodeAll(a.preferHardware ? q.decoderConfigPreferHardware : q.decoderConfigNoPreference, chunks, 33333);
  say(`platform hevc encoder: configured=${configured} encoder-declares=${declared || "nothing"} derived=${q.codec} size=${q.width}x${q.height} coded=${q.codedWidth}x${q.codedHeight} depth=${q.bitDepth} bitrate=${q.bitrate} decodingInfo=[${info(a.decodingInfo)}] isConfigSupported prefer-hardware=${a.preferHardware} no-preference=${a.noPreference} decoded=${r.frames}/${chunks.length}${r.error ? " error=" + r.error : ""} sps=${firstSps(stream)}`);
}

/// browser-sink §4.7 over a report of a learned alternative on the page's wasm decoder and the
/// AV1 floor object `first`, from the answers the platform gave for that object's own query.
function choose(first) {
  const { q, a } = first;
  // The answer for the preference the page configures: prefer-hardware where accepted.
  const supported = a.preferHardware || a.noPreference;
  const d = a.decodingInfo;
  const answer = JSON.parse(platformAnswer(supported, d ? d.smooth : undefined, d ? d.powerEfficient : undefined));
  const report = (energy) => ({
    browser_sink: 1, ua: { [q.codec]: answer }, host: {}, wasm: ["wai.video.int_hyper"], constraints: {}, energy,
    objects: [
      { name: "int_hyper", capability: "wai.video.int_hyper", codec: null, need: "none", params: {}, choice: "clip" },
      { name: "av1", capability: "wai.video.av1", codec: q.codec, need: "none", params: {}, choice: "clip" },
    ],
  });
  const line = (label, r) => {
    const c = JSON.parse(platformChoose(JSON.stringify(r)));
    const t = c.choices ? c.choices[0] : {};
    say(`${label}: av1=[${q.codec} ${JSON.stringify(answer)}] order=${JSON.stringify(t.order)} ranks=${JSON.stringify(t.ranks)} chosen=${t.chosen}`);
  };
  line("platform choose", report({}));
  // Synthetic: no measurement. It exists only to exercise the rule, and is marked so.
  const figure = { joules_micro: 2460000, decodes: 120, window_us: 6000000, class: "OnChipCounter", relative_ppm: 40000, filtering: "Unknown", coverage: "Whole", fabrics: ["cpu", "gpu"], note: "synthetic: no measurement" };
  line("platform choose with a synthetic figure", report({ "wai.video.int_hyper wasm": figure }));
}

window.platformDecode = { done: false, report: "" };
try {
  await init();
  say(`platform: userAgent=${navigator.userAgent}`);
  const paths = await (await fetch("objects.json")).json();
  const first = await av1Objects(paths);
  await hevcRoundTrip();
  if (first) choose(first); else say("platform choose: no AV1 object to choose with");
} catch (e) {
  say(`platform: page error: ${e && e.stack || e}`);
}
window.platformDecode.report = lines.join("\n");
document.getElementById("report").textContent = window.platformDecode.report;
window.platformDecode.done = true;