open-standards/joule-code/joule-code-rs/crates/joule-moq-web/web/moq/media.js

// Media for a WAI track (wai/extensions/moq-streaming-format.md §3): every object is one
// whole WAI1 envelope, whose payload is in the framing SPEC §5 pins for its capability:
//
//   wai.video.vp9    VP9 frames in the IVF frame container (FourCC VP90)
//   wai.video.av1    five u32 (n, h, w, fps_num, fps_den), then each temporal unit as a
//                    u32 length and its low-overhead bitstream (every OBU with its size)
//   wai.audio.opus   an Ogg Opus stream (RFC 7845)
//   wai.video.avc    an access unit of an H.264 Annex B byte stream (a host's screen, WAI
//   wai.video.hevc   remote-desktop §3.2), its parameter sets before an intra-coded picture
//
// A live track carries one object per group (§4.1), so each object here is one encoded
// frame: an IVF file of one frame, an AV1 payload of one temporal unit, an Ogg Opus stream
// of one packet. The manifest's `target` (informational, SPEC §3) states the frame's
// dimensions, its timestamp and whether it is a key frame, which a decoder is told.

const le32 = (v, at, b) => new DataView(b.buffer, b.byteOffset).setUint32(at, v >>> 0, true);
const le16 = (v, at, b) => new DataView(b.buffer, b.byteOffset).setUint16(at, v, true);

export const CAPABILITY_OF = (codec) =>
  codec.startsWith("vp09") ? "wai.video.vp9" : codec.startsWith("av01") ? "wai.video.av1" : codec.startsWith("avc1") ? "wai.video.avc"
    : codec.startsWith("hvc1") || codec.startsWith("hev1") ? "wai.video.hevc" : codec === "opus" ? "wai.audio.opus" : null;

// ---------------------------------------------------------------------------------- IVF

export function ivfOne(frame, width, height, timestampUs) {
  const b = new Uint8Array(32 + 12 + frame.length);
  b.set([0x44, 0x4b, 0x49, 0x46], 0); // DKIF
  le16(0, 4, b); // version
  le16(32, 6, b); // header size
  b.set([0x56, 0x50, 0x39, 0x30], 8); // VP90
  le16(width, 12, b);
  le16(height, 14, b);
  le32(1_000_000, 16, b); // time base: microseconds
  le32(1, 20, b);
  le32(1, 24, b); // one frame
  le32(frame.length, 32, b);
  const dv = new DataView(b.buffer);
  dv.setBigUint64(36, BigInt(Math.max(0, Math.round(timestampUs))), true);
  b.set(frame, 44);
  return b;
}

export function ivfFrames(b) {
  const dv = new DataView(b.buffer, b.byteOffset, b.byteLength);
  if (b.length < 32 || String.fromCharCode(...b.subarray(0, 4)) !== "DKIF") throw new Error("not an IVF file");
  const hl = dv.getUint16(6, true);
  const out = [];
  for (let at = hl; at + 12 <= b.length; ) {
    const n = dv.getUint32(at, true);
    const ts = Number(dv.getBigUint64(at + 4, true));
    if (at + 12 + n > b.length) throw new Error("an IVF frame past its file");
    out.push({ data: b.subarray(at + 12, at + 12 + n), timestamp: ts });
    at += 12 + n;
  }
  return out;
}

// ---------------------------------------------------------------------------- AV1 (SPEC §5)

export function av1One(tu, width, height, fpsNum, fpsDen) {
  const b = new Uint8Array(20 + 4 + tu.length);
  le32(1, 0, b);
  le32(height, 4, b);
  le32(width, 8, b);
  le32(fpsNum, 12, b);
  le32(fpsDen, 16, b);
  le32(tu.length, 20, b);
  b.set(tu, 24);
  return b;
}

export function av1Units(b) {
  const dv = new DataView(b.buffer, b.byteOffset, b.byteLength);
  if (b.length < 20) throw new Error("an AV1 payload shorter than its framing header");
  const n = dv.getUint32(0, true);
  const out = [];
  let at = 20;
  for (let i = 0; i < n; i++) {
    const len = dv.getUint32(at, true);
    if (at + 4 + len > b.length) throw new Error("an AV1 temporal unit past its payload");
    out.push(b.subarray(at + 4, at + 4 + len));
    at += 4 + len;
  }
  if (at !== b.length) throw new Error("an AV1 payload holding more temporal units than it declares");
  return out;
}

// ------------------------------------------------------------------------- Ogg Opus (RFC 7845)

const CRC = (() => {
  const t = new Uint32Array(256);
  for (let i = 0; i < 256; i++) {
    let r = i << 24;
    for (let j = 0; j < 8; j++) r = r & 0x80000000 ? (r << 1) ^ 0x04c11db7 : r << 1;
    t[i] = r >>> 0;
  }
  return t;
})();

function oggCrc(b) {
  let c = 0;
  for (let i = 0; i < b.length; i++) c = ((c << 8) ^ CRC[((c >>> 24) ^ b[i]) & 0xff]) >>> 0;
  return c;
}

function oggPage(packet, { bos = false, eos = false, granule = 0n, serial, seq }) {
  const segs = [];
  let n = packet.length;
  while (n >= 255) { segs.push(255); n -= 255; }
  segs.push(n);
  const b = new Uint8Array(27 + segs.length + packet.length);
  b.set([0x4f, 0x67, 0x67, 0x53], 0); // OggS
  b[4] = 0;
  b[5] = (bos ? 2 : 0) | (eos ? 4 : 0);
  const dv = new DataView(b.buffer);
  dv.setBigInt64(6, granule, true);
  dv.setUint32(14, serial, true);
  dv.setUint32(18, seq, true);
  b[26] = segs.length;
  b.set(segs, 27);
  b.set(packet, 27 + segs.length);
  dv.setUint32(22, oggCrc(b), true);
  return b;
}

// One Opus packet as a whole Ogg Opus stream: OpusHead, OpusTags, the packet. Pre-skip is
// 0: the object continues a live stream whose decoder is already running, and a decoder
// that starts here starts at the packet.
export function oggOpusOne(packet, { channels = 1, inputRate = 48000, samples, serial = 0x77616901 }) {
  const head = new Uint8Array(19);
  head.set([0x4f, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64], 0); // OpusHead
  head[8] = 1;
  head[9] = channels;
  le16(0, 10, head);
  le32(inputRate, 12, head);
  le16(0, 16, head);
  head[18] = 0;
  const vendor = new TextEncoder().encode("wai");
  const tags = new Uint8Array(8 + 4 + vendor.length + 4);
  tags.set([0x4f, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73], 0); // OpusTags
  le32(vendor.length, 8, tags);
  tags.set(vendor, 12);
  le32(0, 12 + vendor.length, tags);
  const pages = [
    oggPage(head, { bos: true, serial, seq: 0 }),
    oggPage(tags, { serial, seq: 1 }),
    oggPage(packet, { eos: true, granule: BigInt(samples), serial, seq: 2 }),
  ];
  const out = new Uint8Array(pages.reduce((n, p) => n + p.length, 0));
  let at = 0;
  for (const p of pages) { out.set(p, at); at += p.length; }
  return out;
}

// The audio packets of an Ogg Opus stream (its two header packets passed over), each page
// checked against its CRC.
export function oggOpusPackets(b) {
  const out = [];
  const dv = new DataView(b.buffer, b.byteOffset, b.byteLength);
  let pending = [];
  let index = 0;
  for (let at = 0; at < b.length; ) {
    if (String.fromCharCode(...b.subarray(at, at + 4)) !== "OggS") throw new Error("not an Ogg page");
    const nseg = b[at + 26];
    const segs = b.subarray(at + 27, at + 27 + nseg);
    const body = segs.reduce((n, x) => n + x, 0);
    const page = b.slice(at, at + 27 + nseg + body);
    const want = dv.getUint32(at + 22, true);
    new DataView(page.buffer).setUint32(22, 0, true);
    if (oggCrc(page) !== want) throw new Error("an Ogg page whose CRC does not match");
    let p = at + 27 + nseg;
    for (const s of segs) {
      pending.push(b.subarray(p, p + s));
      p += s;
      if (s < 255) {
        const n = pending.reduce((k, x) => k + x.length, 0);
        const pk = new Uint8Array(n);
        let o = 0;
        for (const x of pending) { pk.set(x, o); o += x.length; }
        if (index >= 2) out.push(pk);
        index++;
        pending = [];
      }
    }
    at += 27 + nseg + body;
  }
  return out;
}

// ------------------------------------------------------------------------------ envelopes

export function manifestFor(capability, target) {
  const media = capability.startsWith("wai.audio") ? "audio" : "video";
  const kind = capability.split(".").pop();
  // Member order is the writer's and is kept (SPEC §2).
  return JSON.stringify({ wai: "1.0", media, intent: "replicate", model_requirement: { capability, fallback: null }, conditioning: { kind }, target });
}