// Composition in the page (wai/extensions/live-compose.md): the room's participants laid // out on a canvas by wai-live-web's integer layout, from the room's WCD1 descriptor and // its participant set at the instant, and their audio mixed by its integer mixer into an // AudioWorklet. The layout is recomputed on each lifecycle event (§5.8); between events // the participant set does not change, and nor does the layout. // // Drawing spends only on what changed. A tile is drawn when its video has a new frame, at the // next display refresh, and with it whatever lies over it; the whole canvas only after a // re-layout. Nothing new, nothing drawn: a still room costs the page no draws at all, and a // large grid of unsynchronised sources costs one tile per frame and not the whole grid. const MIX_FRAMES = 960; // 20 ms at 48 kHz // The latencies kept for `stats()`: a few seconds of them, so it reports the present. const LATENCIES_KEPT = 512; const overlaps = (a, b) => a[0] < b[0] + b[2] && b[0] < a[0] + a[2] && a[1] < b[1] + b[3] && b[1] < a[1] + a[3]; export class Compositor { constructor(wai, canvas, descriptor, { log = () => {}, bindingOf = null, onError = null, now = () => performance.now() } = {}) { this.wai = wai; this.canvas = canvas; this.descriptorBytes = descriptor; this.desc = JSON.parse(wai.composeDescriptor(descriptor)); if (this.desc.outcome !== "read") throw new Error(`the composition descriptor: ${this.desc.reason}`); const p = this.desc.params; canvas.width = p.width; canvas.height = p.height; this.ctx = canvas.getContext("2d"); const bg = this.desc.background; this.background = `#${bg}`; this.frame = null; // the composition at the last re-layout, as wai-live-web gives it this.layouts = []; // each re-layout: t, set digest, tiles this.video = new Map(); // principal -> latest VideoFrame this.drawn = new Map(); // principal -> the rectangles last drawn this.draws = 0; this.log = log; this.onError = onError; this.now = now; this.started = false; // draws wait for `start()` this.waiting = false; // a draw is waiting for the next display refresh this.whole = true; // the whole canvas is to be drawn: a re-layout, or nothing drawn yet this.changed = new Map(); // principal -> when its newest undrawn frame arrived this.counts = { frames: 0, relayouts: 0, tiles: 0 }; this.latencies = []; // ms from a frame's arrival to its drawing, the newest last // The binding of each frame drawn (WAI browser-sink §10), when the page keeps them. this.bindingOf = bindingOf; this.audio = new Map(); // principal -> { chunks: Int16Array[], at, channels } this.mixed = { blocks: 0, voiced_blocks: 0, peak: 0 }; this.worklet = null; this.workletStats = { played: 0, underruns: 0, peak: 0 }; } relayout(room, t) { const out = JSON.parse(room.frame(this.descriptorBytes, BigInt(t))); if (out.outcome !== "compose") { this.log(`composition refused: ${out.reason} at ${JSON.stringify(out.at)}`); return null; } const f = out.frames[0]; const prev = this.frame; this.frame = f; this.counts.relayouts++; this.whole = true; this.schedule(); if (!prev || prev.set_digest !== f.set_digest) { this.layouts.push({ t: f.t, set_digest: f.set_digest, tiles: f.tiles.map((x) => ({ principal: x.principal, tile: x.tile })) }); this.log(`re-layout at t=${f.t}: ${f.tiles.length} tile(s), set ${f.set_digest.slice(0, 12)}`); } return f; } setVideo(principal, vf) { const old = this.video.get(principal); if (old) old.close(); this.video.set(principal, vf); this.counts.frames++; this.changed.set(principal, this.now()); this.schedule(); } /** Begin drawing: from here a new frame or a re-layout draws at the next display refresh. */ start() { this.started = true; this.schedule(); } /** A draw at the next display refresh, unless one is waiting for it. A hidden document has no * refreshes: it draws nothing until it is shown, and then only the newest frames. */ schedule() { if (!this.started || this.waiting) return; this.waiting = true; requestAnimationFrame(() => { this.waiting = false; if (!this.started) return; // closed while it waited try { this.draw(); } catch (e) { if (this.onError) this.onError(e); else throw e; } }); } draw() { const whole = this.whole; const c = this.ctx; if (whole) { c.fillStyle = this.background; c.fillRect(0, 0, this.canvas.width, this.canvas.height); } const at = this.now(); for (const t of this.changed.values()) this.latencies.push(at - t); if (this.latencies.length > LATENCIES_KEPT) this.latencies.splice(0, this.latencies.length - LATENCIES_KEPT); const changed = this.changed; this.changed = new Map(); this.whole = false; this.draws++; if (!this.frame) return; // Tiles in drawing order: the grid's, then the picture-in-picture window (§5.7). A tile drawn // again covers what lay over it, so that is drawn again too. const drawnNow = []; for (const t of this.frame.tiles) { const vf = this.video.get(t.principal); if (!t.dst || !vf) continue; if (!whole && !changed.has(t.principal) && !drawnNow.some((d) => overlaps(d, t.dst))) continue; drawnNow.push(t.dst); this.counts.tiles++; const [sx, sy, sw, sh] = t.src; const [dx, dy, dw, dh] = t.dst; c.drawImage(vf, sx, sy, sw, sh, dx, dy, dw, dh); const b = this.bindingOf ? this.bindingOf(vf) : null; this.drawn.set(t.principal, { tile: t.tile, src: t.src, dst: t.dst, frame: [vf.displayWidth, vf.displayHeight], binding: b ? b.binding : null, group: b ? b.group : null }); } } /** What drawing spent (WAI's UI budget): counts a test can hold, and the latency from a frame's * arrival to its drawing, which is reported and not held, a shared host's timing being noise. */ stats() { const l = [...this.latencies].sort((a, b) => a - b); const at = (p) => (l.length ? Number(l[Math.min(l.length - 1, Math.floor(l.length * p))].toFixed(2)) : null); return { ...this.counts, draws: this.draws, arrival_to_draw_p50_ms: at(0.5), arrival_to_draw_p95_ms: at(0.95) }; } // A pixel at the centre of each tile: what a test reads to see video was drawn there. samples() { if (!this.frame) return []; return this.frame.tiles.map((t) => { const [x, y, w, h] = t.tile; const d = this.ctx.getImageData(x + (w >> 1), y + (h >> 1), 1, 1).data; return { principal: t.principal, rgb: [d[0], d[1], d[2]] }; }); } // ------------------------------------------------------------------------------ audio /** The audio context the mix plays into, made now if it is not yet. Making one holds the * page's thread until the browser has its audio output, which the first context in a browser * starts: a page that must not stall later (a watch room's, stating twice a slot) makes it * first. `audioOpened`: when it was made (`wall_ms`, the page's `Date.now()`) and how long * making it held the thread (`took_us`). */ openAudio() { if (!this.ac) { const wall = Date.now(); const t = performance.now(); this.ac = new AudioContext({ sampleRate: this.desc.params.sample_rate, latencyHint: "interactive" }); this.audioOpened = { wall_ms: wall, took_us: Math.round((performance.now() - t) * 1000) }; } return this.ac; } async startAudio(url) { this.openAudio(); await this.ac.audioWorklet.addModule(url); this.worklet = new AudioWorkletNode(this.ac, "room-mix", { outputChannelCount: [2] }); this.worklet.port.onmessage = (e) => { this.workletStats = e.data; }; this.worklet.connect(this.ac.destination); try { await this.ac.resume(); } catch { /* resumes when allowed */ } this.mixTimer = setInterval(() => this.mixOnce(), 20); } /** Stop for good, as when the entry it was made for failed and another makes its own: no more * mixing or drawing, its frames released and its audio context closed. */ close() { this.started = false; clearInterval(this.mixTimer); this.mixTimer = null; if (this.worklet) { this.worklet.port.onmessage = null; this.worklet.disconnect(); this.worklet = null; } for (const vf of this.video.values()) vf.close(); this.video.clear(); if (this.ac && this.ac.state !== "closed") this.ac.close().catch(() => {}); } addAudio(principal, audioData) { let v = this.audio.get(principal); if (!v) { v = { chunks: [], channels: audioData.numberOfChannels }; this.audio.set(principal, v); } const n = audioData.numberOfFrames; const ch = audioData.numberOfChannels; const f = new Float32Array(n * ch); audioData.copyTo(f, { planeIndex: 0, format: "f32" }); const s = new Int16Array(n * ch); for (let i = 0; i < s.length; i++) s[i] = Math.max(-32768, Math.min(32767, Math.round(f[i] * 32767))); v.channels = ch; v.chunks.push(s); // At most a second queued per voice. let total = v.chunks.reduce((k, c) => k + c.length, 0); while (total > 48000 * ch && v.chunks.length > 1) total -= v.chunks.shift().length; audioData.close(); } take(v, frames) { const want = frames * v.channels; const out = new Int16Array(want); let at = 0; while (at < want && v.chunks.length) { const c = v.chunks[0]; const n = Math.min(c.length, want - at); out.set(c.subarray(0, n), at); at += n; if (n === c.length) v.chunks.shift(); else v.chunks[0] = c.subarray(n); } return { pcm: out, had: at }; } // One block of the mix (§6): the voices of the layout, each with its gain and pan. mixOnce() { if (!this.frame || !this.worklet) return; const voices = this.frame.voices; const chans = [], gains = [], pans = [], parts = []; let had = 0; for (const v of voices) { const a = this.audio.get(v.principal); if (!a) continue; const t = this.take(a, MIX_FRAMES); had += t.had; chans.push(a.channels); gains.push(v.gain); pans.push(v.pan); parts.push(t.pcm); } if (!parts.length) return; const pcm = new Int16Array(parts.reduce((k, p) => k + p.length, 0)); let at = 0; for (const p of parts) { pcm.set(p, at); at += p.length; } const out = this.wai.mixBlock(this.desc.params.channels, MIX_FRAMES, Uint8Array.from(chans), Uint32Array.from(gains), Int32Array.from(pans), pcm); let peak = 0; for (let i = 0; i < out.length; i++) { const a = Math.abs(out[i]); if (a > peak) peak = a; } this.mixed.blocks++; if (had) this.mixed.voiced_blocks++; if (peak > this.mixed.peak) this.mixed.peak = peak; this.worklet.port.postMessage({ pcm: out }, [out.buffer]); } }