diff --git a/joule-code/joule-code-rs/crates/joule-moq-web/e2e/compose.test.mjs b/joule-code/joule-code-rs/crates/joule-moq-web/e2e/compose.test.mjs new file mode 100644 index 000000000..2b71a2375 --- /dev/null +++ b/joule-code/joule-code-rs/crates/joule-moq-web/e2e/compose.test.mjs @@ -0,0 +1,148 @@ +// The compositor's drawing, without a browser: it spends only on what changed (UI budget). Run with +// `node --test e2e/compose.test.mjs`. +import test from "node:test"; +import assert from "node:assert/strict"; + +const refreshes = []; +globalThis.requestAnimationFrame = (f) => refreshes.push(f); +const refresh = () => refreshes.splice(0).forEach((f) => f()); + +const { Compositor } = await import("../web/room/compose.js"); + +const params = { width: 640, height: 360, fps: 30, sample_rate: 48000 }; +const fakeWai = { composeDescriptor: () => JSON.stringify({ outcome: "read", params, background: "101218" }) }; + +function make() { + const calls = { images: [], fills: 0 }; + const ctx = { + fillRect() { calls.fills++; }, + drawImage(vf, ...r) { calls.images.push({ who: vf.who, dst: r.slice(4) }); }, + set fillStyle(_) {}, + }; + const canvas = { getContext: (kind, opts) => { calls.opts = opts; return ctx; }, width: 0, height: 0 }; + let t = 0; + const c = new Compositor(fakeWai, canvas, new Uint8Array(), { now: () => t }); + return { c, calls, tick: (ms) => { t += ms; } }; +} + +const frame = (who) => ({ who, displayWidth: 16, displayHeight: 9, closed: false, close() { this.closed = true; } }); +const tile = (principal, dst) => ({ principal, tile: dst, src: [0, 0, 16, 9], dst }); +const room = (tiles) => ({ frame: () => JSON.stringify({ outcome: "compose", frames: [{ t: 0, set_digest: tiles.map((x) => x.principal).join(), tiles }] }) }); + +const A = tile("a", [0, 0, 320, 360]); +const B = tile("b", [320, 0, 320, 360]); +const PIP = tile("pip", [260, 200, 120, 90]); // over both A and B, last in drawing order + +test("nothing new draws nothing", () => { + const { c, calls } = make(); + c.relayout(room([A, B]), 0); + c.start(); + refresh(); + const { draws } = c.stats(); + assert.equal(draws, 1, "the first draw, of the layout"); + assert.equal(calls.fills, 1); + for (let i = 0; i < 10; i++) refresh(); + assert.equal(c.stats().draws, draws, "no refresh is even asked for"); + assert.equal(refreshes.length, 0); +}); + +test("nothing is drawn before it starts", () => { + const { c } = make(); + c.relayout(room([A]), 0); + c.setVideo("a", frame("a")); + assert.equal(refreshes.length, 0); + c.start(); + assert.equal(refreshes.length, 1); + refresh(); + assert.equal(c.stats().tiles, 1); +}); + +test("a new frame draws its tile alone, not the canvas", () => { + const { c, calls } = make(); + c.relayout(room([A, B]), 0); + c.start(); + c.setVideo("a", frame("a")); + c.setVideo("b", frame("b")); + refresh(); + calls.images.length = 0; + const fills = calls.fills; + c.setVideo("b", frame("b")); + refresh(); + assert.deepEqual(calls.images.map((d) => d.who), ["b"]); + assert.equal(calls.fills, fills, "the background is not painted again"); +}); + +test("frames between refreshes draw once, the newest, and the others are released", () => { + const { c, calls } = make(); + c.relayout(room([A]), 0); + c.start(); + const first = frame("a"); + c.setVideo("a", first); + c.setVideo("a", frame("a")); + assert.equal(refreshes.length, 1, "one draw asked for"); + assert.equal(first.closed, true); + refresh(); + assert.equal(calls.images.length, 1); + const s = c.stats(); + assert.equal(s.frames, 2); + assert.ok(s.tiles <= s.frames, `${s.tiles} tiles from ${s.frames} frames`); +}); + +test("a tile drawn again brings back what lay over it", () => { + const { c, calls } = make(); + c.relayout(room([A, B, PIP]), 0); + c.start(); + for (const who of ["a", "b", "pip"]) c.setVideo(who, frame(who)); + refresh(); + calls.images.length = 0; + c.setVideo("a", frame("a")); + refresh(); + assert.deepEqual(calls.images.map((d) => d.who), ["a", "pip"], "the window over it, in drawing order; not the tile it doesn't touch"); + calls.images.length = 0; + c.setVideo("pip", frame("pip")); + refresh(); + assert.deepEqual(calls.images.map((d) => d.who), ["pip"], "nothing lies over the window"); +}); + +test("tiles that share a column or a row but don't touch are left alone", () => { + const { c, calls } = make(); + const [TL, BL, TR] = [tile("tl", [0, 0, 320, 180]), tile("bl", [0, 180, 320, 180]), tile("tr", [320, 0, 320, 180])]; + c.relayout(room([TL, BL, TR]), 0); + c.start(); + for (const who of ["tl", "bl", "tr"]) c.setVideo(who, frame(who)); + refresh(); + calls.images.length = 0; + c.setVideo("tl", frame("tl")); + refresh(); + assert.deepEqual(calls.images.map((d) => d.who), ["tl"]); +}); + +test("a re-layout draws the whole canvas", () => { + const { c, calls } = make(); + c.relayout(room([A, B]), 0); + c.start(); + c.setVideo("a", frame("a")); + c.setVideo("b", frame("b")); + refresh(); + calls.images.length = 0; + const fills = calls.fills; + c.relayout(room([tile("a", [0, 0, 640, 360])]), 1); + refresh(); + assert.equal(calls.fills, fills + 1); + assert.deepEqual(calls.images.map((d) => d.who), ["a"]); +}); + +test("the canvas asks for no software rendering", () => { + const { calls } = make(); + assert.ok(!calls.opts || !calls.opts.willReadFrequently, "willReadFrequently is a software canvas, for a test's readback alone"); +}); + +test("arrival to draw is reported from when the frame arrived", () => { + const { c, tick } = make(); + c.relayout(room([A]), 0); + c.start(); + c.setVideo("a", frame("a")); + tick(16); + refresh(); + assert.equal(c.stats().arrival_to_draw_p95_ms, 16); +}); diff --git a/joule-code/joule-code-rs/crates/joule-moq-web/e2e/remote_input.test.mjs b/joule-code/joule-code-rs/crates/joule-moq-web/e2e/remote_input.test.mjs new file mode 100644 index 000000000..086dec4f1 --- /dev/null +++ b/joule-code/joule-code-rs/crates/joule-moq-web/e2e/remote_input.test.mjs @@ -0,0 +1,60 @@ +// What a controller's input waits for: a pointer moving waits for the next refresh, and anything +// else goes at once. Run with `node --test e2e/remote_input.test.mjs`. +import test from "node:test"; +import assert from "node:assert/strict"; + +globalThis.requestAnimationFrame ??= () => 0; +const { RemoteViewer } = await import("../web/room/remote.js"); + +function viewer() { + let t = 0; + const v = { + batch: [], + scheduled: 0, + flushed: 0, + roomNow: () => ++t, + log() {}, + schedule() { this.scheduled++; }, + flush() { this.flushed++; return Promise.resolve(); }, + }; + v.queue = RemoteViewer.prototype.queue; + v.moveTo = RemoteViewer.prototype.moveTo; + return v; +} + +test("a move waits for the next refresh", () => { + const v = viewer(); + v.moveTo({ x: 1, y: 1 }); + assert.deepEqual([v.scheduled, v.flushed], [1, 0]); +}); + +for (const a of [{ type: "key", usage: 4, down: true }, { type: "button", button: 0, down: true }, { type: "wheel", dx: 0, dy: 1 }, { type: "text", text: "a" }]) { + test(`a ${a.type} goes at once, with the moves before it`, () => { + const v = viewer(); + v.moveTo({ x: 1, y: 1 }); + v.queue(a); + assert.equal(v.flushed, 1); + assert.deepEqual(v.batch.map((x) => x.type), ["move", a.type]); + }); +} + +test("moves between refreshes are one action, at the latest place", () => { + const v = viewer(); + v.moveTo({ x: 1, y: 1 }); + const at = v.batch[0].at; + v.moveTo({ x: 2, y: 3 }); + v.moveTo({ x: 5, y: 8 }); + assert.equal(v.batch.length, 1); + assert.deepEqual([v.batch[0].x, v.batch[0].y], [5, 8]); + assert.ok(v.batch[0].at > at, "when it was last there"); +}); + +test("a move after a key is a new action, so the order they happened in stands", () => { + const v = viewer(); + v.moveTo({ x: 1, y: 1 }); + v.queue({ type: "key", usage: 4, down: true }); + v.moveTo({ x: 2, y: 2 }); + assert.deepEqual(v.batch.map((x) => x.type), ["move", "key", "move"]); + assert.equal(v.batch[0].x, 1, "the first move keeps its place"); + assert.equal(v.batch[2].x, 2); +}); diff --git a/joule-code/joule-code-rs/crates/joule-moq-web/e2e/room.e2e.mjs b/joule-code/joule-code-rs/crates/joule-moq-web/e2e/room.e2e.mjs index 2a6ce025f..3fe01d182 100644 --- a/joule-code/joule-code-rs/crates/joule-moq-web/e2e/room.e2e.mjs +++ b/joule-code/joule-code-rs/crates/joule-moq-web/e2e/room.e2e.mjs @@ -341,6 +341,10 @@ async function compareLayout(label, p, fx, dir, expectTiles) { check(`${label}: every tile was drawn into the layout's rectangles`, drawnOk, f.tiles.map((t) => `${t.principal.slice(0, 6)} ${JSON.stringify(t.dst)}`).join(", ")); const bg = [0x10, 0x12, 0x18]; const shown = snap.samples.filter((s) => s.principal !== "" && JSON.stringify(s.rgb) !== JSON.stringify(bg)); + // The UI budget: every draw answers a new frame or a re-layout (the first answers the start). A + // count, which a busy host can't flake; the latency beside it is reported, not held. + const ui = snap.ui; + check(`${label}: drawing spends only on what changed`, ui.draws <= ui.frames + ui.relayouts + 1, JSON.stringify(ui)); check(`${label}: each tile shows video, not the background`, shown.length === f.tiles.length, snap.samples.map((s) => `${s.principal.slice(0, 6)}:${s.rgb}`).join(" ")); return snap; } diff --git a/joule-code/joule-code-rs/crates/joule-moq-web/e2e/run.sh b/joule-code/joule-code-rs/crates/joule-moq-web/e2e/run.sh index 2fb2016d1..25c3cf8fc 100755 --- a/joule-code/joule-code-rs/crates/joule-moq-web/e2e/run.sh +++ b/joule-code/joule-code-rs/crates/joule-moq-web/e2e/run.sh @@ -73,7 +73,7 @@ fi BROWSER_BIN=${BROWSER_BIN:-} step "unit tests of the page's modules" -node --test "$CRATE/web/moq/capsule.test.mjs" "$CRATE/web/moq/media.test.mjs" "$CRATE/web/moq/session.test.mjs" "$CRATE/web/ads/presentation.test.mjs" "$CRATE/web/room/privacy-pass.test.mjs" "$CRATE/web/room/binding.test.mjs" "$CRATE/web/room/cowatch.test.mjs" "$CRATE/web/room/remote.test.mjs" "$CRATE/web/room/markup.test.mjs" "$CRATE/web/room/endpoint.test.mjs" "$CRATE/web/room/termination.test.mjs" +node --test "$CRATE/e2e/compose.test.mjs" "$CRATE/e2e/remote_input.test.mjs" "$CRATE/web/moq/capsule.test.mjs" "$CRATE/web/moq/media.test.mjs" "$CRATE/web/moq/session.test.mjs" "$CRATE/web/ads/presentation.test.mjs" "$CRATE/web/room/privacy-pass.test.mjs" "$CRATE/web/room/binding.test.mjs" "$CRATE/web/room/cowatch.test.mjs" "$CRATE/web/room/remote.test.mjs" "$CRATE/web/room/markup.test.mjs" "$CRATE/web/room/endpoint.test.mjs" "$CRATE/web/room/termination.test.mjs" step "end-to-end" browser=() diff --git a/joule-code/joule-code-rs/crates/joule-moq-web/web/room/app.js b/joule-code/joule-code-rs/crates/joule-moq-web/web/room/app.js index 09e39e8f0..7bb481c65 100644 --- a/joule-code/joule-code-rs/crates/joule-moq-web/web/room/app.js +++ b/joule-code/joule-code-rs/crates/joule-moq-web/web/room/app.js @@ -157,7 +157,12 @@ function log(m) { logLines.push(line); if (logLines.length > 2000) logLines.shift(); const el = $("log"); - if (el) { el.textContent += line + "\n"; el.scrollTop = el.scrollHeight; } + if (el) { + // A text node a line, as many as are kept: one more line never rebuilds what is shown. + el.append(line + "\n"); + if (el.childNodes.length > 2000) el.firstChild.remove(); + el.scrollTop = el.scrollHeight; + } console.log(line); } @@ -554,11 +559,11 @@ async function begin(role, stream) { // A remote desktop room (WAI remote-desktop) has no composition: its host's screen is drawn // on a canvas of its own. if (role !== "clock" && fx.compose) { - compositor = new Compositor(wai, $("canvas"), unhex(fx.compose), { log, bindingOf: (vf) => SideTable.of(vf) }); + compositor = new Compositor(wai, $("canvas"), unhex(fx.compose), { log, bindingOf: (vf) => SideTable.of(vf), onError: fail }); if (DEBUG_BINDING) setInterval(() => { try { renderStates($("binding-debug"), bindingRows()); } catch (e) { fail(e); } }, 500); await compositor.startAudio(new URL("./worklet.js", import.meta.url)); await startTermination(); - setInterval(() => { try { compositor.draw(); } catch (e) { fail(e); } }, Math.round(1000 / compositor.desc.params.fps)); + compositor.start(); } if (role !== "clock") setInterval(sweepMissing, 1000); status("Connecting to the relay…"); @@ -2042,7 +2047,10 @@ function renderTimeline(e) { const p = e.payload || {}; const what = e.kind === "caption" ? `“${p.text}” (${p.origin_label})` : e.kind === "chat" ? p.text : p.detail ? `${(p.subject || "").slice(0, 8)} ${JSON.stringify(p.detail)}` : ""; li.textContent = `t=${(e.t / 1e6).toFixed(3)}s ${e.who} ${e.kind} ${what}`; - $("timeline").appendChild(li); + const list = $("timeline"); + list.appendChild(li); + // What is shown is bounded; `state.timeline` holds the whole of it. + if (list.childElementCount > 1000) list.firstElementChild.remove(); } function renderCaption() { @@ -2097,6 +2105,7 @@ window.roomTest.snapshot = () => ({ layouts: compositor && compositor.layouts, drawn: compositor && Object.fromEntries(compositor.drawn), draws: compositor && compositor.draws, + ui: compositor && compositor.stats(), samples: compositor && compositor.samples(), mixed: compositor && compositor.mixed, worklet: compositor && compositor.workletStats, diff --git a/joule-code/joule-code-rs/crates/joule-moq-web/web/room/compose.js b/joule-code/joule-code-rs/crates/joule-moq-web/web/room/compose.js index 2592f5525..b4967e67a 100644 --- a/joule-code/joule-code-rs/crates/joule-moq-web/web/room/compose.js +++ b/joule-code/joule-code-rs/crates/joule-moq-web/web/room/compose.js @@ -3,11 +3,20 @@ // its participant set at the instant, and their audio mixed by its integer mixer into an // AudioWorklet. The layout is recomputed on each lifecycle event (§5.8); between events // the participant set does not change, and nor does the layout. +// +// Drawing spends only on what changed. A tile is drawn when its video has a new frame, at the +// next display refresh, and with it whatever lies over it; the whole canvas only after a +// re-layout. Nothing new, nothing drawn: a still room costs the page no draws at all, and a +// large grid of unsynchronised sources costs one tile per frame and not the whole grid. const MIX_FRAMES = 960; // 20 ms at 48 kHz +// The latencies kept for `stats()`: a few seconds of them, so it reports the present. +const LATENCIES_KEPT = 512; + +const overlaps = (a, b) => a[0] < b[0] + b[2] && b[0] < a[0] + a[2] && a[1] < b[1] + b[3] && b[1] < a[1] + a[3]; export class Compositor { - constructor(wai, canvas, descriptor, { log = () => {}, bindingOf = null } = {}) { + constructor(wai, canvas, descriptor, { log = () => {}, bindingOf = null, onError = null, now = () => performance.now() } = {}) { this.wai = wai; this.canvas = canvas; this.descriptorBytes = descriptor; @@ -16,7 +25,7 @@ export class Compositor { const p = this.desc.params; canvas.width = p.width; canvas.height = p.height; - this.ctx = canvas.getContext("2d", { willReadFrequently: true }); + this.ctx = canvas.getContext("2d"); const bg = this.desc.background; this.background = `#${bg}`; this.frame = null; // the composition at the last re-layout, as wai-live-web gives it @@ -25,6 +34,14 @@ export class Compositor { this.drawn = new Map(); // principal -> the rectangles last drawn this.draws = 0; this.log = log; + this.onError = onError; + this.now = now; + this.started = false; // draws wait for `start()` + this.waiting = false; // a draw is waiting for the next display refresh + this.whole = true; // the whole canvas is to be drawn: a re-layout, or nothing drawn yet + this.changed = new Map(); // principal -> when its newest undrawn frame arrived + this.counts = { frames: 0, relayouts: 0, tiles: 0 }; + this.latencies = []; // ms from a frame's arrival to its drawing, the newest last // The binding of each frame drawn (WAI browser-sink §10), when the page keeps them. this.bindingOf = bindingOf; this.audio = new Map(); // principal -> { chunks: Int16Array[], at, channels } @@ -42,6 +59,9 @@ export class Compositor { const f = out.frames[0]; const prev = this.frame; this.frame = f; + this.counts.relayouts++; + this.whole = true; + this.schedule(); if (!prev || prev.set_digest !== f.set_digest) { this.layouts.push({ t: f.t, set_digest: f.set_digest, tiles: f.tiles.map((x) => ({ principal: x.principal, tile: x.tile })) }); this.log(`re-layout at t=${f.t}: ${f.tiles.length} tile(s), set ${f.set_digest.slice(0, 12)}`); @@ -53,24 +73,70 @@ export class Compositor { const old = this.video.get(principal); if (old) old.close(); this.video.set(principal, vf); + this.counts.frames++; + this.changed.set(principal, this.now()); + this.schedule(); + } + + /** Begin drawing: from here a new frame or a re-layout draws at the next display refresh. */ + start() { + this.started = true; + this.schedule(); + } + + /** A draw at the next display refresh, unless one is waiting for it. A hidden document has no + * refreshes: it draws nothing until it is shown, and then only the newest frames. */ + schedule() { + if (!this.started || this.waiting) return; + this.waiting = true; + requestAnimationFrame(() => { + this.waiting = false; + try { + this.draw(); + } catch (e) { + if (this.onError) this.onError(e); else throw e; + } + }); } draw() { + const whole = this.whole; const c = this.ctx; - c.fillStyle = this.background; - c.fillRect(0, 0, this.canvas.width, this.canvas.height); + if (whole) { + c.fillStyle = this.background; + c.fillRect(0, 0, this.canvas.width, this.canvas.height); + } + const at = this.now(); + for (const t of this.changed.values()) this.latencies.push(at - t); + if (this.latencies.length > LATENCIES_KEPT) this.latencies.splice(0, this.latencies.length - LATENCIES_KEPT); + const changed = this.changed; + this.changed = new Map(); + this.whole = false; + this.draws++; if (!this.frame) return; - // Tiles in drawing order: the grid's, then the picture-in-picture window (§5.7). + // Tiles in drawing order: the grid's, then the picture-in-picture window (§5.7). A tile drawn + // again covers what lay over it, so that is drawn again too. + const drawnNow = []; for (const t of this.frame.tiles) { const vf = this.video.get(t.principal); if (!t.dst || !vf) continue; + if (!whole && !changed.has(t.principal) && !drawnNow.some((d) => overlaps(d, t.dst))) continue; + drawnNow.push(t.dst); + this.counts.tiles++; const [sx, sy, sw, sh] = t.src; const [dx, dy, dw, dh] = t.dst; c.drawImage(vf, sx, sy, sw, sh, dx, dy, dw, dh); const b = this.bindingOf ? this.bindingOf(vf) : null; this.drawn.set(t.principal, { tile: t.tile, src: t.src, dst: t.dst, frame: [vf.displayWidth, vf.displayHeight], binding: b ? b.binding : null, group: b ? b.group : null }); } - this.draws++; + } + + /** What drawing spent (WAI's UI budget): counts a test can hold, and the latency from a frame's + * arrival to its drawing, which is reported and not held, a shared host's timing being noise. */ + stats() { + const l = [...this.latencies].sort((a, b) => a - b); + const at = (p) => (l.length ? Number(l[Math.min(l.length - 1, Math.floor(l.length * p))].toFixed(2)) : null); + return { ...this.counts, draws: this.draws, arrival_to_draw_p50_ms: at(0.5), arrival_to_draw_p95_ms: at(0.95) }; } // A pixel at the centre of each tile: what a test reads to see video was drawn there. diff --git a/joule-code/joule-code-rs/crates/joule-moq-web/web/room/remote.js b/joule-code/joule-code-rs/crates/joule-moq-web/web/room/remote.js index 05a940e25..cf3007cd9 100644 --- a/joule-code/joule-code-rs/crates/joule-moq-web/web/room/remote.js +++ b/joule-code/joule-code-rs/crates/joule-moq-web/web/room/remote.js @@ -702,8 +702,23 @@ export class RemoteViewer { /** Send the batched actions as one input statement (§5.1), no more than one a display refresh. */ queue(a) { - this.batch.push({ ...a, at: this.roomNow() }); - this.schedule(); + const queued = { ...a, at: this.roomNow() }; + this.batch.push(queued); + // A pointer moving is many actions a refresh, and only its latest place matters soon. A key, a + // button or a wheel is one thing that happened, and waiting for a refresh only delays it: it + // goes now, with whatever moves came before it. A loop of actions in one task still makes one + // statement, since the send starts after the task. + if (a.type === "move") this.schedule(); + else this.flush().catch((e) => this.log(`an input: ${e.message || e}`)); + return queued; + } + + /** The pointer moved. Moves between refreshes are one action, at the pointer's latest place, so + * long as nothing has been queued after it. */ + moveTo(p) { + const last = this.batch[this.batch.length - 1]; + if (last && last.type === "move") Object.assign(last, p, { at: this.roomNow() }); + else this.queue({ type: "move", ...p }); } /** A flush at the next display refresh, unless one is waiting for it. */ @@ -763,8 +778,7 @@ export class RemoteViewer { listen() { const c = this.canvas; c.tabIndex = 0; - let moved = null; - c.addEventListener("pointermove", (e) => { if (!this.may(CLASS.pointer)) return; const p = this.at(e); if (moved) { moved.x = p.x; moved.y = p.y; return; } moved = { type: "move", ...p }; this.queue(moved); requestAnimationFrame(() => { moved = null; }); }); + c.addEventListener("pointermove", (e) => { if (this.may(CLASS.pointer)) this.moveTo(this.at(e)); }); for (const [ev, down] of [["pointerdown", true], ["pointerup", false]]) { c.addEventListener(ev, (e) => { if (!this.may(CLASS.pointer)) return; e.preventDefault(); c.focus(); this.queue({ type: "button", button: e.button, down, ...this.at(e) }); }); }