joule-code/joule-code-rs/crates/joule-moq-web/e2e/compose.test.mjs

@@ -0,0 +1,148 @@
+// The compositor's drawing, without a browser: it spends only on what changed (UI budget). Run with
+// `node --test e2e/compose.test.mjs`.
+import test from "node:test";
+import assert from "node:assert/strict";
+
+const refreshes = [];
+globalThis.requestAnimationFrame = (f) => refreshes.push(f);
+const refresh = () => refreshes.splice(0).forEach((f) => f());
+
+const { Compositor } = await import("../web/room/compose.js");
+
+const params = { width: 640, height: 360, fps: 30, sample_rate: 48000 };
+const fakeWai = { composeDescriptor: () => JSON.stringify({ outcome: "read", params, background: "101218" }) };
+
+function make() {
+  const calls = { images: [], fills: 0 };
+  const ctx = {
+    fillRect() { calls.fills++; },
+    drawImage(vf, ...r) { calls.images.push({ who: vf.who, dst: r.slice(4) }); },
+    set fillStyle(_) {},
+  };
+  const canvas = { getContext: (kind, opts) => { calls.opts = opts; return ctx; }, width: 0, height: 0 };
+  let t = 0;
+  const c = new Compositor(fakeWai, canvas, new Uint8Array(), { now: () => t });
+  return { c, calls, tick: (ms) => { t += ms; } };
+}
+
+const frame = (who) => ({ who, displayWidth: 16, displayHeight: 9, closed: false, close() { this.closed = true; } });
+const tile = (principal, dst) => ({ principal, tile: dst, src: [0, 0, 16, 9], dst });
+const room = (tiles) => ({ frame: () => JSON.stringify({ outcome: "compose", frames: [{ t: 0, set_digest: tiles.map((x) => x.principal).join(), tiles }] }) });
+
+const A = tile("a", [0, 0, 320, 360]);
+const B = tile("b", [320, 0, 320, 360]);
+const PIP = tile("pip", [260, 200, 120, 90]); // over both A and B, last in drawing order
+
+test("nothing new draws nothing", () => {
+  const { c, calls } = make();
+  c.relayout(room([A, B]), 0);
+  c.start();
+  refresh();
+  const { draws } = c.stats();
+  assert.equal(draws, 1, "the first draw, of the layout");
+  assert.equal(calls.fills, 1);
+  for (let i = 0; i < 10; i++) refresh();
+  assert.equal(c.stats().draws, draws, "no refresh is even asked for");
+  assert.equal(refreshes.length, 0);
+});
+
+test("nothing is drawn before it starts", () => {
+  const { c } = make();
+  c.relayout(room([A]), 0);
+  c.setVideo("a", frame("a"));
+  assert.equal(refreshes.length, 0);
+  c.start();
+  assert.equal(refreshes.length, 1);
+  refresh();
+  assert.equal(c.stats().tiles, 1);
+});
+
+test("a new frame draws its tile alone, not the canvas", () => {
+  const { c, calls } = make();
+  c.relayout(room([A, B]), 0);
+  c.start();
+  c.setVideo("a", frame("a"));
+  c.setVideo("b", frame("b"));
+  refresh();
+  calls.images.length = 0;
+  const fills = calls.fills;
+  c.setVideo("b", frame("b"));
+  refresh();
+  assert.deepEqual(calls.images.map((d) => d.who), ["b"]);
+  assert.equal(calls.fills, fills, "the background is not painted again");
+});
+
+test("frames between refreshes draw once, the newest, and the others are released", () => {
+  const { c, calls } = make();
+  c.relayout(room([A]), 0);
+  c.start();
+  const first = frame("a");
+  c.setVideo("a", first);
+  c.setVideo("a", frame("a"));
+  assert.equal(refreshes.length, 1, "one draw asked for");
+  assert.equal(first.closed, true);
+  refresh();
+  assert.equal(calls.images.length, 1);
+  const s = c.stats();
+  assert.equal(s.frames, 2);
+  assert.ok(s.tiles <= s.frames, `${s.tiles} tiles from ${s.frames} frames`);
+});
+
+test("a tile drawn again brings back what lay over it", () => {
+  const { c, calls } = make();
+  c.relayout(room([A, B, PIP]), 0);
+  c.start();
+  for (const who of ["a", "b", "pip"]) c.setVideo(who, frame(who));
+  refresh();
+  calls.images.length = 0;
+  c.setVideo("a", frame("a"));
+  refresh();
+  assert.deepEqual(calls.images.map((d) => d.who), ["a", "pip"], "the window over it, in drawing order; not the tile it doesn't touch");
+  calls.images.length = 0;
+  c.setVideo("pip", frame("pip"));
+  refresh();
+  assert.deepEqual(calls.images.map((d) => d.who), ["pip"], "nothing lies over the window");
+});
+
+test("tiles that share a column or a row but don't touch are left alone", () => {
+  const { c, calls } = make();
+  const [TL, BL, TR] = [tile("tl", [0, 0, 320, 180]), tile("bl", [0, 180, 320, 180]), tile("tr", [320, 0, 320, 180])];
+  c.relayout(room([TL, BL, TR]), 0);
+  c.start();
+  for (const who of ["tl", "bl", "tr"]) c.setVideo(who, frame(who));
+  refresh();
+  calls.images.length = 0;
+  c.setVideo("tl", frame("tl"));
+  refresh();
+  assert.deepEqual(calls.images.map((d) => d.who), ["tl"]);
+});
+
+test("a re-layout draws the whole canvas", () => {
+  const { c, calls } = make();
+  c.relayout(room([A, B]), 0);
+  c.start();
+  c.setVideo("a", frame("a"));
+  c.setVideo("b", frame("b"));
+  refresh();
+  calls.images.length = 0;
+  const fills = calls.fills;
+  c.relayout(room([tile("a", [0, 0, 640, 360])]), 1);
+  refresh();
+  assert.equal(calls.fills, fills + 1);
+  assert.deepEqual(calls.images.map((d) => d.who), ["a"]);
+});
+
+test("the canvas asks for no software rendering", () => {
+  const { calls } = make();
+  assert.ok(!calls.opts || !calls.opts.willReadFrequently, "willReadFrequently is a software canvas, for a test's readback alone");
+});
+
+test("arrival to draw is reported from when the frame arrived", () => {
+  const { c, tick } = make();
+  c.relayout(room([A]), 0);
+  c.start();
+  c.setVideo("a", frame("a"));
+  tick(16);
+  refresh();
+  assert.equal(c.stats().arrival_to_draw_p95_ms, 16);
+});

joule-code/joule-code-rs/crates/joule-moq-web/e2e/remote_input.test.mjs

@@ -0,0 +1,60 @@
+// What a controller's input waits for: a pointer moving waits for the next refresh, and anything
+// else goes at once. Run with `node --test e2e/remote_input.test.mjs`.
+import test from "node:test";
+import assert from "node:assert/strict";
+
+globalThis.requestAnimationFrame ??= () => 0;
+const { RemoteViewer } = await import("../web/room/remote.js");
+
+function viewer() {
+  let t = 0;
+  const v = {
+    batch: [],
+    scheduled: 0,
+    flushed: 0,
+    roomNow: () => ++t,
+    log() {},
+    schedule() { this.scheduled++; },
+    flush() { this.flushed++; return Promise.resolve(); },
+  };
+  v.queue = RemoteViewer.prototype.queue;
+  v.moveTo = RemoteViewer.prototype.moveTo;
+  return v;
+}
+
+test("a move waits for the next refresh", () => {
+  const v = viewer();
+  v.moveTo({ x: 1, y: 1 });
+  assert.deepEqual([v.scheduled, v.flushed], [1, 0]);
+});
+
+for (const a of [{ type: "key", usage: 4, down: true }, { type: "button", button: 0, down: true }, { type: "wheel", dx: 0, dy: 1 }, { type: "text", text: "a" }]) {
+  test(`a ${a.type} goes at once, with the moves before it`, () => {
+    const v = viewer();
+    v.moveTo({ x: 1, y: 1 });
+    v.queue(a);
+    assert.equal(v.flushed, 1);
+    assert.deepEqual(v.batch.map((x) => x.type), ["move", a.type]);
+  });
+}
+
+test("moves between refreshes are one action, at the latest place", () => {
+  const v = viewer();
+  v.moveTo({ x: 1, y: 1 });
+  const at = v.batch[0].at;
+  v.moveTo({ x: 2, y: 3 });
+  v.moveTo({ x: 5, y: 8 });
+  assert.equal(v.batch.length, 1);
+  assert.deepEqual([v.batch[0].x, v.batch[0].y], [5, 8]);
+  assert.ok(v.batch[0].at > at, "when it was last there");
+});
+
+test("a move after a key is a new action, so the order they happened in stands", () => {
+  const v = viewer();
+  v.moveTo({ x: 1, y: 1 });
+  v.queue({ type: "key", usage: 4, down: true });
+  v.moveTo({ x: 2, y: 2 });
+  assert.deepEqual(v.batch.map((x) => x.type), ["move", "key", "move"]);
+  assert.equal(v.batch[0].x, 1, "the first move keeps its place");
+  assert.equal(v.batch[2].x, 2);
+});

joule-code/joule-code-rs/crates/joule-moq-web/e2e/room.e2e.mjs

@@ -341,6 +341,10 @@ async function compareLayout(label, p, fx, dir, expectTiles) {
   check(`${label}: every tile was drawn into the layout's rectangles`, drawnOk, f.tiles.map((t) => `${t.principal.slice(0, 6)} ${JSON.stringify(t.dst)}`).join(", "));
   const bg = [0x10, 0x12, 0x18];
   const shown = snap.samples.filter((s) => s.principal !== "" && JSON.stringify(s.rgb) !== JSON.stringify(bg));
+  // The UI budget: every draw answers a new frame or a re-layout (the first answers the start). A
+  // count, which a busy host can't flake; the latency beside it is reported, not held.
+  const ui = snap.ui;
+  check(`${label}: drawing spends only on what changed`, ui.draws <= ui.frames + ui.relayouts + 1, JSON.stringify(ui));
   check(`${label}: each tile shows video, not the background`, shown.length === f.tiles.length, snap.samples.map((s) => `${s.principal.slice(0, 6)}:${s.rgb}`).join(" "));
   return snap;
 }

joule-code/joule-code-rs/crates/joule-moq-web/e2e/run.sh

@@ -73,7 +73,7 @@ fi
 BROWSER_BIN=${BROWSER_BIN:-}
 
 step "unit tests of the page's modules"
-node --test "$CRATE/web/moq/capsule.test.mjs" "$CRATE/web/moq/media.test.mjs" "$CRATE/web/moq/session.test.mjs" "$CRATE/web/ads/presentation.test.mjs" "$CRATE/web/room/privacy-pass.test.mjs" "$CRATE/web/room/binding.test.mjs" "$CRATE/web/room/cowatch.test.mjs" "$CRATE/web/room/remote.test.mjs" "$CRATE/web/room/markup.test.mjs" "$CRATE/web/room/endpoint.test.mjs" "$CRATE/web/room/termination.test.mjs"
+node --test "$CRATE/e2e/compose.test.mjs" "$CRATE/e2e/remote_input.test.mjs" "$CRATE/web/moq/capsule.test.mjs" "$CRATE/web/moq/media.test.mjs" "$CRATE/web/moq/session.test.mjs" "$CRATE/web/ads/presentation.test.mjs" "$CRATE/web/room/privacy-pass.test.mjs" "$CRATE/web/room/binding.test.mjs" "$CRATE/web/room/cowatch.test.mjs" "$CRATE/web/room/remote.test.mjs" "$CRATE/web/room/markup.test.mjs" "$CRATE/web/room/endpoint.test.mjs" "$CRATE/web/room/termination.test.mjs"
 
 step "end-to-end"
 browser=()

joule-code/joule-code-rs/crates/joule-moq-web/web/room/app.js

@@ -157,7 +157,12 @@ function log(m) {
   logLines.push(line);
   if (logLines.length > 2000) logLines.shift();
   const el = $("log");
-  if (el) { el.textContent += line + "\n"; el.scrollTop = el.scrollHeight; }
+  if (el) {
+    // A text node a line, as many as are kept: one more line never rebuilds what is shown.
+    el.append(line + "\n");
+    if (el.childNodes.length > 2000) el.firstChild.remove();
+    el.scrollTop = el.scrollHeight;
+  }
   console.log(line);
 }
 
@@ -554,11 +559,11 @@ async function begin(role, stream) {
   // A remote desktop room (WAI remote-desktop) has no composition: its host's screen is drawn
   // on a canvas of its own.
   if (role !== "clock" && fx.compose) {
-    compositor = new Compositor(wai, $("canvas"), unhex(fx.compose), { log, bindingOf: (vf) => SideTable.of(vf) });
+    compositor = new Compositor(wai, $("canvas"), unhex(fx.compose), { log, bindingOf: (vf) => SideTable.of(vf), onError: fail });
     if (DEBUG_BINDING) setInterval(() => { try { renderStates($("binding-debug"), bindingRows()); } catch (e) { fail(e); } }, 500);
     await compositor.startAudio(new URL("./worklet.js", import.meta.url));
     await startTermination();
-    setInterval(() => { try { compositor.draw(); } catch (e) { fail(e); } }, Math.round(1000 / compositor.desc.params.fps));
+    compositor.start();
   }
   if (role !== "clock") setInterval(sweepMissing, 1000);
   status("Connecting to the relay…");
@@ -2042,7 +2047,10 @@ function renderTimeline(e) {
   const p = e.payload || {};
   const what = e.kind === "caption" ? `“${p.text}” (${p.origin_label})` : e.kind === "chat" ? p.text : p.detail ? `${(p.subject || "").slice(0, 8)} ${JSON.stringify(p.detail)}` : "";
   li.textContent = `t=${(e.t / 1e6).toFixed(3)}s ${e.who} ${e.kind} ${what}`;
-  $("timeline").appendChild(li);
+  const list = $("timeline");
+  list.appendChild(li);
+  // What is shown is bounded; `state.timeline` holds the whole of it.
+  if (list.childElementCount > 1000) list.firstElementChild.remove();
 }
 
 function renderCaption() {
@@ -2097,6 +2105,7 @@ window.roomTest.snapshot = () => ({
   layouts: compositor && compositor.layouts,
   drawn: compositor && Object.fromEntries(compositor.drawn),
   draws: compositor && compositor.draws,
+  ui: compositor && compositor.stats(),
   samples: compositor && compositor.samples(),
   mixed: compositor && compositor.mixed,
   worklet: compositor && compositor.workletStats,

joule-code/joule-code-rs/crates/joule-moq-web/web/room/compose.js

@@ -3,11 +3,20 @@
 // its participant set at the instant, and their audio mixed by its integer mixer into an
 // AudioWorklet. The layout is recomputed on each lifecycle event (§5.8); between events
 // the participant set does not change, and nor does the layout.
+//
+// Drawing spends only on what changed. A tile is drawn when its video has a new frame, at the
+// next display refresh, and with it whatever lies over it; the whole canvas only after a
+// re-layout. Nothing new, nothing drawn: a still room costs the page no draws at all, and a
+// large grid of unsynchronised sources costs one tile per frame and not the whole grid.
 
 const MIX_FRAMES = 960; // 20 ms at 48 kHz
+// The latencies kept for `stats()`: a few seconds of them, so it reports the present.
+const LATENCIES_KEPT = 512;
+
+const overlaps = (a, b) => a[0] < b[0] + b[2] && b[0] < a[0] + a[2] && a[1] < b[1] + b[3] && b[1] < a[1] + a[3];
 
 export class Compositor {
-  constructor(wai, canvas, descriptor, { log = () => {}, bindingOf = null } = {}) {
+  constructor(wai, canvas, descriptor, { log = () => {}, bindingOf = null, onError = null, now = () => performance.now() } = {}) {
     this.wai = wai;
     this.canvas = canvas;
     this.descriptorBytes = descriptor;
@@ -16,7 +25,7 @@ export class Compositor {
     const p = this.desc.params;
     canvas.width = p.width;
     canvas.height = p.height;
-    this.ctx = canvas.getContext("2d", { willReadFrequently: true });
+    this.ctx = canvas.getContext("2d");
     const bg = this.desc.background;
     this.background = `#${bg}`;
     this.frame = null; // the composition at the last re-layout, as wai-live-web gives it
@@ -25,6 +34,14 @@ export class Compositor {
     this.drawn = new Map(); // principal -> the rectangles last drawn
     this.draws = 0;
     this.log = log;
+    this.onError = onError;
+    this.now = now;
+    this.started = false; // draws wait for `start()`
+    this.waiting = false; // a draw is waiting for the next display refresh
+    this.whole = true; // the whole canvas is to be drawn: a re-layout, or nothing drawn yet
+    this.changed = new Map(); // principal -> when its newest undrawn frame arrived
+    this.counts = { frames: 0, relayouts: 0, tiles: 0 };
+    this.latencies = []; // ms from a frame's arrival to its drawing, the newest last
     // The binding of each frame drawn (WAI browser-sink §10), when the page keeps them.
     this.bindingOf = bindingOf;
     this.audio = new Map(); // principal -> { chunks: Int16Array[], at, channels }
@@ -42,6 +59,9 @@ export class Compositor {
     const f = out.frames[0];
     const prev = this.frame;
     this.frame = f;
+    this.counts.relayouts++;
+    this.whole = true;
+    this.schedule();
     if (!prev || prev.set_digest !== f.set_digest) {
       this.layouts.push({ t: f.t, set_digest: f.set_digest, tiles: f.tiles.map((x) => ({ principal: x.principal, tile: x.tile })) });
       this.log(`re-layout at t=${f.t}: ${f.tiles.length} tile(s), set ${f.set_digest.slice(0, 12)}`);
@@ -53,24 +73,70 @@ export class Compositor {
     const old = this.video.get(principal);
     if (old) old.close();
     this.video.set(principal, vf);
+    this.counts.frames++;
+    this.changed.set(principal, this.now());
+    this.schedule();
+  }
+
+  /** Begin drawing: from here a new frame or a re-layout draws at the next display refresh. */
+  start() {
+    this.started = true;
+    this.schedule();
+  }
+
+  /** A draw at the next display refresh, unless one is waiting for it. A hidden document has no
+   * refreshes: it draws nothing until it is shown, and then only the newest frames. */
+  schedule() {
+    if (!this.started || this.waiting) return;
+    this.waiting = true;
+    requestAnimationFrame(() => {
+      this.waiting = false;
+      try {
+        this.draw();
+      } catch (e) {
+        if (this.onError) this.onError(e); else throw e;
+      }
+    });
   }
 
   draw() {
+    const whole = this.whole;
     const c = this.ctx;
-    c.fillStyle = this.background;
-    c.fillRect(0, 0, this.canvas.width, this.canvas.height);
+    if (whole) {
+      c.fillStyle = this.background;
+      c.fillRect(0, 0, this.canvas.width, this.canvas.height);
+    }
+    const at = this.now();
+    for (const t of this.changed.values()) this.latencies.push(at - t);
+    if (this.latencies.length > LATENCIES_KEPT) this.latencies.splice(0, this.latencies.length - LATENCIES_KEPT);
+    const changed = this.changed;
+    this.changed = new Map();
+    this.whole = false;
+    this.draws++;
     if (!this.frame) return;
-    // Tiles in drawing order: the grid's, then the picture-in-picture window (§5.7).
+    // Tiles in drawing order: the grid's, then the picture-in-picture window (§5.7). A tile drawn
+    // again covers what lay over it, so that is drawn again too.
+    const drawnNow = [];
     for (const t of this.frame.tiles) {
       const vf = this.video.get(t.principal);
       if (!t.dst || !vf) continue;
+      if (!whole && !changed.has(t.principal) && !drawnNow.some((d) => overlaps(d, t.dst))) continue;
+      drawnNow.push(t.dst);
+      this.counts.tiles++;
       const [sx, sy, sw, sh] = t.src;
       const [dx, dy, dw, dh] = t.dst;
       c.drawImage(vf, sx, sy, sw, sh, dx, dy, dw, dh);
       const b = this.bindingOf ? this.bindingOf(vf) : null;
       this.drawn.set(t.principal, { tile: t.tile, src: t.src, dst: t.dst, frame: [vf.displayWidth, vf.displayHeight], binding: b ? b.binding : null, group: b ? b.group : null });
     }
-    this.draws++;
+  }
+
+  /** What drawing spent (WAI's UI budget): counts a test can hold, and the latency from a frame's
+   * arrival to its drawing, which is reported and not held, a shared host's timing being noise. */
+  stats() {
+    const l = [...this.latencies].sort((a, b) => a - b);
+    const at = (p) => (l.length ? Number(l[Math.min(l.length - 1, Math.floor(l.length * p))].toFixed(2)) : null);
+    return { ...this.counts, draws: this.draws, arrival_to_draw_p50_ms: at(0.5), arrival_to_draw_p95_ms: at(0.95) };
   }
 
   // A pixel at the centre of each tile: what a test reads to see video was drawn there.

joule-code/joule-code-rs/crates/joule-moq-web/web/room/remote.js

@@ -702,8 +702,23 @@ export class RemoteViewer {
 
   /** Send the batched actions as one input statement (§5.1), no more than one a display refresh. */
   queue(a) {
-    this.batch.push({ ...a, at: this.roomNow() });
-    this.schedule();
+    const queued = { ...a, at: this.roomNow() };
+    this.batch.push(queued);
+    // A pointer moving is many actions a refresh, and only its latest place matters soon. A key, a
+    // button or a wheel is one thing that happened, and waiting for a refresh only delays it: it
+    // goes now, with whatever moves came before it. A loop of actions in one task still makes one
+    // statement, since the send starts after the task.
+    if (a.type === "move") this.schedule();
+    else this.flush().catch((e) => this.log(`an input: ${e.message || e}`));
+    return queued;
+  }
+
+  /** The pointer moved. Moves between refreshes are one action, at the pointer's latest place, so
+   * long as nothing has been queued after it. */
+  moveTo(p) {
+    const last = this.batch[this.batch.length - 1];
+    if (last && last.type === "move") Object.assign(last, p, { at: this.roomNow() });
+    else this.queue({ type: "move", ...p });
   }
 
   /** A flush at the next display refresh, unless one is waiting for it. */
@@ -763,8 +778,7 @@ export class RemoteViewer {
   listen() {
     const c = this.canvas;
     c.tabIndex = 0;
-    let moved = null;
-    c.addEventListener("pointermove", (e) => { if (!this.may(CLASS.pointer)) return; const p = this.at(e); if (moved) { moved.x = p.x; moved.y = p.y; return; } moved = { type: "move", ...p }; this.queue(moved); requestAnimationFrame(() => { moved = null; }); });
+    c.addEventListener("pointermove", (e) => { if (this.may(CLASS.pointer)) this.moveTo(this.at(e)); });
     for (const [ev, down] of [["pointerdown", true], ["pointerup", false]]) {
       c.addEventListener(ev, (e) => { if (!this.may(CLASS.pointer)) return; e.preventDefault(); c.focus(); this.queue({ type: "button", button: e.button, down, ...this.at(e) }); });
     }