// A remote desktop in a room (WAI extensions/remote-desktop.md), in a page. // // A host publishes its screen as the room's `screen` track and decides every input; this module // is the viewer's part, in two halves: // // - the rules, with nothing of the page: input statements (§5.1, §5.2), floor statements // (§7.2) and latency records (§8.1) written byte for byte, key, text and clipboard actions // sealed to the host's sealing key with HPKE (§5.5, RFC 9180), every statement read (§4.2), // the host's rules (§5.3) and the floor in effect (§7.3) replayed over a room's records, and // the table from a keyboard event's `code` (`uievents-code`) to its HID usage (HID Usage // Tables 1.5 §10). Its tests (remote.test.mjs) run them against the corpus // wai/live-conformance/remote/; // - RemoteViewer, the page's viewer in a room a room service keeps (joule-moq-room with // `[remote]`): the page makes its key itself and asks the service for its grant, as the room // page does for every room, and the viewer reads its control power from that grant, and from // each exchange of it the room's clock signs (live-rooms §4.6). It draws the host's screen, // which the page has verified against its receipts and decoded, shows what the host captures // (its scope, redactions, whether it is paused, its encoder) and who holds the floor, sends // the viewer's pointer, keys, text and pasted clipboard text as input statements under its // grant, once the clock has recorded it joining and only the classes its grant in effect // gives, asks for the floor, and, for an input the host applied, signs a latency record of // the frame the outcome names, presented at the display refresh that showed it (`source` 1). // It sends nothing without a grant. Its keys, text and clipboard text go sealed to the // sealing key the host's screen statement states, and never in the clear: to a host that // states none it sends no key, text or clipboard action. Of another controller's input it // shows that the controller is typing or pasting, never what. // // Everything here is the platform's: WebCrypto's SHA-256, HMAC, Ed25519, X25519 and AES-GCM, a // canvas, pointer and keyboard events, requestAnimationFrame. const subtle = globalThis.crypto.subtle; const enc = new TextEncoder(); export const MAGIC = { screen: "WDS1", input: "WDI1", outcome: "WDO1", floor: "WDF1", latency: "WDL1" }; export const TYPE = { move: 1, button: 2, wheel: 3, key: 4, text: 5, touch: 6, clipboard: 7, sealed: 8 }; export const TYPE_NAME = { 1: "move", 2: "button", 3: "wheel", 4: "key", 5: "text", 6: "touch", 7: "clipboard", 8: "sealed" }; export const CLASS = { pointer: 1, keys: 2, touch: 4, clipboard: 8 }; const CLASS_OF = { 1: 1, 2: 1, 3: 1, 4: 2, 5: 2, 6: 4, 7: 8 }; export const FLOOR = { request: 1, give: 2, release: 3, take: 4 }; export const REFUSAL = { 1: "input_wrong_screen", 2: "input_not_permitted", 3: "input_no_floor", 4: "input_screen_paused", 5: "input_unsupported", 6: "input_out_of_bounds", 7: "input_stale", 8: "input_unsealed", 9: "input_bad_seal" }; /** The actions sealed to a host that states a sealing key (§5.5). */ export const SEALABLE = ["key", "text", "clipboard"]; /** The type an action acts as (§5.2): its own, or for a sealed action the type it seals. */ const actsAs = (a) => (a.type === "sealed" ? a.of : a.type); export const SOURCE = { frameTiming: 1, lightSensor: 2, headless: 3 }; const LATENCY_DOMAIN = enc.encode("wai:remote-latency\x01"); const META = 0x01; const NO_RTT = 0xffffffff; export const hex = (b) => Array.from(b, (x) => x.toString(16).padStart(2, "0")).join(""); export const unhex = (h) => new Uint8Array((h.match(/../g) || []).map((x) => parseInt(x, 16))); export function concat(...parts) { const n = parts.reduce((a, p) => a + p.length, 0); const out = new Uint8Array(n); let o = 0; for (const p of parts) { out.set(p, o); o += p.length; } return out; } export async function sha256(...parts) { return new Uint8Array(await subtle.digest("SHA-256", concat(...parts))); } const le = (n, bytes) => { const out = new Uint8Array(bytes); let v = BigInt(n); if (v < 0n) v += 1n << BigInt(8 * bytes); for (let i = 0; i < bytes; i++) { out[i] = Number(v & 0xffn); v >>= 8n; } return out; }; class Rd { constructor(b, p = 0) { this.b = b; this.p = p; } take(n) { if (this.p + n > this.b.length) throw new Error("short"); const s = this.b.subarray(this.p, this.p + n); this.p += n; return s; } u8() { return this.take(1)[0]; } uint(n) { const s = this.take(n); let v = 0n; for (let i = n - 1; i >= 0; i--) v = (v << 8n) | BigInt(s[i]); return v; } u16() { return Number(this.uint(2)); } i16() { const v = this.u16(); return v >= 0x8000 ? v - 0x10000 : v; } u32() { return Number(this.uint(4)); } u64() { return Number(this.uint(8)); } // every instant here is below 2^53 short() { return this.take(this.u8()); } done() { return this.p === this.b.length; } } const utf8 = new TextDecoder("utf-8", { fatal: true }); // ---------------------------------------------------------------- §5 input /** One action's bytes (§5.2): its type, then its fields. */ function actionBytes(a) { const t = TYPE[a.type]; switch (a.type) { case "move": return concat(Uint8Array.of(t), le(a.x, 2), le(a.y, 2)); case "button": return concat(Uint8Array.of(t, a.button, a.down ? 1 : 0), le(a.x, 2), le(a.y, 2)); case "wheel": return concat(Uint8Array.of(t), le(a.dx, 2), le(a.dy, 2), le(a.x, 2), le(a.y, 2)); case "key": return concat(Uint8Array.of(t), le(a.usage, 2), Uint8Array.of(a.down ? 1 : 0, a.modifiers || 0)); case "text": { const b = enc.encode(a.text); return concat(Uint8Array.of(t, b.length), b); } case "touch": return concat(Uint8Array.of(t, a.id, a.phase), le(a.x, 2), le(a.y, 2)); case "clipboard": { const b = enc.encode(a.text); return concat(Uint8Array.of(t), le(b.length, 2), b); } case "sealed": return concat(Uint8Array.of(t, TYPE[a.of]), le(a.length, 2), unhex(a.commit)); default: throw new Error(`no action ${a.type}`); } } /** An input statement from `magic` through its last action (§5.1). */ function clearPart(i) { if (i.actions.length < 1 || i.actions.length > 32) throw new Error("1 to 32 actions"); return concat(enc.encode(MAGIC.input), Uint8Array.of(1), unhex(i.host), unhex(i.screen), Uint8Array.of(i.actions.length), ...i.actions.map((a) => concat(le(a.dt_us, 4), actionBytes(a)))); } /** An input statement (§5.1): `{ host, screen (hex), actions: [{ dt_us, type, … }], seal?: { enc, * sealed } (hex) }`, its seal after its last action (§5.5). */ export function inputPayload(i) { const c = clearPart(i); return i.seal ? concat(c, unhex(i.seal.enc), unhex(i.seal.sealed)) : c; } // ---------------------------------------------------------------- §5.5 sealed actions const SEAL_INFO = enc.encode("wai:remote-seal\x01"); const COMMIT_DOMAIN = enc.encode("wai:remote-commit\x01"); const SALT_LEN = 16; const TEXT_PAD = 64; const be16 = (n) => Uint8Array.of(n >> 8, n & 0xff); const KEM = concat(enc.encode("KEM"), be16(0x0020)); const SUITE = concat(enc.encode("HPKE"), be16(0x0020), be16(0x0001), be16(0x0001)); const HPKE_V1 = enc.encode("HPKE-v1"); const NONE = new Uint8Array(0); // HMAC-SHA256. An empty key is HMAC's key of zero bytes, which HMAC pads to the hash's block with // zero bytes as it pads every short key (RFC 2104 §2): the same as 32 zero bytes, which is also // HKDF-Extract's salt when none is given (RFC 5869 §2.2). WebCrypto takes no empty key. async function hmac(key, data) { const k = await subtle.importKey("raw", key.length ? key : new Uint8Array(32), { name: "HMAC", hash: "SHA-256" }, false, ["sign"]); return new Uint8Array(await subtle.sign("HMAC", k, data)); } // RFC 9180 §4: LabeledExtract and LabeledExpand over HKDF-SHA256 (RFC 5869 §2.2, §2.3). const labeledExtract = (suite, salt, label, ikm) => hmac(salt, concat(HPKE_V1, suite, enc.encode(label), ikm)); async function labeledExpand(suite, prk, label, info, n) { const labeled = concat(be16(n), HPKE_V1, suite, enc.encode(label), info); let out = NONE, t = NONE; for (let i = 1; out.length < n; i++) { t = await hmac(prk, concat(t, labeled, Uint8Array.of(i))); out = concat(out, t); } return out.subarray(0, n); } const b64u = (s) => Uint8Array.from(atob(s.replace(/-/g, "+").replace(/_/g, "/")), (c) => c.charCodeAt(0)); /** WebCrypto's X25519 private key for a 32-byte scalar (RFC 8410's PKCS #8 form), and its public * key. */ export async function x25519Key(sk) { const pkcs8 = concat(Uint8Array.of(0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x6e, 0x04, 0x22, 0x04, 0x20), sk); const privateKey = await subtle.importKey("pkcs8", pkcs8, { name: "X25519" }, true, ["deriveBits"]); return { privateKey, publicKey: b64u((await subtle.exportKey("jwk", privateKey)).x) }; } /** DeriveKeyPair of DHKEM(X25519, HKDF-SHA256) (RFC 9180 §7.1.3): `{ privateKey, publicKey, * secret }`. */ export async function deriveKeyPair(ikm) { const prk = await labeledExtract(KEM, NONE, "dkp_prk", ikm); const secret = await labeledExpand(KEM, prk, "sk", NONE, 32); return { ...(await x25519Key(secret)), secret }; } /** HPKE's single-shot SealBase (RFC 9180 §6.1) with DHKEM(X25519, HKDF-SHA256), HKDF-SHA256 and * AES-128-GCM, its ephemeral key derived from `ikmE`: `{ enc, ct }`. WebCrypto refuses a public key * whose shared secret is all zero bytes, as §7.1.4 has it. */ export async function sealBase(pkR, info, aad, pt, ikmE) { const e = await deriveKeyPair(ikmE); const pub = await subtle.importKey("raw", pkR, { name: "X25519" }, false, []); const dh = new Uint8Array(await subtle.deriveBits({ name: "X25519", public: pub }, e.privateKey, 256)); const eae = await labeledExtract(KEM, NONE, "eae_prk", dh); const shared = await labeledExpand(KEM, eae, "shared_secret", concat(e.publicKey, pkR), 32); const ctx = concat(Uint8Array.of(0), await labeledExtract(SUITE, NONE, "psk_id_hash", NONE), await labeledExtract(SUITE, NONE, "info_hash", info)); const secret = await labeledExtract(SUITE, shared, "secret", NONE); const key = await subtle.importKey("raw", await labeledExpand(SUITE, secret, "key", ctx, 16), "AES-GCM", false, ["encrypt"]); const nonce = await labeledExpand(SUITE, secret, "base_nonce", ctx, 12); const ct = new Uint8Array(await subtle.encrypt({ name: "AES-GCM", iv: nonce, additionalData: aad, tagLength: 128 }, key, pt)); return { enc: e.publicKey, ct }; } /** A key, text or clipboard action's opening (§5.5): its salt, then its fields, a text's padded to * 64 bytes and a clipboard text's with no length before it. */ function openingOf(salt, a) { if (a.type === "key") return concat(salt, le(a.usage, 2), Uint8Array.of(a.down ? 1 : 0, a.modifiers || 0)); const b = enc.encode(a.text); if (a.type === "text") { if (b.length < 1 || b.length > TEXT_PAD) throw new Error("a text of 1 to 64 bytes"); const pad = new Uint8Array(1 + TEXT_PAD); pad[0] = b.length; pad.set(b, 1); return concat(salt, pad); } if (b.length > 0xffff) throw new Error("a clipboard text of at most 65535 bytes"); return concat(salt, b); } /** A sealed action's commitment (§5.5): SHA-256 over the domain, the room, the author, the host, * the screen statement, the action's index, its `dt_us`, the type it seals and its opening. */ export function commitment({ room, author, host, screen, k, dt_us, of, opening }) { return sha256(COMMIT_DOMAIN, unhex(room), unhex(author), unhex(host), unhex(screen), Uint8Array.of(k), le(dt_us, 4), Uint8Array.of(TYPE[of]), opening); } /** Seal an input (§5.5): `o` is `{ room, author, host, screen, sealKey (hex), actions }`; each * key, text and clipboard action becomes a sealed action with its commitment, the rest stay in * the clear, and the openings go into one HPKE seal to `sealKey`. `o.salt(k, a)` and * `o.ikm(clearPart)` give the salts and the ephemeral key's input keying material; by default * both are fresh random bytes (a corpus gives fixed ones). The statement, and its bytes. */ export async function sealInput(o) { const salt = o.salt || (() => crypto.getRandomValues(new Uint8Array(SALT_LEN))); const ikm = o.ikm || (() => crypto.getRandomValues(new Uint8Array(32))); const openings = []; const actions = []; for (const [k, a] of o.actions.entries()) { if (!SEALABLE.includes(a.type)) { actions.push(a); continue; } const opening = openingOf(await salt(k, a), a); openings.push(opening); const length = a.type === "clipboard" ? enc.encode(a.text).length : 0; const commit = hex(await commitment({ room: o.room, author: o.author, host: o.host, screen: o.screen, k, dt_us: a.dt_us, of: a.type, opening })); actions.push({ dt_us: a.dt_us, type: "sealed", of: a.type, length, commit }); } const i = { host: o.host, screen: o.screen, actions }; if (openings.length) { const clear = clearPart(i); const s = await sealBase(unhex(o.sealKey), concat(SEAL_INFO, unhex(o.room), unhex(o.host)), concat(unhex(o.author), clear), concat(...openings), await ikm(clear)); i.seal = { enc: hex(s.enc), sealed: hex(s.ct) }; } return { input: i, payload: inputPayload(i) }; } /** Split a text into pieces of at most `n` bytes of UTF-8 each, never inside a character: a text * action carries 1 to 64 bytes (§5.2). */ export function textPieces(text, n = TEXT_PAD) { const out = []; let cur = ""; for (const ch of text) { if (enc.encode(cur + ch).length > n) { out.push(cur); cur = ""; } cur += ch; } if (cur) out.push(cur); return out; } /** A floor statement (§7.2). */ export function floorPayload(f) { return concat(enc.encode(MAGIC.floor), Uint8Array.of(1), unhex(f.host), Uint8Array.of(FLOOR[f.action] || f.action), f.to ? unhex(f.to) : new Uint8Array(32)); } /** A latency record's bytes from `magic` through `rtt_us` (§8.1). */ export function latencyBody(l) { return concat( enc.encode(MAGIC.latency), Uint8Array.of(1), unhex(l.room), unhex(l.viewer), unhex(l.host), unhex(l.input), Uint8Array.of(l.action), unhex(l.outcome), unhex(l.track), le(l.group, 8), le(l.object, 8), unhex(l.content), le(l.input_at, 8), le(l.presented_at, 8), Uint8Array.of(l.source), le(l.latency_us, 4), le(l.rtt_us, 4), ); } /** The record signed with Ed25519 under the viewer's key (§8.1): 335 bytes. */ export async function signLatency(l, privateKey) { const body = latencyBody(l); const sig = new Uint8Array(await subtle.sign({ name: "Ed25519" }, privateKey, concat(LATENCY_DOMAIN, body))); return concat(body, sig); } /** WebCrypto's key for a 32-byte Ed25519 seed (RFC 8410's PKCS #8 form), and its public key. */ export async function keyFromSeed(seed) { const pkcs8 = concat(Uint8Array.of(0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x04, 0x22, 0x04, 0x20), seed); const privateKey = await subtle.importKey("pkcs8", pkcs8, { name: "Ed25519" }, true, ["sign"]); const jwk = await subtle.exportKey("jwk", privateKey); const publicKey = Uint8Array.from(atob(jwk.x.replace(/-/g, "+").replace(/_/g, "/")), (c) => c.charCodeAt(0)); return { privateKey, publicKey }; } function readAction(r) { const t = r.u8(); const bool = (b) => { if (b > 1) throw new Error("bool"); return b === 1; }; switch (t) { case 1: return { type: "move", x: r.u16(), y: r.u16() }; case 2: { const button = r.u8(); const down = bool(r.u8()); if (button > 4) throw new Error("button"); return { type: "button", button, down, x: r.u16(), y: r.u16() }; } case 3: { const dx = r.i16(); const dy = r.i16(); if (!dx && !dy) throw new Error("wheel"); return { type: "wheel", dx, dy, x: r.u16(), y: r.u16() }; } case 4: { const usage = r.u16(); const down = bool(r.u8()); const modifiers = r.u8(); if (usage < 4 || usage > 0xe7 || modifiers & ~0x0f) throw new Error("key"); return { type: "key", usage, down, modifiers }; } case 5: { const b = r.short(); if (b.length < 1 || b.length > 64) throw new Error("text"); return { type: "text", text: utf8.decode(b) }; } case 6: { const id = r.u8(); const phase = r.u8(); if (phase < 1 || phase > 4) throw new Error("phase"); return { type: "touch", id, phase, x: r.u16(), y: r.u16() }; } case 7: { const b = r.take(r.u16()); return { type: "clipboard", text: utf8.decode(b), bytes: b.length }; } case 8: { const of = TYPE_NAME[r.u8()]; const length = r.u16(); if (!SEALABLE.includes(of) || (of !== "clipboard" && length)) throw new Error("sealed"); return { type: "sealed", of, length, commit: hex(r.take(32)) }; } default: throw new Error("type"); } } const rect = (r) => ({ x: r.u16(), y: r.u16(), w: r.u16(), h: r.u16() }); /** A statement of this extension (§4.1, §4.2), or null for another `meta` payload; one that does * not read is `{ kind: "malformed" }`. */ export function readStatement(payload) { const magic = new TextDecoder().decode(payload.subarray(0, 4)); const kind = Object.keys(MAGIC).find((k) => MAGIC[k] === magic); if (!kind) return null; try { if (payload[4] !== 1) throw new Error("version"); const r = new Rd(payload, 5); let s; if (kind === "screen") { s = { kind, prev: hex(r.take(32)), state: r.u8(), track: hex(r.take(32)), width: r.u16(), height: r.u16(), fps: r.u16(), scope: r.u8(), region: rect(r) }; s.redactions = Array.from({ length: r.u8() }, () => rect(r)); Object.assign(s, { pointer: r.u8(), accepts: r.u8(), clip_max: r.u16(), max_age_ms: r.u32(), floor: r.u8() }); const tok = (b) => new TextDecoder().decode(b); Object.assign(s, { capability: tok(r.short()), codec: tok(r.short()), encoder: tok(r.short()), api: r.u8(), hardware: r.u8() }); // §3.3: the statement ends after `hardware`, or after the host's sealing key. s.seal = payload.length - r.p === 32 ? hex(r.take(32)) : null; } else if (kind === "input") { s = { kind, host: hex(r.take(32)), screen: hex(r.take(32)), actions: [] }; const n = r.u8(); if (n < 1 || n > 32) throw new Error("count"); for (let i = 0; i < n; i++) { const dt_us = r.u32(); if (s.actions.length && dt_us > s.actions[s.actions.length - 1].dt_us) throw new Error("order"); s.actions.push({ dt_us, ...readAction(r) }); } // §5.5: a statement with a sealed action ends with its seal, exactly as long as its openings // and the tag. const sealed = s.actions.filter((a) => a.type === "sealed"); if (sealed.length) { const want = 16 + sealed.reduce((n, a) => n + SALT_LEN + (a.of === "key" ? 4 : a.of === "text" ? 1 + TEXT_PAD : a.length), 0); const e = r.take(32); if (payload.length - r.p !== want) throw new Error("seal"); s.seal = { enc: hex(e), sealed: hex(r.take(want)) }; } } else if (kind === "outcome") { s = { kind, input: hex(r.take(32)), outcome: r.u8(), reason: r.u8(), applied_at: r.u64(), captured_at: r.u64(), content: hex(r.take(32)), rtt_us: r.u32() }; if (s.outcome < 1 || s.outcome > 2 || s.reason > 9) throw new Error("outcome"); } else if (kind === "floor") { s = { kind, host: hex(r.take(32)), action: r.u8(), to: hex(r.take(32)) }; if (s.action < 1 || s.action > 4 || (s.to !== "0".repeat(64)) !== (s.action === FLOOR.give)) throw new Error("floor"); } else { s = { kind, room: hex(r.take(32)), viewer: hex(r.take(32)), host: hex(r.take(32)), input: hex(r.take(32)), action: r.u8(), outcome: hex(r.take(32)), track: hex(r.take(32)), group: r.u64(), object: r.u64(), content: hex(r.take(32)), input_at: r.u64(), presented_at: r.u64(), source: r.u8(), latency_us: r.u32(), rtt_us: r.u32() }; s.sig = r.take(64); if (s.source < 1 || s.source > 3) throw new Error("source"); } if (!r.done()) throw new Error("bytes after"); return s; } catch { return { kind: "malformed" }; } } /** A WEV1 record's fields (live-events §3.1), read as far as this module needs them. */ export function readEvent(b) { const r = new Rd(b, 5); const ev = { room: hex(r.take(32)), principal: hex(r.take(32)), seq: r.u64(), prev: hex(r.take(32)), t: r.u64() }; ev.anchor = r.u8() === 1 ? { track: hex(r.take(32)), group: r.u64(), object: r.u64() } : null; ev.kind = r.u8(); ev.payload = r.take(r.u16()); ev.body = b.subarray(0, r.p); ev.sig = r.take(64); return ev; } /** A WRG1 grant's principal, powers and attributes (live-rooms §4.1, §4.4), and its digest. */ export async function readGrant(b) { const r = new Rd(b, 5); const g = { room: hex(r.take(32)), principal: hex(r.take(32)), prev: hex(r.take(32)), effective: r.u64(), nbf: r.u64(), exp: r.u64() }; g.role = r.u8(); g.publish = r.u8(); g.subscribe = r.u8(); g.max_layer = r.u8(); g.powers = r.u8(); r.take(1 + 16 + 16 + 65 + 8 + 8 + 3); r.take(r.u8()); const no = r.u8(); for (let i = 0; i < no; i++) r.short(); const attrs = new Rd(r.take(r.u16())); g.attrs = {}; while (!attrs.done()) { const k = new TextDecoder().decode(attrs.short()); g.attrs[k] = new TextDecoder().decode(attrs.take(attrs.u16())); } g.digest = hex(await sha256(b.subarray(0, r.p))); return g; } /** §3.1: the classes of a grant's control power, 0 for none; null when malformed. */ export function controlOf(g) { const v = g.attrs.remote_control; if (v === undefined) return 0; return /^[1-9][0-9]?$/.test(v) && Number(v) <= 15 ? Number(v) : null; } /** §3.2: the anchor `track` of a host's screen, layer 0. */ export async function screenTrack(room, host) { return hex(await sha256(enc.encode(`${room}-${host}-screen`), Uint8Array.of(0), enc.encode("0"))); } // ---------------------------------------------------------------- §5.3 the rules /** The host's rules (§5.3) for input statement `i` at `t`, decided at `n`, at a place where the * host's screen in effect is `screen` (with its event's `digest`), the author's control power * `control`, and whether it holds the floor: 0 when applied, else the refusal's number. Rule 9 * reads a seal, which only the host's opening key opens: it is made here only where the screen * states no sealing key ([`sealUnknown`] names the inputs it cannot be made for). */ export function decide(i, { me, screen, digest, control, holdsFloor }, t, n) { if (i.host !== me || !screen || i.screen !== digest) return 1; const classes = i.actions.reduce((c, a) => c | CLASS_OF[TYPE[actsAs(a)]], 0); if (classes & ~control) return 2; if (screen.floor === 2 && !holdsFloor) return 3; if (screen.state === 2) return 4; if (i.actions.some((a) => !((screen.accepts >> (TYPE[actsAs(a)] - 1)) & 1))) return 5; for (const a of i.actions) { const clip = a.type === "clipboard" ? a.bytes : a.type === "sealed" && a.of === "clipboard" ? a.length : 0; if (clip > screen.clip_max || ("x" in a && (a.x >= screen.width || a.y >= screen.height))) return 6; } if (n - (t - i.actions[0].dt_us) > screen.max_age_ms * 1000) return 7; if (screen.seal && i.actions.some((a) => SEALABLE.includes(a.type))) return 8; if (i.seal && !screen.seal) return 9; return 0; } /** Whether rule 9 of §5.3 is the host's alone to make for input `i` at its place: it is sealed, to * a screen that states a sealing key. */ export const sealUnknown = (i, { screen }) => !!(i.seal && screen && screen.seal); /** §7.2 for floor statement `f` at its place (`{ author, roomKey, control, direct, hostHasScreen, * holder, toControl }`): null when accepted, else the reason. A refused floor statement changes * nothing (§7.3). */ export function floorRule(f, p) { if (!p.hostHasScreen) return "floor_not_permitted"; const steward = p.roomKey || p.author === f.host || p.direct; if (f.action === FLOOR.request) return p.roomKey || !p.control ? "floor_not_permitted" : null; if (f.action === FLOOR.give) { if (!(steward || p.holder === p.author)) return "floor_not_permitted"; return f.to === f.host || !p.toControl ? "floor_not_eligible" : null; } if (f.action === FLOOR.release) return p.holder !== p.author ? "floor_not_holder" : null; return steward ? null : "floor_not_permitted"; } /** Every chain of a room in the merged order (live-rooms §9.2), each record read. */ function merged(chains, roomKeys) { const next = chains.map(() => 0); const out = []; for (;;) { let best = -1, key = null; chains.forEach((c, k) => { if (next[k] >= c.length) return; const ev = c[next[k]]; const kk = [ev.t, roomKeys.includes(ev.principal) ? 0 : 1, k]; if (!key || kk[0] < key[0] || (kk[0] === key[0] && (kk[1] < key[1] || (kk[1] === key[1] && kk[2] < key[2])))) { best = k; key = kk; } }); if (best < 0) return out; out.push(chains[best][next[best]++]); } } /** A room's inputs and what the rules give each (§5.3 at its place, §7.3), with the outcome its * host stated: `{ digest, author, outcome: "applied" | "refused" | "unanswered", stated, rule }`, * where `rule` is what the rules give (0 applied, else the refusal's number) once an outcome * names the input's deciding instant. `room` is a corpus vector's form. */ export async function replay(room, clock, authority) { return (await replayRoom(room, clock, authority)).inputs; } /** [`replay`]'s inputs, with the floor: each change of a host's floor in effect (`{ host, t, * holder }`), and each floor statement §7.2 refuses, which changes nothing (`{ digest, author, * host, t, action, reason }`). */ export async function replayRoom(room, clock, authority) { const roomKeys = [clock, authority]; const grants = {}; for (const g of room.grants) { const r = await readGrant(unhex(g)); grants[r.digest] = r; } const chains = room.chains.map((c) => c.map((h) => { const b = unhex(h); return { ...readEvent(b), bytes: b }; })); for (const c of chains) for (const ev of c) ev.digest = hex(await sha256(ev.body)); const inEffect = {}, hosts = {}, inputs = [], outcomes = [], floor = [], floorRefused = []; const control = (p) => (inEffect[p] && grants[inEffect[p]] ? controlOf(grants[inEffect[p]]) || 0 : 0); const direct = (p) => !!(inEffect[p] && grants[inEffect[p]] && grants[inEffect[p]].powers & 2); const lose = (p, t) => { for (const [k, h] of Object.entries(hosts)) if (h.holder === p) { h.holder = null; floor.push({ host: k, t, holder: null }); } }; for (const ev of merged(chains, roomKeys)) { if (roomKeys.includes(ev.principal) && ev.kind >= 0x10) { const subject = hex(ev.payload.subarray(0, 32)); const detail = ev.payload.subarray(32); if (ev.kind === 0x10) inEffect[subject] = hex(detail.subarray(0, 32)); else if (ev.kind === 0x14) { inEffect[subject] = hex(detail.subarray(32, 64)); if (!control(subject)) lose(subject, ev.t); } else if (ev.kind === 0x11) { delete inEffect[subject]; lose(subject, ev.t); if (hosts[subject] && hosts[subject].holder) { hosts[subject].holder = null; floor.push({ host: subject, t: ev.t, holder: null }); } } continue; } if (ev.kind !== META) continue; const s = readStatement(ev.payload); if (!s) continue; if (s.kind === "screen") { const h = hosts[ev.principal] || (hosts[ev.principal] = { holder: null }); h.screen = s; h.digest = ev.digest; } else if (s.kind === "floor") { // §7.2 at its place: a statement it refuses is stated and changes nothing (§7.3). const h = hosts[s.host]; const why = floorRule(s, { author: ev.principal, roomKey: roomKeys.includes(ev.principal), control: control(ev.principal), direct: direct(ev.principal), hostHasScreen: !!(h && h.screen), holder: h ? h.holder : null, toControl: control(s.to), }); if (why) floorRefused.push({ digest: ev.digest, author: ev.principal, host: s.host, t: ev.t, action: Object.keys(FLOOR).find((k) => FLOOR[k] === s.action), reason: why }); else if (s.action === FLOOR.give) { h.holder = s.to; floor.push({ host: s.host, t: ev.t, holder: s.to }); } else if (s.action === FLOOR.release || (s.action === FLOOR.take && h.holder)) { h.holder = null; floor.push({ host: s.host, t: ev.t, holder: null }); } } else if (s.kind === "input") { const h = hosts[s.host] || {}; inputs.push({ digest: ev.digest, author: ev.principal, t: ev.t, input: s, place: { me: s.host, screen: h.screen, digest: h.digest, control: control(ev.principal), holdsFloor: h.holder === ev.principal } }); } else if (s.kind === "outcome") { outcomes.push({ t: ev.t, anchor: ev.anchor, ...s }); } } const results = inputs.map((x) => { const o = outcomes.find((y) => y.input === x.digest); if (!o) return { digest: x.digest, author: x.author, outcome: "unanswered", rule: null }; return { digest: x.digest, author: x.author, outcome: o.outcome === 1 ? "applied" : "refused", stated: o.outcome === 1 ? 0 : o.reason, rule: decide(x.input, x.place, x.t, o.t), sealUnknown: sealUnknown(x.input, x.place), anchor: o.anchor }; }); return { inputs: results, floor, floorRefused }; } // ---------------------------------------------------------------- keys /** HID Keyboard/Keypad usage IDs (HID Usage Tables 1.5 §10) of the `uievents-code` values. */ export const HID = { KeyA: 0x04, KeyB: 0x05, KeyC: 0x06, KeyD: 0x07, KeyE: 0x08, KeyF: 0x09, KeyG: 0x0a, KeyH: 0x0b, KeyI: 0x0c, KeyJ: 0x0d, KeyK: 0x0e, KeyL: 0x0f, KeyM: 0x10, KeyN: 0x11, KeyO: 0x12, KeyP: 0x13, KeyQ: 0x14, KeyR: 0x15, KeyS: 0x16, KeyT: 0x17, KeyU: 0x18, KeyV: 0x19, KeyW: 0x1a, KeyX: 0x1b, KeyY: 0x1c, KeyZ: 0x1d, Digit1: 0x1e, Digit2: 0x1f, Digit3: 0x20, Digit4: 0x21, Digit5: 0x22, Digit6: 0x23, Digit7: 0x24, Digit8: 0x25, Digit9: 0x26, Digit0: 0x27, Enter: 0x28, Escape: 0x29, Backspace: 0x2a, Tab: 0x2b, Space: 0x2c, Minus: 0x2d, Equal: 0x2e, BracketLeft: 0x2f, BracketRight: 0x30, Backslash: 0x31, Semicolon: 0x33, Quote: 0x34, Backquote: 0x35, Comma: 0x36, Period: 0x37, Slash: 0x38, CapsLock: 0x39, F1: 0x3a, F2: 0x3b, F3: 0x3c, F4: 0x3d, F5: 0x3e, F6: 0x3f, F7: 0x40, F8: 0x41, F9: 0x42, F10: 0x43, F11: 0x44, F12: 0x45, Insert: 0x49, Home: 0x4a, PageUp: 0x4b, Delete: 0x4c, End: 0x4d, PageDown: 0x4e, ArrowRight: 0x4f, ArrowLeft: 0x50, ArrowDown: 0x51, ArrowUp: 0x52, ControlLeft: 0xe0, ShiftLeft: 0xe1, AltLeft: 0xe2, MetaLeft: 0xe3, ControlRight: 0xe4, ShiftRight: 0xe5, AltRight: 0xe6, MetaRight: 0xe7, }; /** The modifiers of a keyboard event (§5.2): bit 0 shift, 1 control, 2 alt, 3 meta. */ export const modifiersOf = (e) => (e.shiftKey ? 1 : 0) | (e.ctrlKey ? 2 : 0) | (e.altKey ? 4 : 0) | (e.metaKey ? 8 : 0); // ---------------------------------------------------------------- the viewer /** The page's remote desktop viewer. `o`: `{ room, me (hex), host (hex), clock (hex), grant (the * hex record the room service issued for this page's key), key ({ privateKey }: that key, made in * the page), grantOf(digest) (the hex record of a grant of the room, read again from the service), * roomNow, unixNow (seconds since 1970, on the room's clock; the page's own by default), * emit(kind, payload), canvas, panel, log }`, and for tests `salt(k, a)` and `ikm(clearPart)`, * the random bytes a seal draws (fresh random bytes by default). */ export class RemoteViewer { constructor(o) { Object.assign(this, o); this.screen = null; this.screenDigest = null; this.holder = null; this.outcomes = new Map(); this.sent = new Map(); this.chunks = new Map(); this.frames = new Map(); this.counts = { sent: 0, applied: 0, refused: {}, latencies: 0, unsent: 0 }; this.batch = []; this.flushing = false; // What other controllers are doing, by principal: `{ what: "typing" | "pasting", at }` (the // page's own clock, ms). Never what they typed: their key, text and clipboard actions are // sealed to the host. this.activity = new Map(); // Every record the room accepted and handed to this viewer, as its bytes (the last 4000). this.received = []; // Whether the room's clock has recorded this page joining, and its grants in effect, each from // its instant: `{ at, digest, control, nbf, exp }`. this.joined = false; this.grants = []; if (!this.unixNow) this.unixNow = () => Math.floor(Date.now() / 1000); } async start() { if (this.grant) { const g = await readGrant(unhex(this.grant)); if (g.principal !== this.me) throw new Error("the grant is not for this page's key"); this.grants = [{ at: 0, digest: g.digest, control: controlOf(g) || 0, nbf: g.nbf, exp: g.exp }]; } this.track = await screenTrack(this.room, this.host); if (this.canvas) this.listen(); this.render(); } /** This page's grant in effect at room-clock `t`: the last whose instant is at or before `t`. */ grantAt(t) { let g = null; for (const x of this.grants) if (x.at <= t) g = x; return g; } /** The classes of the control power of this page's grant in effect at room-clock `t`. */ controlAt(t) { const g = this.grantAt(t); return g ? g.control : 0; } /** The classes this page may send now: its grant in effect's while that grant is good (from its * `nbf` to before its `exp`, live-rooms §4.3), and none without one. */ get control() { const g = this.grantAt(this.roomNow()); const now = this.unixNow(); return g && g.nbf <= now && now < g.exp ? g.control : 0; } /** Whether this page may send now: it holds a grant, and the clock has recorded it joining. */ mayEmit() { return !!this.grant && this.grants.length > 0 && this.joined; } /** A record of the room's clock about this page: its join, an exchange of its grant (whose new * record is read again from the service), its leave. */ async onClock(ev) { if (ev.kind < 0x10) return; const subject = hex(ev.payload.subarray(0, 32)); if (subject !== this.me) return; const detail = ev.payload.subarray(32); if (ev.kind === 0x10) this.joined = true; else if (ev.kind === 0x11) this.joined = false; else if (ev.kind === 0x14) { const digest = hex(detail.subarray(32, 64)); const rec = this.grantOf ? await this.grantOf(digest) : null; if (!rec) return this.log && this.log(`a grant exchange names a grant the room does not list: ${digest.slice(0, 12)}`); const g = await readGrant(unhex(rec)); this.grants.push({ at: ev.t, digest, control: controlOf(g) || 0, nbf: g.nbf, exp: g.exp }); this.grant = rec; } this.render(); } /** A record the page's room accepted. */ onRecord(bytes) { this.received.push(bytes); if (this.received.length > 4000) this.received.shift(); const ev = readEvent(bytes); if (ev.principal === this.clock) { this.onClock(ev).catch((e) => this.log && this.log(`the clock's record: ${e.message || e}`)); return; } // Another controller's input: that it is typing or pasting, from the types its actions act as. if (ev.principal !== this.host && ev.principal !== this.me && ev.kind === META) { const s = readStatement(ev.payload); if (s && s.kind === "input") { const types = s.actions.map(actsAs); const what = types.includes("clipboard") ? "pasting" : types.includes("key") || types.includes("text") ? "typing" : null; if (what) { this.activity.set(ev.principal, { what, at: Date.now() }); this.render(); setTimeout(() => this.render(), 3100); } } return; } if (ev.principal !== this.host || ev.kind !== META) return; const s = readStatement(ev.payload); if (!s || s.kind === "malformed") return; sha256(ev.body).then((d) => { const digest = hex(d); if (s.kind === "screen") { this.screen = s; this.screenDigest = digest; } else if (s.kind === "floor" && s.action !== FLOOR.request) this.holder = s.action === FLOOR.give ? s.to : null; else if (s.kind === "outcome") { this.outcomes.set(s.input, { digest, s, anchor: ev.anchor }); if (this.sent.has(s.input)) { if (s.outcome === 1) { this.counts.applied++; this.measure(s.input).catch((e) => this.log(`a latency record: ${e.message || e}`)); } else { const r = REFUSAL[s.reason]; this.counts.refused[r] = (this.counts.refused[r] || 0) + 1; } } } this.render(); }); } /** The page verified group `group` of the screen track (content hash `content`) and gave its * envelope's chunk to the decoder at `timestamp`. */ chunk(group, timestamp, content) { this.chunks.set(timestamp, { group, content }); if (this.chunks.size > 600) this.chunks.delete(this.chunks.keys().next().value); } /** A decoded frame of the host's screen: drawn, and the refresh that shows it noted. */ frame(vf) { const c = this.chunks.get(vf.timestamp); if (this.canvas) { if (this.canvas.width !== vf.displayWidth) { this.canvas.width = vf.displayWidth; this.canvas.height = vf.displayHeight; } this.canvas.getContext("2d").drawImage(vf, 0, 0); } vf.close(); if (!c) return; // Presented at the start of the display refresh after the one it was drawn in (§8.3). requestAnimationFrame(() => requestAnimationFrame(() => { this.frames.set(c.group, { content: c.content, at: this.roomNow() }); if (this.frames.size > 600) this.frames.delete(this.frames.keys().next().value); for (const [input, w] of this.sent) if (w.waiting && this.outcomes.get(input)?.anchor?.group === c.group) this.measure(input).catch(() => {}); })); } async measure(input) { const w = this.sent.get(input); const o = this.outcomes.get(input); if (!w || !o || !o.anchor || w.measured) return; const f = this.frames.get(o.anchor.group); if (!f) { w.waiting = true; return; } w.measured = true; const inputAt = w.t - w.input.actions[0].dt_us; const l = { room: this.room, viewer: this.me, host: this.host, input, action: 0, outcome: o.digest, track: o.anchor.track, group: o.anchor.group, object: o.anchor.object, content: f.content, input_at: inputAt, presented_at: Math.max(f.at, inputAt), source: SOURCE.frameTiming, latency_us: Math.max(f.at, inputAt) - inputAt, rtt_us: this.rtt ?? NO_RTT, }; await this.emit(META, await signLatency(l, this.key.privateKey)); this.counts.latencies++; this.lastLatency = l.latency_us; this.render(); } /** Send the batched actions as one input statement (§5.1), no more than one a display refresh. */ queue(a) { const queued = { ...a, at: this.roomNow() }; this.batch.push(queued); // A pointer moving is many actions a refresh, and only its latest place matters soon. A key, a // button or a wheel is one thing that happened, and waiting for a refresh only delays it: it // goes now, with whatever moves came before it. A loop of actions in one task still makes one // statement, since the send starts after the task. if (a.type === "move") this.schedule(); else this.flush().catch((e) => this.log(`an input: ${e.message || e}`)); return queued; } /** The pointer moved. Moves between refreshes are one action, at the pointer's latest place, so * long as nothing has been queued after it. */ moveTo(p) { const last = this.batch[this.batch.length - 1]; if (last && last.type === "move") Object.assign(last, p, { at: this.roomNow() }); else this.queue({ type: "move", ...p }); } /** A flush at the next display refresh, unless one is waiting for it. */ schedule() { if (this.flushing) return; this.flushing = true; const frame = globalThis.requestAnimationFrame || ((f) => setTimeout(f, 16)); frame(() => this.flush().catch((e) => this.log(`an input: ${e.message || e}`))); } /** Send what is batched. Sends run one after another, in the order they were asked for: a * seal takes WebCrypto's time, and a key's release must not reach the host before its press. */ flush() { this.flushing = false; const run = (this.sending || Promise.resolve()).then(() => this.flushNow()); this.sending = run.catch(() => {}); return run; } async flushNow() { const acts = this.batch.splice(0, 32); if (!acts.length || !this.screenDigest) return; // Nothing is sent without a grant, before the clock has recorded this page joining, or of a // class the grant in effect does not give. if (!this.mayEmit() || acts.some((a) => !(this.control & CLASS_OF[TYPE[a.type]]))) { this.counts.unsent += acts.length; return; } // Keys, text and clipboard text go sealed to the host's sealing key, never in the clear: to a // host that states none, they are not sent (§5.5). const sealKey = this.screen && this.screen.seal; if (!sealKey && acts.some((a) => SEALABLE.includes(a.type))) { this.counts.unsent += acts.length; this.counts.unsealed = (this.counts.unsealed || 0) + 1; this.render(); return; } const t = this.roomNow(); const i = { host: this.host, screen: this.screenDigest, actions: acts.map(({ at, ...a }) => ({ dt_us: Math.max(0, t - at), ...a })) }; // Every dt_us at most the one before it: the actions in the order they happened. for (let k = 1; k < i.actions.length; k++) i.actions[k].dt_us = Math.min(i.actions[k].dt_us, i.actions[k - 1].dt_us); const payload = i.actions.some((a) => SEALABLE.includes(a.type)) ? (await sealInput({ room: this.room, author: this.me, host: this.host, screen: this.screenDigest, sealKey, actions: i.actions, salt: this.salt, ikm: this.ikm })).payload : inputPayload(i); const r = await this.emit(META, payload); if (r && r.digest) this.sent.set(r.digest, { t: r.t, input: i }); this.counts.sent++; // What came while this was sent goes at the next refresh, its actions' instants as they were. if (this.batch.length) this.schedule(); this.render(); } async requestFloor() { if (this.mayEmit() && this.control) await this.emit(META, floorPayload({ host: this.host, action: "request" })); } async releaseFloor() { if (this.mayEmit() && this.holder === this.me) await this.emit(META, floorPayload({ host: this.host, action: "release" })); } /** Frame coordinates of a pointer event on the canvas. */ at(e) { const b = this.canvas.getBoundingClientRect(); const w = this.screen ? this.screen.width : this.canvas.width, h = this.screen ? this.screen.height : this.canvas.height; return { x: Math.max(0, Math.min(w - 1, Math.floor(((e.clientX - b.left) * w) / b.width))), y: Math.max(0, Math.min(h - 1, Math.floor(((e.clientY - b.top) * h) / b.height))) }; } may(cls) { return this.mayEmit() && (this.control & cls) !== 0; } listen() { const c = this.canvas; c.tabIndex = 0; c.addEventListener("pointermove", (e) => { if (this.may(CLASS.pointer)) this.moveTo(this.at(e)); }); for (const [ev, down] of [["pointerdown", true], ["pointerup", false]]) { c.addEventListener(ev, (e) => { if (!this.may(CLASS.pointer)) return; e.preventDefault(); c.focus(); this.queue({ type: "button", button: e.button, down, ...this.at(e) }); }); } c.addEventListener("contextmenu", (e) => e.preventDefault()); c.addEventListener("wheel", (e) => { if (!this.may(CLASS.pointer)) return; e.preventDefault(); const unit = e.deltaMode === 1 ? 40 : e.deltaMode === 2 ? 800 : 1; const clamp = (v) => Math.max(-32768, Math.min(32767, Math.round(v * unit))); const dx = clamp(e.deltaX), dy = clamp(e.deltaY); if (dx || dy) this.queue({ type: "wheel", dx, dy, ...this.at(e) }); }, { passive: false }); for (const [ev, down] of [["keydown", true], ["keyup", false]]) { c.addEventListener(ev, (e) => { if (!this.may(CLASS.keys) || e.isComposing) return; const usage = HID[e.code]; if (usage === undefined) return; e.preventDefault(); this.queue({ type: "key", usage, down, modifiers: modifiersOf(e) }); }); } c.addEventListener("compositionend", (e) => { if (this.may(CLASS.keys) && e.data) for (const text of textPieces(e.data)) this.queue({ type: "text", text }); }); c.addEventListener("paste", (e) => { if (!this.may(CLASS.clipboard) || !this.screen || !this.screen.clip_max) return; const text = e.clipboardData.getData("text/plain"); if (text && enc.encode(text).length <= this.screen.clip_max) { e.preventDefault(); this.queue({ type: "clipboard", text }); } }); } render() { if (!this.panel) return; const s = this.screen; const what = !s ? "waiting for the host's screen statement" : `${s.state === 2 ? "paused" : "live"} · ${s.width}×${s.height} · ${["", "a whole display", "one window", "a region of a display"][s.scope]} · ${s.redactions.length} redaction(s) · ${s.floor === 2 ? "floor" : "open"} · encoder ${s.encoder}`; const floor = !s || s.floor !== 2 ? "" : this.holder === this.me ? "You hold the floor." : this.holder ? `The floor is held by ${this.holder.slice(0, 8)}.` : "No one holds the floor."; const power = !this.grant ? "No grant." : this.control ? "Your grant lets you control." : "Your grant lets you watch."; const refused = Object.entries(this.counts.refused).map(([k, v]) => `${k} ${v}`).join(", "); // Who is typing or pasting now: never what (their actions are sealed to the host). const now = Date.now(); const busy = [...this.activity].filter(([, x]) => now - x.at < 3000).map(([p, x]) => `${p.slice(0, 8)} is ${x.what}…`).join(" "); const sealed = !s ? "" : s.seal ? "Keys and text go sealed to the host." : "The host takes no sealed keys: keys and text are not sent."; this.panel.textContent = `${what}. ${power} ${floor} ${sealed} Sent ${this.counts.sent}, applied ${this.counts.applied}${refused ? `, refused: ${refused}` : ""}${this.lastLatency != null ? `; last latency ${(this.lastLatency / 1000).toFixed(1)} ms` : ""}.${busy ? ` ${busy}` : ""}`; } }