open-standards/wai/telemetry-conformance/cml/differential.mjs

// The delivery-telemetry corpus against an independent open-source CMCD and CMSD
// library (extensions/delivery-telemetry.md §9.3).
//
// For each observation it builds the report's members from the extension's tables
// (§3, §4) here, not from wai-rs's output; encodes them with the library in the mode the
// event fixes, and the header shards for Request mode; requires the bytes of
// expected/cmcd.json; reads the expected strings back with the library's dictionary,
// query and header readers and requires the members; and validates every report, and
// the Event-mode reports together, requiring no error. CMSD-Static and CMSD-Dynamic
// are encoded and read back the same way. Exit status 0 only if everything agrees.
//
//   npm ci && node differential.mjs <telemetry-conformance dir>

import fs from 'node:fs';
import path from 'node:path';
import { encodeCmcd, toCmcdHeaders, decodeCmcd, fromCmcdQuery, fromCmcdHeaders, validateCmcd, validateCmcdEvents } from '@svta/cml-cmcd';
import { encodeCmsdStatic, encodeCmsdDynamic, decodeCmsdStatic, decodeCmsdDynamic } from '@svta/cml-cmsd';
import { SfItem, SfToken } from '@svta/cml-structured-field-values';

const dir = path.resolve(process.argv[2] ?? '..');
const read = (p) => JSON.parse(fs.readFileSync(path.join(dir, p), 'utf8'));
const P = 'science.transaction.wai-';
const fails = [];
const fail = (s) => fails.push(s);

// §4.1: the custom keys' header shards (Request mode).
const CUSTOM_HEADERS = {
  'CMCD-Object': ['b', 'tx', 'ch', 'cap', 'ri', 'l', 'lt', 'p', 'rcpt'].map((k) => P + k),
  'CMCD-Request': [P + 'lc'],
  'CMCD-Session': [P + 'coh'],
};

// §3.1's event table.
const EVENT = {
  response: ['rr'], start: ['ps', 's'], present: ['ps', 'p'], 'stall-start': ['ps', 'r'], 'stall-end': ['ps', 'p'],
  abandon: ['ps', 'q'], end: ['ps', 'e'], switch: ['ce', null, 'wai-switch'], capable: ['ce', null, 'wai-capable'],
  'layer-change': ['ce', null, 'wai-layer-change'], 'frame-drop': ['ce', null, 'wai-frame-drop'],
};

const pathToken = (p) => (p === 'primary' ? 'p' : 'f');

// An observation's report members, from the extension's tables.
function build(o) {
  if (o.event === 'present' && !o.present.first) return null;
  const mode = o.event === 'request' ? 'request' : 'event';
  const c = { v: 2, sid: o.sid, sn: o.sn, st: o.st, sf: o.sf };
  const put = (k, v) => { if (v !== null && v !== undefined) c[k] = v; };
  const custom = (k, v) => { if (v !== null && v !== undefined) c[P + k] = String(v); };
  put('cid', o.cid);
  custom('coh', o.cohort);
  if (o.metrics) {
    put('msd', o.metrics.msd_ms);
    if (o.metrics.bsa !== null) c.bsa = [o.metrics.bsa];
    if (o.metrics.bsda_ms !== null) c.bsda = [o.metrics.bsda_ms];
    put('dfa', o.metrics.dfa);
  }
  const ob = o.object;
  if (ob) {
    custom('b', ob.counted); custom('tx', ob.transferred); custom('ch', ob.content_hash); custom('cap', ob.capability);
    custom('ri', ob.rendition); custom('l', ob.layer); custom('lt', ob.layers_total);
    custom('p', ob.path === null ? null : pathToken(ob.path)); custom('rcpt', ob.receipt);
    put('ot', ob.ot); put('d', ob.d_ms); put('dl', ob.dl_ms); custom('lc', ob.layer_cap);
    put('rc', ob.rc); put('ttfb', ob.ttfb_ms); put('ttlb', ob.ttlb_ms); put('url', ob.url);
  }
  if (mode === 'event') {
    const [e, sta, cen] = EVENT[o.event];
    c.e = e; c.ts = o.ts_ms;
    put('sta', sta ?? null); put('cen', cen ?? null);
    if (o.present) { custom('u', o.present.unit); custom('pl', o.present.layers); custom('late', o.present.late_ms); custom('dec', o.present.decode_us); }
    if (o.switch) { custom('u', o.switch.unit); custom('p', pathToken(o.switch.to)); custom('sw', o.switch.reason); }
    if (o.capable) { custom('co', o.capable.outcome); custom('cr', o.capable.reason); custom('cf', o.capable.classical_floor); }
    if (o.layer_change) { custom('lc', o.layer_change.to); custom('lcf', o.layer_change.from); }
    custom('clm', o.claim);
  }
  return { mode, c };
}

// Two members agree: the library reads Tokens as strings and inner lists as arrays.
function same(a, b) {
  const norm = (v) => (v instanceof SfToken ? v.description ?? String(v) : typeof v === 'symbol' ? v.description : v);
  const ka = Object.keys(a).sort(), kb = Object.keys(b).sort();
  if (JSON.stringify(ka) !== JSON.stringify(kb)) return false;
  return ka.every((k) => JSON.stringify(norm(a[k])) === JSON.stringify(norm(b[k])));
}

const errors = (r) => (r.issues ?? []).filter((i) => i.severity === 'error');

const expected = read('expected/cmcd.json');
const events = [];
for (const f of fs.readdirSync(path.join(dir, 'observations')).sort()) {
  const name = f.replace(/\.json$/, '');
  const o = read(`observations/${f}`);
  const x = expected[name];
  if (!x) { fail(`${name}: not in expected/cmcd.json`); continue; }
  const b = build(o);
  if (b === null) {
    if (x.mode !== 'none' || x.report !== null) fail(`${name}: not reported, expected says ${x.mode}`);
    continue;
  }
  if (x.mode !== b.mode) { fail(`${name}: mode ${b.mode}, expected ${x.mode}`); continue; }
  const enc = encodeCmcd(b.c, { version: 2, reportingMode: b.mode });
  if (enc !== x.report) fail(`${name}: the library writes\n    ${enc}\n  expected\n    ${x.report}`);
  const dec = decodeCmcd(x.report);
  if (!same(dec, b.c)) fail(`${name}: the expected report reads back as ${JSON.stringify(dec)}`);
  const v = validateCmcd(dec, { reportingMode: b.mode });
  if (errors(v).length) fail(`${name}: ${JSON.stringify(errors(v))}`);
  if (b.mode === 'request') {
    const h = toCmcdHeaders(b.c, { version: 2, customHeaderMap: CUSTOM_HEADERS });
    if (JSON.stringify(Object.fromEntries(Object.entries(h).sort())) !== JSON.stringify(x.headers)) {
      fail(`${name}: the library's header shards are ${JSON.stringify(h)}`);
    }
    if (!same(fromCmcdQuery('CMCD=' + encodeURIComponent(x.report)), dec)) fail(`${name}: the query form reads back otherwise`);
    if (!same(fromCmcdHeaders(x.headers), dec)) fail(`${name}: the header shards read back otherwise`);
  } else {
    events.push(x.report);
  }
}
const ev = validateCmcdEvents(events.join('\n'));
if (errors(ev).length || ev.valid === false) fail(`event reports: ${JSON.stringify(ev.issues)}`);

// CMSD: the extension's §6 members, keys and parameters in bytewise order.
const cmsdExpected = read('expected/cmsd.json');
for (const f of fs.readdirSync(path.join(dir, 'cmsd')).sort()) {
  const name = f.replace(/\.json$/, '');
  const v = read(`cmsd/${f}`);
  if (name.startsWith('static')) {
    const m = {};
    const put = (k, x) => { if (x !== null) m[k] = x; };
    put('at', v.at); put('d', v.d); put('ht', v.ht); put('nor', v.nor); put('nrr', v.nrr); put('ot', v.ot);
    put(P + 'ch', v.content_hash); put(P + 'l', v.layer === null ? null : String(v.layer));
    put(P + 'lt', v.layers_total === null ? null : String(v.layers_total));
    put(P + 'op', v.op === null ? null : new SfToken(v.op)); put(P + 'rcpt', v.receipt);
    put('sf', v.sf); put('st', v.st); if (v.su) m.su = true;
    const enc = encodeCmsdStatic(m);
    if (enc !== cmsdExpected[name]) fail(`cmsd ${name}: the library writes ${enc}`);
    if (!same(decodeCmsdStatic(cmsdExpected[name]), m)) fail(`cmsd ${name}: reads back otherwise`);
  } else {
    const items = v.map((e) => {
      const p = {};
      if (e.du) p.du = true;
      for (const k of ['etp', 'mb', 'rd', 'rtt']) if (e[k] !== null) p[k] = e[k];
      if (e.drop !== null) p[P + 'drop'] = String(e.drop);
      return new SfItem(e.server, p);
    });
    const enc = encodeCmsdDynamic(items);
    if (enc !== cmsdExpected[name]) fail(`cmsd ${name}: the library writes ${enc}`);
    const back = decodeCmsdDynamic(cmsdExpected[name]);
    if (back.length !== items.length || back.some((b, i) => b.value !== items[i].value || !same(b.params, items[i].params))) {
      fail(`cmsd ${name}: reads back otherwise: ${JSON.stringify(back)}`);
    }
  }
}

if (fails.length) {
  for (const f of fails) console.log('FAIL', f);
  process.exit(1);
}
console.log(`telemetry: every report agrees with the independent library (${Object.keys(expected).length} observations, ${events.length} Event-mode reports, ${Object.keys(cmsdExpected).length} CMSD fields)`);