d9839467e2
So a reason left out of §13's table is caught on this side too, as the Rust test that §13 lists exactly the reference's codes catches it. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1110 lines
50 KiB
Python
1110 lines
50 KiB
Python
#!/usr/bin/env python3
|
|
"""Second implementation of extensions/opportunity-report.md (wai.delivery.opportunity).
|
|
|
|
Written from the extension's text and sharing no code with wai-rs: it reads every opportunity
|
|
report (WOR1), statement check (WCK1), SCTE 35 section (§5) and reported statement (§9) itself;
|
|
verifies contracts, runs and signed revenue-share statements with revenue_share_verify.py, and
|
|
rooms (their events, grants, policy and the schedule of breaks their cues make) with
|
|
live_ads_verify.py; and recomputes every break, attribution, line, citation and flag.
|
|
|
|
python3 -I opportunity_report_verify.py <live-conformance/opportunity>
|
|
python3 -I opportunity_report_verify.py --document FILE
|
|
|
|
With a corpus directory it gives every vector of vectors.json its outcome and compares it with the
|
|
one the corpus states, requires every reason of §13 and every code it reuses to refuse a vector,
|
|
requires every mutated copy of the corpus to fail, and drops each of its own checks in turn and
|
|
requires the corpus to catch the weakened verifier. With --document it prints the outcome of one
|
|
report's or check's verification document, status 0 only when it is accepted. Standard library
|
|
only.
|
|
"""
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, HERE)
|
|
sys.path.insert(0, os.path.join(HERE, "..", "claim-conformance"))
|
|
import revenue_share_verify as RSV # noqa: E402
|
|
import live_ads_verify as LAV # noqa: E402
|
|
import live_rooms_verify as LRV # noqa: E402
|
|
import live_events_verify as LEV # noqa: E402
|
|
import ad_presentation_verify as APV # noqa: E402
|
|
|
|
EXTENSION = os.path.join(HERE, "..", "extensions", "opportunity-report.md")
|
|
|
|
REPORT_MAGIC, CHECK_MAGIC = b"WOR1", b"WCK1" # §7.1, §10.1
|
|
REPORT_DOMAIN = b"wai:opportunity-report\x01" # §7.4
|
|
CHECK_DOMAIN = b"wai:statement-check\x01" # §10.5
|
|
CITES_DOMAIN = b"wai:opportunity-cites\x01" # §7.3
|
|
CUED, SCHEDULED = 1, 2 # §4.3
|
|
CLASS_NAMES = {CUED: "cued", SCHEDULED: "scheduled"}
|
|
RUN_KIND, CUE_KIND, POLICY_KIND = 10, 12, 13 # §7.3
|
|
PERIOD, DAY, PROGRAMME = 1, 2, 3 # §10.2
|
|
SCOPE_NAMES = {PERIOD: "period", DAY: "day", PROGRAMME: "programme"}
|
|
FLAGS = ["unproved", "above", "below", "impressions_above_signed", "impressions_below_signed", "revenue_above_signed",
|
|
"revenue_below_signed"] # §10.3
|
|
FORMS = {"json": 1, "csv": 2}
|
|
FORM_NAMES = {1: "json", 2: "csv"}
|
|
BASES = {"gross": 1, "share": 2}
|
|
BASIS_NAMES = {1: "gross", 2: "share"}
|
|
WHOLE = 10 ** 6
|
|
U64 = 2 ** 64 - 1
|
|
DAY_US = 86_400_000_000
|
|
ZERO = bytes(32)
|
|
ROOM_CARRIER = 1 # revenue-share §3.1
|
|
OPENING = (0x22, 0x30, 0x32, 0x34, 0x36) # §5.2, cue-injection §6.2
|
|
PLACEMENT = (0x34, 0x36)
|
|
SUB_SEGMENTED = (0x30, 0x32, 0x34, 0x36, 0x38, 0x3A, 0x44, 0x46) # §5.1 step 7
|
|
REUSED = ["truncated", "malformed", "unknown_version", "bad_key", "bad_signature", "contract_mismatch", "outside_contract",
|
|
"delivery_gap", "break_overlap", "outside_window", "asset_unmapped", "line_unsourced", "citation_mismatch"]
|
|
|
|
SKIP = set()
|
|
Refuse = RSV.Refuse
|
|
# Pure steps, held once per input: a room's live-ads verification, a run's, a signed statement's.
|
|
_ROOMS, _RUNS, _STATEMENTS = {}, {}, {}
|
|
|
|
|
|
def memo(cache, value, f):
|
|
k = json.dumps(value, sort_keys=True)
|
|
if k not in cache:
|
|
cache[k] = f()
|
|
return cache[k]
|
|
|
|
|
|
def sha(*parts):
|
|
h = hashlib.sha256()
|
|
for p in parts:
|
|
h.update(p)
|
|
return h.digest()
|
|
|
|
|
|
def ed_verify(key, msg, sig):
|
|
return RSV.ed_verify(key, msg, sig)
|
|
|
|
|
|
def lowhex(v):
|
|
"""A byte field of a document: lowercase hexadecimal, or None."""
|
|
if not isinstance(v, str) or len(v) % 2 or any(c not in "0123456789abcdef" for c in v):
|
|
return None
|
|
return bytes.fromhex(v)
|
|
|
|
|
|
# ---------------------------------------------------------------- §5 the section a cue carries
|
|
|
|
class Bad(Exception):
|
|
pass
|
|
|
|
|
|
def crc32(b):
|
|
"""ISO/IEC 13818-1 Annex A: 0x04C11DB7, preset all ones, no reflection, no final inversion."""
|
|
c = 0xFFFFFFFF
|
|
for x in b:
|
|
c ^= x << 24
|
|
for _ in range(8):
|
|
c = ((c << 1) ^ 0x04C11DB7) & 0xFFFFFFFF if c & 0x80000000 else (c << 1) & 0xFFFFFFFF
|
|
return c
|
|
|
|
|
|
class Bits:
|
|
"""Big-endian bit fields of b, up to the byte `end`; running out is Bad."""
|
|
|
|
def __init__(self, b, start, end):
|
|
self.b, self.at, self.end = b, start * 8, end
|
|
|
|
def take(self, n):
|
|
if self.at + n > self.end * 8:
|
|
raise Bad()
|
|
v = 0
|
|
for _ in range(n):
|
|
v = (v << 1) | ((self.b[self.at // 8] >> (7 - self.at % 8)) & 1)
|
|
self.at += 1
|
|
return v
|
|
|
|
def skip(self, nbytes):
|
|
if self.at // 8 + nbytes > self.end:
|
|
raise Bad()
|
|
self.at += 8 * nbytes
|
|
|
|
|
|
def splice_time(r):
|
|
if r.take(1):
|
|
r.take(6)
|
|
r.take(33)
|
|
else:
|
|
r.take(7)
|
|
|
|
|
|
def segmentation(s, start, end):
|
|
"""SCTE 35-1 Table 20 after its identifier, filling [start, end) exactly."""
|
|
r = Bits(s, start, end)
|
|
d = {"event_id": r.take(32), "cancelled": r.take(1) == 1, "duration": None, "type": None, "sub": None}
|
|
r.take(7)
|
|
if d["cancelled"]:
|
|
if r.at // 8 != end:
|
|
raise Bad()
|
|
return d
|
|
program, flagged = r.take(1), r.take(1)
|
|
r.take(6) # delivery_not_restricted_flag and the restriction bits
|
|
if not program:
|
|
r.skip(6 * r.take(8))
|
|
if flagged:
|
|
d["duration"] = r.take(40)
|
|
r.take(8) # segmentation_upid_type
|
|
r.skip(r.take(8))
|
|
d["type"] = r.take(8)
|
|
r.take(8) # segment_num
|
|
r.take(8) # segments_expected
|
|
left = end - r.at // 8
|
|
if left == 2 and d["type"] in SUB_SEGMENTED:
|
|
d["sub"] = (r.take(8), r.take(8))
|
|
elif left != 0:
|
|
raise Bad()
|
|
return d
|
|
|
|
|
|
def nothing(command):
|
|
return {"command": command, "insert": None, "id": None, "duration": None, "slots": None}
|
|
|
|
|
|
def read_section(s):
|
|
"""§5.1, for what §5.2 says a section states; Bad is cue_section_invalid."""
|
|
n = len(s)
|
|
if n < 20:
|
|
raise Bad()
|
|
crc_at = n - 4
|
|
if crc32(s[:crc_at]) != int.from_bytes(s[crc_at:], "big") and "section-crc" not in SKIP:
|
|
raise Bad()
|
|
if s[3] != 0:
|
|
raise Bad()
|
|
if s[4] >> 7:
|
|
return nothing(None)
|
|
cmd_len = ((s[11] & 0x0F) << 8) | s[12]
|
|
cmd = s[13]
|
|
if cmd_len == 0xFFF and cmd not in (0x00, 0x05, 0x06):
|
|
return nothing(cmd)
|
|
end = crc_at if cmd_len == 0xFFF else 14 + cmd_len
|
|
if end > crc_at:
|
|
raise Bad()
|
|
r = Bits(s, 14, end)
|
|
st = nothing(cmd)
|
|
if cmd == 0x05: # splice_insert()
|
|
st["id"] = r.take(32)
|
|
cancel = r.take(1) == 1
|
|
r.take(7)
|
|
out = False
|
|
if not cancel:
|
|
out, program, flagged, immediate = r.take(1) == 1, r.take(1), r.take(1), r.take(1)
|
|
r.take(4)
|
|
if program and not immediate:
|
|
splice_time(r)
|
|
if not program:
|
|
for _ in range(r.take(8)):
|
|
r.take(8)
|
|
if not immediate:
|
|
splice_time(r)
|
|
if flagged:
|
|
r.take(7)
|
|
st["duration"] = r.take(33)
|
|
r.take(16)
|
|
r.take(8)
|
|
r.take(8)
|
|
st["insert"] = (cancel, out)
|
|
elif cmd == 0x06: # time_signal()
|
|
splice_time(r)
|
|
elif cmd != 0x00:
|
|
r.at = end * 8
|
|
cmd_end = r.at // 8
|
|
if cmd_len != 0xFFF and cmd_end != end:
|
|
raise Bad()
|
|
if cmd_end + 2 > crc_at:
|
|
raise Bad()
|
|
p = cmd_end + 2
|
|
if p + int.from_bytes(s[cmd_end:p], "big") != crc_at:
|
|
raise Bad()
|
|
segs = []
|
|
while p < crc_at:
|
|
if p + 2 > crc_at:
|
|
raise Bad()
|
|
tag, ln = s[p], s[p + 1]
|
|
if p + 2 + ln > crc_at:
|
|
raise Bad()
|
|
if tag == 0x02 and ln >= 4 and s[p + 2:p + 6] == b"CUEI":
|
|
segs.append(segmentation(s, p + 6, p + 2 + ln))
|
|
p += 2 + ln
|
|
live = [d for d in segs if not d["cancelled"]]
|
|
if cmd == 0x06:
|
|
opening = next((d for d in live if d["type"] in OPENING), None)
|
|
if opening is not None:
|
|
st["id"], st["duration"] = opening["event_id"], opening["duration"]
|
|
ppo = next((d for d in live if d["type"] in PLACEMENT), None)
|
|
if ppo is not None and ppo["sub"] is not None and ppo["sub"][1] > 0:
|
|
st["slots"] = ppo["sub"][1]
|
|
return st
|
|
|
|
|
|
def agrees(st, cue):
|
|
"""§5.3: the section and its opening cue's summary describe one splice."""
|
|
if st["insert"] is not None and (st["insert"][0] or not st["insert"][1]):
|
|
return False
|
|
if st["id"] is not None and st["id"] != cue["cue_id"]:
|
|
return False
|
|
return st["duration"] is None or st["duration"] // 90 == cue["duration_ms"]
|
|
|
|
|
|
def command_name(c):
|
|
return {None: "encrypted", 0x05: "splice_insert", 0x06: "time_signal", 0x00: "splice_null"}.get(c, "other")
|
|
|
|
|
|
# ---------------------------------------------------------------- §3 the evidence
|
|
|
|
def the_contract(doc):
|
|
"""§8.2 steps 1 and 2."""
|
|
c = RSV.contract_at(RSV.unhex(doc["contract"]), ["contract"])
|
|
for k, h in enumerate(doc["contracts"]):
|
|
o = RSV.contract_at(RSV.unhex(h), ["contracts", k])
|
|
if o["digest"] != c["digest"] and RSV.overlaps(o, c) and "contract-overlap" not in SKIP:
|
|
raise Refuse("contract_overlap", ["contracts", k])
|
|
return c
|
|
|
|
|
|
def the_runs(c, runs):
|
|
"""§8.2 step 4, as revenue-share §8.2 step 4."""
|
|
out = []
|
|
for k, o in enumerate(runs):
|
|
try:
|
|
r = RSV.read_run(o)
|
|
except RSV.NoRead:
|
|
raise Refuse("malformed", ["run", k])
|
|
if "run-verify" not in SKIP and not memo(_RUNS, o, lambda: RSV.run_verifies(r)):
|
|
raise Refuse("run_invalid", ["run", k])
|
|
if "run-foreign" not in SKIP and (r["channel_id"].encode() != c["channel"] or r["signer_pubkey"] != c["channel_signer"]):
|
|
raise Refuse("run_foreign", ["run", k])
|
|
if "run-overlap" not in SKIP and any(r["window_start_ms"] < x["window_end_ms"] and x["window_start_ms"] < r["window_end_ms"]
|
|
for x in out):
|
|
raise Refuse("run_overlap", ["run", k])
|
|
out.append(r)
|
|
return out
|
|
|
|
|
|
def room_reads(v):
|
|
"""§8.1: a room's object."""
|
|
if not isinstance(v, dict) or set(v) != {"descriptor", "grants", "key_log", "chains", "policy"}:
|
|
return False
|
|
if lowhex(v["descriptor"]) is None or lowhex(v["policy"]) is None:
|
|
return False
|
|
if not isinstance(v["grants"], list) or any(lowhex(g) is None for g in v["grants"]):
|
|
return False
|
|
if v["key_log"] is not None and not isinstance(v["key_log"], str):
|
|
return False
|
|
return isinstance(v["chains"], list) and all(isinstance(c, list) and all(lowhex(e) is not None for e in c) for c in v["chains"])
|
|
|
|
|
|
def room_outcome(v):
|
|
"""live-ads §9.2 steps 1 to 3 over a room with no receipts: live_ads_verify.py's outcome."""
|
|
holder = {"creatives": [], "issuers": []}
|
|
APV.held(holder)
|
|
lv = dict(v, pt0_ms=0, receipts=[], pods=[], claims=[], creatives=[], issuers=[], name="room")
|
|
return LAV.ads(lv, judge=lambda vv, d, p: APV.Judge(holder, vv, p))[0]
|
|
|
|
|
|
def the_rooms(c, rooms):
|
|
"""§8.2 step 5: each verified as live-ads §9.2 steps 1 to 3 verify one, with no receipts."""
|
|
out = []
|
|
for k, v in enumerate(rooms):
|
|
at = ["room", k]
|
|
if not room_reads(v):
|
|
raise Refuse("malformed", at)
|
|
got = memo(_ROOMS, v, lambda: room_outcome(v))
|
|
if got["outcome"] == "gap":
|
|
raise Refuse("delivery_gap", at)
|
|
if got["outcome"] != "accept":
|
|
raise Refuse(got["reason"], at + got["at"])
|
|
room_id = bytes.fromhex(got["room"])
|
|
if (ROOM_CARRIER, room_id) not in c["carriers"] and "room-foreign" not in SKIP:
|
|
raise Refuse("room_foreign", at)
|
|
if any(x["id"] == room_id for x in out) and "room-repeated" not in SKIP:
|
|
raise Refuse("room_repeated", at)
|
|
policy = LAV.read_policy(bytes.fromhex(v["policy"]))
|
|
events = {}
|
|
for chain in v["chains"]:
|
|
for h in chain:
|
|
ev = LEV.read_record(bytes.fromhex(h))
|
|
events[ev["digest"]] = ev
|
|
out.append({"id": room_id, "t0": LRV.read_descriptor(bytes.fromhex(v["descriptor"]))["t0"], "slots": policy["slots"],
|
|
"policy": sha(policy["body"]), "breaks": got["breaks"], "events": events})
|
|
return out
|
|
|
|
|
|
# ---------------------------------------------------------------- §4, §6 breaks
|
|
|
|
def share_of(c, party, split):
|
|
return next((p for k, p in RSV.parties(c, split) if k == party), None)
|
|
|
|
|
|
def attribute(c, run, rid, e, at):
|
|
"""§6: an entry's asset by the rule, and its split; the between split for none."""
|
|
asset = RSV.asset_of(run["schedule"], e, c["rule"])
|
|
split = c["between"]
|
|
if asset is not None:
|
|
split = RSV.split_of(c, asset.encode())
|
|
if split is None:
|
|
if "unmapped" not in SKIP:
|
|
raise Refuse("asset_unmapped", at)
|
|
split = c["between"]
|
|
return rid, e, asset, split
|
|
|
|
|
|
def place(runs, t):
|
|
"""§6: the run whose window holds t, and its entry whose span does."""
|
|
for k, r in enumerate(runs):
|
|
if r["window_start_ms"] * 1000 <= t < r["window_end_ms"] * 1000:
|
|
for e, x in enumerate(r["schedule"]):
|
|
if x["start_ms"] * 1000 <= t < (x["start_ms"] + x["duration_ms"]) * 1000:
|
|
return k, e
|
|
return None
|
|
return None
|
|
|
|
|
|
def the_breaks(c, start, end, runs, ids, rooms):
|
|
"""§8.2 steps 6 and 7."""
|
|
out, starts = [], []
|
|
for k, room in enumerate(rooms):
|
|
mine = []
|
|
for i, b in enumerate(room["breaks"]):
|
|
at = ["room", k, "break", i]
|
|
s = room["t0"] + b["t"]
|
|
if s > U64:
|
|
continue
|
|
starts.append(s)
|
|
if not (start <= s < end) and "period" not in SKIP:
|
|
continue
|
|
cue = LEV.read_payload(0x02, room["events"][bytes.fromhex(b["digest"])]["payload"], 65535)
|
|
st = None
|
|
if cue["form"] == 1:
|
|
try:
|
|
st = read_section(cue["scte"])
|
|
except Bad:
|
|
if "section-read" not in SKIP:
|
|
raise Refuse("cue_section_invalid", at)
|
|
if st is not None and not agrees(st, cue) and "section-agree" not in SKIP:
|
|
raise Refuse("cue_section_mismatch", at)
|
|
duration = b["duration_ms"] if b["duration_ms"] is not None else b["avail_ms"]
|
|
if "break-overlap" not in SKIP and any(o["start"] < s + duration * 1000 and s < o["start"] + o["duration_ms"] * 1000 for o in out):
|
|
raise Refuse("break_overlap", at)
|
|
where = place(runs, s)
|
|
if where is None:
|
|
if "window" in SKIP:
|
|
continue
|
|
raise Refuse("outside_window", at)
|
|
rid, e, asset, split = attribute(c, runs[where[0]], ids[where[0]], where[1], at)
|
|
if st is not None and st["slots"] is not None and "slots-section" not in SKIP:
|
|
slots, source, pol = st["slots"], "section", None
|
|
else:
|
|
slots, source, pol = room["slots"], "policy", room["policy"]
|
|
cited = {(CUE_KIND, bytes.fromhex(b["digest"])), (RUN_KIND, rid)}
|
|
if b["closed_by"] is not None:
|
|
cited.add((CUE_KIND, bytes.fromhex(b["closed_by"])))
|
|
if pol is not None:
|
|
cited.add((POLICY_KIND, pol))
|
|
mine.append({"class": CUED, "room": k, "index": i, "taken": None, "cue": b["digest"], "start": s, "duration_ms": duration,
|
|
"open": b["type"] == "out" and b["closed_by"] is None, "slots": slots, "slots_from": source,
|
|
"section": None if st is None else {"command": command_name(st["command"]), "id": st["id"],
|
|
"duration": st["duration"], "slots": st["slots"]},
|
|
"run": rid, "entry": e, "asset": asset, "split": split, "cited": cited})
|
|
out += mine
|
|
seen = set()
|
|
for k in sorted(range(len(runs)), key=lambda j: runs[j]["window_start_ms"]):
|
|
r = runs[k]
|
|
for e in r["aired"]:
|
|
x = r["schedule"][e]
|
|
if x["kind"] != "ad":
|
|
continue
|
|
key = (x["asset_id"], x["start_ms"], x["duration_ms"])
|
|
if key in seen and "dedupe" not in SKIP:
|
|
continue
|
|
seen.add(key)
|
|
s, z = x["start_ms"] * 1000, (x["start_ms"] + x["duration_ms"]) * 1000
|
|
if any(s <= t < z for t in starts) and "cued-test" not in SKIP:
|
|
continue
|
|
if not (start <= s < end) and "period" not in SKIP:
|
|
continue
|
|
rid, e2, asset, split = attribute(c, r, ids[k], e, ["run", k, "entry", e])
|
|
out.append({"class": SCHEDULED, "room": None, "index": None, "taken": {"run": k, "entry": e}, "cue": None, "start": s,
|
|
"duration_ms": x["duration_ms"], "open": False, "slots": None, "slots_from": None, "section": None,
|
|
"run": rid, "entry": e2, "asset": asset, "split": split, "cited": {(RUN_KIND, rid)}})
|
|
return out
|
|
|
|
|
|
# ---------------------------------------------------------------- §7 the report
|
|
|
|
def cites_digest(cited):
|
|
return sha(CITES_DOMAIN, *[bytes([k]) + i for k, i in sorted(cited)])
|
|
|
|
|
|
def read_report(b):
|
|
"""§7.5, or Refuse at `report`."""
|
|
at = ["report"]
|
|
bad = RSV.head(b, REPORT_MAGIC)
|
|
if bad:
|
|
raise Refuse(bad, at)
|
|
r = RSV.R(b, 5)
|
|
try:
|
|
rep = {"issuer": r.take(32), "contract": r.take(32), "party": r.take(32), "start": r.u(8), "end": r.u(8)}
|
|
rep["rooms"] = [r.take(32) for _ in range(r.u(2))]
|
|
rep["runs"] = [r.take(32) for _ in range(r.u(2))]
|
|
rep["lines"] = []
|
|
for _ in range(r.u(2)):
|
|
rep["lines"].append({"split": r.u(2), "class": r.u(1), "breaks": r.u(8), "break_ms": r.u(8), "slots": r.u(8),
|
|
"unslotted": r.u(8), "ppm": r.u(4), "cited": r.u(4), "cites": r.take(32)})
|
|
rep["totals"] = (r.u(8), r.u(8), r.u(8))
|
|
rep["sig"] = r.take(64)
|
|
except ValueError:
|
|
raise Refuse("truncated", at)
|
|
if r.p != len(b):
|
|
raise Refuse("malformed", at)
|
|
keys = [(x["split"], x["class"]) for x in rep["lines"]]
|
|
rules = (rep["start"] < rep["end"] and rep["issuer"] != ZERO and rep["party"] != ZERO and RSV.ascending(rep["rooms"])
|
|
and RSV.ascending(rep["runs"]) and RSV.ascending(keys)
|
|
and all(x["class"] in (1, 2) and x["breaks"] > 0 and x["cited"] > 0 and x["unslotted"] <= x["breaks"]
|
|
and 1 <= x["ppm"] <= WHOLE for x in rep["lines"]))
|
|
if not rules and "report-rules" not in SKIP:
|
|
raise Refuse("opportunity_report_invalid", at)
|
|
rep["body"] = b[:-64]
|
|
rep["digest"] = sha(rep["body"])
|
|
return rep
|
|
|
|
|
|
def groups_of(breaks):
|
|
g = {}
|
|
for b in breaks:
|
|
x = g.setdefault((b["split"], b["class"]), {"breaks": 0, "break_ms": 0, "slots": 0, "unslotted": 0, "cited": set()})
|
|
x["breaks"] += 1
|
|
x["break_ms"] += b["duration_ms"]
|
|
if b["slots"] is None:
|
|
x["unslotted"] += 1
|
|
else:
|
|
x["slots"] += b["slots"]
|
|
x["cited"] |= b["cited"]
|
|
return g
|
|
|
|
|
|
def lines_of(c, party, breaks):
|
|
"""§7.2, §7.3: the party's lines and totals, or None for a figure past its field."""
|
|
out = []
|
|
for (split, cls), g in sorted(groups_of(breaks).items()):
|
|
ppm = share_of(c, party, split)
|
|
if ppm is None:
|
|
continue
|
|
out.append({"split": split, "class": cls, "breaks": g["breaks"], "break_ms": g["break_ms"], "slots": g["slots"],
|
|
"unslotted": g["unslotted"], "ppm": ppm, "cited": len(g["cited"]), "cites": cites_digest(g["cited"]),
|
|
"cited_records": sorted(g["cited"])})
|
|
totals = (sum(x["breaks"] for x in out), sum(x["break_ms"] for x in out), sum(x["slots"] for x in out))
|
|
if any(x[f] > U64 for x in out for f in ("break_ms", "slots")) or any(t > U64 for t in totals):
|
|
return None, None
|
|
return out, totals
|
|
|
|
|
|
LINE_FIELDS = ("split", "class", "breaks", "break_ms", "slots", "unslotted", "ppm", "cited", "cites")
|
|
|
|
|
|
def verify_report(doc):
|
|
"""§8.2: (outcome, held) where held is what a check reads of an accepted report."""
|
|
try:
|
|
if not isinstance(doc, dict) or set(doc) != {"contract", "contracts", "report", "runs", "rooms"}:
|
|
raise Refuse("malformed", [])
|
|
if not all(isinstance(doc[k], list) for k in ("contracts", "runs", "rooms")):
|
|
raise Refuse("malformed", [])
|
|
if lowhex(doc["contract"]) is None or lowhex(doc["report"]) is None or any(lowhex(h) is None for h in doc["contracts"]):
|
|
raise Refuse("malformed", [])
|
|
c = the_contract(doc) # steps 1, 2
|
|
rep = read_report(bytes.fromhex(doc["report"])) # step 3
|
|
if "binding" not in SKIP and (rep["contract"] != c["digest"] or (rep["party"] != c["operator"] and rep["party"] not in c["owners"])):
|
|
raise Refuse("contract_mismatch", ["report"])
|
|
if "contract-period" not in SKIP and not (c["start"] <= rep["start"] and rep["end"] <= c["end"]):
|
|
raise Refuse("outside_contract", ["report"])
|
|
if "report-signature" not in SKIP:
|
|
bad = ed_verify(rep["issuer"], REPORT_DOMAIN + rep["body"], rep["sig"])
|
|
if bad:
|
|
raise Refuse(bad, ["report"])
|
|
runs = the_runs(c, doc["runs"]) # step 4
|
|
ids = [RSV.run_identity(r) for r in runs]
|
|
rooms = the_rooms(c, doc["rooms"]) # step 5
|
|
breaks = the_breaks(c, rep["start"], rep["end"], runs, ids, rooms) # steps 6, 7
|
|
groups = groups_of(breaks) # step 8
|
|
for x in rep["lines"]:
|
|
g = groups.get((x["split"], x["class"]))
|
|
holds = x["split"] < len(c["splits"]) and share_of(c, rep["party"], x["split"]) is not None
|
|
if "unsourced" not in SKIP and (g is None or not holds or x["breaks"] > g["breaks"]):
|
|
raise Refuse("line_unsourced", ["report"])
|
|
lines, totals = lines_of(c, rep["party"], breaks)
|
|
if lines is None:
|
|
raise Refuse("opportunity_report_mismatch", ["report"])
|
|
for x in rep["lines"]:
|
|
mine = [y for y in lines if (y["split"], y["class"]) == (x["split"], x["class"])]
|
|
if mine and "citations" not in SKIP and (x["cited"] != mine[0]["cited"] or x["cites"] != mine[0]["cites"]):
|
|
raise Refuse("citation_mismatch", ["report"])
|
|
same = (rep["rooms"] == sorted(r["id"] for r in rooms) and rep["runs"] == sorted(ids)
|
|
and [{k: x[k] for k in LINE_FIELDS} for x in rep["lines"]] == [{k: y[k] for k in LINE_FIELDS} for y in lines]
|
|
and rep["totals"] == totals)
|
|
if not same and "mismatch" not in SKIP:
|
|
raise Refuse("opportunity_report_mismatch", ["report"])
|
|
except Refuse as e:
|
|
return {"outcome": "refuse", "reason": e.reason, "at": e.at}, None
|
|
counted = [b for b in breaks if share_of(c, rep["party"], b["split"]) is not None]
|
|
return accepted_report(c, rep, breaks, lines), {"contract": c, "report": rep, "counted": counted}
|
|
|
|
|
|
def accepted_report(c, rep, breaks, lines):
|
|
"""§8.3."""
|
|
return {
|
|
"outcome": "accept",
|
|
"report": rep["digest"].hex(),
|
|
"contract": rep["contract"].hex(),
|
|
"issuer": rep["issuer"].hex(),
|
|
"party": rep["party"].hex(),
|
|
"role": "operator" if rep["party"] == c["operator"] else "owner",
|
|
"start": rep["start"],
|
|
"end": rep["end"],
|
|
"rooms": [d.hex() for d in rep["rooms"]],
|
|
"runs": [d.hex() for d in rep["runs"]],
|
|
"breaks": [{"class": CLASS_NAMES[b["class"]], "room": b["room"], "index": b["index"], "taken": b["taken"], "cue": b["cue"],
|
|
"start": b["start"], "duration_ms": b["duration_ms"], "open": b["open"], "slots": b["slots"],
|
|
"slots_from": b["slots_from"], "section": b["section"], "run": b["run"].hex(), "entry": b["entry"],
|
|
"asset": b["asset"], "split": b["split"], "cites": [[k, d.hex()] for k, d in sorted(b["cited"])]} for b in breaks],
|
|
"lines": [{"split": y["split"], "class": CLASS_NAMES[y["class"]], "breaks": y["breaks"], "break_ms": y["break_ms"],
|
|
"slots": y["slots"], "unslotted": y["unslotted"], "ppm": y["ppm"],
|
|
"cites": [[k, d.hex()] for k, d in y["cited_records"]]} for y in lines],
|
|
"totals": {"breaks": rep["totals"][0], "break_ms": rep["totals"][1], "slots": rep["totals"][2]},
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------- §9 the reported statement
|
|
|
|
DATE = re.compile(r"(\d{4})-(\d{2})-(\d{2})")
|
|
|
|
|
|
def day_start(s):
|
|
"""§2: the start of a day `YYYY-MM-DD` from 1970 to 9999, in microseconds, or None."""
|
|
m = DATE.fullmatch(s) if isinstance(s, str) and s.isascii() else None
|
|
if m is None:
|
|
return None
|
|
y, mo, d = (int(x) for x in m.groups())
|
|
leap = (y % 4 == 0 and y % 100 != 0) or y % 400 == 0
|
|
days = [31, 29 if leap else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
|
|
if not (1970 <= y <= 9999 and 1 <= mo <= 12 and 1 <= d <= days[mo - 1]):
|
|
return None
|
|
ordinal = sum(366 if (x % 4 == 0 and x % 100 != 0) or x % 400 == 0 else 365 for x in range(1970, y)) + sum(days[:mo - 1]) + d - 1
|
|
return ordinal * DAY_US
|
|
|
|
|
|
def an_int(v):
|
|
return isinstance(v, int) and not isinstance(v, bool) and 0 <= v <= U64
|
|
|
|
|
|
def utf8(s):
|
|
try:
|
|
return s.encode("utf-8")
|
|
except UnicodeEncodeError:
|
|
return None
|
|
|
|
|
|
def from_json(text):
|
|
try:
|
|
o = json.loads(text)
|
|
except ValueError:
|
|
return None
|
|
need = {"channel", "start", "end", "currency", "basis", "impressions", "revenue"}
|
|
if not isinstance(o, dict) or not need <= set(o) or not set(o) <= need | {"days", "programmes"}:
|
|
return None
|
|
if not all(isinstance(o[k], str) for k in ("channel", "currency", "basis")) or not all(an_int(o[k]) for k in ("start", "end", "impressions", "revenue")):
|
|
return None
|
|
if o["basis"] not in BASES:
|
|
return None
|
|
r = {"channel": utf8(o["channel"]), "start": o["start"], "end": o["end"], "currency": utf8(o["currency"]), "basis": BASES[o["basis"]],
|
|
"impressions": o["impressions"], "revenue": o["revenue"], "days": [], "programmes": []}
|
|
for name, key in (("days", "day"), ("programmes", "asset")):
|
|
items = o.get(name, [])
|
|
if not isinstance(items, list):
|
|
return None
|
|
for x in items:
|
|
if not isinstance(x, dict) or set(x) != {key, "impressions", "revenue"} or not isinstance(x[key], str):
|
|
return None
|
|
if not an_int(x["impressions"]) or not an_int(x["revenue"]):
|
|
return None
|
|
if name == "days":
|
|
start = day_start(x["day"])
|
|
if start is None:
|
|
return None
|
|
r["days"].append({"day": x["day"], "start": start, "impressions": x["impressions"], "revenue": x["revenue"]})
|
|
else:
|
|
r["programmes"].append({"asset": utf8(x["asset"]), "impressions": x["impressions"], "revenue": x["revenue"]})
|
|
if r["channel"] is None or r["currency"] is None or any(p["asset"] is None for p in r["programmes"]):
|
|
return None
|
|
return r
|
|
|
|
|
|
def csv_records(text):
|
|
"""RFC 4180 records, each ended by CRLF or LF, the last line break optional; None when not."""
|
|
b, n, p, out = text, len(text), 0, []
|
|
if n == 0:
|
|
return None
|
|
while True:
|
|
rec = []
|
|
while True:
|
|
f = []
|
|
if p < n and b[p] == '"':
|
|
p += 1
|
|
while True:
|
|
if p >= n:
|
|
return None
|
|
if b[p] == '"':
|
|
if p + 1 < n and b[p + 1] == '"':
|
|
f.append('"')
|
|
p += 2
|
|
else:
|
|
p += 1
|
|
break
|
|
else:
|
|
f.append(b[p])
|
|
p += 1
|
|
else:
|
|
while p < n and b[p] not in ",\r\n":
|
|
if b[p] == '"':
|
|
return None
|
|
f.append(b[p])
|
|
p += 1
|
|
rec.append("".join(f))
|
|
if p < n and b[p] == ",":
|
|
p += 1
|
|
continue
|
|
break
|
|
out.append(rec)
|
|
if p == n:
|
|
return out
|
|
if b[p] == "\r" and p + 1 < n and b[p + 1] == "\n":
|
|
p += 2
|
|
elif b[p] == "\n":
|
|
p += 1
|
|
else:
|
|
return None
|
|
if p == n:
|
|
return out
|
|
|
|
|
|
def csv_int(s):
|
|
if not re.fullmatch(r"0|[1-9][0-9]*", s or "", re.ASCII):
|
|
return None
|
|
v = int(s)
|
|
return v if v <= U64 else None
|
|
|
|
|
|
def from_csv(text):
|
|
recs = csv_records(text)
|
|
if recs is None or any(len(r) != 4 for r in recs) or not recs or recs[0] != ["scope", "key", "impressions", "revenue"]:
|
|
return None
|
|
rows = recs[1:]
|
|
meta = {}
|
|
for name in ("channel", "start", "end", "currency", "basis"):
|
|
if not rows or rows[0][0] != name or rows[0][2] != "" or rows[0][3] != "":
|
|
return None
|
|
meta[name] = rows[0][1]
|
|
rows = rows[1:]
|
|
if not rows or rows[0][0] != "period" or rows[0][1] != "":
|
|
return None
|
|
period, rows = rows[0], rows[1:]
|
|
start, end, imps, rev = csv_int(meta["start"]), csv_int(meta["end"]), csv_int(period[2]), csv_int(period[3])
|
|
if None in (start, end, imps, rev) or meta["basis"] not in BASES:
|
|
return None
|
|
r = {"channel": meta["channel"].encode(), "start": start, "end": end, "currency": meta["currency"].encode(),
|
|
"basis": BASES[meta["basis"]], "impressions": imps, "revenue": rev, "days": [], "programmes": []}
|
|
in_programmes = False
|
|
for row in rows:
|
|
i, v = csv_int(row[2]), csv_int(row[3])
|
|
if i is None or v is None:
|
|
return None
|
|
if row[0] == "day" and not in_programmes:
|
|
ds = day_start(row[1])
|
|
if ds is None:
|
|
return None
|
|
r["days"].append({"day": row[1], "start": ds, "impressions": i, "revenue": v})
|
|
elif row[0] == "programme":
|
|
in_programmes = True
|
|
r["programmes"].append({"asset": row[1].encode(), "impressions": i, "revenue": v})
|
|
else:
|
|
return None
|
|
return r
|
|
|
|
|
|
def read_reported(form, text):
|
|
"""§9: the reported statement of a form, held to §9.3's rules, or None (reported_invalid)."""
|
|
r = from_json(text) if form == 1 else from_csv(text)
|
|
if r is None:
|
|
return None
|
|
rules = (1 <= len(r["channel"]) <= 1024 and len(r["currency"]) == 3 and all(0x41 <= x <= 0x5A for x in r["currency"])
|
|
and r["start"] < r["end"]
|
|
and all(r["start"] <= d["start"] and d["start"] + DAY_US <= r["end"] for d in r["days"])
|
|
and all(a["start"] < b["start"] for a, b in zip(r["days"], r["days"][1:]))
|
|
and all(1 <= len(p["asset"]) <= 1024 for p in r["programmes"])
|
|
and all(a["asset"] < b["asset"] for a, b in zip(r["programmes"], r["programmes"][1:])))
|
|
if not rules and "reported-rules" not in SKIP:
|
|
return None
|
|
return r
|
|
|
|
|
|
# ---------------------------------------------------------------- §10 the check
|
|
|
|
def read_check(b):
|
|
"""§10.6, or Refuse at `check`."""
|
|
at = ["check"]
|
|
bad = RSV.head(b, CHECK_MAGIC)
|
|
if bad:
|
|
raise Refuse(bad, at)
|
|
r = RSV.R(b, 5)
|
|
try:
|
|
c = {"issuer": r.take(32), "report": r.take(32), "reported": r.take(32), "form": r.u(1), "statement": r.take(32),
|
|
"floor": r.u(4), "ceiling": r.u(4), "lines": []}
|
|
for _ in range(r.u(2)):
|
|
x = {"scope": r.u(1), "key": r.text()}
|
|
for f in ("breaks", "break_ms", "slots", "unslotted", "impressions", "revenue", "fill_ppm", "revenue_per_slot"):
|
|
x[f] = r.u(8)
|
|
signed = r.u(1)
|
|
if signed not in (0, 1):
|
|
raise Refuse("statement_check_invalid", at)
|
|
x["signed"] = (r.u(8), r.u(8)) if signed else None
|
|
x["flags"] = r.u(1)
|
|
c["lines"].append(x)
|
|
c["sig"] = r.take(64)
|
|
except ValueError:
|
|
raise Refuse("truncated", at)
|
|
if r.p != len(b):
|
|
raise Refuse("malformed", at)
|
|
|
|
def key_ok(x):
|
|
if x["scope"] == PERIOD:
|
|
return x["key"] == b""
|
|
try:
|
|
k = x["key"].decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
return False
|
|
if x["scope"] == DAY:
|
|
return day_start(k) is not None
|
|
return x["scope"] == PROGRAMME and 1 <= len(x["key"]) <= 1024
|
|
|
|
ls = c["lines"]
|
|
rules = (c["issuer"] != ZERO and c["form"] in (1, 2) and c["floor"] <= WHOLE and c["ceiling"] >= WHOLE
|
|
and bool(ls) and ls[0]["scope"] == PERIOD and sum(1 for x in ls if x["scope"] == PERIOD) == 1
|
|
and RSV.ascending([(x["scope"], x["key"]) for x in ls])
|
|
and all(key_ok(x) and (x["signed"] is None or x["scope"] == PERIOD) and x["flags"] < 0x80 for x in ls))
|
|
if not rules and "check-rules" not in SKIP:
|
|
raise Refuse("statement_check_invalid", at)
|
|
c["body"] = b[:-64]
|
|
c["digest"] = sha(c["body"])
|
|
return c
|
|
|
|
|
|
def ratio(a, m, b):
|
|
return 0 if b == 0 else min(a * m // b, U64)
|
|
|
|
|
|
def tally(bs):
|
|
return (len(bs), min(sum(b["duration_ms"] for b in bs), U64), min(sum(b["slots"] for b in bs if b["slots"] is not None), U64),
|
|
sum(1 for b in bs if b["slots"] is None))
|
|
|
|
|
|
def check_lines(counted, reported, signed, floor, ceiling):
|
|
"""§10.2 to §10.4."""
|
|
def line(scope, key, t, imps, rev, sig):
|
|
return {"scope": scope, "key": key, "breaks": t[0], "break_ms": t[1], "slots": t[2], "unslotted": t[3], "impressions": imps,
|
|
"revenue": rev, "fill_ppm": ratio(imps, WHOLE, t[2]), "revenue_per_slot": ratio(rev, 1, t[2]), "signed": sig, "flags": 0}
|
|
out = [line(PERIOD, b"", tally(counted), reported["impressions"], reported["revenue"], signed)]
|
|
for d in reported["days"]:
|
|
out.append(line(DAY, d["day"].encode(), tally([b for b in counted if d["start"] <= b["start"] < d["start"] + DAY_US]),
|
|
d["impressions"], d["revenue"], None))
|
|
for p in reported["programmes"]:
|
|
out.append(line(PROGRAMME, p["asset"], tally([b for b in counted if b["asset"] is not None and b["asset"].encode() == p["asset"]]),
|
|
p["impressions"], p["revenue"], None))
|
|
rate, period_slots = out[0]["fill_ppm"], out[0]["slots"]
|
|
for x in out:
|
|
f = 0
|
|
if x["impressions"] > 0 and x["breaks"] == 0 and "flag-unproved" not in SKIP:
|
|
f |= 1
|
|
if x["scope"] != PERIOD and x["slots"] > 0 and period_slots > 0 and "flag-bounds" not in SKIP:
|
|
if x["fill_ppm"] * WHOLE > rate * ceiling:
|
|
f |= 2
|
|
if x["fill_ppm"] * WHOLE < rate * floor:
|
|
f |= 4
|
|
if x["signed"] is not None and "flag-signed" not in SKIP:
|
|
si, sr = x["signed"]
|
|
f |= (8 if x["impressions"] > si else 0) | (16 if x["impressions"] < si else 0)
|
|
f |= (32 if x["revenue"] > sr else 0) | (64 if x["revenue"] < sr else 0)
|
|
x["flags"] = f
|
|
return out
|
|
|
|
|
|
CHECK_LINE_FIELDS = ("scope", "key", "breaks", "break_ms", "slots", "unslotted", "impressions", "revenue", "fill_ppm", "revenue_per_slot",
|
|
"signed", "flags")
|
|
|
|
|
|
def verify_check(doc):
|
|
"""§11.2."""
|
|
try:
|
|
if not isinstance(doc, dict) or set(doc) != {"report", "reported", "statement", "check"}:
|
|
raise Refuse("malformed", [])
|
|
rp = doc["reported"]
|
|
if not isinstance(rp, dict) or set(rp) != {"form", "text"} or rp["form"] not in FORMS or not isinstance(rp["text"], str):
|
|
raise Refuse("malformed", [])
|
|
cb = lowhex(doc["check"])
|
|
if cb is None or not (doc["statement"] is None or isinstance(doc["statement"], dict)):
|
|
raise Refuse("malformed", [])
|
|
got, held = verify_report(doc["report"]) # step 1
|
|
if got["outcome"] != "accept":
|
|
raise Refuse(got["reason"], ["report"] + got["at"])
|
|
c, rep = held["contract"], held["report"]
|
|
form = FORMS[rp["form"]]
|
|
reported = read_reported(form, rp["text"]) # step 2
|
|
if reported is None:
|
|
raise Refuse("reported_invalid", ["reported"])
|
|
if "reported-foreign" not in SKIP and (reported["channel"] != c["channel"] or reported["currency"] != c["currency"]
|
|
or (reported["start"], reported["end"]) != (rep["start"], rep["end"])):
|
|
raise Refuse("reported_foreign", ["reported"])
|
|
statement = None
|
|
if doc["statement"] is not None: # step 3
|
|
sg = memo(_STATEMENTS, doc["statement"], lambda: RSV.verify(doc["statement"]))
|
|
if sg["outcome"] != "accept":
|
|
raise Refuse(sg["reason"], ["statement"] + sg["at"])
|
|
statement = RSV.read_statement(bytes.fromhex(doc["statement"]["statement"]))
|
|
if "statement-foreign" not in SKIP and (statement["contract"] != rep["contract"] or statement["party"] != rep["party"]
|
|
or (statement["start"], statement["end"]) != (rep["start"], rep["end"])):
|
|
raise Refuse("statement_foreign", ["statement"])
|
|
ch = read_check(cb) # step 4
|
|
if "check-signature" not in SKIP:
|
|
bad = ed_verify(ch["issuer"], CHECK_DOMAIN + ch["body"], ch["sig"])
|
|
if bad:
|
|
raise Refuse(bad, ["check"])
|
|
signed = None
|
|
if statement is not None:
|
|
imps = min(sum(x["impressions"] for x in statement["lines"]), U64)
|
|
rev = min(sum(x["revenue"] for x in statement["lines"]), U64) if reported["basis"] == 1 else statement["total"]
|
|
signed = (imps, rev)
|
|
lines = check_lines(held["counted"], reported, signed, ch["floor"], ch["ceiling"])
|
|
same = (ch["report"] == rep["digest"] and ch["reported"] == sha(rp["text"].encode()) and ch["form"] == form
|
|
and ch["statement"] == (ZERO if statement is None else statement["digest"])
|
|
and [{k: x[k] for k in CHECK_LINE_FIELDS} for x in ch["lines"]] == [{k: y[k] for k in CHECK_LINE_FIELDS} for y in lines])
|
|
if not same and "check-mismatch" not in SKIP:
|
|
raise Refuse("statement_check_mismatch", ["check"])
|
|
except Refuse as e:
|
|
return {"outcome": "refuse", "reason": e.reason, "at": e.at}
|
|
return {
|
|
"outcome": "accept",
|
|
"check": ch["digest"].hex(),
|
|
"issuer": ch["issuer"].hex(),
|
|
"report": ch["report"].hex(),
|
|
"party": rep["party"].hex(),
|
|
"start": rep["start"],
|
|
"end": rep["end"],
|
|
"reported": ch["reported"].hex(),
|
|
"form": FORM_NAMES[ch["form"]],
|
|
"basis": BASIS_NAMES[reported["basis"]],
|
|
"statement": None if statement is None else statement["digest"].hex(),
|
|
"floor_ppm": ch["floor"],
|
|
"ceiling_ppm": ch["ceiling"],
|
|
"lines": [{"scope": SCOPE_NAMES[x["scope"]], "key": x["key"].decode("utf-8"), "breaks": x["breaks"], "break_ms": x["break_ms"],
|
|
"slots": x["slots"], "unslotted": x["unslotted"], "impressions": x["impressions"], "revenue": x["revenue"],
|
|
"fill_ppm": x["fill_ppm"], "revenue_per_slot": x["revenue_per_slot"],
|
|
"signed": None if x["signed"] is None else {"impressions": x["signed"][0], "revenue": x["signed"][1]},
|
|
"flags": [n for i, n in enumerate(FLAGS) if x["flags"] & (1 << i)]} for x in ch["lines"]],
|
|
}
|
|
|
|
|
|
def verify(doc):
|
|
"""A report's or a check's document: a check's holds `reported`."""
|
|
if isinstance(doc, dict) and "reported" in doc:
|
|
return verify_check(doc)
|
|
return verify_report(doc)[0]
|
|
|
|
|
|
# ---------------------------------------------------------------- the corpus
|
|
|
|
def document(doc, v):
|
|
"""A vector as its verification document, its parts resolved by name."""
|
|
def report_doc(p):
|
|
return {"contract": doc["contracts"][p["contract"]], "contracts": [doc["contracts"][n] for n in p["contracts"]],
|
|
"report": p["report"], "runs": None if p["runs"] is None else [doc["runs"][n] for n in p["runs"]],
|
|
"rooms": [doc["rooms"][n] for n in p["rooms"]]}
|
|
if v["kind"] == "report":
|
|
return report_doc(v["report"])
|
|
s = v["statement"]
|
|
sd = None if s is None else {"contract": doc["contracts"][s["contract"]], "contracts": [], "statement": s["statement"],
|
|
"runs": [doc["runs"][n] for n in s["runs"]], "ledgers": [doc["ledgers"][n] for n in s["ledgers"]]}
|
|
return {"report": report_doc(v["report"]), "reported": doc["reported"][v["reported"]], "statement": sd, "check": v["check"]}
|
|
|
|
|
|
def check(doc, quiet=False):
|
|
"""The failures: every vector whose stated outcome is not the one this verifier reaches."""
|
|
fails = []
|
|
for v in doc.get("vectors", []):
|
|
try:
|
|
got = verify(document(doc, v))
|
|
except Exception as e: # a crash on a mutated copy is a failure to verify it
|
|
got = {"outcome": "crash", "error": "%s: %s" % (type(e).__name__, e)}
|
|
ok = got == v.get("outcome")
|
|
if not quiet:
|
|
what = got["outcome"] + (" %s at %s" % (got["reason"], got["at"]) if got["outcome"] == "refuse" else "")
|
|
print(("PASS" if ok else "FAIL") + " %s: %s" % (v.get("name"), what))
|
|
if not ok:
|
|
fails.append(v.get("name"))
|
|
return fails
|
|
|
|
|
|
def reasons_listed():
|
|
"""§13's table, in order."""
|
|
text = open(EXTENSION, encoding="utf-8").read()
|
|
at = text.index("\n## 13. Reasons")
|
|
end = text.find("\n## ", at + 1)
|
|
return [m.group(1) for m in re.finditer(r"^\| `([a-z_]+)` \|", text[at:end], re.M)]
|
|
|
|
|
|
def flip_hex(h, at):
|
|
b = bytearray(bytes.fromhex(h))
|
|
b[at % len(b)] ^= 0x01
|
|
return b.hex()
|
|
|
|
|
|
def mutations(doc):
|
|
"""Copies of the corpus, one defect each, that a verifier must reject."""
|
|
out = []
|
|
names = [v["name"] for v in doc["vectors"]]
|
|
|
|
def mutate(label, f):
|
|
m = copy.deepcopy(doc)
|
|
f(m)
|
|
out.append((label, m))
|
|
|
|
for k, v in enumerate(doc["vectors"]):
|
|
if v["outcome"]["outcome"] != "accept":
|
|
continue
|
|
if v["kind"] == "report":
|
|
for at in (6, 40, 100, 140, -90, -70, -1):
|
|
mutate("%s: report byte %d flipped" % (v["name"], at), lambda m, k=k, at=at: m["vectors"][k]["report"].update(
|
|
report=flip_hex(m["vectors"][k]["report"]["report"], at)))
|
|
mutate("%s: its stated totals raised" % v["name"], lambda m, k=k: m["vectors"][k]["outcome"]["totals"].update(
|
|
breaks=m["vectors"][k]["outcome"]["totals"]["breaks"] + 1))
|
|
if v["outcome"]["breaks"]:
|
|
mutate("%s: its first break moved a second" % v["name"], lambda m, k=k: m["vectors"][k]["outcome"]["breaks"][0].update(
|
|
start=m["vectors"][k]["outcome"]["breaks"][0]["start"] + 1_000_000))
|
|
mutate("%s: its first break's slots changed" % v["name"], lambda m, k=k: m["vectors"][k]["outcome"]["breaks"][0].update(
|
|
slots=7))
|
|
if v["report"]["rooms"]:
|
|
mutate("%s: its last room dropped" % v["name"], lambda m, k=k: m["vectors"][k]["report"]["rooms"].pop())
|
|
else:
|
|
for at in (40, 104, 150, -80, -66, -1):
|
|
mutate("%s: check byte %d flipped" % (v["name"], at), lambda m, k=k, at=at: m["vectors"][k].update(
|
|
check=flip_hex(m["vectors"][k]["check"], at)))
|
|
mutate("%s: a line's flags changed in its outcome" % v["name"], lambda m, k=k: m["vectors"][k]["outcome"]["lines"][-1].update(
|
|
flags=["above"]))
|
|
if v["statement"] is not None:
|
|
mutate("%s: its signed statement left out" % v["name"], lambda m, k=k: m["vectors"][k].update(statement=None))
|
|
for name, at in (("main", -1), ("main", 40), ("two_rooms", 60)):
|
|
mutate("the contract %s: byte %d flipped" % (name, at), lambda m, name=name, at=at: m["contracts"].update(
|
|
{name: flip_hex(doc["contracts"][name], at)}))
|
|
mutate("the run of the first hour: its signature flipped",
|
|
lambda m: m["runs"]["hour_1"].update(sig=flip_hex(doc["runs"]["hour_1"]["sig"], 3)))
|
|
mutate("the run of the second hour: its first programme renamed",
|
|
lambda m: m["runs"]["hour_2"]["schedule"][0].update(asset_id="a/ep-1"))
|
|
mutate("the main room: an event of its producer's flipped",
|
|
lambda m: m["rooms"]["main"]["chains"][1].__setitem__(5, flip_hex(doc["rooms"]["main"]["chains"][1][5], 130)))
|
|
mutate("the main room: its policy flipped", lambda m: m["rooms"]["main"].update(policy=flip_hex(doc["rooms"]["main"]["policy"], 70)))
|
|
mutate("owner B's reported statement: one impression more",
|
|
lambda m: m["reported"]["b_programmes"].update(text=doc["reported"]["b_programmes"]["text"].replace('"impressions":14', '"impressions":15')))
|
|
mutate("owner A's CSV: its line breaks made LF",
|
|
lambda m: m["reported"]["a_days"].update(text=doc["reported"]["a_days"]["text"].replace("\r\n", "\n")))
|
|
for k, v in enumerate(doc["vectors"]):
|
|
if v["outcome"]["outcome"] == "refuse":
|
|
mutate("%s: its stated reason changed" % v["name"], lambda m, k=k, v=v: m["vectors"][k]["outcome"].update(
|
|
reason="malformed" if v["outcome"]["reason"] != "malformed" else "truncated"))
|
|
break
|
|
assert len(names) == len(set(names)), "vector names are unique"
|
|
return out
|
|
|
|
|
|
CHECKS = ["contract-overlap", "report-rules", "binding", "contract-period", "report-signature", "run-verify", "run-foreign", "run-overlap",
|
|
"room-foreign", "room-repeated", "period", "section-crc", "section-read", "section-agree", "break-overlap", "window", "unmapped",
|
|
"slots-section", "dedupe", "cued-test", "unsourced", "citations", "mismatch", "reported-rules", "reported-foreign",
|
|
"statement-foreign", "check-rules", "check-signature", "check-mismatch", "flag-unproved", "flag-bounds", "flag-signed"]
|
|
|
|
|
|
def self_tests():
|
|
"""The standard's sample (SCTE 35-1 §14.1, as cue-injection §5.7 holds it) reads as §5 reads it."""
|
|
sample = bytes.fromhex("fc3034000000000000fffff00506fe72bd0050001e021c435545494800008e7fcf0001a599b00808000000002ca0a18a3402009ac9d17e")
|
|
st = read_section(sample)
|
|
fails = []
|
|
if (st["command"], st["id"], st["duration"], st["slots"]) != (0x06, 0x4800008E, 307 * 90_000, None):
|
|
fails.append("the standard's sample does not state its opportunity")
|
|
damaged = bytearray(sample)
|
|
damaged[-1] ^= 1
|
|
try:
|
|
read_section(bytes(damaged))
|
|
fails.append("a section with a damaged CRC reads")
|
|
except Bad:
|
|
pass
|
|
print("%s the standard's SCTE 35 sample reads, and a damaged copy does not" % ("PASS" if not fails else "FAIL"))
|
|
return fails
|
|
|
|
|
|
def main(corpus_dir):
|
|
doc = json.load(open(os.path.join(corpus_dir, "vectors.json"), encoding="utf-8"))
|
|
fails = self_tests()
|
|
fails += check(doc)
|
|
files = sorted(os.listdir(corpus_dir))
|
|
if files != ["README.md", "vectors.json"]:
|
|
fails.append("the corpus holds other files: %s" % files)
|
|
seen = {v["outcome"].get("reason") for v in doc["vectors"]}
|
|
listed = reasons_listed()
|
|
missing = [r for r in listed + REUSED if r not in seen]
|
|
if missing:
|
|
fails.append("no vector is refused %s" % missing)
|
|
print("%s every reason of §13 and every reused code refuses a vector (%d listed)" % ("PASS" if not missing else "FAIL", len(listed)))
|
|
# And no refusal is unaccounted for: each is §13's, a code §13 reuses, or a code a text it
|
|
# delegates to (revenue-share, live-ads) raises there.
|
|
delegated = set(RSV.reasons_listed()) | set(RSV.REUSED) | set(LAV.REASONS)
|
|
unexplained = sorted(r for r in seen if r is not None and r not in listed and r not in REUSED and r not in delegated)
|
|
if unexplained:
|
|
fails.append("refusals §13 does not account for: %s" % unexplained)
|
|
print("%s every refusal of the corpus is a reason of §13, a code it reuses, or one a text it delegates to raises" % (
|
|
"PASS" if not unexplained else "FAIL"))
|
|
muts = mutations(doc)
|
|
weak = [name for name, m in muts if not check(m, quiet=True)]
|
|
print("%s every mutated copy of the corpus fails (%d copies)" % ("PASS" if not weak else "FAIL", len(muts)))
|
|
fails += ["the mutated copy %r verifies" % w for w in weak]
|
|
undetected = []
|
|
for c in CHECKS:
|
|
SKIP.add(c)
|
|
try:
|
|
if not check(doc, quiet=True):
|
|
undetected.append(c)
|
|
finally:
|
|
SKIP.discard(c)
|
|
print("%s the corpus catches the verifier with each of its checks dropped (%d checks)" % ("PASS" if not undetected else "FAIL", len(CHECKS)))
|
|
fails += ["dropping the check %r goes unnoticed" % c for c in undetected]
|
|
for f in fails:
|
|
print("FAIL", f)
|
|
return 1 if fails else 0
|
|
|
|
|
|
def check_document(path):
|
|
doc = json.load(open(path, encoding="utf-8"))
|
|
got = verify(doc)
|
|
print(json.dumps(got, indent=1))
|
|
return 0 if got["outcome"] == "accept" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) == 3 and sys.argv[1] == "--document":
|
|
sys.exit(check_document(sys.argv[2]))
|
|
if len(sys.argv) != 2:
|
|
print("usage: opportunity_report_verify.py <live-conformance/opportunity> | --document FILE", file=sys.stderr)
|
|
sys.exit(2)
|
|
sys.exit(main(sys.argv[1]))
|