open-standards/wai/encode-conformance/encode_verify.py

#!/usr/bin/env python3
"""Independent verifier of the platform-encode corpus (extensions/platform-encode.md).

Written from the extension's text, and from the passages it cites: SPEC §2 (the `WAI1`
envelope) and §7 *Signatures*; jwp-receipts §2.1 (an object's content hash) and §2.4.5 (the
energy rules); energy-measurement §2 (the acquisition classes); packaging §6.2 (the stage
index, `WSIX`); contribution §2 (how a record is read); and ISO/IEC 14496-10 and 23008-2 for
the Annex B byte stream, NAL unit headers, the sequence parameter sets and the start of the
slice header. It shares no code with wai-rs or wai-pack and never runs them. It verifies
Ed25519 signatures with ../claim-conformance/_ed25519.py (WAI's strict rule) and content
hashes with ../claim-conformance/_blake3.py; everything else is here, in integers:

- the record's strict reading (§6.1), its signature and digest (§6.2) and §7 steps 3 to 7;
- the Annex B stream (§4.3), in its order: item 1 and item 2's count over the whole stream,
  then items 3 to 6 access unit by access unit, then item 7: NAL units, access units, IDR and
  random access points, parameter sets, 8-bit 4:2:0 progressive single layer, no reordering
  (with the top field order count of ISO/IEC 14496-10 §8.2.1.1), the SPS's cropped size,
  profile, level, tier and range (§7 step 8);
- the held source (§4.1, step 9): it reads, its digest, size, frame count, rate and range;
- an encoded rendition against its package (§9.1, §9.3, step 10): the receipt, the stage
  index's pieces and base entries, each floor object's content hash, its exact manifest
  bytes and its payload re-framed from the held stream.

It runs every case of cases.json and rendition/cases.json and requires its outcome; requires
every record reason of §12 to appear in the corpus; then mutates copies of the corpus and
requires every copy to fail, and drops each of its checks in turn and requires the corpus
(or, where the committed corpus has no case for it, a self-test mutation) to catch the
weakened verifier.

    python3 -I encode_verify.py <encode-conformance dir>

Exit status 0 only if every case matches, every mutation is refused as expected and every
dropped check is caught. Python 3.8 or later; standard library only.
"""

import hashlib
import json  # reads the corpus's own case lists only; records are read by read_json below
import os
import re
import shutil
import sys
import tempfile

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(HERE, "..", "claim-conformance"))
import _ed25519 as ED  # noqa: E402
from _blake3 import blake3 as _blake3_raw  # noqa: E402

SPEC_PATH = os.path.join(HERE, "..", "extensions", "platform-encode.md")

# ---------------------------------------------------------------- reasons (§12)

ACCEPT = "accept"
MALFORMED = "record_malformed"
BAD_KEY = "bad_key"
BAD_SIG = "bad_signature"
CAPABILITY = "encode-capability"
ID_INCOMPLETE = "encode-identity-incomplete"
ID_INCONSISTENT = "encode-identity-inconsistent"
SETTINGS = "encode-settings-invalid"
PLAN = "encode-plan-invalid"
ENERGY_INVALID = "encode-energy-invalid"
ENERGY_COVERAGE = "encode-energy-coverage"
STREAM_MISMATCH = "encode-stream-mismatch"
STREAM_MALFORMED = "encode-stream-malformed"
FRAME_COUNT = "encode-frame-count"
KEYFRAME = "encode-keyframe-mismatch"
PARAMS = "encode-parameter-sets"
UNSUPPORTED = "encode-stream-unsupported"
REORDERED = "encode-stream-reordered"
SPS_MISMATCH = "encode-sps-mismatch"
SOURCE_MISMATCH = "encode-source-mismatch"
PACKAGE_MISMATCH = "encode-package-mismatch"
RECORD_REASONS = [MALFORMED, BAD_KEY, BAD_SIG, CAPABILITY, ID_INCOMPLETE, ID_INCONSISTENT, SETTINGS, PLAN,
                  ENERGY_INVALID, ENERGY_COVERAGE, STREAM_MISMATCH, STREAM_MALFORMED, FRAME_COUNT, KEYFRAME,
                  PARAMS, UNSUPPORTED, REORDERED, SPS_MISMATCH, SOURCE_MISMATCH, PACKAGE_MISMATCH]

AVC_CAP, HEVC_CAP = "wai.video.avc", "wai.video.hevc"
AVC, HEVC = "avc", "hevc"
MAX_INT = (1 << 53) - 1
U32 = (1 << 32) - 1
MAX_DEPTH = 32

# ---------------------------------------------------------------- the checks, by name
#
# Each check can be dropped (part 4b) to show that the corpus catches a verifier without it.
# A check that is never observable in an outcome is not listed: the 32-level depth limit,
# for one, since no member of §6.1 nests deeper than three levels, so exact members refuse
# any deeper record with the same reason.

CHECKS = [
    ("json-duplicate", "§6.1: no member named twice in one object"),
    ("json-number", "§6.1: integers unsigned, no fraction or exponent, at most 2^53-1"),
    ("member-types", "§6.1: each member's type and values (incl. no boolean, terms, hex, enums)"),
    ("exact-members", "§6.1: no member added"),
    ("date-time", "§6.1: issued_at a date-time"),
    ("signature", "§6.2: the Ed25519 signature at all"),
    ("small-order-key", "SPEC §7 step 2: the key is not of small order"),
    ("capability", "§7 step 3"),
    ("identity-blank", "§7 step 4: name/version/device/driver not blank"),
    ("identity-api", "§7 step 4: api one of §3's"),
    ("identity-library", "§7 step 4: library_sha256 exactly for software"),
    ("identity-software-hw", "§7 step 4: software is hardware no"),
    ("identity-software-avc", "§7 step 4: software is wai.video.avc"),
    ("identity-hw-engine", "§7 step 4: va and gpu-driver are hardware yes"),
    ("settings-size", "§7 step 5: the rung's width and height even, 16..8192"),
    ("settings-source-size", "§7 step 5: the source's width and height even, 2..16384"),
    ("settings-rate", "§7 step 5: fps members non-zero, at most 2^32-1"),
    ("settings-kbps", "§7 step 5: kbps 1..1000000"),
    ("settings-frames", "§7 step 5: at least one source frame"),
    ("settings-resample", "§7 step 5: resample none exactly at the source's size"),
    ("plan-order", "§7 step 6: plan starts at 0 and increases"),
    ("plan-range", "§7 step 6: last entry below source.frames"),
    ("plan-pieces", "§7 step 6: the package's pieces against the plan"),
    ("energy-nonzero", "§8.1 / jwp §2.4.5 rule 1"),
    ("energy-class", "§8.1 / jwp §2.4.5 rule 2: a known class, not HwShunt"),
    ("energy-evidence", "§8.1 / jwp §2.4.5 rule 2: the class's evidence, whole, incl. the floor"),
    ("energy-coverage-value", "§8.1 / jwp §2.4.5 rule 3: Whole or Partial"),
    ("energy-attribution", "§8.1: task, rail or bracket"),
    ("energy-fabrics", "§8.1: fabrics known, at least one, in order, none twice"),
    ("energy-engine", "§8.1: a hardware encode counting no engine is Partial"),
    ("stream-digest", "§7 step 8: SHA-256 and length"),
    ("annexb", "§4.3 item 1: Annex B syntax, every SPS and slice-header start reads"),
    ("au-trailing", "§4.3 item 2: no access unit holding no slice after the last picture"),
    ("au-count", "§4.3 item 2, step 8: one access unit per frame, stream.units"),
    ("keyframes", "§4.3 item 3: IDR exactly at the plan, no other RAP"),
    ("idr-mix", "§4.3 item 3: an access unit's slices all IDR or none"),
    ("parameter-sets", "§4.3 item 4: IDR access units' parameter sets, no foreign ones"),
    ("sps-single", "§4.3 item 4: no SPS other than the stream's first"),
    ("unsupported", "§4.3 item 5"),
    ("reorder-field", "§4.3 item 6: sps_max_num_reorder_pics / max_num_reorder_frames"),
    ("reorder-poc", "§4.3 item 6: AVC picture order count increases"),
    ("sps-size", "§4.3 item 7: cropped size"),
    ("sps-range", "§4.3 item 7: AVC VUI range"),
    ("sps-profile-level-tier", "§7 step 8: profile, level, tier"),
    ("source-digest", "§7 step 9: the source's digest"),
    ("source-geometry", "§7 step 9: the source's width, height and frame count"),
    ("source-rate-range", "§7 step 9: the source's frame rate and range"),
    ("receipt", "§9.3: receipt SHA-256"),
    ("index-pieces", "§9.3: stage index pieces' starts and frames"),
    ("index-motif", "§9.3: each piece's base entry is the record's motif"),
    ("object-present", "§9.3: each piece's object present"),
    ("object-hash", "§9.3: content hash is the record's object"),
    ("object-manifest", "§9.3: exact §9.1 manifest"),
    ("object-payload", "§9.3: payload re-framed from the held stream"),
]
CHECK_NAMES = {name for name, _ in CHECKS}
DROP = set()


def on(name):
    if name not in CHECK_NAMES:
        raise AssertionError("unknown check " + name)
    return name not in DROP


class Refuse(Exception):
    def __init__(self, code, detail=""):
        super().__init__(code)
        self.code = code
        self.detail = detail


# ---------------------------------------------------------------- hashing, with a cache
# The self-test runs the same bytes many times; a cache keyed by the whole input changes nothing.

_B3, _ED = {}, {}


def blake3(data):
    v = _B3.get(data)
    if v is None:
        v = _B3[data] = _blake3_raw(data)
    return v


def ed_verify(pk, msg, sig, skip):
    key = (pk, msg, sig, skip)
    if key not in _ED:
        _ED[key] = ED.verify(pk, msg, sig, skip)
    return _ED[key]


def sha256(b):
    return hashlib.sha256(b).digest()


# ---------------------------------------------------------------- strict JSON (§6.1)


class Malformed(Exception):
    pass


_PLAIN = re.compile(r'[^"\\\x00-\x1f]+')
_NUMBER = re.compile(r'-?(?:0|[1-9][0-9]*)(\.[0-9]+)?([eE][+-]?[0-9]+)?')
_HEX4 = re.compile(r'[0-9a-fA-F]{4}')
_SHORT = {"b": "\b", "f": "\f", "n": "\n", "r": "\r", "t": "\t"}


class _Reader:
    def __init__(self, text):
        self.t, self.i = text, 0

    def ws(self):
        t, i = self.t, self.i
        while i < len(t) and t[i] in " \t\n\r":
            i += 1
        self.i = i

    def value(self, depth):
        t = self.t
        if self.i >= len(t):
            raise Malformed("the text ends where a value was expected")
        c = t[self.i]
        if c == "{":
            return self.obj(depth + 1)
        if c == "[":
            return self.arr(depth + 1)
        if c == '"':
            return self.string()
        if c == "-" or "0" <= c <= "9":
            return self.number()
        for word, val in (("null", None), ("true", True), ("false", False)):
            if t.startswith(word, self.i):
                self.i += len(word)
                if val is not None and on("member-types"):
                    raise Malformed("a boolean")
                return val
        raise Malformed("no JSON value at offset %d" % self.i)

    def obj(self, depth):
        if depth > MAX_DEPTH:
            raise Malformed("nested deeper than 32 levels")
        self.i += 1
        out = {}
        self.ws()
        if self.t.startswith("}", self.i):
            self.i += 1
            return out
        while True:
            self.ws()
            if not self.t.startswith('"', self.i):
                raise Malformed("a member name that is not a string")
            k = self.string()
            self.ws()
            if not self.t.startswith(":", self.i):
                raise Malformed("no ':' after a member name")
            self.i += 1
            self.ws()
            v = self.value(depth)
            if k in out and on("json-duplicate"):
                raise Malformed("member %r named twice" % k)
            out[k] = v
            self.ws()
            if self.t.startswith(",", self.i):
                self.i += 1
                continue
            if self.t.startswith("}", self.i):
                self.i += 1
                return out
            raise Malformed("expected ',' or '}' at offset %d" % self.i)

    def arr(self, depth):
        if depth > MAX_DEPTH:
            raise Malformed("nested deeper than 32 levels")
        self.i += 1
        out = []
        self.ws()
        if self.t.startswith("]", self.i):
            self.i += 1
            return out
        while True:
            self.ws()
            out.append(self.value(depth))
            self.ws()
            if self.t.startswith(",", self.i):
                self.i += 1
                continue
            if self.t.startswith("]", self.i):
                self.i += 1
                return out
            raise Malformed("expected ',' or ']' at offset %d" % self.i)

    def hex4(self, at):
        h = self.t[at:at + 4]
        if not _HEX4.fullmatch(h):
            raise Malformed("a \\u escape without four hex digits")
        return int(h, 16)

    def string(self):
        t, i, parts = self.t, self.i + 1, []
        while True:
            m = _PLAIN.match(t, i)
            if m:
                parts.append(m.group())
                i = m.end()
            if i >= len(t):
                raise Malformed("an unterminated string")
            c = t[i]
            if c == '"':
                self.i = i + 1
                return "".join(parts)
            if c != "\\":
                raise Malformed("an unescaped control character in a string")
            e = t[i + 1:i + 2]
            if e in ('"', "\\", "/"):
                parts.append(e)
                i += 2
            elif e and e in _SHORT:
                parts.append(_SHORT[e])
                i += 2
            elif e == "u":
                cp = self.hex4(i + 2)
                i += 6
                if 0xD800 <= cp < 0xDC00:
                    if t[i:i + 2] != "\\u":
                        raise Malformed("a lone surrogate")
                    lo = self.hex4(i + 2)
                    if not 0xDC00 <= lo < 0xE000:
                        raise Malformed("a lone surrogate")
                    cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00)
                    i += 6
                elif 0xDC00 <= cp < 0xE000:
                    raise Malformed("a lone surrogate")
                parts.append(chr(cp))
            else:
                raise Malformed("an invalid escape")

    def number(self):
        m = _NUMBER.match(self.t, self.i)
        if not m:
            raise Malformed("not a number")
        tok = m.group()
        self.i = m.end()
        if m.group(1) or m.group(2):
            if on("json-number"):
                raise Malformed("a number with a fraction or an exponent: " + tok)
            return float(tok)
        v = int(tok)
        if on("json-number") and (tok.startswith("-") or v > MAX_INT):
            raise Malformed("an integer with a sign or above 2^53-1: " + tok)
        return v


def read_json(data):
    """RFC 8259 as §6.1 restricts it, or Malformed."""
    try:
        text = data.decode("utf-8")
    except UnicodeDecodeError:
        raise Malformed("not UTF-8")
    r = _Reader(text)
    r.ws()
    if not r.t.startswith("{", r.i):
        raise Malformed("not a JSON object")
    v = r.value(0)
    r.ws()
    if r.i != len(text):
        raise Malformed("text after the record")
    return v


# ---------------------------------------------------------------- JCS (RFC 8785)

_JESC = {'"': '\\"', "\\": "\\\\", "\b": "\\b", "\f": "\\f", "\n": "\\n", "\r": "\\r", "\t": "\\t"}


def jcs_str(s):
    out = ['"']
    for ch in s:
        e = _JESC.get(ch)
        if e is not None:
            out.append(e)
        elif ord(ch) < 0x20:
            out.append("\\u%04x" % ord(ch))
        else:
            out.append(ch)
    out.append('"')
    return "".join(out)


def jcs(v):
    if v is None:
        return "null"
    if v is True:
        return "true"
    if v is False:
        return "false"
    if isinstance(v, int):
        return str(v)
    if isinstance(v, float):  # only reachable with json-number dropped
        return repr(v)
    if isinstance(v, str):
        return jcs_str(v)
    if isinstance(v, list):
        return "[" + ",".join(jcs(x) for x in v) + "]"
    if isinstance(v, dict):
        keys = sorted(v, key=lambda k: k.encode("utf-16-be"))
        return "{" + ",".join(jcs_str(k) + ":" + jcs(v[k]) for k in keys) + "}"
    raise ValueError("a value JCS does not write")


# ---------------------------------------------------------------- the record (§6.1)

_TERM = re.compile(r"[a-z0-9._-]{1,64}")
_HEXRE = {32: re.compile(r"[0-9a-f]{64}"), 64: re.compile(r"[0-9a-f]{128}")}
_DATETIME = re.compile(r"([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})(?:\.[0-9]{1,9})?Z")


def is_datetime(s):
    m = _DATETIME.fullmatch(s)
    if not m:
        return False
    y, mo, d, h, mi, se = (int(m.group(k)) for k in range(1, 7))
    if not (1 <= mo <= 12 and h <= 23 and mi <= 59 and se <= 60):
        return False
    leap = y % 4 == 0 and (y % 100 != 0 or y % 400 == 0)
    return 1 <= d <= [31, 29 if leap else 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][mo - 1]


def _nullable(t):
    return ("null|", t)


def _enum(*vals):
    return ("enum",) + vals


def _array(t):
    return ("array", t)


ENCODER = {"api": "term", "name": "str1024", "version": "str1024", "hardware": _enum("yes", "no"),
           "device": "str1024", "driver": "str1024", "library_sha256": _nullable("hex32")}
SETTINGS_T = {"width": "int", "height": "int", "fps_num": "int", "fps_den": "int", "kbps": "int",
              "range": _enum("video", "full"), "resample": "term"}
SOURCE_T = {"frames": "int", "width": "int", "height": "int", "sha256": "hex32"}
STREAM_T = {"sha256": "hex32", "bytes": "int", "units": "int", "profile_idc": "int", "level_idc": "int",
            "tier": _enum("main", "high")}
UNCERTAINTY_T = {"relative_ppm": "int", "window_us": "int"}
ENERGY_T = {"joules_micro": "int", "energy_provenance": "string", "energy_uncertainty": _nullable(UNCERTAINTY_T),
            "energy_counter_filtering": _nullable("string"), "energy_calibration_ref": _nullable("string"),
            "coverage": "string", "counter": "term", "attribution": "string", "fabrics": _array("string")}
PIECE_T = {"start": "int", "frames": "int", "motif": "hex32", "object": "hex32"}
PACKAGE_T = {"receipt_sha256": "hex32", "rung": "term", "pieces": _array(PIECE_T)}
RECORD_T = {"kind": _enum("wai-encode"), "capability": "string", "encoder": ENCODER, "settings": SETTINGS_T,
            "source": SOURCE_T, "idr": _array("int"), "stream": STREAM_T, "energy": _nullable(ENERGY_T),
            "wall_us": "int", "package": _nullable(PACKAGE_T), "issued_at": "datetime",
            "signer_pubkey": "hex32", "sig": "hex64"}


def check_type(v, t, at):
    if isinstance(t, dict):
        if not isinstance(v, dict):
            raise Malformed(at + ": not an object")
        missing = [k for k in t if k not in v]
        if missing:
            raise Malformed(at + ": missing " + ", ".join(missing))
        extra = sorted(k for k in v if k not in t)
        if extra and on("exact-members"):
            raise Malformed(at + ": a member added: " + ", ".join(extra))
        for k in t:
            check_type(v[k], t[k], at + "." + k)
        return
    if isinstance(t, tuple):
        if t[0] == "null|":
            if v is not None:
                check_type(v, t[1], at)
            return
        if t[0] == "array":
            if not isinstance(v, list):
                raise Malformed(at + ": not an array")
            for i, x in enumerate(v):
                check_type(x, t[1], "%s[%d]" % (at, i))
            return
        if t[0] == "enum":
            if on("member-types") and not (isinstance(v, str) and v in t[1:]):
                raise Malformed(at + ": not one of " + ", ".join(t[1:]))
            return
    if t == "datetime":
        if on("date-time") and not (isinstance(v, str) and is_datetime(v)):
            raise Malformed(at + ": not a date-time")
        return
    if not on("member-types"):
        return
    ok = {
        "int": lambda: type(v) is int,
        "string": lambda: isinstance(v, str),
        "str1024": lambda: isinstance(v, str) and len(v.encode("utf-8")) <= 1024,
        "term": lambda: isinstance(v, str) and _TERM.fullmatch(v) is not None,
        "hex32": lambda: isinstance(v, str) and _HEXRE[32].fullmatch(v) is not None,
        "hex64": lambda: isinstance(v, str) and _HEXRE[64].fullmatch(v) is not None,
    }[t]()
    if not ok:
        raise Malformed("%s: not a %s" % (at, t))


def read_record(data):
    """§7 step 1."""
    try:
        rec = read_json(data)
        check_type(rec, RECORD_T, "record")
    except Malformed as e:
        raise Refuse(MALFORMED, str(e))
    return rec


SIG_DOMAIN = b"wai:video-encode\x01"
DIGEST_DOMAIN = b"wai:video-encode-record\x01"


def signed_bytes(rec):
    return SIG_DOMAIN + jcs({k: v for k, v in rec.items() if k != "sig"}).encode("utf-8")


def record_digest(rec):
    """§6.2."""
    return sha256(DIGEST_DOMAIN + jcs(rec).encode("utf-8"))


def check_signature(rec):
    """§7 step 2, SPEC §7 *Signatures*."""
    if not on("signature"):
        return
    skip = () if on("small-order-key") else ("small-order-key",)
    r = ed_verify(bytes.fromhex(rec["signer_pubkey"]), signed_bytes(rec), bytes.fromhex(rec["sig"]), skip)
    if r == "bad_key":
        raise Refuse(BAD_KEY, "the signer's key fails SPEC §7 step 1 or 2")
    if r is not None:
        raise Refuse(BAD_SIG, "the signature fails SPEC §7")


# Unicode White_Space (PropList.txt).
WHITE_SPACE = frozenset(chr(c) for c in [0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x20, 0x85, 0xA0, 0x1680]
                        + list(range(0x2000, 0x200B)) + [0x2028, 0x2029, 0x202F, 0x205F, 0x3000])
APIS = ("platform", "va", "gpu-driver", "software")


def blank(s):
    return all(ch in WHITE_SPACE for ch in s)


def check_record_body(rec):
    """§7 steps 3 to 7."""
    cap = rec["capability"]
    if on("capability") and cap not in (AVC_CAP, HEVC_CAP):
        raise Refuse(CAPABILITY, cap)
    enc = rec["encoder"]
    if on("identity-blank"):
        for m in ("name", "version", "device", "driver"):
            if blank(enc[m]):
                raise Refuse(ID_INCOMPLETE, "encoder.%s is blank" % m)
    api = enc["api"]
    if on("identity-api") and api not in APIS:
        raise Refuse(ID_INCONSISTENT, "api %r is none of §3's" % api)
    if on("identity-library") and (enc["library_sha256"] is not None) != (api == "software"):
        raise Refuse(ID_INCONSISTENT, "library_sha256 present otherwise than exactly for software")
    if api == "software":
        if on("identity-software-hw") and enc["hardware"] != "no":
            raise Refuse(ID_INCONSISTENT, "a software encode stated hardware")
        if on("identity-software-avc") and cap != AVC_CAP:
            raise Refuse(ID_INCONSISTENT, "a software encode of " + cap)
    if on("identity-hw-engine") and api in ("va", "gpu-driver") and enc["hardware"] != "yes":
        raise Refuse(ID_INCONSISTENT, "a %s encode stated hardware no" % api)

    s, src = rec["settings"], rec["source"]
    if on("settings-size"):
        for name in ("width", "height"):
            if s[name] % 2 or not 16 <= s[name] <= 8192:
                raise Refuse(SETTINGS, "settings.%s %d is not even and from 16 to 8192" % (name, s[name]))
    if on("settings-source-size"):
        for name in ("width", "height"):
            if src[name] % 2 or not 2 <= src[name] <= 16384:
                raise Refuse(SETTINGS, "source.%s %d is not even and from 2 to 16384" % (name, src[name]))
    if on("settings-rate"):
        for name in ("fps_num", "fps_den"):
            if not 1 <= s[name] <= U32:
                raise Refuse(SETTINGS, "%s %d" % (name, s[name]))
    if on("settings-kbps") and not 1 <= s["kbps"] <= 1000000:
        raise Refuse(SETTINGS, "kbps %d" % s["kbps"])
    if on("settings-frames") and src["frames"] < 1:
        raise Refuse(SETTINGS, "no source frame")
    if on("settings-resample"):
        same = (s["width"], s["height"]) == (src["width"], src["height"])
        want = "none" if same else "wai.resample.two-tap"
        if s["resample"] != want:
            raise Refuse(SETTINGS, "resample %r where the rung's size %s the source's" %
                         (s["resample"], "is" if same else "is not"))

    plan, frames = rec["idr"], src["frames"]
    if on("plan-order") and (not plan or plan[0] != 0 or any(b <= a for a, b in zip(plan, plan[1:]))):
        raise Refuse(PLAN, "the plan does not start at 0 and increase")
    if on("plan-range") and plan and plan[-1] >= frames:
        raise Refuse(PLAN, "the plan's last entry is not below source.frames")
    pkg = rec["package"]
    if pkg is not None and on("plan-pieces"):
        pieces = pkg["pieces"]
        if [p["start"] for p in pieces] != plan:
            raise Refuse(PLAN, "the pieces' starts are not the plan")
        end = 0
        for p in pieces:
            if p["frames"] < 1 or p["start"] != end:
                raise Refuse(PLAN, "a piece of no frame, or not starting where the one before ended")
            end = p["start"] + p["frames"]
        if end != frames:
            raise Refuse(PLAN, "the pieces' frames do not sum to source.frames")

    if rec["energy"] is not None:
        check_energy(rec["energy"], enc["hardware"] == "yes")


CLASSES = ("CalibratedInstrument", "OnChipCounter", "HwShunt", "ModelBased", "Estimator")
FABRICS = ("cpu", "gpu", "media", "npu")


def check_energy(e, hardware):
    """§8.1: jwp-receipts §2.4.5 rules 1 to 3, then this extension's members, then coverage."""
    j = e["joules_micro"]
    if on("energy-nonzero") and j == 0:
        raise Refuse(ENERGY_INVALID, "a zero figure")
    cls = e["energy_provenance"]
    if on("energy-class"):
        if cls not in CLASSES:
            raise Refuse(ENERGY_INVALID, "no acquisition class: %r" % cls)
        if cls == "HwShunt":
            raise Refuse(ENERGY_INVALID, "the legacy HwShunt class")
    if on("energy-evidence"):
        u, filt, cal = e["energy_uncertainty"], e["energy_counter_filtering"], e["energy_calibration_ref"]
        if cls in ("CalibratedInstrument", "OnChipCounter"):
            if u is None:
                raise Refuse(ENERGY_INVALID, "a hardware class without its declared uncertainty")
            ppm, win = u["relative_ppm"], u["window_us"]
            if ppm == 0 or win == 0 or ppm > U32:
                raise Refuse(ENERGY_INVALID, "an undeclared uncertainty")
            if j * ppm < 288676:
                raise Refuse(ENERGY_INVALID, "below the resolution floor: %d x %d < 288676" % (j, ppm))
            if cls == "OnChipCounter":
                if filt not in ("On", "Off", "Unknown") or cal is not None:
                    raise Refuse(ENERGY_INVALID, "OnChipCounter's evidence")
            else:
                if cal is None or blank(cal) or filt is not None:
                    raise Refuse(ENERGY_INVALID, "CalibratedInstrument's evidence")
        elif u is not None or filt is not None or cal is not None:
            raise Refuse(ENERGY_INVALID, "evidence on a class that carries none: " + str(cls))
    if on("energy-coverage-value") and e["coverage"] not in ("Whole", "Partial"):
        raise Refuse(ENERGY_INVALID, "coverage %r" % e["coverage"])
    if on("energy-attribution") and e["attribution"] not in ("task", "rail", "bracket"):
        raise Refuse(ENERGY_INVALID, "attribution %r" % e["attribution"])
    if on("energy-fabrics"):
        f = e["fabrics"]
        if not f or any(x not in FABRICS for x in f) or any(FABRICS.index(a) >= FABRICS.index(b)
                                                              for a, b in zip(f, f[1:])):
            raise Refuse(ENERGY_INVALID, "fabrics %r" % (f,))
    if on("energy-engine") and hardware and not ({"gpu", "media"} & set(e["fabrics"])) \
            and e["coverage"] != "Partial":
        raise Refuse(ENERGY_COVERAGE, "a hardware encode's figure counting no engine is not Partial")


# ---------------------------------------------------------------- Annex B (§4.3)


class Unreadable(Exception):
    pass


def unescape(ebsp):
    """The RBSP of a NAL unit's payload: each emulation_prevention_three_byte removed."""
    out, zeros = bytearray(), 0
    for x in ebsp:
        if zeros >= 2 and x == 3:
            zeros = 0
            continue
        out.append(x)
        zeros = zeros + 1 if x == 0 else 0
    return bytes(out)


def escape(rbsp):
    out, zeros = bytearray(), 0
    for x in rbsp:
        if zeros >= 2 and x <= 3:
            out.append(3)
            zeros = 0
        out.append(x)
        zeros = zeros + 1 if x == 0 else 0
    return bytes(out)


class Bits:
    def __init__(self, data):
        self.b, self.pos, self.n, self.fields = data, 0, len(data) * 8, {}

    def u(self, k, name=None):
        if self.pos + k > self.n:
            raise Unreadable("ran out of bits")
        start, v = self.pos, 0
        for _ in range(k):
            v = (v << 1) | ((self.b[self.pos >> 3] >> (7 - (self.pos & 7))) & 1)
            self.pos += 1
        if name:
            self.fields[name] = (start, self.pos)
        return v

    def ue(self, name=None):
        start, z = self.pos, 0
        while self.u(1) == 0:
            z += 1
            if z > 31:
                raise Unreadable("an Exp-Golomb code longer than 32 bits")
        v = (1 << z) - 1 + self.u(z)
        if name:
            self.fields[name] = (start, self.pos)
        return v

    def se(self):
        k = self.ue()
        return (k + 1) // 2 if k & 1 else -(k // 2)


def need(cond, what):
    if not cond:
        raise Unreadable(what)


def split_annexb(b):
    """§4.3 item 1: the NAL units of a byte stream as (bytes, defect) pairs in order; a pair
    (None, defect) where the byte-stream syntax breaks, after which nothing is read."""
    n, i = len(b), 0
    while i < n and b[i] == 0:
        i += 1
    if i < 2 or i >= n or b[i] != 1:
        return [(None, "the stream does not begin with zero bytes and a start code prefix")]
    i += 1
    out = []
    while True:
        ends = [e for e in (b.find(b"\x00\x00\x00", i), b.find(b"\x00\x00\x01", i)) if e >= 0]
        j = min(ends) if ends else n
        out.append((b[i:j], None))
        if j == n:
            return out
        k = j
        while k < n and b[k] == 0:
            k += 1
        if k == n:
            return out
        if b[k] != 1:
            out.append((None, "bytes after a NAL unit's end that are not a start code"))
            return out
        i = k + 1


AVC_AU_START = {6, 7, 8, 9, 14, 15, 16, 17, 18}                              # ISO/IEC 14496-10 §7.4.1.2.3
HEVC_AU_START = {32, 33, 34, 35, 39} | set(range(41, 45)) | set(range(48, 56))  # 23008-2 §7.4.2.4.4
PS_TYPES = {AVC: (7, 8), HEVC: (32, 33, 34)}
SPS_TYPE = {AVC: 7, HEVC: 33}
PPS_TYPE = {AVC: 8, HEVC: 34}


class Nal:
    def __init__(self, raw, codec, defect):
        self.raw, self.defect = raw, defect
        self.type, self.layer, self.tid1, self.ref_idc = None, 0, 1, 0
        self.vcl = self.slice = False
        self.first, self.hdr = None, 1 if codec == AVC else 2
        if raw is None:
            return
        if not raw:
            self.defect = "an empty NAL unit"
            return
        if b"\x00\x00\x02" in raw:
            self.defect = "a NAL unit holding 00 00 02"
        if raw[0] & 0x80:
            self.defect = self.defect or "a NAL unit with its forbidden bit set"
        if codec == AVC:
            self.type, self.ref_idc = raw[0] & 31, (raw[0] >> 5) & 3
            self.vcl, self.slice = 1 <= self.type <= 5, self.type in (1, 2, 5)
        else:
            if len(raw) < 2:
                self.defect = "an HEVC NAL unit shorter than its header"
                return
            self.type = (raw[0] >> 1) & 63
            self.layer = ((raw[0] & 1) << 5) | (raw[1] >> 3)
            self.tid1 = raw[1] & 7
            if self.tid1 == 0:
                self.defect = self.defect or "nuh_temporal_id_plus1 0"
            self.vcl = self.slice = self.type < 32
        if self.slice:
            self.head = unescape(raw[self.hdr:self.hdr + 96])
            try:
                r = Bits(self.head)
                self.first = r.ue() == 0 if codec == AVC else r.u(1) == 1
            except Unreadable:
                self.defect = self.defect or "a slice header that does not read"

    def is_idr(self, codec):
        return self.type == 5 if codec == AVC else self.type in (19, 20)


def group_aus(nals, codec):
    """§4.3 item 2: access units. One begins at the first of the non-VCL types the standard
    lists after the previous picture's slices, or at a slice whose first_mb_in_slice is 0
    (AVC) or whose first_slice_segment_in_pic_flag is 1 (HEVC)."""
    starters = AVC_AU_START if codec == AVC else HEVC_AU_START
    aus, cur, cur_vcl = [], None, False
    for nal in nals:
        if nal.vcl and nal.first:
            starts = cur is None or cur_vcl
        elif nal.type is not None and not nal.vcl and nal.type in starters:
            starts = cur is None or cur_vcl
        else:
            starts = cur is None
        if starts:
            cur, cur_vcl = [], False
            aus.append(cur)
        cur.append(nal)
        cur_vcl = cur_vcl or nal.vcl
    return aus


def read_aus(b, codec):
    return group_aus([Nal(raw, codec, d) for raw, d in split_annexb(b)], codec)


AVC_HIGH = {100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135}


def avc_scaling_list(r, size):
    last = nxt = 8
    for _ in range(size):
        if nxt != 0:
            nxt = (last + r.se() + 256) % 256
        last = last if nxt == 0 else nxt


def avc_hrd(r):
    cpb = r.ue()
    need(cpb <= 31, "cpb_cnt_minus1")
    r.u(4)
    r.u(4)
    for _ in range(cpb + 1):
        r.ue()
        r.ue()
        r.u(1)
    r.u(20)


def avc_vui(r):
    v = {"full": None, "reorder": None}
    if r.u(1):
        if r.u(8) == 255:
            r.u(32)
    if r.u(1):
        r.u(1)
    if r.u(1):
        r.u(3)
        v["full"] = r.u(1, "video_full_range_flag")
        if r.u(1):
            r.u(24)
    if r.u(1):
        r.ue()
        r.ue()
    if r.u(1):
        r.u(32)
        r.u(32)
        r.u(1)
    nal = r.u(1)
    if nal:
        avc_hrd(r)
    vcl = r.u(1)
    if vcl:
        avc_hrd(r)
    if nal or vcl:
        r.u(1)
    r.u(1)
    if r.u(1, "bitstream_restriction_flag"):
        r.u(1)
        for _ in range(4):
            r.ue()
        v["reorder"] = r.ue("max_num_reorder_frames")
        r.ue()
    return v


def crop_units(chroma, sep):
    if chroma == 0 or sep:
        return 1, 1
    return (2, 2) if chroma == 1 else (2, 1) if chroma == 2 else (1, 1)


def avc_sps(raw):
    """ISO/IEC 14496-10 §7.3.2.1.1, read to its end (with the VUI, Annex E)."""
    r = Bits(unescape(raw[1:]))
    s = {"profile_idc": r.u(8, "profile_idc")}
    r.u(8)
    s["level_idc"] = r.u(8, "level_idc")
    s["tier"] = "main"
    s["id"] = r.ue("sps_id")
    need(s["id"] <= 31, "seq_parameter_set_id")
    s["chroma"], s["sep"], s["bdl"], s["bdc"] = 1, 0, 8, 8
    if s["profile_idc"] in AVC_HIGH:
        s["chroma"] = r.ue("chroma_format_idc")
        need(s["chroma"] <= 3, "chroma_format_idc")
        if s["chroma"] == 3:
            s["sep"] = r.u(1)
        s["bdl"], s["bdc"] = 8 + r.ue("bit_depth_luma_minus8"), 8 + r.ue("bit_depth_chroma_minus8")
        need(s["bdl"] <= 14 and s["bdc"] <= 14, "bit depth")
        r.u(1)
        if r.u(1):
            for i in range(8 if s["chroma"] != 3 else 12):
                if r.u(1):
                    avc_scaling_list(r, 16 if i < 6 else 64)
    s["log2_max_frame_num"] = r.ue() + 4
    need(s["log2_max_frame_num"] <= 16, "log2_max_frame_num_minus4")
    s["poc_type"] = r.ue("pic_order_cnt_type")
    need(s["poc_type"] <= 2, "pic_order_cnt_type")
    s["dpoaz"] = 0
    if s["poc_type"] == 0:
        s["log2_max_poc_lsb"] = r.ue("log2_max_pic_order_cnt_lsb_minus4") + 4
        need(s["log2_max_poc_lsb"] <= 16, "log2_max_pic_order_cnt_lsb_minus4")
    elif s["poc_type"] == 1:
        s["dpoaz"] = r.u(1)
        r.se()
        r.se()
        cyc = r.ue()
        need(cyc <= 255, "num_ref_frames_in_pic_order_cnt_cycle")
        for _ in range(cyc):
            r.se()
    r.ue()
    r.u(1)
    wmb, hmu = r.ue() + 1, r.ue() + 1
    s["frame_mbs_only"] = r.u(1, "frame_mbs_only_flag")
    if not s["frame_mbs_only"]:
        r.u(1)
    r.u(1)
    crop = [0, 0, 0, 0]
    if r.u(1, "frame_cropping_flag"):
        crop = [r.ue(), r.ue(), r.ue(), r.ue()]
    s["vui"] = avc_vui(r) if r.u(1) else None
    cx, cy = crop_units(s["chroma"], s["sep"])
    cy *= 2 - s["frame_mbs_only"]
    s["width"] = 16 * wmb - cx * (crop[0] + crop[1])
    s["height"] = 16 * (2 - s["frame_mbs_only"]) * hmu - cy * (crop[2] + crop[3])
    s["fields"] = r.fields
    return s


def avc_pps(raw):
    r = Bits(unescape(raw[1:]))
    p = {"id": r.ue(), "sps_id": r.ue()}
    need(p["id"] <= 255 and p["sps_id"] <= 31, "parameter set id")
    return p


def hevc_ptl(r, max_sub):
    r.u(2)
    tier = r.u(1, "general_tier_flag")
    profile = r.u(5, "general_profile_idc")
    r.u(32)
    r.u(4)
    r.u(43)
    r.u(1)
    level = r.u(8, "general_level_idc")
    flags = [(r.u(1), r.u(1)) for _ in range(max_sub)]
    if max_sub > 0:
        for _ in range(max_sub, 8):
            r.u(2)
    for p, lv in flags:
        if p:
            r.u(88)
        if lv:
            r.u(8)
    return profile, "high" if tier else "main", level


def hevc_sps(raw):
    """ISO/IEC 23008-2 §7.3.2.2.1, read as §4.3 item 1 reads it: through the sub-layer ordering
    information (sps_max_latency_increase_plus1). Nothing after it is read, so no defect after
    it is one §4.3 sees."""
    r = Bits(unescape(raw[2:]))
    s = {}
    r.u(4)
    s["max_sub"] = r.u(3)
    need(s["max_sub"] <= 6, "sps_max_sub_layers_minus1")
    r.u(1)
    s["profile_idc"], s["tier"], s["level_idc"] = hevc_ptl(r, s["max_sub"])
    s["id"] = r.ue()
    need(s["id"] <= 15, "sps_seq_parameter_set_id")
    s["chroma"] = r.ue("chroma_format_idc")
    need(s["chroma"] <= 3, "chroma_format_idc")
    s["sep"] = r.u(1) if s["chroma"] == 3 else 0
    w, h = r.ue(), r.ue()
    win = [0, 0, 0, 0]
    if r.u(1, "conformance_window_flag"):
        win = [r.ue(), r.ue(), r.ue(), r.ue()]
    s["bdl"], s["bdc"] = 8 + r.ue("bit_depth_luma_minus8"), 8 + r.ue("bit_depth_chroma_minus8")
    need(s["bdl"] <= 16 and s["bdc"] <= 16, "bit depth")
    need(r.ue() <= 12, "log2_max_pic_order_cnt_lsb_minus4")
    info = r.u(1)
    for _ in range(0 if info else s["max_sub"], s["max_sub"] + 1):
        r.ue()
        s["reorder"] = r.ue("sps_max_num_reorder_pics")
        r.ue()
    r.fields["end"] = (r.pos, r.pos)
    sw, sh = crop_units(s["chroma"], s["sep"])
    s["width"], s["height"] = w - sw * (win[0] + win[1]), h - sh * (win[2] + win[3])
    s["fields"] = r.fields
    return s


def hevc_pps(raw):
    r = Bits(unescape(raw[2:]))
    p = {"id": r.ue(), "sps_id": r.ue()}
    need(p["id"] <= 63 and p["sps_id"] <= 15, "parameter set id")
    return p


def slice_head(nal, codec, sps_tab, pps_tab, cur):
    """The start of the slice header §4.3 reads. HEVC: its first bit, read with the NAL unit;
    the SPS is the one in force. AVC: first_mb_in_slice, then slice_type and
    pic_parameter_set_id to find the SPS, and for pic_order_cnt_type 0 the fields up to
    pic_order_cnt_lsb (14496-10 §7.3.3), no further. None where an AVC slice's parameter sets
    were never sent."""
    if codec == HEVC:
        return {"sps": cur, "nal": nal}
    r = Bits(nal.head)
    r.ue()
    need(r.ue() <= 9, "slice_type")
    pid = r.ue()
    need(pid <= 255, "pic_parameter_set_id")
    pps = pps_tab.get(pid)
    sps = sps_tab.get(pps["sps_id"]) if pps else None
    if sps is None:
        return None
    h = {"sps": sps, "nal": nal, "lsb": None}
    if sps["poc_type"] == 0:
        if sps["sep"]:
            r.u(2)
        r.u(sps["log2_max_frame_num"])
        if not sps["frame_mbs_only"] and r.u(1):
            r.u(1)
        if nal.type == 5:
            r.ue()
        h["lsb"] = r.u(sps["log2_max_poc_lsb"])
    return h


def check_stream(b, codec, frames, plan, width, height, full):
    """§4.3: item 1 and item 2's count over the whole stream; then items 3 to 6 access unit by
    access unit, each access unit's in that order; then item 7. The first that fails is the
    reason. Returns the access units and the stream's sequence parameter sets."""
    aus = read_aus(b, codec)
    sps_t, pps_t = SPS_TYPE[codec], PPS_TYPE[codec]
    sps_tab, pps_tab, cur = {}, {}, None
    au_heads, au_sps, all_sps = [], [], []

    def fail(code, i, what):
        raise Refuse(code, ("access unit %d: " % i if i is not None else "") + what)

    # item 1, over the whole stream: the byte stream; every SPS and slice header start reads
    for i, au in enumerate(aus):
        heads, sps_here = [], []
        for nal in au:
            if nal.defect:
                if on("annexb"):
                    fail(STREAM_MALFORMED, i, nal.defect)
                if nal.type is None:
                    continue
            if nal.layer != 0:
                continue
            if nal.type == pps_t:
                try:
                    p = avc_pps(nal.raw) if codec == AVC else hevc_pps(nal.raw)
                    pps_tab[p["id"]] = p
                except Unreadable:
                    pass  # §4.3 reads no PPS; a slice naming this one finds none (item 4)
                continue
            try:
                if nal.type == sps_t:
                    s = avc_sps(nal.raw) if codec == AVC else hevc_sps(nal.raw)
                    sps_tab[s["id"]] = cur = s
                    sps_here.append(s)
                    all_sps.append((nal.raw, s))
                elif nal.slice and nal.first is not None:
                    heads.append(slice_head(nal, codec, sps_tab, pps_tab, cur))
            except Unreadable as e:
                if on("annexb"):
                    fail(STREAM_MALFORMED, i, "a %s that does not read (%s)" % (
                        "slice header's start" if nal.slice else "sequence parameter set", e))
        au_heads.append(heads)
        au_sps.append(sps_here)
    if on("au-trailing"):
        for i, au in enumerate(aus):
            if not any(n.vcl for n in au):
                fail(STREAM_MALFORMED, i, "NAL units after the last picture begin an access unit with no slice")
    # item 2's count, over the whole stream
    if on("au-count") and len(aus) != frames:
        raise Refuse(FRAME_COUNT, "%d access units for the source's %d frames" % (len(aus), frames))

    # items 3 to 6, access unit by access unit
    planset = set(plan)
    first_ps, first_sps_raw, sps_seen = None, all_sps[0][0] if all_sps else None, False
    prev_msb = prev_lsb = 0
    last_poc = None
    for i, au in enumerate(aus):
        heads, vcl = au_heads[i], [n for n in au if n.vcl]
        # item 3
        idr = [n.is_idr(codec) for n in vcl]
        if on("idr-mix") and any(idr) and not all(idr):
            fail(STREAM_MALFORMED, i, "an access unit mixing IDR and other slices")
        au_idr = bool(vcl) and all(idr)
        if on("keyframes"):
            if codec == HEVC:
                other = [n.type for n in vcl if 16 <= n.type <= 23 and n.type not in (19, 20)]
                if other:
                    fail(KEYFRAME, i, "a random access point other than an IDR (nal_unit_type %d)" % other[0])
            if au_idr and i not in planset:
                fail(KEYFRAME, i, "an IDR access unit off the plan")
            if not au_idr and i in planset:
                fail(KEYFRAME, i, "a planned picture that is not an IDR access unit")
        # item 4
        sps_seen = sps_seen or bool(au_sps[i])
        if on("parameter-sets"):
            ps = [n.raw for n in au if n.type in PS_TYPES[codec]]
            if au_idr:
                if first_ps is None:
                    if not set(PS_TYPES[codec]) <= {n.type for n in au}:
                        fail(PARAMS, i, "the first IDR access unit does not carry its parameter sets")
                    first_ps = ps
                elif ps != first_ps:
                    fail(PARAMS, i, "an IDR access unit's parameter sets are not the first IDR's")
            elif any(first_ps is None or x not in first_ps for x in ps):
                fail(PARAMS, i, "parameter set bytes the first IDR access unit does not carry")
        if on("sps-single") and any(n.raw != first_sps_raw for n in au if n.type == sps_t):
            fail(PARAMS, i, "a sequence parameter set other than the stream's first")
        if on("parameter-sets"):
            if not sps_seen:
                fail(PARAMS, i, "an access unit before the first sequence parameter set")
            if any(h is None for h in heads):
                fail(PARAMS, i, "a slice whose parameter sets were never sent")
        actives = au_sps[i] + [h["sps"] for h in heads if h and h["sps"]]
        # item 5
        if on("unsupported"):
            for n in au:
                if codec == HEVC and n.layer != 0:
                    fail(UNSUPPORTED, i, "nuh_layer_id %d" % n.layer)
                if codec == AVC and n.type in (14, 15, 20, 21):
                    fail(UNSUPPORTED, i, "a NAL unit of another layer (nal_unit_type %d)" % n.type)
            for s in actives:
                if s["chroma"] != 1 or s["sep"] or s["bdl"] != 8 or s["bdc"] != 8:
                    fail(UNSUPPORTED, i, "not 8-bit 4:2:0 in one colour plane")
                if codec == AVC and not s["frame_mbs_only"]:
                    fail(UNSUPPORTED, i, "frame_mbs_only_flag 0")
                if codec == AVC and s["poc_type"] not in (0, 2):
                    fail(UNSUPPORTED, i, "pic_order_cnt_type %d" % s["poc_type"])
        # item 6
        if on("reorder-field"):
            for s in actives:
                if codec == HEVC and s["reorder"] != 0:
                    fail(REORDERED, i, "sps_max_num_reorder_pics %d" % s["reorder"])
                if codec == AVC and s["vui"] and s["vui"]["reorder"] not in (None, 0):
                    fail(REORDERED, i, "max_num_reorder_frames %d" % s["vui"]["reorder"])
        h0 = heads[0] if heads and heads[0] else None
        if codec == AVC and h0 and h0["sps"]["poc_type"] == 0:
            # the top field order count, PicOrderCntMsb + pic_order_cnt_lsb (§8.2.1.1)
            if au_idr:
                prev_msb = prev_lsb = 0
            lsb, half = h0["lsb"], 1 << (h0["sps"]["log2_max_poc_lsb"] - 1)
            if lsb < prev_lsb and prev_lsb - lsb >= half:
                msb = prev_msb + 2 * half
            elif lsb > prev_lsb and lsb - prev_lsb > half:
                msb = prev_msb - 2 * half
            else:
                msb = prev_msb
            poc = msb + lsb
            if on("reorder-poc") and not au_idr and last_poc is not None and poc <= last_poc:
                fail(REORDERED, i, "top field order count %d after %d" % (poc, last_poc))
            last_poc = poc
            if h0["nal"].ref_idc:
                prev_msb, prev_lsb = msb, lsb
    # item 7
    for _, s in all_sps:
        if on("sps-size") and (s["width"], s["height"]) != (width, height):
            fail(SPS_MISMATCH, None, "the SPS's cropped picture is %dx%d, not %dx%d" %
                 (s["width"], s["height"], width, height))
        if on("sps-range") and codec == AVC and s["vui"] and s["vui"]["full"] is not None \
                and s["vui"]["full"] != full:
            fail(SPS_MISMATCH, None, "video_full_range_flag %d" % s["vui"]["full"])
    return aus, [s for _, s in all_sps]


def check_stream_held(rec, b):
    """§7 step 8."""
    st = rec["stream"]
    if on("stream-digest") and (sha256(b).hex() != st["sha256"] or len(b) != st["bytes"]):
        raise Refuse(STREAM_MISMATCH, "the stream's SHA-256 or length is not the record's")
    codec = AVC if rec["capability"] == AVC_CAP else HEVC
    s = rec["settings"]
    aus, sps_list = check_stream(b, codec, rec["source"]["frames"], rec["idr"], s["width"], s["height"],
                                  1 if s["range"] == "full" else 0)
    if on("au-count") and len(aus) != st["units"]:
        raise Refuse(FRAME_COUNT, "%d access units, stream.units %d" % (len(aus), st["units"]))
    if on("sps-profile-level-tier"):
        for sps in sps_list:
            got = (sps["profile_idc"], sps["level_idc"], sps["tier"])
            if got != (st["profile_idc"], st["level_idc"], st["tier"]):
                raise Refuse(SPS_MISMATCH, "the SPS's profile, level, tier %r are not the record's" % (got,))
    return aus


# ---------------------------------------------------------------- the source (§4.1)


def read_y4m(b):
    """§4.1: a source's width, height, frame count, frame rate, range and digest, or
    Unreadable where it is not such a source."""
    nl = b.find(b"\n")
    need(nl > 0, "no header line")
    head = b[:nl].split(b" ")
    need(head[0] == b"YUV4MPEG2", "not YUV4MPEG2")
    tags = {}
    for tok in head[1:]:
        need(len(tok) >= 1, "an empty header token")
        t = tok[:1].decode("latin-1")
        need(t not in tags or t in "X", "a tag twice")
        tags.setdefault(t, []).append(tok[1:])

    def dec(v):
        need(re.fullmatch(rb"[0-9]{1,9}", v) is not None, "not a decimal")
        return int(v)
    need("W" in tags and "H" in tags and "F" in tags, "no W, H or F")
    w, h = dec(tags["W"][0]), dec(tags["H"][0])
    need(w % 2 == 0 and h % 2 == 0 and 0 < w <= 16384 and 0 < h <= 16384, "a side odd, zero or past 16384")
    f = tags["F"][0].split(b":")
    need(len(f) == 2, "a frame rate")
    num, den = dec(f[0]), dec(f[1])
    need(num > 0 and den > 0, "a frame rate with a zero")
    need(tags.get("I", [b"p"])[0] in (b"p", b"?"), "not progressive")
    need(tags.get("C", [b"420"])[0] in (b"420", b"420jpeg", b"420paldv", b"420mpeg2"), "not 8-bit 4:2:0")
    size = w * h + 2 * (w // 2) * (h // 2)
    d, pos, frames = hashlib.sha256(), nl + 1, 0
    while pos < len(b):
        need(b.startswith(b"FRAME", pos), "a frame without its header")
        e = b.find(b"\n", pos)
        need(e >= 0 and (e == pos + 5 or b[pos + 5:pos + 6] == b" "), "a frame header")
        need(e + 1 + size <= len(b), "a truncated frame")
        d.update(b[e + 1:e + 1 + size])
        pos, frames = e + 1 + size, frames + 1
    need(frames >= 1, "no frame")
    full = any(v == b"COLORRANGE=FULL" for v in tags.get("X", []))
    return {"width": w, "height": h, "frames": frames, "fps": (num, den), "full": full, "digest": d.digest()}


def check_source(rec, b):
    """§7 step 9: the source reads as §4.1; its digest; its width, height and frame count the
    record's `source`; its frame rate and range the settings'."""
    try:
        y = read_y4m(b)
    except Unreadable as e:
        raise Refuse(SOURCE_MISMATCH, "not a §4.1 source: %s" % e)
    src, s = rec["source"], rec["settings"]
    if on("source-digest") and y["digest"].hex() != src["sha256"]:
        raise Refuse(SOURCE_MISMATCH, "the source's digest is not source.sha256")
    if on("source-geometry") and (y["width"], y["height"], y["frames"]) != (src["width"], src["height"], src["frames"]):
        raise Refuse(SOURCE_MISMATCH, "the source is %dx%d, %d frames; the record says %dx%d, %d" % (
            y["width"], y["height"], y["frames"], src["width"], src["height"], src["frames"]))
    if on("source-rate-range"):
        num, den = y["fps"]
        if (num, den) != (s["fps_num"], s["fps_den"]):  # as the header writes them, not as a ratio
            raise Refuse(SOURCE_MISMATCH, "the source's rate %d:%d is not the settings'" % (num, den))
        if y["full"] != (s["range"] == "full"):
            raise Refuse(SOURCE_MISMATCH, "the source's range is not the settings'")


# ---------------------------------------------------------------- the rendition (§9)

ROLES, TIERS, MEDIA = 7, 3, 7


def pad8(n):
    return (n + 7) // 8 * 8


def read_wsix(b):
    """packaging §6.2: header, pieces, entries and strings, or Unreadable (the reader's reason)."""
    def le(o, n):
        return int.from_bytes(b[o:o + n], "little")
    need(b[:4] == b"WSIX", "index-magic")
    need(len(b) >= 6, "index-truncated")
    need(le(4, 2) == 1, "index-version")
    need(len(b) >= 160, "index-truncated")
    n_p, n_e, n_s, n_c = le(16, 4), le(20, 4), le(24, 4), le(28, 4)
    at_e = 160 + 32 * n_p
    at_c = at_e + 128 * n_e
    at_plan = at_c + 16 * n_c
    at_s = at_plan + pad8(n_c * n_p)
    need(at_s <= len(b), "index-truncated")
    raw, pos = [], at_s
    for _ in range(n_s):
        need(pos + 2 <= len(b) and pos + 2 + le(pos, 2) <= len(b), "index-truncated")
        raw.append(b[pos + 2:pos + 2 + le(pos, 2)])
        pos += 2 + le(pos, 2)
    end = at_s + pad8(pos - at_s)
    need(end <= len(b), "index-truncated")
    P = [b[160 + 32 * k:192 + 32 * k] for k in range(n_p)]
    E = [b[at_e + 128 * k:at_e + 128 * (k + 1)] for k in range(n_e)]
    C = [b[at_c + 16 * k:at_c + 16 * (k + 1)] for k in range(n_c)]
    zeros = [b[146:160], b[at_plan + n_c * n_p:at_s], b[pos:end]] + [p[28:32] for p in P] + \
        [e[58:60] + e[108:112] + e[120:128] for e in E] + [c[8:16] for c in C]
    need(not any(any(z) for z in zeros), "index-reserved")
    need(le(6, 2) < MEDIA, "index-media-class")
    for e in E:
        need(e[54] < ROLES, "index-role")
        need(e[56] < TIERS, "index-tier")
    strings = []
    for s in raw:
        try:
            strings.append(s.decode("utf-8"))
        except UnicodeDecodeError:
            raise Unreadable("index-string")
    need(all(strings), "index-string")
    idx = [le(144, 2)] + [int.from_bytes(e[52:54], "little") for e in E] + [int.from_bytes(c[0:2], "little") for c in C]
    need(all(k < n_s for k in idx), "index-string")
    need(end == len(b), "index-trailing")
    return {
        "tb": (le(8, 4), le(12, 4)), "first_piece": le(96, 8), "component": strings[le(144, 2)],
        "pieces": [(int.from_bytes(p[0:8], "little"), int.from_bytes(p[8:12], "little")) for p in P],
        "entries": [{"hash": e[0:32], "piece": int.from_bytes(e[40:48], "little"),
                     "cap": strings[int.from_bytes(e[52:54], "little")], "role": e[54], "layer": e[55]} for e in E],
    }


def content_hash(b):
    """jwp-receipts §2.1."""
    return blake3(b"moq-jwp:object\x01" + b)


def settings_digest(rec):
    """§9.1: BLAKE3("wai:render-settings\\x01" || JCS(S))."""
    enc = rec["encoder"]
    S = {"source": "encoded", "api": enc["api"], "hardware": enc["hardware"], "device": enc["device"],
         "driver": enc["driver"], "library_sha256": enc["library_sha256"], "stream_sha256": rec["stream"]["sha256"]}
    return blake3(b"wai:render-settings\x01" + jcs(S).encode("utf-8"))


def rendition_manifest(rec, piece, motif_cap):
    """§9.1: the manifest's exact bytes for this record and piece."""
    cap, enc, s = rec["capability"], rec["encoder"], rec["settings"]
    text = ('{"wai":"1.0","media":"video","intent":"replicate","model_requirement":{"capability":%s},'
            '"conditioning":{"kind":%s},"derived_from":{"motif":%s,"motif_capability":%s,"renderer":%s,'
            '"settings":%s},"target":{"frames":%d,"h":%d,"w":%d}}') % (
        jcs_str(cap), jcs_str("avc" if cap == AVC_CAP else "hevc"), jcs_str(piece["motif"]), jcs_str(motif_cap),
        jcs_str(enc["name"] + "@" + enc["version"]), jcs_str(settings_digest(rec).hex()),
        piece["frames"], s["height"], s["width"])
    return text.encode("utf-8")


def rendition_payload(aus, piece):
    """§9.1: the piece's access units' NAL units, each after 00 00 00 01."""
    return b"".join(b"\x00\x00\x00\x01" + n.raw for au in aus[piece["start"]:piece["start"] + piece["frames"]]
                    for n in au if n.raw)


def read_envelope(b):
    """SPEC §2: (manifest, payload) of one WAI1 envelope and nothing else, or None."""
    if len(b) < 12 or b[:4] != b"WAI1":
        return None
    ml = int.from_bytes(b[4:8], "little")
    if 8 + ml + 4 > len(b):
        return None
    pl = int.from_bytes(b[8 + ml:12 + ml], "little")
    if 12 + ml + pl != len(b):
        return None
    return b[8:8 + ml], b[12 + ml:]


def check_package(rec, aus, pkg):
    """§9.3, every failure encode-package-mismatch."""
    p = rec["package"]

    def fail(what):
        raise Refuse(PACKAGE_MISMATCH, what)
    if on("receipt") and sha256(pkg["receipt"]).hex() != p["receipt_sha256"]:
        fail("the receipt file's SHA-256 is not receipt_sha256")
    try:
        ix = read_wsix(pkg["index"])
    except Unreadable as e:
        fail("the stage index does not read (%s)" % e)
    if ix["component"] != pkg["component"]:
        fail("the stage index is of component %r" % ix["component"])
    pieces = p["pieces"]
    if on("index-pieces"):
        if len(ix["pieces"]) != len(pieces):
            fail("the stage index has %d pieces, the record %d" % (len(ix["pieces"]), len(pieces)))
        # A video component's timebase unit is one frame (packaging §2): compared directly.
        for k, ((start, dur), pc) in enumerate(zip(ix["pieces"], pieces)):
            if (start, dur) != (pc["start"], pc["frames"]):
                fail("piece %d: the stage index's start or duration is not the record's" % k)
    caps = []
    for k, pc in enumerate(pieces):
        base = [e for e in ix["entries"] if e["piece"] == ix["first_piece"] + k and e["role"] == 0 and e["layer"] == 0]
        if on("index-motif") and (len(base) != 1 or base[0]["hash"].hex() != pc["motif"]):
            fail("piece %d: its base entry is not the record's motif" % k)
        caps.append(base[0]["cap"] if base else "")
    for k, pc in enumerate(pieces):
        path = os.path.join(pkg["objects"], pc["object"] + ".wai")
        if not os.path.isfile(path):
            if on("object-present"):
                fail("piece %d: its object is not present" % k)
            continue
        with open(path, "rb") as f:
            data = f.read()
        if on("object-hash") and content_hash(data).hex() != pc["object"]:
            fail("piece %d: its object's content hash is not the record's" % k)
        env = read_envelope(data)
        if env is None:
            if on("object-manifest") or on("object-payload"):
                fail("piece %d: its object is not one WAI1 envelope" % k)
            continue
        if on("object-manifest") and env[0] != rendition_manifest(rec, pc, caps[k]):
            fail("piece %d: its manifest is not §9.1's" % k)
        if on("object-payload") and env[1] != rendition_payload(aus, pc):
            fail("piece %d: its payload is not the stream's access units re-framed" % k)


# ---------------------------------------------------------------- §7, in order


def verify(rec_bytes, stream=None, source=None, pkg=None):
    """§7: (outcome, steps made, detail, record). A refusal's detail starts with its step."""
    made, rec, step = [], None, 1
    try:
        rec = read_record(rec_bytes)
        made.append(1)
        step = 2
        check_signature(rec)
        made.append(2)
        step = 3
        check_record_body(rec)  # steps 3 to 7; the reason names which
        made += [3, 4, 5, 6, 7]
        aus = None
        if stream is not None:
            step = 8
            aus = check_stream_held(rec, stream)
            made.append(8)
        if source is not None:
            step = 9
            check_source(rec, source)
            made.append(9)
        if pkg is not None and rec["package"] is not None and aus is not None:
            step = 10
            check_package(rec, aus, pkg)
            made.append(10)
        return ACCEPT, made, "", rec
    except Refuse as e:
        return e.code, made, "step %s: %s" % (STEP_OF.get(e.code, step), e.detail), rec
    except Exception as e:  # only a weakened verifier gets here
        return "crash:" + type(e).__name__, made, str(e), rec


STEP_OF = {MALFORMED: 1, BAD_KEY: 2, BAD_SIG: 2, CAPABILITY: 3, ID_INCOMPLETE: 4, ID_INCONSISTENT: 4,
           SETTINGS: 5, PLAN: 6, ENERGY_INVALID: 7, ENERGY_COVERAGE: 7, SOURCE_MISMATCH: 9, PACKAGE_MISMATCH: 10}


def steps_text(made):
    out, run = [], []
    for s in made + [None]:
        if run and s == run[-1] + 1:
            run.append(s)
            continue
        if run:
            out.append(str(run[0]) if len(run) == 1 else "%d-%d" % (run[0], run[-1]))
        run = [s] if s is not None else []
    return ",".join(out)


# ---------------------------------------------------------------- the corpus


def readb(path):
    with open(path, "rb") as f:
        return f.read()


class Corpus:
    def __init__(self, root):
        self.root = os.path.abspath(root)
        with open(os.path.join(self.root, "cases.json")) as f:
            self.cases = json.load(f)["cases"]
        self.rdir = os.path.join(self.root, "rendition")
        with open(os.path.join(self.rdir, "cases.json")) as f:
            r = json.load(f)
        self.rcases, self.component = r["cases"], r["component"]
        self.package = os.path.normpath(os.path.join(self.rdir, r["package"]))

    def run(self, case):
        def opt(k):
            return readb(os.path.join(self.root, case[k])) if case.get(k) else None
        return verify(readb(os.path.join(self.root, case["record"])), opt("stream"), opt("source"))

    def record_path(self, rcase):
        return os.path.join(self.rdir, rcase["dir"], rcase["rung"] + ".encode.json")

    def run_rendition(self, rcase):
        pkg = {"receipt": readb(os.path.join(self.package, "package.receipt.json")),
               "index": readb(os.path.join(self.package, "index-%s.wsix" % self.component)),
               "objects": os.path.join(self.rdir, rcase["dir"], "objects"), "component": self.component}
        out = verify(readb(self.record_path(rcase)), readb(os.path.join(self.rdir, rcase["stream"])), None, pkg)
        rec = out[3]
        if out[0] == ACCEPT and (rec["package"]["rung"] != rcase["rung"]
                                 or not rcase["stream"].endswith(rcase["rung"] + (".h264" if rec["capability"] == AVC_CAP else ".h265"))):
            return "layout: rung id or file names disagree", out[1], "", rec
        return out

    def everything(self):
        """(name, part, outcome, expect, made, detail) for every case."""
        rows = []
        for c in self.cases:
            o, made, d, _ = self.run(c)
            rows.append((c["name"], 1, o, c["expect"], made, d))
        for c in self.rcases:
            o, made, d, _ = self.run_rendition(c)
            rows.append((c["name"], 2, o, c["expect"], made, d))
        return rows


def pins(corpus):
    """§10.1: each fixture stream is its manifest's SHA-256."""
    bad = []
    for d in (corpus.root, corpus.rdir):
        with open(os.path.join(d, "streams.json")) as f:
            for name, digest in json.load(f).items():
                if sha256(readb(os.path.join(d, "streams", name))).hex() != digest:
                    bad.append(os.path.relpath(os.path.join(d, "streams", name), corpus.root))
    return bad


def spec_reasons():
    """§12's table: (record reasons, producer reasons), read from the extension's text."""
    with open(SPEC_PATH, encoding="utf-8") as f:
        text = f.read()
    sec = text.split("\n## 12. Reasons", 1)[1].split("\n## ", 1)[0]
    rec, prod = [], []
    for line in sec.splitlines():
        cells = line.split("|")
        if len(cells) < 4 or not cells[1].strip().startswith("`"):
            continue
        codes = re.findall(r"`([^`]+)`", cells[1])
        (prod if "producer" in cells[2] else rec).extend(codes)
    return rec, prod


# ---------------------------------------------------------------- self-test: re-signing under a fresh key
#
# A record names its own signer (§6.2, §13: a relying party trusts the signer or not), so a
# copy re-signed under another key is still a record, and its outcome must not change unless
# a member did. The self-test signs with a key of its own; the controls below show that
# re-signing alone changes no outcome.

SELF_SEED = hashlib.sha256(b"encode_verify.py self-test signer").digest()


def _mul(k, p):
    return ED._double_mul(k, p, 0, ED.IDENTITY)


def resign(rec):
    rec = dict(rec)
    h = hashlib.sha512(SELF_SEED).digest()
    a = (int.from_bytes(h[:32], "little") & ((1 << 254) - 8)) | (1 << 254)
    A = ED.encode(_mul(a, ED.B))
    rec["signer_pubkey"] = A.hex()
    msg = signed_bytes(rec)
    r = int.from_bytes(hashlib.sha512(h[32:] + msg).digest(), "little") % ED.L
    R = ED.encode(_mul(r, ED.B))
    k = int.from_bytes(hashlib.sha512(R + A + msg).digest(), "little") % ED.L
    rec["sig"] = (R + ((r + k * a) % ED.L).to_bytes(32, "little")).hex()
    return rec


def write_record(path, rec):
    with open(path, "wb") as f:
        f.write(jcs(rec).encode("utf-8"))


def ue_bits(v):
    x = v + 1
    return "0" * (x.bit_length() - 1) + format(x, "b")


def rewrite_bits(nal, hdr, start, end, bits):
    """A NAL unit whose RBSP has bits [start, end) replaced, re-terminated and re-escaped."""
    rb = unescape(nal[hdr:])
    s = "".join(format(x, "08b") for x in rb)
    s = s[:s.rfind("1")]
    s = s[:start] + bits + s[end:] + "1"
    s += "0" * (-len(s) % 8)
    return nal[:hdr] + escape(int(s, 2).to_bytes(len(s) // 8, "big"))


def truncate(path):
    b = readb(path)
    with open(path, "wb") as f:
        f.write(b[:-1])


def was(v, want, bits):
    if v != want:
        raise AssertionError("the field is written %s, not %s" % (v, want))
    return bits


def frame(nals):
    return b"".join(b"\x00\x00\x00\x01" + n for n in nals)


def au_lists(b, codec):
    return [[n.raw for n in au] for au in read_aus(b, codec)]


def flat(aus):
    return [n for au in aus for n in au]


# ---------------------------------------------------------------- self-test: the mutations
#
# Each mutation: (name, part, target case, description, apply(ctx), expected outcome).
# apply edits a fresh copy of the corpus in place; ctx gives its paths.


class Ctx:
    def __init__(self, corpus, case):
        self.c, self.case = corpus, case

    def path(self, rel):
        return os.path.join(self.c.root, rel)

    def rpath(self, rel):
        return os.path.join(self.c.rdir, rel)

    def edit_text(self, rel, old, new, count=1):
        p = self.path(rel)
        t = readb(p)
        if old not in t:
            raise AssertionError("mutation pattern %r not in %s" % (old, rel))
        with open(p, "wb") as f:
            f.write(t.replace(old, new, count))

    def flip(self, path, at):
        b = bytearray(readb(path))
        b[at if at >= 0 else len(b) + at] ^= 0x01
        with open(path, "wb") as f:
            f.write(bytes(b))

    def record(self):
        return read_json(readb(self.path(self.case["record"])))

    def put_record(self, rec):
        write_record(self.path(self.case["record"]), resign(rec))

    def rebind(self, new_stream, **members):
        """Write the stream, and the record re-bound to it (and to `members`), re-signed."""
        sp = self.path(self.case["stream"])
        with open(sp, "wb") as f:
            f.write(new_stream)
        rec = self.record()
        rec["stream"] = dict(rec["stream"], sha256=sha256(new_stream).hex(), bytes=len(new_stream))
        for k, v in members.items():
            rec[k] = v
        self.put_record(rec)

    def stream_aus(self):
        codec = AVC if self.case["stream"].endswith(".h264") else HEVC
        return au_lists(readb(self.path(self.case["stream"])), codec), codec


def _hex_flip(t, key):
    at = t.index(key) + len(key) + 10
    c = t[at:at + 1]
    return t[:at] + (b"0" if c != b"0" else b"1") + t[at + 1:]


def m_text(fn):
    def apply(x):
        p = x.path(x.case["record"])
        t = fn(readb(p))
        with open(p, "wb") as f:
            f.write(t)
    return apply


def m_stream_edit(fn, **members):
    """Edit the access units of the target's stream, then re-bind the record to it."""
    def apply(x):
        aus, codec = x.stream_aus()
        x.rebind(frame(flat(fn(aus, codec))), **members)
    return apply


def m_record(fn):
    def apply(x):
        rec = x.record()
        fn(rec)
        x.put_record(rec)
    return apply


def m_rrecord(fn):
    def apply(x):
        p = x.c.record_path(x.case)
        rec = read_json(readb(p))
        fn(x, rec)
        write_record(p, resign(rec))
    return apply


def nal_type(n, codec):
    return n[0] & 31 if codec == AVC else (n[0] >> 1) & 63


def drop_type(au_index, t):
    def fn(aus, codec):
        aus[au_index] = [n for n in aus[au_index] if nal_type(n, codec) != t]
        return aus
    return fn


def retype(au_index, t):
    def fn(aus, codec):
        out = []
        for n in aus[au_index]:
            if codec == AVC and n[0] & 31 in (1, 5):
                n = bytes([(n[0] & 0xE0) | t]) + n[1:]
            elif codec == HEVC and (n[0] >> 1) & 63 < 32:
                n = bytes([(n[0] & 0x81) | (t << 1)]) + n[1:]
            out.append(n)
        aus[au_index] = out
        return aus
    return fn


def swap(a, b):
    def fn(aus, codec):
        aus[a], aus[b] = aus[b], aus[a]
        return aus
    return fn


def slice_byte(au_index, fn_byte):
    def fn(aus, codec):
        out = []
        for n in aus[au_index]:
            if (codec == AVC and n[0] & 31 in (1, 5)) or (codec == HEVC and (n[0] >> 1) & 63 < 32):
                n = fn_byte(n)
            out.append(n)
        aus[au_index] = out
        return aus
    return fn


def sps_field(field, bits_of, only_au=None, through=None):
    """Rewrite one field of the SPS, or the fields from `field` to `through`, in every copy
    (or in the one of access unit `only_au`); bits_of gets the first field's bits as written."""
    def fn(aus, codec):
        hdr, st = (1, 7) if codec == AVC else (2, 33)
        for k, au in enumerate(aus):
            if only_au is not None and k != only_au:
                continue
            for j, n in enumerate(au):
                if nal_type(n, codec) == st:
                    s = avc_sps(n) if codec == AVC else hevc_sps(n)
                    if field not in s["fields"]:
                        raise AssertionError("the SPS has no field " + field)
                    a, b = s["fields"][field][0], s["fields"][through or field][1]
                    _, e1 = s["fields"][field]
                    rb = "".join(format(v, "08b") for v in unescape(n[hdr:]))
                    au[j] = rewrite_bits(n, hdr, a, b, bits_of(rb[a:e1]))
        return aus
    return fn


def second_sps(aus, codec):
    """AVC: beside each SPS a copy whose seq_parameter_set_id is 1, so every IDR access unit
    carries the same two SPSs and only item 4's one-SPS rule refuses the stream."""
    for au in aus:
        out = []
        for n in au:
            out.append(n)
            if n[0] & 31 == 7:
                a, b = avc_sps(n)["fields"]["sps_id"]
                out.append(rewrite_bits(n, 1, a, b, ue_bits(1)))
        au[:] = out
    return aus


def hevc_sps_cut(aus, codec):
    """HEVC: every SPS cut after its sub-layer ordering information and re-terminated."""
    for au in aus:
        for j, n in enumerate(au):
            if (n[0] >> 1) & 63 == 33:
                au[j] = rewrite_bits(n, 2, hevc_sps(n)["fields"]["end"][0], 1 << 30, "")
    return aus


def mixed_idr(aus, codec):
    """AVC: a copy of access unit 6's slice, its first_mb_in_slice made 1, appended to the
    IDR access unit 5, so that access unit holds an IDR and a non-IDR slice."""
    p = [n for n in aus[6] if n[0] & 31 == 1][0]
    aus[5] = aus[5] + [rewrite_bits(p, 1, 0, 1, ue_bits(1))]
    return aus


def build_mutations():
    A, H = "accept-platform-avc", "accept-platform-hevc"
    M = []

    def add(name, part, target, what, apply, expect):
        M.append((name, part, target, what, apply, expect))
    # --- the record's text (part 1) ---
    add("record-sig-digit", 1, A, "flip a hex digit of sig", m_text(lambda t: _hex_flip(t, b'"sig":"')), BAD_SIG)
    add("record-key-noncanonical", 1, A, "signer_pubkey y >= p",
        m_text(lambda t: re.sub(rb'"signer_pubkey":"[0-9a-f]{64}"', b'"signer_pubkey":"' + b"f" * 62 + b'7f"', t)), BAD_KEY)
    add("record-key-digit", 1, A, "flip a hex digit of signer_pubkey",
        m_text(lambda t: _hex_flip(t, b'"signer_pubkey":"')), BAD_SIG)
    add("record-duplicate-member", 1, A, "name kind twice", m_text(lambda t: b'{"kind":"wai-encode",' + t[1:]), MALFORMED)
    add("record-truncated", 1, A, "drop the record's last byte", m_text(lambda t: t[:-1]), MALFORMED)
    add("record-exponent", 1, A, "wall_us written 1e3", m_text(lambda t: t.replace(b'"wall_us":1000', b'"wall_us":1e3')), MALFORMED)
    add("record-leading-zero", 1, A, "wall_us written 01000", m_text(lambda t: t.replace(b'"wall_us":1000', b'"wall_us":01000')), MALFORMED)
    add("record-minus-zero", 1, A, "wall_us written -0", m_text(lambda t: t.replace(b'"wall_us":1000', b'"wall_us":-0')), MALFORMED)
    add("record-too-big", 1, A, "wall_us 2^53", m_text(lambda t: t.replace(b'"wall_us":1000', b'"wall_us":9007199254740992')), MALFORMED)
    add("record-nested-33", 1, A, "package nested 33 deep",
        m_text(lambda t: t.replace(b'"package":null', b'"package":' + b"[" * 33 + b"]" * 33)), MALFORMED)
    add("record-lone-surrogate", 1, A, "a lone surrogate in encoder.device",
        m_text(lambda t: t.replace(b'"corpus device"', b'"\\ud800"')), MALFORMED)
    add("record-raw-tab", 1, A, "an unescaped tab in encoder.device",
        m_text(lambda t: t.replace(b'"corpus device"', b'"corpus\tdevice"')), MALFORMED)
    add("record-upper-hex", 1, A, "source.sha256 in uppercase hex",
        m_text(lambda t: re.sub(rb'("source":\{[^}]*"sha256":")([0-9a-f]{64})', lambda m: m.group(1) + m.group(2).upper(), t)), MALFORMED)
    add("record-bad-date", 1, A, "issued_at on 31 February",
        m_text(lambda t: t.replace(b'2026-10-25T12:00:00Z', b'2026-02-31T12:00:00Z')), MALFORMED)
    add("record-ten-fraction-digits", 1, A, "issued_at with ten fraction digits",
        m_text(lambda t: t.replace(b'2026-10-25T12:00:00Z', b'2026-10-25T12:00:00.0000000000Z')), MALFORMED)
    add("record-missing-member", 1, A, "drop wall_us", m_text(lambda t: t.replace(b',"wall_us":1000', b'')), MALFORMED)
    # --- files the record binds (part 1) ---
    add("stream-byte-flip", 1, A, "flip a byte inside the stream",
        lambda x: x.flip(x.path(x.case["stream"]), 20000), STREAM_MISMATCH)
    add("stream-truncated", 1, A, "drop the stream's last byte",
        lambda x: truncate(x.path(x.case["stream"])), STREAM_MISMATCH)
    add("source-byte-flip", 1, A, "flip a byte of the source's last frame",
        lambda x: x.flip(x.path(x.case["source"]), -1), SOURCE_MISMATCH)
    add("source-interlaced", 1, A, "the source header says It",
        lambda x: x.edit_text(x.case["source"], b" Ip ", b" It "), SOURCE_MISMATCH)
    add("source-frame-truncated", 1, A, "drop the source's last byte",
        lambda x: truncate(x.path(x.case["source"])), SOURCE_MISMATCH)
    add("source-other-size", 1, A, "the source header says W128 H288 (the same bytes, so the same digest)",
        lambda x: x.edit_text(x.case["source"], b"W256 H144", b"W128 H288"), SOURCE_MISMATCH)
    add("source-other-rate", 1, A, "the source header says F25:1",
        lambda x: x.edit_text(x.case["source"], b" F30:1 ", b" F25:1 "), SOURCE_MISMATCH)
    add("source-same-rate-unreduced", 1, A, "the source header says F60:2 (the settings' 30/1, unreduced)",
        lambda x: x.edit_text(x.case["source"], b" F30:1 ", b" F60:2 "), SOURCE_MISMATCH)
    add("source-full-range", 1, A, "the source header says XCOLORRANGE=FULL",
        lambda x: x.edit_text(x.case["source"], b" C420jpeg\n", b" C420jpeg XCOLORRANGE=FULL\n"), SOURCE_MISMATCH)
    # --- streams re-bound under the self-test key (part 1) ---
    add("avc-drop-sps-au5", 1, A, "drop the SPS of IDR access unit 5", m_stream_edit(drop_type(5, 7)), PARAMS)
    add("avc-drop-pps-au9", 1, A, "drop the PPS of IDR access unit 9", m_stream_edit(drop_type(9, 8)), PARAMS)
    add("avc-drop-ps-au0", 1, A, "drop the SPS and PPS of access unit 0",
        m_stream_edit(lambda a, c: drop_type(0, 8)(drop_type(0, 7)(a, c), c)), PARAMS)
    add("avc-idr-to-p-au5", 1, A, "access unit 5's IDR slice made nal_unit_type 1", m_stream_edit(retype(5, 1)), KEYFRAME)
    add("avc-p-to-idr-au3", 1, A, "access unit 3's slice made nal_unit_type 5", m_stream_edit(retype(3, 5)), KEYFRAME)
    add("avc-mixed-idr-au5", 1, A, "an IDR access unit with a non-IDR slice", m_stream_edit(mixed_idr), STREAM_MALFORMED)
    add("avc-swap-au6-au7", 1, A, "swap access units 6 and 7", m_stream_edit(swap(6, 7)), REORDERED)
    add("avc-insert-000002", 1, A, "00 00 02 inside access unit 4's slice",
        m_stream_edit(slice_byte(4, lambda n: n[:40] + b"\x00\x00\x02" + n[40:])), STREAM_MALFORMED)
    add("avc-forbidden-bit", 1, A, "access unit 2's slice with its forbidden bit set",
        m_stream_edit(slice_byte(2, lambda n: bytes([n[0] | 0x80]) + n[1:])), STREAM_MALFORMED)
    add("avc-garbage-first", 1, A, "a byte before the first start code",
        lambda x: x.rebind(b"\x07" + readb(x.path(x.case["stream"]))), STREAM_MALFORMED)
    add("avc-truncated-sps", 1, A, "every SPS cut to 4 bytes",
        m_stream_edit(lambda a, c: [[n[:4] if n[0] & 31 == 7 else n for n in au] for au in a]), STREAM_MALFORMED)
    add("avc-drop-last-au", 1, A, "drop the last access unit", m_stream_edit(lambda a, c: a[:-1]), FRAME_COUNT)
    add("avc-extra-au", 1, A, "repeat the last access unit", m_stream_edit(lambda a, c: a + [a[-1]]), FRAME_COUNT)
    add("avc-trailing-sei", 1, A, "a copy of access unit 0's SEI after the last picture",
        m_stream_edit(lambda a, c: a + [[n for n in a[0] if n[0] & 31 == 6]]), STREAM_MALFORMED)
    add("avc-second-sps", 1, A, "every IDR access unit also carries an SPS of id 1", m_stream_edit(second_sps), PARAMS)
    add("avc-drop-au3", 1, A, "drop access unit 3 (count before the plan)", m_stream_edit(lambda a, c: a[:3] + a[4:]), FRAME_COUNT)
    add("avc-foreign-pps-au2", 1, A, "access unit 2 carries PPS bytes the first IDR's do not",
        m_stream_edit(lambda a, c: a[:2] + [[a[0][1] + b"\x00\x80"] + a[2]] + a[3:]), PARAMS)
    add("avc-sps-level", 1, A, "level_idc 13 in every SPS", m_stream_edit(sps_field("level_idc", lambda v: was(v, format(12, "08b"), format(13, "08b")))), SPS_MISMATCH)
    add("avc-sps-level-au5", 1, A, "level_idc 13 in access unit 5's SPS only",
        m_stream_edit(sps_field("level_idc", lambda v: was(v, format(12, "08b"), format(13, "08b")), only_au=5)), PARAMS)
    add("avc-sps-chroma-422", 1, A, "chroma_format_idc 2", m_stream_edit(sps_field("chroma_format_idc", lambda v: was(v, ue_bits(1), ue_bits(2)))), UNSUPPORTED)
    add("avc-sps-10-bit", 1, A, "bit_depth_luma_minus8 2", m_stream_edit(sps_field("bit_depth_luma_minus8", lambda v: ue_bits(2))), UNSUPPORTED)
    add("avc-sps-interlaced", 1, A, "frame_mbs_only_flag 0 (with mb_adaptive_frame_field_flag 0)",
        m_stream_edit(sps_field("frame_mbs_only_flag", lambda v: was(v, "1", "00"))), UNSUPPORTED)
    add("avc-sps-poc-type-1", 1, A, "pic_order_cnt_type 1 (its fields 0)",
        m_stream_edit(sps_field("pic_order_cnt_type", lambda v: was(v, ue_bits(0), ue_bits(1) + "1" + "1" + "1" + ue_bits(0)),
                                through="log2_max_pic_order_cnt_lsb_minus4")), UNSUPPORTED)
    add("avc-sps-cropped", 1, A, "frame cropping of 2 rows (bottom offset 1)",
        m_stream_edit(sps_field("frame_cropping_flag", lambda v: was(v, "0", "1" + "1" + "1" + "1" + ue_bits(1)))), SPS_MISMATCH)
    add("avc-vui-full-range", 1, "accept-software-avc", "video_full_range_flag 1 in a video-range rung",
        m_stream_edit(sps_field("video_full_range_flag", lambda v: was(v, "0", "1"))), SPS_MISMATCH)
    add("avc-vui-reorder", 1, "accept-software-avc", "max_num_reorder_frames 1",
        m_stream_edit(sps_field("max_num_reorder_frames", lambda v: was(v, ue_bits(0), ue_bits(1)))), REORDERED)
    add("hevc-temporal-id-zero", 1, H, "nuh_temporal_id_plus1 0 in access unit 3's slice",
        m_stream_edit(slice_byte(3, lambda n: n[:1] + bytes([n[1] & 0xF8]) + n[2:])), STREAM_MALFORMED)
    add("hevc-layer-au3", 1, H, "nuh_layer_id 1 in access unit 3's slice",
        m_stream_edit(slice_byte(3, lambda n: n[:1] + bytes([(n[1] & 0x07) | 0x08]) + n[2:])), UNSUPPORTED)
    add("hevc-cra-au6", 1, H, "access unit 6's slice made CRA (21)", m_stream_edit(retype(6, 21)), KEYFRAME)
    add("hevc-bla-au7", 1, H, "access unit 7's slice made BLA (16)", m_stream_edit(retype(7, 16)), KEYFRAME)
    add("hevc-idr-to-trail-au9", 1, H, "IDR access unit 9's slice made TRAIL_R", m_stream_edit(retype(9, 1)), KEYFRAME)
    add("hevc-drop-vps-au5", 1, H, "drop the VPS of IDR access unit 5", m_stream_edit(drop_type(5, 32)), PARAMS)
    add("hevc-sps-reorder", 1, H, "sps_max_num_reorder_pics 1",
        m_stream_edit(sps_field("sps_max_num_reorder_pics", lambda v: was(v, ue_bits(0), ue_bits(1)))), REORDERED)
    add("hevc-sps-10-bit", 1, H, "bit_depth_luma_minus8 2", m_stream_edit(sps_field("bit_depth_luma_minus8", lambda v: ue_bits(2))), UNSUPPORTED)
    add("hevc-sps-window", 1, H, "a conformance window of 2 rows",
        m_stream_edit(sps_field("conformance_window_flag", lambda v: was(v, "0", "1" + "1" + "1" + "1" + ue_bits(1)))), SPS_MISMATCH)
    add("hevc-sps-cut", 1, H, "(control) every SPS cut after its sub-layer ordering information",
        m_stream_edit(hevc_sps_cut), ACCEPT)
    add("hevc-sps-tier", 1, H, "general_tier_flag 1", m_stream_edit(sps_field("general_tier_flag", lambda v: was(v, "0", "1"))), SPS_MISMATCH)
    # --- record members re-signed under the self-test key (part 1) ---
    add("resign-units", 1, A, "stream.units 11", m_record(lambda r: r["stream"].update(units=11)), FRAME_COUNT)
    add("resign-plan", 1, A, "idr [0,5,8]", m_record(lambda r: r.update(idr=[0, 5, 8])), KEYFRAME)
    add("resign-plan-unsorted", 1, A, "idr [0,9,5]", m_record(lambda r: r.update(idr=[0, 9, 5])), PLAN)
    add("resign-profile", 1, A, "stream.profile_idc 77", m_record(lambda r: r["stream"].update(profile_idc=77)), SPS_MISMATCH)
    add("resign-tier-avc", 1, A, "stream.tier high for AVC", m_record(lambda r: r["stream"].update(tier="high")), SPS_MISMATCH)
    add("resign-hevc-tier", 1, H, "stream.tier high", m_record(lambda r: r["stream"].update(tier="high")), SPS_MISMATCH)
    add("resign-range-full", 1, "accept-software-avc", "settings.range full",
        m_record(lambda r: r["settings"].update(range="full")), SPS_MISMATCH)
    add("resign-capability-hevc", 1, A, "capability wai.video.hevc over the AVC stream",
        m_record(lambda r: r.update(capability=HEVC_CAP)), STREAM_MALFORMED)
    add("resign-software-hevc", 1, H, "a software encode of HEVC",
        m_record(lambda r: r["encoder"].update(api="software", hardware="no", library_sha256="00" * 32)), ID_INCONSISTENT)
    add("resign-software-hardware", 1, "accept-software-avc", "a software encode stating hardware yes",
        m_record(lambda r: r["encoder"].update(hardware="yes")), ID_INCONSISTENT)
    add("resign-gpu-driver-no", 1, "accept-gpu-driver-avc", "a gpu-driver encode stating hardware no",
        m_record(lambda r: r["encoder"].update(hardware="no")), ID_INCONSISTENT)
    add("resign-blank-nbsp", 1, A, "encoder.version a no-break space and an ideographic space",
        m_record(lambda r: r["encoder"].update(version=" 　")), ID_INCOMPLETE)
    add("resign-leap-second", 1, A, "(control) issued_at 23:59:60", m_record(lambda r: r.update(issued_at="2026-12-31T23:59:60Z")), ACCEPT)
    add("resign-second-61", 1, A, "issued_at second 61", m_record(lambda r: r.update(issued_at="2026-12-31T23:59:61Z")), MALFORMED)
    add("resign-source-16384", 1, "accept-record-only", "(control) source.width 16384, resample two-tap",
        m_record(lambda r: (r["source"].update(width=16384), r["settings"].update(resample="wai.resample.two-tap"))), ACCEPT)
    add("resign-source-16386", 1, "accept-record-only", "source.width 16386",
        m_record(lambda r: (r["source"].update(width=16386), r["settings"].update(resample="wai.resample.two-tap"))), SETTINGS)
    add("resign-fps-zero", 1, A, "settings.fps_den 0", m_record(lambda r: r["settings"].update(fps_den=0)), SETTINGS)
    add("resign-fps-u32", 1, A, "settings.fps_num 2^32", m_record(lambda r: r["settings"].update(fps_num=1 << 32)), SETTINGS)
    add("resign-kbps-max", 1, A, "settings.kbps 1000001", m_record(lambda r: r["settings"].update(kbps=1000001)), SETTINGS)
    add("resign-size-8194", 1, A, "settings.width 8194 (resample two-tap)",
        m_record(lambda r: r["settings"].update(width=8194, resample="wai.resample.two-tap")), SETTINGS)
    add("resign-frames-zero", 1, A, "source.frames 0", m_record(lambda r: r["source"].update(frames=0)), SETTINGS)
    add("resign-resample-other", 1, "accept-software-avc-128x72", "resample bilinear",
        m_record(lambda r: r["settings"].update(resample="bilinear")), SETTINGS)
    add("resign-resample-two-tap-same", 1, A, "resample two-tap at the source's size",
        m_record(lambda r: r["settings"].update(resample="wai.resample.two-tap")), SETTINGS)

    def energy(**kw):
        e = {"attribution": "task", "counter": "task", "coverage": "Partial", "energy_calibration_ref": None,
             "energy_counter_filtering": "Unknown", "energy_provenance": "OnChipCounter",
             "energy_uncertainty": {"relative_ppm": 50000, "window_us": 400000}, "fabrics": ["cpu"],
             "joules_micro": 213000}
        e.update(kw)
        return m_record(lambda r: r.update(energy=e))
    add("resign-energy-control-ok", 1, A, "(control) a valid figure: Partial, cpu", energy(), ACCEPT)
    add("resign-energy-zero-model", 1, A, "a zero ModelBased figure (no evidence, so only rule 1 refuses it)",
        energy(joules_micro=0, energy_provenance="ModelBased", energy_uncertainty=None,
               energy_counter_filtering=None), ENERGY_INVALID)
    add("resign-energy-hwshunt-bare", 1, A, "HwShunt with no evidence", energy(
        energy_provenance="HwShunt", energy_uncertainty=None, energy_counter_filtering=None), ENERGY_INVALID)
    add("resign-energy-unknown-class", 1, A, "class 'Guess'", energy(energy_provenance="Guess"), ENERGY_INVALID)
    add("resign-energy-filtering-null", 1, A, "OnChipCounter with filtering null",
        energy(energy_counter_filtering=None), ENERGY_INVALID)
    add("resign-energy-window-zero", 1, A, "window_us 0",
        energy(energy_uncertainty={"relative_ppm": 50000, "window_us": 0}), ENERGY_INVALID)
    add("resign-energy-floor-exact", 1, A, "(control) joules x ppm = 288676 exactly",
        energy(joules_micro=288676, energy_uncertainty={"relative_ppm": 1, "window_us": 1}), ACCEPT)
    add("resign-energy-floor-below", 1, A, "joules x ppm = 288675",
        energy(joules_micro=288675, energy_uncertainty={"relative_ppm": 1, "window_us": 1}), ENERGY_INVALID)
    add("resign-energy-calibrated-blank", 1, A, "CalibratedInstrument with a blank reference",
        energy(energy_provenance="CalibratedInstrument", energy_counter_filtering=None,
               energy_calibration_ref="  "), ENERGY_INVALID)
    add("resign-energy-model-evidence", 1, A, "ModelBased with an uncertainty",
        energy(energy_provenance="ModelBased", energy_counter_filtering=None), ENERGY_INVALID)
    add("resign-energy-coverage", 1, A, "coverage 'Most'", energy(coverage="Most"), ENERGY_INVALID)
    add("resign-energy-attribution", 1, A, "attribution 'guess'", energy(attribution="guess"), ENERGY_INVALID)
    add("resign-energy-fabrics-empty", 1, A, "fabrics []", energy(fabrics=[]), ENERGY_INVALID)
    add("resign-energy-fabrics-twice", 1, A, "fabrics [gpu, gpu]", energy(fabrics=["gpu", "gpu"]), ENERGY_INVALID)
    add("resign-energy-fabrics-unknown", 1, A, "fabrics [cpu, dsp]", energy(fabrics=["cpu", "dsp"]), ENERGY_INVALID)
    add("resign-energy-whole-gpu-va", 1, "accept-va-1-avc", "(control) va, Whole, fabrics [gpu]",
        energy(coverage="Whole", fabrics=["gpu"]), ACCEPT)
    add("resign-energy-whole-npu-va", 1, "accept-va-1-avc", "va, Whole, fabrics [cpu, npu]",
        energy(coverage="Whole", fabrics=["cpu", "npu"]), ENERGY_COVERAGE)
    add("resign-energy-whole-software", 1, "accept-software-avc", "(control) software, Whole, fabrics [cpu]",
        energy(coverage="Whole"), ACCEPT)
    # --- the rendition (part 2) ---
    RA, RH, RS = "accept-platform-avc", "accept-platform-hevc", "accept-software-avc"
    add("object-byte-flip", 2, RH, "flip the last byte of piece 2's object",
        lambda x: x.flip(_object_path(x, 2), -1), PACKAGE_MISMATCH)
    add("object-manifest-byte", 2, RA, "flip a byte of piece 0's manifest",
        lambda x: x.flip(_object_path(x, 0), 20), PACKAGE_MISMATCH)
    add("object-deleted", 2, RS, "delete piece 4's object", lambda x: os.remove(_object_path(x, 4)), PACKAGE_MISMATCH)
    add("objects-swapped", 2, RA, "swap the files of pieces 1 and 3", _swap_objects(1, 3), PACKAGE_MISMATCH)
    add("receipt-byte-flip", 2, RA, "flip a byte of the package receipt",
        lambda x: x.flip(os.path.join(x.c.package, "package.receipt.json"), 30), PACKAGE_MISMATCH)
    add("index-duration", 2, RA, "piece 1's duration 24 in the stage index", _index_poke(160 + 32 * 1 + 8, 24, 4), PACKAGE_MISMATCH)
    add("index-start", 2, RA, "piece 2's start 86 in the stage index", _index_poke(160 + 32 * 2, 86, 8), PACKAGE_MISMATCH)
    add("index-base-hash", 2, RA, "piece 3's base entry hash changed", _index_base_hash(3), PACKAGE_MISMATCH)
    add("index-reserved", 2, RA, "a reserved header byte of the stage index set", _index_poke(150, 1, 1), PACKAGE_MISMATCH)
    add("rendition-stream-flip", 2, RA, "flip a byte inside the held stream",
        lambda x: x.flip(x.rpath(x.case["stream"]), 30000), STREAM_MISMATCH)
    add("resign-object-renamed", 2, RA, "the record names a copy of piece 0's object under another hash",
        m_rrecord(_rename_object(0)), PACKAGE_MISMATCH)
    add("resign-object-payload", 2, RA, "piece 0's payload changed, the object renamed to its new hash",
        m_rrecord(_repayload(0)), PACKAGE_MISMATCH)
    add("resign-pieces-shifted", 2, RA, "the record's piece 1 starts at 59",
        m_rrecord(lambda x, r: r["package"]["pieces"][1].update(start=59)), PLAN)
    add("resign-pieces-sum", 2, RA, "the record's last piece one frame short",
        m_rrecord(lambda x, r: r["package"]["pieces"][4].update(frames=39)), PLAN)
    add("resign-renderer", 2, RA, "encoder.version 1.1 in the record",
        m_rrecord(lambda x, r: r["encoder"].update(version="1.1")), PACKAGE_MISMATCH)
    add("resign-driver", 2, RS, "encoder.driver changed (bound through settings)",
        m_rrecord(lambda x, r: r["encoder"].update(driver="corpus driver 2")), PACKAGE_MISMATCH)
    return M


def _object_path(x, k):
    rec = read_json(readb(x.c.record_path(x.case)))
    return os.path.join(x.c.rdir, x.case["dir"], "objects", rec["package"]["pieces"][k]["object"] + ".wai")


def _swap_objects(a, b):
    def apply(x):
        pa, pb = _object_path(x, a), _object_path(x, b)
        da, db = readb(pa), readb(pb)
        with open(pa, "wb") as f:
            f.write(db)
        with open(pb, "wb") as f:
            f.write(da)
    return apply


def _index_poke(at, value, n):
    def apply(x):
        p = os.path.join(x.c.package, "index-%s.wsix" % x.c.component)
        b = bytearray(readb(p))
        b[at:at + n] = value.to_bytes(n, "little")
        with open(p, "wb") as f:
            f.write(bytes(b))
    return apply


def _index_base_hash(k):
    def apply(x):
        p = os.path.join(x.c.package, "index-%s.wsix" % x.c.component)
        b = bytearray(readb(p))
        ix = read_wsix(bytes(b))
        n_p = len(ix["pieces"])
        e = [i for i, en in enumerate(ix["entries"]) if en["piece"] == k and en["role"] == 0][0]
        b[160 + 32 * n_p + 128 * e] ^= 1
        with open(p, "wb") as f:
            f.write(bytes(b))
    return apply


def _rename_object(k):
    def fn(x, r):
        pc = r["package"]["pieces"][k]
        src = os.path.join(x.c.rdir, x.case["dir"], "objects", pc["object"] + ".wai")
        other = hashlib.sha256(pc["object"].encode()).hexdigest()
        shutil.copyfile(src, os.path.join(x.c.rdir, x.case["dir"], "objects", other + ".wai"))
        pc["object"] = other
    return fn


def _repayload(k):
    def fn(x, r):
        pc = r["package"]["pieces"][k]
        d = os.path.join(x.c.rdir, x.case["dir"], "objects")
        b = bytearray(readb(os.path.join(d, pc["object"] + ".wai")))
        b[-1] ^= 1
        pc["object"] = content_hash(bytes(b)).hex()
        with open(os.path.join(d, pc["object"] + ".wai"), "wb") as f:
            f.write(bytes(b))
    return fn


def copy_corpus(corpus, into):
    """A copy of the corpus and the package it names, at the same relative places."""
    base = os.path.commonpath([corpus.root, corpus.package])
    for d in (corpus.root, corpus.package):
        shutil.copytree(d, os.path.join(into, os.path.relpath(d, base)))
    return Corpus(os.path.join(into, os.path.relpath(corpus.root, base)))


def run_target(c, part, target):
    if part == 1:
        case = next(x for x in c.cases if x["name"] == target)
        return c.run(case)
    case = next(x for x in c.rcases if x["name"] == target)
    return c.run_rendition(case)


# ---------------------------------------------------------------- main


def main(argv):
    if len(argv) != 2:
        print(__doc__)
        return 2
    corpus = Corpus(argv[1])
    ok_all = True
    print("encode_verify.py: %s" % corpus.root)
    bad_pins = pins(corpus)
    print("fixture pins (streams.json, rendition/streams.json): %s" % ("ok" if not bad_pins else "MISMATCH " + ", ".join(bad_pins)))
    ok_all &= not bad_pins

    # parts 1 and 2
    rows = corpus.everything()
    baseline = {}
    for part, title in ((1, "records, §7 steps 1-9"), (2, "rendition, §7 and §9.3")):
        print("\npart %d: %s" % (part, title))
        n = good = acc = 0
        for name, p, o, e, made, d in rows:
            if p != part:
                continue
            n += 1
            match = o == e
            good += match
            acc += e == ACCEPT
            baseline[(p, name)] = o
            print("  %-4s %-30s %-28s %s" % ("ok" if match else "FAIL", name, o,
                                             "made " + steps_text(made) if o == ACCEPT else d))
            if not match:
                print("       expected %s" % e)
        ok_all &= good == n
        print("part %d: %d/%d cases match (%d accept, %d refused)" % (part, good, n, acc, n - acc))
    for c in corpus.cases:
        if c["expect"] == ACCEPT:
            _, _, _, rec = corpus.run(c)
            print("  record digest (§6.2) %-28s %s" % (c["name"], record_digest(rec).hex()))

    # part 3
    print("\npart 3: coverage of §12")
    try:
        spec_rec, spec_prod = spec_reasons()
    except (OSError, IndexError) as e:
        spec_rec, spec_prod = None, None
        print("  could not read §12 from %s (%s)" % (SPEC_PATH, e))
        ok_all = False
    if spec_rec is not None:
        if sorted(spec_rec) != sorted(RECORD_REASONS):
            print("  the verifier's reasons differ from §12's record reasons: %s" %
                  sorted(set(spec_rec) ^ set(RECORD_REASONS)))
            ok_all = False
        expects = {c["expect"] for c in corpus.cases} | {c["expect"] for c in corpus.rcases}
        missing = [r for r in spec_rec if r not in expects]
        stray = sorted(e for e in expects if e != ACCEPT and e not in spec_rec)
        for r in missing:
            print("  NOT COVERED: %s" % r)
        for r in stray:
            print("  an expectation that is no record reason of §12: %s" % r)
        print("part 3: %d/%d record reasons of §12 appear in the corpus (producer reasons, not record reasons: %s)"
              % (len(spec_rec) - len(missing), len(spec_rec), ", ".join(spec_prod)))
        ok_all &= not missing and not stray

    # part 4a: mutations
    print("\npart 4a: mutated copies of the corpus")
    tmp = tempfile.mkdtemp(prefix="encode-verify-")
    try:
        # controls: re-signing alone, and re-framing the stream alone, change no outcome
        controls_ok = True
        ctl = copy_corpus(corpus, os.path.join(tmp, "control"))
        for case in ctl.cases:
            if case["expect"] != ACCEPT:
                continue
            x = Ctx(ctl, case)
            if case.get("stream"):
                aus, _ = x.stream_aus()
                x.rebind(frame(flat(aus)))
            else:
                x.put_record(x.record())
        for case in ctl.rcases:
            if case["expect"] == ACCEPT:
                p = ctl.record_path(case)
                write_record(p, resign(read_json(readb(p))))
        crow = ctl.everything()
        for name, p, o, e, made, d in crow:
            if e == ACCEPT and o != ACCEPT:
                controls_ok = False
                print("  CONTROL FAIL %s: re-signed copy gives %s (%s)" % (name, o, d))
        n_ctl = sum(1 for r in crow if r[3] == ACCEPT)
        print("  controls: %d accepted cases re-signed under the self-test key (streams re-framed with "
              "4-byte start codes and re-bound): %s" % (n_ctl, "all still accept" if controls_ok else "FAIL"))
        ok_all &= controls_ok

        muts = build_mutations()
        built, m_ok = [], 0
        for k, (name, part, target, what, apply, expect) in enumerate(muts):
            c = copy_corpus(corpus, os.path.join(tmp, "m%03d" % k))
            case = next(x for x in (c.cases if part == 1 else c.rcases) if x["name"] == target)
            try:
                apply(Ctx(c, case))
                o, made, d, _ = run_target(c, part, target)
            except Exception as e:
                o, made, d = "build-error:" + type(e).__name__, [], str(e)
            orig = baseline[(part, target)]
            good = o == expect and (o != orig or expect == ACCEPT)
            m_ok += good
            if good:
                built.append((name, part, target, c, expect))
            print("  %-4s %-30s %-28s on %-26s %s%s" % ("ok" if good else "FAIL", name, o, target, what,
                                                         "" if good else "  [expected %s; %s]" % (expect, d)))
        n_real = sum(1 for m in muts if m[5] != ACCEPT)
        print("part 4a: %d/%d mutations give their expected outcome (%d refusals, %d accepting controls); "
              "every refused copy stops matching its original outcome" % (m_ok, len(muts), n_real, len(muts) - n_real))
        ok_all &= m_ok == len(muts)

        # part 4b: dropped checks
        print("\npart 4b: each check dropped in turn")
        caught_all, by_corpus = 0, 0
        gaps = []
        for name, what in CHECKS:
            DROP.clear()
            DROP.add(name)
            hit_c = [r[0] for r in corpus.everything() if r[2] != r[3]]
            hit_m = []
            for mname, part, target, c, expect in built:
                o = run_target(c, part, target)[0]
                if o != expect:
                    hit_m.append(mname)
            DROP.clear()
            caught = bool(hit_c or hit_m)
            caught_all += caught
            by_corpus += bool(hit_c)
            if not hit_c:
                gaps.append(name)
            print("  %-4s %-24s corpus: %-44s mutations: %s" % (
                "ok" if caught else "MISS", name, (", ".join(hit_c[:2]) + (" +%d" % (len(hit_c) - 2) if len(hit_c) > 2 else "")) or "-",
                (", ".join(hit_m[:2]) + (" +%d" % (len(hit_m) - 2) if len(hit_m) > 2 else "")) or "-"))
        print("part 4b: %d/%d dropped checks caught (%d by the committed corpus, %d only by self-test mutations: %s)" % (
            caught_all, len(CHECKS), by_corpus, len(CHECKS) - by_corpus, ", ".join(gaps) or "none"))
        ok_all &= caught_all == len(CHECKS)
    finally:
        shutil.rmtree(tmp, ignore_errors=True)
    print("\n%s" % ("PASS" if ok_all else "FAIL"))
    return 0 if ok_all else 1


if __name__ == "__main__":
    sys.exit(main(sys.argv))