diff --git a/.github/workflows/wai-byte-equality.yml b/.github/workflows/wai-byte-equality.yml index a4f938924..6f0f8ec05 100644 --- a/.github/workflows/wai-byte-equality.yml +++ b/.github/workflows/wai-byte-equality.yml @@ -14,7 +14,8 @@ name: wai cross-architecture byte-equality # (`wai.video.int_hyper`) whose I-frame comes out of a learned conditional codec, # and the MLIC++-topology codec (`wai.neural.int_mlicpp`), whose entropy network runs # before every checkerboard pass and must give every platform the encoder's scale -# bucket for every element. Video is the strictest leg — inter-frame decode is a +# bucket for every element. The binary payloads SPEC §5 pins (WIV1, WIH1) are parsed +# with the wire parser a sink runs and decoded to the same digests. Video is the strictest leg — inter-frame decode is a # feedback loop, so one divergent half-pel rounding does not stay one byte, it # predicts the next frame and compounds for the rest of the clip. # @@ -38,6 +39,7 @@ on: - 'wai/wai-rs/src/int_transform.rs' - 'wai/wai-rs/src/int_hyper_synth.rs' - 'wai/wai-rs/src/int_mlicpp.rs' + - 'wai/wai-rs/src/int_payload/wire.rs' # The fast kernels int_synth, int_transform and int_hyper_synth dispatch to # by default: the goldens here decode through them. - 'wai/wai-rs/src/int_kernels.rs' @@ -62,6 +64,7 @@ on: - 'wai/wai-rs/src/int_transform.rs' - 'wai/wai-rs/src/int_hyper_synth.rs' - 'wai/wai-rs/src/int_mlicpp.rs' + - 'wai/wai-rs/src/int_payload/wire.rs' # The fast kernels int_synth, int_transform and int_hyper_synth dispatch to # by default: the goldens here decode through them. - 'wai/wai-rs/src/int_kernels.rs' diff --git a/.github/workflows/wai-conformance.yml b/.github/workflows/wai-conformance.yml index 4862d0178..4dbad796a 100644 --- a/.github/workflows/wai-conformance.yml +++ b/.github/workflows/wai-conformance.yml @@ -30,6 +30,9 @@ on: - 'wai/glossary/**' - 'wai/SPEC.md' - 'wai/REGISTRY.md' + # A library test checks SPEC §5's pinned integer payloads against this + # extension's sections, so an edit of it is checked here too. + - 'wai/extensions/integer-payloads.md' - 'wai/wai-web/**' - '.github/workflows/wai-conformance.yml' pull_request: @@ -49,6 +52,9 @@ on: - 'wai/glossary/**' - 'wai/SPEC.md' - 'wai/REGISTRY.md' + # A library test checks SPEC §5's pinned integer payloads against this + # extension's sections, so an edit of it is checked here too. + - 'wai/extensions/integer-payloads.md' - 'wai/wai-web/**' - '.github/workflows/wai-conformance.yml' @@ -153,7 +159,16 @@ jobs: # decode allocates anything sized from it. # tests/int_audio.rs and tests/int_splat.rs decode the committed wai.audio.int_codec and # wai.splat.int_codec vectors against the hashes the Python integer reference computed at - # export, and check each vector's metadata. Cargo.toml declares them, and the other + # export, and check each vector's metadata. `--lib` runs int_payload's unit tests: each + # binary payload (WIV1, WIH1, WIA1, WIS1) against the JSON it is converted from, under + # every kernel policy, and decode_int for every integer capability in this build, which + # has no `neural`. tests/int_payload.rs refuses every fixture of int-payload-conformance + # with its code, every strict prefix of every payload, and runs a byte-change sweep in + # which nothing may panic; tests/int_payload_preflight.rs (its own binary, a counting + # allocator) checks that a header past the caps is refused before anything sized from it + # is allocated; tests/int_payload_cost.rs (its own binary, a live-bytes allocator) checks + # that int_payload::declared_cost bounds what each decode holds, and, in the release + # step only, decodes a 2^30-byte clip a 3 KB payload codes. Cargo.toml declares them, and the other # feature-gated targets named here, with required-features, so if this step ever loses a # feature, cargo stops with an error instead of compiling an empty binary that reports ok. # Each committed vector's decoded output is also checked against the SHA-256 in @@ -163,10 +178,24 @@ jobs: # wai.generate.spectral vector, which `generate` enables. - name: Fast integer kernels ≡ reference kernels (overflow checks on) working-directory: wai/wai-rs - run: cargo test --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate + run: cargo test --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test int_payload --test int_payload_preflight --test int_payload_cost - name: Fast integer kernels ≡ reference kernels (release, overflow checks off) working-directory: wai/wai-rs - run: cargo test --release --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate + run: cargo test --release --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test int_payload --test int_payload_preflight --test int_payload_cost + # The integer decode path builds under each of its features alone, so a sink can take + # one medium without the others, and for wasm32 (a browser sink); `int_payload` arms a + # medium only under its feature, and none needs `neural`. + - name: Each integer feature alone, and wasm32 (unix) + if: matrix.os != 'windows' + shell: bash + working-directory: wai/wai-rs + run: | + set -euo pipefail + for f in neural_int video_int audio_int splat_int; do + cargo check --lib --no-default-features --features "$f" + done + rustup target add --toolchain 1.98.0 wasm32-unknown-unknown + cargo check --lib --target wasm32-unknown-unknown --no-default-features --features video_int,audio_int,splat_int # carriage: every registered capability round-trips byte-exact. cli_wrap_intent and # cli_registry run the `wai` binary, which builds without the codecs feature: the # intent rule of `wai wrap`, its refusal of candidate capabilities (SPEC §5), its media @@ -237,7 +266,7 @@ jobs: working-directory: wai/wai-rs run: | set -euo pipefail - F=world,audio_scene,haptic,splat4d,score,provenance,neural_int,film,quantum,quantum_ops,oer,receipt_claims,key_discovery + F=world,audio_scene,haptic,splat4d,score,provenance,neural_int,film,quantum,quantum_ops,oer,receipt_claims,key_discovery,video_int,audio_int,splat_int v() { echo "::group::$1"; cargo run -q --no-default-features --features "$F" --bin "$2" -- verify "$3"; echo "::endgroup::"; } v worlds wai_world_vectors ../world-conformance v spatial-audio wai_audio_vectors ../audio-conformance @@ -255,6 +284,7 @@ jobs: v claims wai_claim_vectors ../claim-conformance v pin wai_pin_vectors ../pin-conformance v signers wai_signer_vectors ../signer-conformance + v int-payload wai_int_payload_vectors ../int-payload-conformance # The pin corpus's second implementation: Python standard library only, sharing no # code with wai-rs. One leg is enough — it checks digests and listing rules, which # do not depend on the architecture — and the macOS host has python3. @@ -283,7 +313,7 @@ jobs: shell: pwsh working-directory: wai/wai-rs run: | - $F = "world,audio_scene,haptic,splat4d,score,provenance,neural_int,film,quantum,quantum_ops,oer,receipt_claims,key_discovery" + $F = "world,audio_scene,haptic,splat4d,score,provenance,neural_int,film,quantum,quantum_ops,oer,receipt_claims,key_discovery,video_int,audio_int,splat_int" function v($grp, $bin, $path) { Write-Output "::group::$grp" cargo run -q --no-default-features --features $F --bin $bin -- verify $path @@ -306,6 +336,7 @@ jobs: v claims wai_claim_vectors ../claim-conformance v pin wai_pin_vectors ../pin-conformance v signers wai_signer_vectors ../signer-conformance + v int-payload wai_int_payload_vectors ../int-payload-conformance # The generated v2 vocabulary (wai/vocab/v2.*) is checked as a consumer uses it: # sample manifests, typed and untyped, WAI1 and WAI2, are expanded through the @@ -335,11 +366,12 @@ jobs: # differential.py (both verifiers' verdicts on generated inputs the corpus does not hold, # under a fixed seed, so a run is reproducible), jwp_verify.py (every JWP vector), and # signer_verify.py (every key-discovery vector and the canonical fixture, with its - # verdicts required to equal wai_signer_vectors' line for line). + # verdicts required to equal wai_signer_vectors' line for line), and + # int_payload_verify.py (the integer payloads and their refusal fixtures). # Each exits non-zero on any failure or any difference, and the step fails with it. # One platform: what they check is the reading of the text, not the architecture. independent-verifiers: - name: independent verifiers · claims, JWP and key discovery (Python) + name: independent verifiers · claims, JWP, key discovery and integer payloads (Python) runs-on: linux steps: - uses: actions/checkout@v5 @@ -371,6 +403,15 @@ jobs: shell: bash run: cargo build --no-default-features --features key_discovery --bin wai_claim_vectors --bin wai_jwp_vectors --bin wai_signer_vectors + # The integer payload corpus's second implementation (int-payload-conformance/README): + # it parses every payload with the extension's rules and check order, converts every + # JSON source independently and requires the committed bytes, decodes the WIV1 + # keyframe-0 payloads in full to their sidecars' SHA-256, and checks every refusal + # fixture. Standard library only, so it needs nothing installed. + - name: int_payload_verify.py + working-directory: wai + shell: bash + run: python3 int-payload-conformance/int_payload_verify.py int-payload-conformance - name: claim_verify.py, differential.py, jwp_verify.py, signer_verify.py working-directory: wai shell: bash diff --git a/.github/workflows/wai-energy-receipts.yml b/.github/workflows/wai-energy-receipts.yml index c994a08a6..f9ef8ac30 100644 --- a/.github/workflows/wai-energy-receipts.yml +++ b/.github/workflows/wai-energy-receipts.yml @@ -70,6 +70,8 @@ on: - 'wai/extensions/**' - 'wai/SPEC.md' - 'wai/CHANGELOG.md' + # tests/video_receipt.rs reads the WIV1 refusal fixtures. + - 'wai/int-payload-conformance/**' - '.github/workflows/wai-energy-receipts.yml' pull_request: paths: @@ -79,6 +81,8 @@ on: - 'wai/extensions/**' - 'wai/SPEC.md' - 'wai/CHANGELOG.md' + # tests/video_receipt.rs reads the WIV1 refusal fixtures. + - 'wai/int-payload-conformance/**' - '.github/workflows/wai-energy-receipts.yml' jobs: diff --git a/.github/workflows/wai-neural-gates.yml b/.github/workflows/wai-neural-gates.yml index 4952426eb..07fccb2c1 100644 --- a/.github/workflows/wai-neural-gates.yml +++ b/.github/workflows/wai-neural-gates.yml @@ -16,7 +16,10 @@ name: WAI neural sink gates # committed int_mlicpp vector and the int_synth corpus: the form-less and set-form # pins through the strict gate, the int_synth pin over the spec it reads, the # several-file capabilities refusing a form-less pin, a pinned decode reading only -# the bytes it verified, a WAI2 envelope whose pins pick the held model, the strict +# the bytes it verified, an unpinned decode at a sink with a key log checked over +# every identity of what it read once and reading only that (or, holding the +# capability's sets to §7.9, the part §7.9 authorised), an encrypted envelope inert at the +# sink (SPEC §6.4 step 2), a WAI2 envelope whose pins pick the held model, the strict # rule read for the capability decoded through (a sound fallback behind an unsound # primary), SPEC §4 step 6's order of reasons after a set_incomplete decode, and the # replicate_* reason the strict WAI2 policy reports for an entry it leaves out, @@ -43,6 +46,7 @@ on: - 'wai/wai-rs/src/ffi.rs' - 'wai/wai-rs/src/ffi_neural.rs' - 'wai/wai-rs/tests/ffi_neural_pin.rs' + - 'wai/wai-rs/tests/neural_cost_limit.rs' - 'wai/wai-rs/src/int_synth.rs' - 'wai/wai-rs/src/ort_session.rs' - 'wai/neural-conformance/vectors/**' @@ -50,6 +54,17 @@ on: - 'wai/wai-rs/tests/neural_native_decode.rs' - 'wai/wai-rs/src/int_mlicpp.rs' - 'wai/wai-rs/tests/vectors/int_mlicpp/**' + # The integer capabilities the neural sink delegates to int_payload::decode_int, + # and the vectors its tests decode through it. + - 'wai/wai-rs/src/int_payload/**' + - 'wai/wai-rs/src/int_video.rs' + - 'wai/wai-rs/src/int_hyper_synth.rs' + - 'wai/wai-rs/src/int_audio.rs' + - 'wai/wai-rs/src/int_splat.rs' + - 'wai/wai-rs/tests/vectors/int_hyper/**' + - 'wai/wai-rs/tests/vectors/int_video/**' + - 'wai/wai-rs/tests/vectors/int_audio/**' + - 'wai/wai-rs/tests/vectors/int_splat/**' - 'wai/wai-rs/Cargo.toml' - 'wai/wai-rs/Cargo.lock' - '.github/workflows/wai-neural-gates.yml' @@ -68,6 +83,7 @@ on: - 'wai/wai-rs/src/ffi.rs' - 'wai/wai-rs/src/ffi_neural.rs' - 'wai/wai-rs/tests/ffi_neural_pin.rs' + - 'wai/wai-rs/tests/neural_cost_limit.rs' - 'wai/wai-rs/src/int_synth.rs' - 'wai/wai-rs/src/ort_session.rs' - 'wai/neural-conformance/vectors/**' @@ -75,6 +91,17 @@ on: - 'wai/wai-rs/tests/neural_native_decode.rs' - 'wai/wai-rs/src/int_mlicpp.rs' - 'wai/wai-rs/tests/vectors/int_mlicpp/**' + # The integer capabilities the neural sink delegates to int_payload::decode_int, + # and the vectors its tests decode through it. + - 'wai/wai-rs/src/int_payload/**' + - 'wai/wai-rs/src/int_video.rs' + - 'wai/wai-rs/src/int_hyper_synth.rs' + - 'wai/wai-rs/src/int_audio.rs' + - 'wai/wai-rs/src/int_splat.rs' + - 'wai/wai-rs/tests/vectors/int_hyper/**' + - 'wai/wai-rs/tests/vectors/int_video/**' + - 'wai/wai-rs/tests/vectors/int_audio/**' + - 'wai/wai-rs/tests/vectors/int_splat/**' - 'wai/wai-rs/Cargo.toml' - 'wai/wai-rs/Cargo.lock' - '.github/workflows/wai-neural-gates.yml' @@ -115,8 +142,40 @@ jobs: pinned_decode_reads_only_verified_bytes wai2_decode_multi_picks_the_verified_rendition \ strict_gate_reads_the_capability_it_decodes_through set_incomplete_at_step_3_outranks_the_fallback_refusal \ decode_multi_strict_reports_replicate_reasons strict_gate_ignores_a_held_capability_it_cannot_dispatch \ + int_hyper_decodes_through_the_neural_dispatch int_synth_check_5_covers_the_spec_a_model_directory_holds \ + an_integer_capability_without_a_decoder_moves_on_to_the_fallback \ + a_listing_that_does_not_match_the_files_read_authorises_nothing \ + a_pinned_set_revoked_under_its_file_digest_is_refused_through_a_listing_pin \ + a_granted_listing_authorises_only_the_files_it_names \ + an_encrypted_envelope_is_inert_at_the_reference_sink \ + a_revoke_only_key_log_checks_and_reads_the_set_once \ 2>&1 | tee "$RUNNER_TEMP/neural-gates.log" - grep -q "test result: ok. 12 passed; 0 failed" "$RUNNER_TEMP/neural-gates.log" + grep -q "test result: ok. 20 passed; 0 failed" "$RUNNER_TEMP/neural-gates.log" + # The integer video, audio and splat capabilities through the neural dispatch, which + # delegates them to int_payload::decode_int: each decodes to Decoded::Int with its + # committed SHA-256, and wai.video.int_motion is advertised with ModelRegistry::builtin. + - name: The neural sink decodes the integer video, audio and splat payloads + working-directory: wai/wai-rs + shell: bash + run: | + set -euo pipefail + cargo test --no-default-features --features neural,video_int,audio_int,splat_int --test neural_native_decode -- \ + --exact integer_media_decode_through_the_neural_dispatch \ + 2>&1 | tee "$RUNNER_TEMP/neural-int.log" + grep -q "test result: ok. 1 passed; 0 failed" "$RUNNER_TEMP/neural-int.log" + # The registry's cost limit refuses an integer payload over it before any symbol is + # decoded (cost_over_limit) and moves on to the next entry; the C ABI refuses an + # integer video it has no output kind for before decoding it; and the declared + # cost over a registry's set, and through the C ABI given the set's path, is the + # library's. Its own binary: a live-bytes allocator measures each refusal. + - name: The neural sink applies the declared cost before decoding + working-directory: wai/wai-rs + shell: bash + run: | + set -euo pipefail + cargo test --no-default-features --features neural,ffi,video_int,audio_int,splat_int --test neural_cost_limit \ + 2>&1 | tee "$RUNNER_TEMP/neural-cost.log" + grep -q "test result: ok. 3 passed; 0 failed" "$RUNNER_TEMP/neural-cost.log" - name: A key log refuses a revoked pinned set at the neural sink (SPEC §3.1 check 5) working-directory: wai/wai-rs shell: bash diff --git a/.github/workflows/wai-web-wasm.yml b/.github/workflows/wai-web-wasm.yml index 23a141843..cd648b6c9 100644 --- a/.github/workflows/wai-web-wasm.yml +++ b/.github/workflows/wai-web-wasm.yml @@ -13,7 +13,8 @@ name: wai-web wasm32 # - demo/verify_pack.mjs: the browser writers `pack` and `packMulti` refuse # every candidate the export lists, in every place an envelope can name one; # - a byte comparison of the demo's staged int_synth copies with the corpus; -# - the crate's wasm_tests under wasm-bindgen-test-runner. +# - the crate's wasm_tests under wasm-bindgen-test-runner, and its native unit +# tests on the host. # # The full-scale int_hyper wasm decode needs vectors regenerated from a # pretrained model; wai-sota-vectors.yml runs it after regenerating them. @@ -85,10 +86,29 @@ jobs: CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER=wasm-bindgen-test-runner \ cargo test --locked --target wasm32-unknown-unknown 2>&1 | tee "$RUNNER_TEMP/wai-web-wasm.log" grep -q "test result: ok. 4 passed; 0 failed" "$RUNNER_TEMP/wai-web-wasm.log" + # The crate's native unit tests, run on the host: the pure-Rust halves of the + # bindings, which wasm_tests does not run. They decode the committed integer + # payloads to their sidecars' SHA-256 (int_payload::tests) and check that the + # writers `pack` and `packMulti` refuse a payload that is not the format SPEC §5 + # pins its integer capability to (tests). The count check turns a test that + # stopped being compiled into a failure instead of a silent "0 passed". + - name: wai-web native unit tests + shell: bash + working-directory: wai/wai-web + run: | + set -euo pipefail + cargo test --locked --lib 2>&1 | tee "$RUNNER_TEMP/wai-web-native.log" + grep -q "test result: ok. 8 passed; 0 failed" "$RUNNER_TEMP/wai-web-native.log" - name: wasm32 int_synth decode equals the frozen native corpus shell: bash working-directory: wai/wai-web run: node demo/verify_int_synth_wasm.mjs ./pkg-node/wai_web.js ../neural-conformance + # decodeWiv1, decodeWih1, decodeWia1 and decodeWis1 on the committed integer + # payloads give the SHA-256 their sidecars state, and a JSON form is refused. + - name: wasm32 integer payload decode equals the native sidecars + shell: bash + working-directory: wai/wai-web + run: node demo/verify_int_payload_wasm.mjs ./pkg-node/wai_web.js ../wai-rs/tests/vectors - name: The browser menu is registered and names no candidate shell: bash working-directory: wai/wai-web diff --git a/wai/.gitattributes b/wai/.gitattributes index 9eb1db2f4..6cb0347ff 100644 --- a/wai/.gitattributes +++ b/wai/.gitattributes @@ -1,3 +1,9 @@ # REGISTRY.md carries a table `wai registry export` generates, which a library test # compares byte for byte (as registry/ and vocab/v2.* are): no line-ending conversion. REGISTRY.md -text +# The binary payloads of the integer codecs (extensions/integer-payloads.md) beside +# their JSON sources: pinned by SHA-256, so never converted. +*.wiv1 -text +*.wih1 -text +*.wia1 -text +*.wis1 -text diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index e9b1ef611..bd77e07a8 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -6,6 +6,184 @@ wire format may change and such changes are recorded here. ## v0.4.0 — current +### The integer codecs' payloads pinned in place: `WIV1`, `WIH1`, `WIA1`, `WIS1` + +- **Pinned in place** (SPEC §5, *Payloads of the WAI-native integer codecs*, and the + new extension [integer-payloads](extensions/integer-payloads.md)): the payload of + `wai.video.int_motion` is `WIV1` with keyframe 0; of `wai.video.int_hyper`, `WIV1` + with keyframe 1, a nested `WIH1`; of `wai.neural.int_hyper`, `WIH1`; of + `wai.audio.int_codec`, `WIA1`, which carries the sample rate; of + `wai.splat.int_codec`, `WIS1`, which carries each attribute's presentation range. + These strings' rows named a decoder and no byte layout; three of them + (`wai.video.int_motion`, `wai.audio.int_codec`, `wai.splat.int_codec`) also named + the tool that wrote a JSON bitstream, which SPEC §5 called the bytes the codec + normally emits. So the pin changes what these strings carry. SPEC §8 gains + *Completing a registration*, which permits it for a WAI-native string whose row + fixed no byte layout and lists the envelopes it makes non-conforming; from this + revision their formats are fixed as SPEC §8 states. No new string is registered. + `codecs::PINNED_INT_PAYLOADS` is the one table of strings and magics. +- **The JSON is a converter input, never a payload.** The encode and export tools + still write JSON; `int_payload::{wiv1,wih1,wia1,wis1}_from_json` convert it and + state what it left implicit (an absent `mv_precision` is 1; the first frame carries + no motion vectors; a short motion-vector list is padded with `(0, 0)`; the audio + sample rate comes from the export's `audio.json`; a splat range must be exactly a + binary32 value; the JSON `afrac` must be the model's; the channel count must match + the medians, tables and offsets). *Envelopes this makes + non-conforming:* a `WAI1` or `WAI2` that carries anything but the pinned format + under these strings, a tool's JSON output among them. It still parses; a sink + refuses it at decode (a JSON form with `bad_magic`) and does not try the fallback. + Every emit path refuses to write it: `Wai::to_emit_bytes` and + `WaiMulti::to_emit_bytes` (so the C packers `wai_envelope_pack` and + `wai_envelope_pack_multi`, `WAI_ERR_INVALID_ARG`), `wai-web`'s `pack` and + `packMulti`, and `wai wrap`. They run `container::check_int_payload_framing`, which + parses the payload with the wire parser a sink runs and the capability's keyframe + kind, so it checks every parse rule of the extension's §2–§6 and leaves to the + decode only what needs a parameter set or the streams (§7); `wai inspect` reports + what it finds. A refused emit is the new `EmitError::IntPayload`. An encrypted + envelope's payload is ciphertext (SPEC §6), so the emit paths skip it and a sink + checks the plaintext after decrypting (SPEC §6.4); `wai inspect` says so. No + committed envelope or conformance corpus carried such a payload. The reference + dispatchers (`Wai::dispatch_verified`, `WaiMulti::pick`, and so the neural sink, + its C ABI and `wai-web`'s `dispatchCapability`) implement SPEC §6.4 step 2: they + open no scheme, so an envelope whose manifest declares an encryption other than + `"none"` is inert at them (`unsupported_encryption`; `container::Inert::Encrypted`, + `container::Skip::Encrypted`, `neural::DecodeError::Encrypted`), and a payload sent + in the clear under a claim of encryption is not decoded as plaintext. A `WAI2` + manifest's `security` member (now stated in SPEC §7.1) must be a security object to + be written (`EmitError::Security`, `container::manifest_v2_security`); one that is + not is read as encrypted. +- **One integer decode path.** `int_payload::decode_int` decodes the strings above + and `wai.neural.int_synth` and `wai.neural.int_mlicpp` from parameter files held in + memory, with no ONNX runtime and no filesystem, natively and for `wasm32`, through a + dispatchable capability only. A refusal is one code from a closed list + ([integer-payloads](extensions/integer-payloads.md) §8). `int_payload::resolve` + verifies a pin over sets held in memory and runs SPEC §3.1's check 5 for a pinned + decode and over the set an unpinned decode reads. The neural sink delegates every + integer capability to it: integer video, PCM and splat attributes come back as + `neural::Decoded::Int`, never converted; `ModelRegistry::builtin` advertises + `wai.video.int_motion`, which reads no set; the C ABI refuses an `Int` output. A + capability this build has no integer decoder for (`unsupported_capability`) moves + dispatch on to the fallback, as SPEC §4 treats a capability a sink cannot decode. + Check 5 covers every identity of the bytes a decode reads (SPEC §3.1): after a pin + verifies, it runs again over the listing of the members read and a one-file set's + file digest (`int_payload::read_identities`), so a set revoked under one identity + is not reached through a pin naming it by another; and a listing beside an + unpinned set counts only when it matches the files read. Outside the integer path, + at a sink that holds a key log, an unpinned decode reads what it can read once — + the registered file of a one-file set, or every file below the directory its + decoder reads from (`pin::read_dir_set`) — and check 5 runs over every digest a pin + over it could name: the file's, or the directory's derived listing + (`pin::listing_of_dir`) and each file's, and each listing beside them that matches. + The decode reads only those bytes, never the path again; where the key log holds + the capability's sets to jwp-receipts §7.9, only the part §7.9 authorised (the + whole, the files a granted listing names, or one granted file). So a revoked set + is refused with no listing beside it, a granted listing authorises the files it + names and nothing else, a file changed after the check is not read, and a file the + decode needs outside the set is `set_incomplete` (SPEC §4, *Module authorisation*). + `pin::PinAuthority` gains `requires_grant`, with a default. + *Breaking:* `neural::Decoded` gains the variant `Int` and is now `#[non_exhaustive]`, + so a `match` on it outside this crate needs a wildcard arm; `neural::ParamSource` + gains `UnpinnedRead` (an unpinned integer set read into memory once) and + `Checked` (what an unpinned decode at a sink with a key log reads, read once) and is + `#[non_exhaustive]` too; `int_payload::IntParams` is opaque, built by + `int_payload::resolve` (or `IntParams::none()`). +- **Checked decoders.** `int_video`, `int_audio` and `int_splat` decode a typed + stream that the binary and JSON forms both read into. Every size is capped at + `2^26` elements (a `WIV1` clip at `2^30` bytes and `2^20` frames, its motion-vector + field at `2^26` elements) the same on every target, every network's shapes are + planned before anything is allocated, every latent and layer is checked for + exactness, and a stream that runs out is refused at the first symbol it cannot + give, so a build with overflow checks and one without accept the same bytes. A + frame, a clip or a latent grows only as its symbols are decoded, to its declared + size and no further, so a short stream stops early; `int_video` holds frames as + RGB8. A stream that carries its symbols is decoded in full, and a few kilobytes can + code a clip at the cap, so the header bounds the work: a sink reads the declared + cost before decoding (`int_payload::declared_cost`, `container::declared_size`, the + C ABI's `wai_int_payload_declared`, `wai-web`'s `intPayloadCost`) and applies its own + size policy (integer-payloads §7, *Memory and work*). The cost counts the networks' + multiply-accumulates layer by layer, exactly (`DeclaredCost::macs`, a `u128`; + `macs` on `intPayloadCost`, a `BigInt`). The reference sink applies one: + `neural::ModelRegistry::with_cost_limit` refuses a payload whose declared cost + exceeds the limit before any symbol is decoded, with the new code `cost_over_limit` + (`int_payload::check_cost`, `neural::DecodeError::OverCostLimit`), and dispatch moves + on as for a capability the sink cannot decode (§8). `neural::declared_cost` reads the + cost over the set a registry resolved, pinned or not. The C ABI's + `wai_int_payload_declared` takes the set's path (or null for the header alone) and + gives the working bytes and the multiply-accumulates too (`wai_int_declared_t` + gains `working_bytes`, `macs_lo`, `macs_hi` and `cost_known`); its neural decode + entry points no longer decode integer video, PCM or splat attributes, which they + have no output kind for, and refuse them before decoding. A `WIA1`'s latent may be no + longer than its `t_out` needs (`geometry_mismatch`). The JSON reader + (`int_video::VideoJson`) caps every size before it pads a motion-vector list, and + reads a field that does not fit the target's `usize` as absent rather than + truncating it. In the browser, `decodeWiv1` keeps the decoder's frames + (`DecodedVideo.frame(i)`) instead of copying the clip into one buffer. +- **One refusal order.** The extension states the order of a decode's checks + exactly (§7): the set's members, the set against the payload, every network + planned with its caps, the shapes, then each stream symbol by symbol, a run-out + before the value it would give. Networks are planned layer by layer in 64-bit + arithmetic (`int_hyper_synth::plan`, the audio synthesis's plan), a size past 64 + bits being past the cap, so a 32-bit and a 64-bit sink refuse a malformed network + with the same code; a planned network's parameter lengths are checked before any + symbol is decoded, and a convolution's padded side is capped. A set's entropy + tables obey §2's length rule (3 to 4097 entries). The parser reads a table of one + per channel table by table. A `WIV1` keyframe's planned size is checked against the + clip's `h × w` before any of the keyframe's symbols is decoded, so + `int_payload::declared_cost` and the decode refuse the same bytes with the same + code; the Rust verifier checks that for every fixture. Fixtures pin each case. +- **One header for a one-frame clip.** A `WIV1` of one frame codes no motion + vector, so its `mv_precision` and `block` must be 1 (`field_range` otherwise); the + converters write them so. The extension's §2 says a payload's hash identifies the + payload, not its output (unused table entries, say, code the same clip in other + bytes), and that one output is identified by its decoded digest (a clip's + `clip_hash`). +- **Canonical streams.** After its last symbol, each rANS stream of these payloads + must have been read to its last word and end in the state `2^31`, where an encoder + starts (`rans::RansDecoder::is_canonical_end`); otherwise the payload is refused + with the new code `stream_noncanonical`. An unread word or another starting state + coded the same symbols in other bytes. Every committed stream, the JSON forms' + included, ends canonically. `int_hyper_synth` gains + `decode_conditional_checked`; `int_synth` refuses a side of `2^64` or more instead of + shifting by it. +- **Parameter sets.** SPEC §5 names the members the integer decoders read: + `g_s.whm`, `h_s.whm` and `entropy.json` for `wai.neural.int_hyper` and + `wai.video.int_hyper`. `wai.audio.int_codec` (`model.wam`) and + `wai.splat.int_codec` (`g_s.whm`) are registered as one file, which widens their + registrations (SPEC §8). `wai registry export` writes nothing new: no exported field + records a set's shape. A `WHM1` or `WAM1` network holds nothing after its last layer + (`IntModel::from_bytes` and `AudioModel::from_bytes` refuse trailing bytes; every + committed network has none), so one network has one file and one digest. +- **Receipts.** A `wai.video.int_motion` receipt identifies its `WIV1` payload + (`VideoReceipt::seal_decode_wiv1`, `verify_reconstruction_wiv1`), and + `wai_video_meter` seals over it. The entry points that read the JSON form are + deprecated; on a conforming JSON bitstream their output is unchanged, so every + receipt sealed over a committed one, and every pinned digest of one, still + verifies. They now read the JSON through `int_video::VideoJson` and the checked + decoder, so they refuse JSON the old decoder took: a stream that runs out (it + decoded on from the exhausted state) or does not end canonically, a table that + breaks the table rules, `q` outside 1..=65535, an `mv_precision` other than 1 or 2, + other than one motion-vector list per frame, vectors on the first frame or more + vectors than blocks, a vector outside `i16`, a zero size, and a size past the caps. jwp-receipts §2.1's object content hash moves to its own + module and feature (`jwp_hash`), which `receipt_claims` enables; + `claim_receipt::object_content_hash` re-exports it, and no corpus changed. +- **JSON readers that refuse.** `IntModel::try_from_json` and + `CondBitstream::try_from_value` read the conditional codec's debug JSON without + panicking; `wai-web`'s `decodeIntHyper` and `decodeIntHyperBin` read it through + them, `CondEntropy::try_from_value` and `decode_conditional_checked`, so an empty + model or bitstream, or sizes past the caps, are refused. `int_video::VideoJson` + refuses a frame whose motion vectors are not a list. +- **Browser.** `wai-web` decodes the four formats (`decodeWiv1`, `decodeWih1`, + `decodeWia1`, `decodeWis1`); the entry points that read the JSON forms are for + debugging and conversion. +- **Conformance.** The payloads sit beside their JSON sources in + `wai-rs/tests/vectors/`, written by `wai_int_payload_vectors`, with refusal fixtures + and an index in `int-payload-conformance/`; `int_payload_verify.py` checks them + independently in Python's standard library and decodes the `WIV1` keyframe-0 + payloads in full; `byte-exact-conformance` parses and decodes the `WIV1` and `WIH1` + payloads on every leg of its matrix. CI runs the new tests in both build profiles, + the generator in the corpus loop, the verifier, a `cargo check` of each integer + feature alone, and a `wasm32` check. + ### The registry as one table, candidate capabilities, the codec glossary under test, and the public text retrofit - **Candidate capabilities** (SPEC §5): a registered string whose payload format no diff --git a/wai/README.md b/wai/README.md index 667dfdd2c..aa0d59a36 100644 --- a/wai/README.md +++ b/wai/README.md @@ -111,6 +111,11 @@ hash: a deployer can require `intent=replicate` to dispatch only to these. | `wai.audio.int_codec` | audio — learned 1-D transform, integer synthesis → int16 PCM | byte-exact (hash-portable) | | `wai.splat.int_codec` | 3D — learned Gaussian-splat-attribute transform, integer synthesis | byte-exact (hash-portable) | +Their payloads are binary formats SPEC §5 pins — `WIV1` for the integer video +codecs, `WIH1`, `WIA1` and `WIS1` +([`extensions/integer-payloads.md`](extensions/integer-payloads.md)). The JSON the +encode and export tools write is converted to them, never carried. + ## Interactive & temporal content (extensions) Beyond the codec menu, WAI carries content that consumes input or moves diff --git a/wai/REGISTRY.md b/wai/REGISTRY.md index 20695f0e2..3f7f47bbd 100644 --- a/wai/REGISTRY.md +++ b/wai/REGISTRY.md @@ -49,7 +49,12 @@ Two further tables hold strings that look like capabilities and are not: ## Status - **registered** — the payload format (for a record or derivation, the - definition) is pinned. + definition) is pinned. The WAI-native integer codecs `wai.video.int_motion`, + `wai.video.int_hyper`, `wai.neural.int_hyper`, `wai.audio.int_codec` and + `wai.splat.int_codec` were registered with a decoder and no byte layout (the first, + the fourth and the fifth also with the tool that wrote their bitstreams); v0.4.0 + completes their registrations by pinning their formats (SPEC §8, *Completing a + registration*; SPEC §5, *Payloads of the WAI-native integer codecs*). - **candidate** — the string is registered but no revision has pinned its payload format. An encoder MUST NOT emit a candidate, and a sink MUST treat it as unsupported (SPEC §5, *Candidate capabilities*). The reference implementation diff --git a/wai/SPEC.md b/wai/SPEC.md index 04ce24cb2..582086957 100644 --- a/wai/SPEC.md +++ b/wai/SPEC.md @@ -249,6 +249,14 @@ order, and the first that fails is the reason reported: is offline or has no clock it trusts, a grant outside its window or for another capability — or the sink holds no grant for the set. The sink reports §7.9's check and reason with it. A sink that holds no key log skips this check. + Check 5 covers every identity of the bytes the decode reads, not only the + pin's: once checks 6 to 8 have found the set, the sink runs this check again + over each digest a pin over those bytes could name — the listing of the files + the decode reads, and, for a capability registered with a one-file set, that + file's SHA-256 — so a set refused under one identity is not reached through a + pin that names it by another (a set-form pin over a listing of one revoked file, + or a listing that adds a file the decode does not read). Authorisation needs + one of them to be held; the pin's own digest, which passed above, is one. 6. `prior_not_held`: the sink holds no parameter set for the capability whose digest, in the pin's form, equals `sha256`; a file the sink cannot read is not held. A sink that holds the set a well-formed `supersedes` names — a file when @@ -414,7 +422,12 @@ pin does not resolve. An envelope that pins no set is decoded against the set th sink holds for `cap`, and a sink that holds a key log runs check 5 over that set too, identified by each digest a pin over it could name: it does not decode `cap` for that envelope when any of them is revoked, or, for a capability it holds to -§7.9, when §7.9 holds the set under none of them. Either way +§7.9, when §7.9 holds the set under none of them. It then decodes from the bytes +check 5 ran over alone, as a pinned decode reads only what its pin verified (§3.1): +read once, so a file changed after the check is not what the decode reads, and, for +a capability it holds to §7.9, only the part §7.9 holds (the set, the files a held +listing names, or one held file); a file the decode needs outside them is +`set_incomplete`. Either way the algorithm above runs unchanged, so a sink MUST NOT report a file inert because of module authorisation while a fallback it can decode without that module remains, and an encoder emitting a capability whose decode needs a module MUST declare such a @@ -422,11 +435,16 @@ fallback (§3). A sink configured with no key log holds no module under §7.9, s runs no module obtained apart from an envelope, and it does not run check 5. The reference dispatcher in `wai-rs` runs no module obtained apart from an envelope; it runs check 5 when the sink is configured with a key log (`key_log::SinkKeyLog`, -feature `key_discovery`, given to `pin::verify_with` or to -`neural::ModelRegistry::with_key_log`, which also runs it before an unpinned -decode): the revocation part for every capability a pin applies to, and §7.9's -authorisation for the capabilities the sink holds to it. `wai-web` and the C ABI -take no key log. +feature `key_discovery`, given to `pin::verify_with`, to `int_payload::resolve` +(which runs it for a pinned decode and over the files an unpinned integer decode +reads) or to `neural::ModelRegistry::with_key_log`, which also runs it before an +unpinned decode, over the bytes that decode then reads, read once: an integer +decode's members, and otherwise the registered file of a one-file set, or every +file below the directory the decoder reads from, identified by the listing +`pin::listing_of_dir` derives, each file's digest and each listing beside them that +matches): the revocation part for +every capability a pin applies to, and §7.9's authorisation for the capabilities the +sink holds to it. `wai-web` and the C ABI take no key log. --- @@ -437,9 +455,12 @@ codec or a WAI-native class. The mandatory floor and the recommended set are royalty-free; every payload and companion capability carries a licence class (see the end of this section), and one that is not royalty-free is never a default. The "payload format" column is the -on-the-wire payload bytes; in every case it is the canonical bytes -that codec normally emits (a JXL file, an Opus stream, etc.) — WAI -does not wrap or re-frame them. +on-the-wire payload bytes. For a standard codec it is the canonical +bytes that codec normally emits (a JXL file, an Opus stream, etc.) — +WAI does not wrap or re-frame them. For a WAI-native class it is the +format the class's registration or extension defines; for the +WAI-native integer codecs, the format *Payloads of the WAI-native +integer codecs* (below) pins. The registry is also published in machine-readable form, generated from the reference implementation: [`registry/capabilities.json`](registry/capabilities.json) @@ -502,8 +523,10 @@ Every conforming sink SHOULD implement: Each names a model the sink MAY have installed. Payload is a compact conditioning representation specific to the capability (latent tokens, -prompt embeddings, etc.). The standard does NOT specify the byte -layout of these payloads — the capability owner does. +prompt embeddings, etc.). The standard does not specify the byte +layout of these payloads — the capability owner does — except where this +section pins one: the integer codecs' payloads are pinned below +(*Payloads of the WAI-native integer codecs*). Each row names the model and the operating point its own publication reports (bitrate, sample rate) or the property WAI's integer construction @@ -515,11 +538,15 @@ sink-supplied. A capability that decodes against a parameter set (§3.1) is registered with that set's shape. **One file:** `wai.neural.int_synth` (its model specification, `int_synth.json`); `wai.neural.int_mlicpp` (its `WMB1` bundle, read from a set as -`model.wmb`); and `wai.neural.encodec32`, `wai.neural.dac`, `wai.neural.mimi`, +`model.wmb`); `wai.audio.int_codec` (its `WAM1` synthesis, read from a set as +`model.wam`); `wai.splat.int_codec` (its `WHM1` synthesis, read from a set as +`g_s.whm`); and `wai.neural.encodec32`, `wai.neural.dac`, `wai.neural.mimi`, `wai.neural.wavtokenizer`, `wai.neural.snac` and `wai.neural.video_bmshj2018` (a self-contained ONNX model; one that keeps tensors in external files is several files). **Several files:** every other capability that decodes against a parameter -set. A later revision MAY register a capability of several files as one file when +set; `wai.neural.int_hyper` and `wai.video.int_hyper` read `g_s.whm`, `h_s.whm` and +`entropy.json` from theirs. `wai.audio.int_codec` and `wai.splat.int_codec` were +registered as several files until this revision named their one-file formats. A later revision MAY register a capability of several files as one file when it defines a one-file format that holds the whole set; that widens the registration and makes no conforming envelope non-conforming (§8). @@ -540,7 +567,7 @@ registration and makes no conforming envelope non-conforming (§8). | `wai.neural.bmshj2018` | bmshj2018-factorized + rANS (byte-exact CompressAI port) | about 0.26 bpp at 29 dB PSNR (quality 3) | CompressAI | | `wai.neural.mbt2018_mean` | mean-scale hyperprior (Gaussian-conditional entropy) | the conditional-entropy family (arXiv 1809.02736) | `neural/hyperprior.rs` | | `wai.neural.int_synth` | integer fixed-point synthesis (all-i64 transposed-conv, no float/ONNX) | bit-identical on every machine — soundly backs `intent = replicate` | `neural/int_synth.rs` (QAT weights sink-supplied) | -| `wai.neural.int_hyper` | integer fixed-point **conditional** codec — whole bitstream→image path integer (integer hyper-decoder σ → integer-σ entropy → integer synthesis) | the integer-entropy contract fully assembled: a PTQ of mbt2018_mean, bit-identical on every machine, whose integer decode CI holds within 1 dB of the same model's float decode on a pinned photograph | `int_hyper_synth.rs` (tables sink-supplied) | +| `wai.neural.int_hyper` | integer fixed-point **conditional** codec — whole bitstream→image path integer (integer hyper-decoder σ → integer-σ entropy → integer synthesis) | the integer-entropy contract fully assembled: a PTQ of mbt2018_mean, bit-identical on every machine, whose integer decode CI holds within 1 dB of the same model's float decode on a pinned photograph | payload `WIH1`; `int_hyper_synth.rs` (tables sink-supplied) | | `wai.neural.int_mlicpp` | integer fixed-point **MLIC++ entropy topology** — `K` channel slices of the latent, each decoded as a checkerboard anchor pass then a non-anchor pass, with an entropy network over concat[latent decoded so far, hyper] re-run before every pass to give σ‖μ; hyper-decoder, entropy network and synthesis all integer | the channel-autoregressive × checkerboard entropy model at **decode-equivalence**: the encoder and every decoder compute the same σ, so the same distribution for every element, at any thread count; bit-identical on every machine. The committed conformance vector has synthetic weights, so it demonstrates decode-equivalence and states no quality figure | `int_mlicpp.rs` (one-file model bundle sink-supplied; `tools/wai_int_mlicpp_export.py`) | | `wai.image.jpegai` | JPEG AI (ISO/IEC 6048-1:2025) codestream → RGB — integer entropy front end (mANS, with an integer scale-index derivation) + 16-bit-PTQ integer hyper-decoder, context-model and synthesis networks + dequantisation, latent refinement, post-filters (the eICCI CNN among them) and colour conversion in IEEE-754 float with a fixed evaluation order. Scope: one configuration — 256×256 4:4:4, `model_id` 0 (tools_0), `beta_displacement_log` 0, the high-operation-point synthesis, the eICCI model pair the weight bundle carries, and the other header conditions in [determinism-tiers](extensions/determinism-tiers.md) §5; both decode paths refuse any other codestream. Within it, a sink without the integer weight bundle, or in a non-default floating-point environment, takes a float ONNX decode instead. Only checkpoint constants are sink-supplied | **entropy-consistency** (registry class `NeuralEntropyExact`): the entropy decode, including the scale-index derivation, is integer on both decode paths, so the decoded symbols reproduce on every machine. The exact decoder also pins one reconstruction to RGB, reproducible across machines under the build and floating-point conditions in [determinism-tiers](extensions/determinism-tiers.md) §5–§6 (it checks the floating-point environment before decoding) — a property of that decoder, not of the capability, so `wai.image.jpegai` does not back `replicate` (§7). On one reference codestream the exact decoder stays within max\|Δ\| ≤ 2 of the reference decoder (recorded max\|Δ\| = 1 on ~4.4% of samples); evidence in [determinism-tiers](extensions/determinism-tiers.md) §6. **Downgraded in v0.4.0:** this row previously claimed decode-equivalence, and the registry classed the capability `StandardDefined` | `jpegai_decode.rs` (+ `jpegai_{bitstream,mans,pipeline,synthesis,icci}.rs`) | | `wai.generate.spectral` | deterministic integer GENERATIVE synthesis (`intent=create`) — prompt+seed → integer-LUT spectral composition, no float/ONNX | byte-identical on every machine, pinned to (model, prompt, seed, steps); the create-side of "conformance is a hash" (generation-equivalence) | `generate.rs` (small pinned model bundle) | @@ -559,8 +586,8 @@ registration and makes no conforming envelope non-conforming (§8). | `wai.neural.video_bmshj2018` | per-frame bmshj2018 | per-frame image coding; decodable in a browser on WebGPU | CompressAI | | `wai.neural.dcvc_rt` | DCVC-RT (2025), inter-frame with reference-feature propagation | inter-frame neural video; its paper reports real-time 1080p and 4K decode on a GPU | research code, `neural/video_dcvc.rs` | | `wai.neural.dcvc_fm` | DCVC-FM (feature modulation), wide single-model quality range (RGB + YUV) | a wide single-model quality range; shares the inter-frame route | research code, `neural/video_dcvc.rs` | -| `wai.video.int_motion` | DETERMINISTIC INTEGER inter-frame — I-frame + integer block motion-compensated P-frames + integer-quantized residual over rANS, no float/ONNX; motion is whole-pel or **half-pel** (`mv_precision`), the latter predicting at the half-grid by integer bilinear interpolation (`(a+b+1)>>1`, `(a+b+c+d+2)>>2`) so real footage — which pans a non-integer number of pixels per frame — costs fewer residual bits while staying byte-exact | byte-identical frame-by-frame on every machine (*frame-state-equivalence*, a per-frame portable hash); the temporal extension of the integer contract, where a float inter-frame decode desyncs down the GOP | `int_video.rs` (whole-pel `tools/wai_int_video_encode.py`; half-pel from real footage `tools/wai_int_video_real_encode.py`) | -| `wai.video.int_hyper` | NEURAL-KEYFRAME inter-frame — the I-frame is the LEARNED integer codec `int_hyper` decoded byte-exact, the P-frames are integer block-MC residuals on top | a learned keyframe you can put in a receipt + integer inter-frames; whole decode integer, byte-identical everywhere | `int_video::decode_pframes` ∘ `int_hyper_synth::decode_conditional` | +| `wai.video.int_motion` | DETERMINISTIC INTEGER inter-frame — I-frame + integer block motion-compensated P-frames + integer-quantized residual over rANS, no float/ONNX; motion is whole-pel or **half-pel** (`mv_precision`), the latter predicting at the half-grid by integer bilinear interpolation (`(a+b+1)>>1`, `(a+b+c+d+2)>>2`) so real footage — which pans a non-integer number of pixels per frame — costs fewer residual bits while staying byte-exact | byte-identical frame-by-frame on every machine (*frame-state-equivalence*, a per-frame portable hash); the temporal extension of the integer contract, where a float inter-frame decode desyncs down the GOP | payload `WIV1`, keyframe 0; `int_video.rs` (the encode tools write a JSON form a converter reads) | +| `wai.video.int_hyper` | NEURAL-KEYFRAME inter-frame — the I-frame is the LEARNED integer codec `int_hyper` decoded byte-exact, the P-frames are integer block-MC residuals on top | a learned keyframe you can put in a receipt + integer inter-frames; whole decode integer, byte-identical everywhere | payload `WIV1`, keyframe 1 (a nested `WIH1`); `int_video::decode_pframes_typed` ∘ `int_hyper_synth::decode_conditional_checked` | | `wai.video.manifest` | ATTESTED PER-SESSION MANIFEST — the OTT delivery layer, not a codec: a personalised HLS manifest (ads stitched at `EXT-X-DISCONTINUITY`, blackouts + geo enforced) rendered as a **pure function of the session *class*** `(content, entitlement, geo, blackout set, ad pod, window)` rather than the viewer, and sealed in an Ed25519 receipt that binds the JCP grant (capability, joule ceiling, `funds_ceiling` = ad budget), the composition, the stitched ads and the policy applied | **manifest-equivalence**: `manifest_hash == hash(render(segments))` and `session_class_hash` are recomputable by anyone holding the bytes, so the ads, blackouts and compose-work stop being dashboard claims; class-determinism means one generation serves every viewer of the class, and `uj_per_viewer` is the amortised energy that a 1:1-unique generator cannot report below its per-viewer cost. Only the measurement — `joules_micro`, its acquisition class and the class's evidence — is measured-attested: signed, and not re-measurable | `wai_video_manifest.rs` | | `wai.video.ssai` | ATTESTED SERVER-SIDE AD INSERTION — a **deterministic** ad decision (greedy by price within the avail duration and the funds remaining) plus a per-impression Ed25519 receipt binding the candidate pod, the manifest it belongs to, the served `creative_hash`es and the VAST quartile each reached | **spend-equivalence**: `budget_spent_micros` is recomputed from the *billable* impressions and rejected if it exceeds the grant's `funds_ceiling` — an overspent receipt fails verification; every creative and quartile is checkable, so a buyer and seller reconcile impressions from the same bytes instead of two logs that only a dashboard joins. Only the measurement — `joules_micro`, its acquisition class and the class's evidence — is measured-attested: signed, and not re-measurable | `wai_video_ssai.rs` | | `wai.video.keys` | ATTESTED CONTENT-KEY RELEASE — the DRM licensing decision as a record: every key release, and every refusal, is an Ed25519 receipt binding the KID, the DRM system, the entitlement demanded and the window enforced, with a BLAKE3 `key_commit` proving *which* key was released without carrying it | **decision-equivalence**: `verify()` re-runs the release policy over the receipt's own `(grant capability, entitlement, issued, expiry)` and rejects any misreported outcome, and enforces granted-iff-committed — so "keys went only to entitled, unexpired sessions" is checkable against the signed log, not asserted. Only the measurement — `joules_micro`, its acquisition class and the class's evidence — is measured-attested: signed, and not re-measurable | `wai_video_keys.rs` | @@ -657,12 +684,12 @@ standard. See the linked extension for each. | `wai.splat.spz` | SPZ (an open-source Gaussian-splat format) | asset, content-addressed; presentation only | | `wai.audio.scene` | WSCN scene (objects + trajectories) | mixdown-equivalence · [spatial-audio](extensions/spatial-audio.md) | | `wai.audio.score` | WSCR container (note events) | music as instructions; sink synthesizes, mixdown-equivalence · [score](extensions/score.md) | -| `wai.audio.int_codec` | LEARNED 1-D transform codec — integer transposed-conv synthesis + integer IGDN over a factorized-rANS latent (model + bitstream from `tools/wai_int_audio_export.py`) | **sample-equivalence**: int16 PCM byte-identical on every machine (a portable PCM hash); the audio analog of `wai.neural.int_hyper`, where a float learned-audio decode desyncs on ~1e-5 round-off. Pure i64, no float/ONNX (`int_audio.rs`) | +| `wai.audio.int_codec` | `WIA1` — a LEARNED 1-D transform codec: integer transposed-conv synthesis + integer IGDN over a factorized-rANS latent (`tools/wai_int_audio_export.py` writes the model and a JSON form a converter reads) | **sample-equivalence**: int16 PCM byte-identical on every machine (a portable PCM hash); the audio analog of `wai.neural.int_hyper`, where a float learned-audio decode desyncs on ~1e-5 round-off. Pure i64, no float/ONNX (`int_audio.rs`) | | `wai.haptic.signal` | WHAP envelope (intensity/sharpness keyframes) | sample-equivalence · [haptics](extensions/haptics.md) | | `wai.feed.state` | WWLD, sim `wai.world.sim.feed0` | replay-equivalence · [state-feeds](extensions/state-feeds.md) | | `wai.avatar.pose` | WWLD, sim `wai.world.sim.skeletal0` | pose-equivalence · [avatars](extensions/avatars.md) | | `wai.splat.4d` | W4DS container (per-Gaussian keyframed tracks) | frame-state-equivalence · [volumetric](extensions/volumetric.md) | -| `wai.splat.int_codec` | LEARNED Gaussian-splat ATTRIBUTE codec — a learned transform over the `[14,S,S]` attribute image (pos/scale/rot/color/opacity), integer synthesis (int_hyper machinery, 14 channels) over a factorized-rANS latent (`int_splat.rs`; model + bitstream from `tools/wai_int_splat_export.py`) | **splat-equivalence**: the decoded attribute tensor is byte-identical on every machine (a portable cloud hash) — the int_hyper of 3D, where a float splat-attribute decode desyncs. The float rasterizer stays sink-supplied presentation; the *attributes* are the contract. Pure i64, no float/ONNX | +| `wai.splat.int_codec` | `WIS1` — a LEARNED Gaussian-splat ATTRIBUTE codec: a learned transform over the `[14,S,S]` attribute image (pos/scale/rot/color/opacity), integer synthesis (int_hyper machinery, 14 channels) over a factorized-rANS latent (`int_splat.rs`; `tools/wai_int_splat_export.py` writes the model and a JSON form a converter reads) | **splat-equivalence**: the decoded attribute tensor is byte-identical on every machine (a portable cloud hash) — the int_hyper of 3D, where a float splat-attribute decode desyncs. The float rasterizer stays sink-supplied presentation; the *attributes* are the contract. Pure i64, no float/ONNX | | `wai.film.linear` | WFLM container (sealed world + camera track + frame schedule) | film-equivalence · [linear-film](extensions/linear-film.md) | | `wai.film.reel` | WREL container (sealed world + a sequence of shots = camera track + schedule each) | reel-equivalence (film-equivalence with cuts) · [linear-film](extensions/linear-film.md) | | `wai.ternary.tt` | WQTT container (balanced-ternary MPS cores — packed trits + integer per-column scales) | a **quantum-inspired** (tensor-network) transform ported from the ternary/MPS fabric with ONE change: the per-column scale is an **integer**, so the whole decode is `i64`/`i128` and the reconstruction is byte-identical on every machine — **reconstruction-equivalence** (a portable recon hash), receiptable. Scope: a **provenance** transform, measured at 13.3 dB SNR for 3.8× with six residual layers (`Lossy` tier; the extension's §5). Encoder-side float SVD only · [ternary-tt](extensions/ternary-tt.md) | @@ -681,6 +708,38 @@ grant's joule ceiling (gate ⊗ meter ⊗ receipt in one object). WAI owns the receipt; the sink's control system runs the hardware: [`extensions/quantum-ops.md`](extensions/quantum-ops.md). +### Payloads of the WAI-native integer codecs + +This revision pins a payload format to each WAI-native integer codec whose row named +its decoder and no byte layout. The rows of `wai.video.int_motion`, +`wai.audio.int_codec` and `wai.splat.int_codec` also named the tool that wrote a +bitstream, a JSON document, which this section's opening called the bytes the codec +normally emits; those of `wai.neural.int_hyper` and `wai.video.int_hyper` named their +decoders only. The pin completes these registrations as §8 permits (*Completing a +registration*). [integer-payloads](extensions/integer-payloads.md) §3–§6 defines +each layout and its decode, §7 the checks a sink makes, and §8 the closed list of +codes it refuses with. The rows above keep their headings. + +| capability | payload format | parameter set | +|---|---|---| +| `wai.video.int_motion` | `WIV1`, keyframe 0 | none | +| `wai.video.int_hyper` | `WIV1`, keyframe 1 (a nested `WIH1`) | several files: `g_s.whm`, `h_s.whm`, `entropy.json` | +| `wai.neural.int_hyper` | `WIH1` | several files: `g_s.whm`, `h_s.whm`, `entropy.json` | +| `wai.audio.int_codec` | `WIA1` | one file: `model.wam` | +| `wai.splat.int_codec` | `WIS1` | one file: `g_s.whm` | + +From this revision on, the format of each is fixed as §8 states. The JSON the encode +and export tools write is a converter input +([integer-payloads](extensions/integer-payloads.md) §10), never a payload: an encoder +MUST NOT carry it under these strings, and a sink refuses it (`bad_magic`). + +*Envelopes the pin makes non-conforming* (they still parse): a `WAI1` whose +`capability` or `fallback` is one of these strings and whose payload is not that +string's format, a tool's JSON form among them; and a `WAI2` with a component of one +of these strings whose bytes are not its format. A sink refuses such a payload when it +decodes it, and does not try the fallback +([integer-payloads](extensions/integer-payloads.md) §11). + ### Immersive audio, cinema, volumetric, haptics, HDR (registered, sink-supplied) The same "name the capability, the sink supplies the decoder" pattern extends @@ -999,7 +1058,9 @@ A confidentiality-aware sink MUST, before running §4: 2. Else, if the sink does not implement the named scheme, or holds no key for `security.kid`, the file is **inert at this sink** — return a clear "no key / unsupported encryption" error. It MUST NOT feed - ciphertext to a codec. + ciphertext to a codec. (The reference dispatchers in `wai-rs` implement + no scheme and report `unsupported_encryption`, for a `WAI2` envelope + against every entry.) 3. Else recover the plaintext payload via the named scheme (for `mls-sframe`: SFrame `open` under the member's base key), then run §4 on the recovered plaintext. @@ -1529,7 +1590,10 @@ by which capability happens to be installed. } ``` Each entry has its own `capability` (the dispatch key) and `kind` - (the codec id matching the bytes in the corresponding payload). An entry MAY + (the codec id matching the bytes in the corresponding payload). The manifest + MAY carry `security` (§6.1), which covers every payload: a sink applies §6.4 + before step 2 of the selection below, and an encoder writes only a security + object there. An entry MAY carry `prior`, the pin (§3.1) of the parameter set its payload was coded against. Two entries MAY name one capability with different pins: the same content coded against two parameter sets, such as a set and its successor: @@ -1592,7 +1656,8 @@ A v1.1-aware sink MUST: which step 2 never picks), `missing_capability`, `component_digest_mismatch`, a pin reason (§3.1), or, where a deployer policy restricts step 2 to the entries §7's `replicate` rule admits, the `replicate_*` reason the rule gives for an entry - it leaves out. + it leaves out; an envelope inert under §6.4 step 2 names that reason for every + entry. A sink MUST NOT decode one entry's payload against the parameter set another entry pins. The envelope's `intent` applies to the rendition the sink picks, and @@ -1765,7 +1830,18 @@ envelope that uses any string a later revision registers. whole envelope where an earlier reader ignored the member. - New capabilities (§5) MAY be added in any MINOR revision. Existing capabilities MUST NOT be redefined — once a capability string is - registered for a codec, its payload format is fixed forever. + registered for a codec, its payload format is fixed forever; for a registration + completed as the next item says, from the revision that completes it. +- **Completing a registration.** A WAI-native string whose §5 row named the decoder + that reads its payload and fixed no byte layout (naming, at most, a tool that wrote + it) has a registration with no payload format to hold. A MINOR revision MAY + complete such a registration by pinning a byte layout, which is not a + redefinition. Envelopes that carried another form under the string, a tool's + output among them, become non-conforming; they still parse, and the completing + revision lists them in §5. From that revision on, the string's payload format is + fixed as the item above states. §5, *Payloads of the WAI-native integer codecs*, + completes the registrations of `wai.video.int_motion`, `wai.video.int_hyper`, + `wai.neural.int_hyper`, `wai.audio.int_codec` and `wai.splat.int_codec` this way. - **Editions.** A registration that names an edition (or version) of its standard carries that edition. A registration that names none carries the edition of its named standard current on the date it was registered, and no diff --git a/wai/byte-exact-conformance/src/int_payload_golden.rs b/wai/byte-exact-conformance/src/int_payload_golden.rs new file mode 100644 index 000000000..817684bce --- /dev/null +++ b/wai/byte-exact-conformance/src/int_payload_golden.rs @@ -0,0 +1,118 @@ +//! Cross-architecture byte-equality for the binary payloads SPEC §5 pins +//! (`extensions/integer-payloads.md`): `WIV1` (`wai.video.int_motion` with keyframe 0, +//! `wai.video.int_hyper` with keyframe 1) and `WIH1` (`wai.neural.int_hyper`). +//! +//! The goldens next door decode the JSON forms the encode tools write. These parse the +//! committed binary payloads with the wire parser a sink runs (`int_payload/wire.rs`, +//! `#[path]`-included and self-contained) and decode them through the checked typed +//! decoders, so every leg of the matrix checks the normative parse as well as the +//! decode. Each must reproduce the same SHA-256 digests the JSON forms do, from the +//! vectors' `sha256.json` sidecars, and the payloads themselves are pinned by their +//! SHA-256. + +#![cfg(test)] + +use std::path::PathBuf; + +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::int_hyper_synth::{decode_conditional_checked, CondEntropy, CondStream, IntModel}; +use crate::int_payload_wire::{parse_wih1, parse_wiv1, Wih1, Wiv1}; +use crate::int_video::{decode_pframes_typed, decode_typed, VideoStream}; + +const INT_VIDEO_WIV1_SHA256: &str = "3bbf0a864d9528e73aff036d1a636893f839d3f6c5f31dde7824a59e6c116565"; +const VIDEO_REAL_WIV1_SHA256: &str = "efce5ada256cb3443a673ee0c10aa4e0a8ce4a372102cd918627e5eec746880f"; +const VIDEO_NK_WIV1_SHA256: &str = "d2c62da643328bfbdf9eab129283322e9077a491c5ddd269464577f51a958bc8"; +const TINY_WIH1_SHA256: &str = "6d8a195a5a138452292b55d70a575d1fbb4372fd8614b3b821b676e5090c3c17"; + +fn vectors() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../wai-rs/tests/vectors") +} +fn read(rel: &str) -> Vec { + let p = vectors().join(rel); + std::fs::read(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())) +} +fn jload(rel: &str) -> Value { + serde_json::from_slice(&read(rel)).unwrap_or_else(|e| panic!("parse {rel}: {e}")) +} +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn stream<'a>(v: &'a Wiv1<'a>) -> VideoStream<'a> { + VideoStream { + n_frames: v.n_frames as usize, h: v.h as usize, w: v.w as usize, block: v.block as usize, + q: v.q as i64, offset: v.offset, mv_precision: v.mv_precision, cdf: &v.cdf, mvs: &v.mvs, rans: v.rans, + } +} + +/// Every frame's SHA-256 and the clip's against the sidecar entry `key`. +fn assert_clip(frames: &[Vec], key: &str) -> String { + let side = jload("int_video/sha256.json"); + let want = side[key]["frames_sha256"].as_array().expect("frames_sha256"); + assert_eq!(want.len(), frames.len(), "{key}: frame count"); + let mut all = Vec::new(); + for (i, f) in frames.iter().enumerate() { + let d = Sha256::digest(f); + assert_eq!(hex(&d), want[i].as_str().unwrap(), "{key}: frame {i} diverged"); + all.extend_from_slice(&d); + } + let clip = hex(&Sha256::digest(&all)); + assert_eq!(clip, side[key]["clip_sha256"].as_str().unwrap(), "{key}: clip"); + clip +} + +/// The tiny conditional codec's keyframe, from its committed set (the compact `WHM1` +/// networks and the entropy tables). +fn keyframe(k: &Wih1<'_>) -> (Vec, usize, usize) { + let g_s = IntModel::from_bytes(&read("int_hyper/tiny_g_s.bin")).expect("g_s"); + let h_s = IntModel::from_bytes(&read("int_hyper/tiny_h_s.bin")).expect("h_s"); + let ent = CondEntropy::try_from_value(&jload("int_hyper/tiny_entropy.json")).expect("entropy tables"); + let s = CondStream { + m: k.m as usize, sh: k.sh as usize, sw: k.sw as usize, n_z: k.n_z as usize, zh: k.zh as usize, zw: k.zw as usize, + z_rans: k.z_rans, rans: k.rans, + }; + decode_conditional_checked(&g_s, &h_s, &ent, &s).expect("the keyframe decodes") +} + +/// `WIV1`, keyframe 0: the synthetic whole-pel clip and the half-pel real footage. +#[test] +fn wiv1_keyframe_0_payloads_are_byte_identical() { + for (bin, key, payload_sha, clip_sha) in [ + ("int_video/int_video_bitstream.wiv1", "int_video", INT_VIDEO_WIV1_SHA256, + "5afd6477d09525b7db4485027588ba3bc43de3d29c996eb57f2d3274bf8309f9"), + ("int_video/video_real_bitstream.wiv1", "video_real", VIDEO_REAL_WIV1_SHA256, + "e6724ca214694e012c3e2b49df4e116a0a7e150435a12d7b144560ce2d5c7bdf"), + ] { + let p = read(bin); + assert_eq!(hex(&Sha256::digest(&p)), payload_sha, "{key}: the committed payload"); + let v = parse_wiv1(&p, Some(0)).expect("the payload parses"); + let clip = decode_typed(&stream(&v)).expect("decode must succeed on every architecture"); + assert_eq!(assert_clip(&clip.frames, key), clip_sha); + } +} + +/// `WIV1`, keyframe 1: the nested `WIH1` keyframe, then integer P-frames on top. +#[test] +fn wiv1_keyframe_1_payload_is_byte_identical() { + let p = read("int_video/video_nk_bitstream.wiv1"); + assert_eq!(hex(&Sha256::digest(&p)), VIDEO_NK_WIV1_SHA256); + let v = parse_wiv1(&p, Some(1)).expect("the payload parses"); + let k = parse_wih1(v.wih1.expect("keyframe 1 nests a WIH1")).expect("the keyframe parses"); + let (frame0, h, w) = keyframe(&k); + assert_eq!((h, w), (v.h as usize, v.w as usize), "the keyframe is the clip's size"); + let clip = decode_pframes_typed(&frame0, &stream(&v)).expect("integer P-frames"); + assert_eq!(assert_clip(&clip.frames, "video_nk"), "4a81f0c94ccd938a7ca80df203303d5735d9dfcdfa150da155d9f2e4ca283e8f"); +} + +/// `WIH1` alone: the learned keyframe's image. +#[test] +fn wih1_payload_is_byte_identical() { + let p = read("int_hyper/tiny_bitstream.wih1"); + assert_eq!(hex(&Sha256::digest(&p)), TINY_WIH1_SHA256); + let (rgb, h, w) = keyframe(&parse_wih1(&p).expect("the payload parses")); + assert_eq!((h, w), (16, 16)); + assert_eq!(hex(&Sha256::digest(&rgb)), "dce7282d46f88f03d5dc0fd42f325756d310fe1fc982b6e1792a1fbfbeb482cf"); + assert_eq!(hex(&Sha256::digest(&rgb)), jload("int_hyper/sha256.json")["tiny"]["sha256"].as_str().unwrap()); +} diff --git a/wai/byte-exact-conformance/src/lib.rs b/wai/byte-exact-conformance/src/lib.rs index 4088a4701..b442bccc9 100644 --- a/wai/byte-exact-conformance/src/lib.rs +++ b/wai/byte-exact-conformance/src/lib.rs @@ -69,7 +69,14 @@ pub mod int_hyper_synth; #[path = "../../wai-rs/src/int_mlicpp.rs"] pub mod int_mlicpp; +// The wire parser of the integer payloads SPEC §5 pins (WIV1, WIH1, WIA1, WIS1). +// Self-contained — std's Vec only, nothing from the crate — so the matrix parses the +// committed binary payloads with the parser a sink runs. +#[path = "../../wai-rs/src/int_payload/wire.rs"] +pub mod int_payload_wire; + mod golden; +mod int_payload_golden; mod int_video_golden; mod neural_keyframe_golden; mod jpegai_golden; diff --git a/wai/extensions/integer-payloads.md b/wai/extensions/integer-payloads.md new file mode 100644 index 000000000..b3ebb8d63 --- /dev/null +++ b/wai/extensions/integer-payloads.md @@ -0,0 +1,559 @@ +# WAI Extension: Integer Payloads + +> Status: Draft. Pins the payload formats of the WAI-native integer codecs that +> [SPEC.md §5](../SPEC.md) registers without one (*Payloads of the WAI-native integer +> codecs*): `WIV1` for `wai.video.int_motion` and `wai.video.int_hyper`, `WIH1` for +> `wai.neural.int_hyper`, `WIA1` for `wai.audio.int_codec` and `WIS1` for +> `wai.splat.int_codec`. Defines each layout, its decode, the checks a sink makes and +> the closed list of refusal codes; names each capability's parameter-set members; +> and makes the JSON the encode and export tools write a converter input, never a +> payload. Reference impl: `wai-rs` — `int_payload::{wire, decode_int, resolve}`, +> `int_payload::{wiv1_from_json, wih1_from_json, wia1_from_json, wis1_from_json}`, +> `codecs::PINNED_INT_PAYLOADS`, `container::check_int_payload_framing`. +> Keywords **MUST**, **MUST NOT**, **SHOULD**, **MAY** are RFC 2119/8174. + +## 1. Scope + +| capability | payload | parameter set (§9) | +|---|---|---| +| `wai.video.int_motion` | `WIV1` with keyframe 0 (§3) | none | +| `wai.video.int_hyper` | `WIV1` with keyframe 1, a nested `WIH1` (§3) | several files: `entropy.json`, `g_s.whm`, `h_s.whm` | +| `wai.neural.int_hyper` | `WIH1` (§4) | several files: `entropy.json`, `g_s.whm`, `h_s.whm` | +| `wai.audio.int_codec` | `WIA1` (§5) | one file: `model.wam` | +| `wai.splat.int_codec` | `WIS1` (§6) | one file: `g_s.whm` | + +**Why this pin is permitted.** Until this revision, these capabilities' SPEC §5 rows +named the decoder that read their payload, and no byte layout. The rows of +`wai.video.int_motion`, `wai.audio.int_codec` and `wai.splat.int_codec` also named the +encode or export tool that wrote a bitstream, a JSON document; those of +`wai.neural.int_hyper` and `wai.video.int_hyper` named their decoders only. SPEC §5 +called every row's payload the bytes its codec normally emits, which for these rows +was the tools' JSON. So this revision changes what these strings carry. SPEC §8 +permits it under *Completing a registration*: a revision may complete the +registration of a WAI-native string whose row named a decoder and fixed no byte layout +by pinning one, and it lists the envelopes that become non-conforming (§11; SPEC §5). +SPEC §8's rule that a registered string's payload format is fixed forever applies to +these strings from this revision on. The encode and export tools still write JSON +that carries the same fields; it is a converter input (§10). + +`wai.neural.int_synth` and `wai.neural.int_mlicpp` keep the payload formats SPEC §5 +gives them. A sink decodes them and the five strings above (`wai.video.int_motion`, +`wai.video.int_hyper`, `wai.neural.int_hyper`, `wai.audio.int_codec`, +`wai.splat.int_codec`) through one path (`int_payload::decode_int`), and §8 names the +codes it reports for those two. + +## 2. Common rules + +**Encoding.** Every multi-byte field is little-endian. Offsets are in bytes. + +**Framing.** A payload starts with its four-byte magic and a version byte. This +revision defines version 1 for every format. Every reserved field is zero. Nothing +follows the last section. + +**rANS streams.** A stream is a `u32` length `L` followed by `L` bytes. `L` is a +multiple of 4 and at least 8. It is decoded as follows (precision 16, with a bypass +escape of 4-bit groups): + +- The state `x` starts as `(w1 << 32) | w0`, from the stream's first two `u32` + words. *Renormalising*: if `x < 2^31`, then `x = (x << 32) | w`, where `w` is the + next unread word; if no word is left, the stream has **run out** and `x` is left + as it is. +- `bits(n)`: `v = x mod 2^n`, `x = x >> n`, renormalise, return `v`. +- Decoding a symbol against a table `cdf` of length `n` with offset `o`: let + `cum = x mod 2^16` and `s` be the largest index with `cdf[s] ≤ cum` + (`0 ≤ s ≤ n − 2`); then `x = (cdf[s+1] − cdf[s]) · (x >> 16) + cum − cdf[s]`, and + renormalise. If `s < n − 2`, the value is `s`. If `s = n − 2` (the escape): let + `v = bits(4)` and `count = v`; then, at most 8 times, while `v = 15` and the stream + has not run out, `v = bits(4)` and `count = count + v`; then `count = min(count, 8)` + and `raw = Σ_{j < count} bits(4) << 4j`. The value is `−(raw >> 1) − 1` if `raw` + is odd and `(raw >> 1) + (n − 2)` if it is even, saturated to `i32`. The symbol + is `value + o`, saturated to `i32`. +- A decode stops at the first symbol during whose decode its stream runs out, and + refuses the payload (`stream_exhausted`): the value that symbol would give is not + used, and nothing after it is decoded. A stream that never runs out decodes every + symbol, so a conforming payload is unaffected. +- After the last symbol a decode reads from a stream, the stream ends **canonically**: + every word of it has been read, and the state is `2^31`. Otherwise the payload is + refused (`stream_noncanonical`). An encoder starts its state at `2^31` and writes + the words it flushes and no others, so its streams end this way. A word after the + last symbol's, or a state the encoder did not start from, codes the same symbols in + other bytes; the rule closes both, so neither can give one decode several content + hashes. A stream that codes no symbol (the residual stream of a one-frame `WIV1` + with keyframe 1) is the state `2^31` alone: the words `2^31` and `0`. + +**One clip, several payloads.** These rules fix how a payload is read, not the +only payload of a decoded output: tables that differ in entries the stream never +selects (a symbol the clip does not use, say), or a different escape layout, code +the same output in other bytes. A payload's hash therefore identifies the payload, +not what it decodes to. Where one output must have one identity, it is identified by +the digest of its decoded output: for a clip, its frame-state-equivalence digests +(SPEC §7, a video receipt's `clip_hash`); for the other formats, their capabilities' +equivalence digests. + +**CDF tables.** A table is `n` entries of `u32` with `3 ≤ n ≤ 4097`, `cdf[0] = 0`, +`cdf[n − 1] = 65536` and every entry greater than the one before. These rules are +what keep the symbol search and the state update above in range. + +**Order of the checks.** A parser checks, in this order, and refuses with the first +that fails: + +1. the magic: the capability's (§1), or else `format_capability_mismatch` if it is + another of the four magics, or `bad_magic`; +2. the version byte: present (`stream_length`), and 1 (`version`); +3. the fixed header: present (`stream_length`); +4. the reserved fields (`reserved_nonzero`); +5. the header's fields, in the order the format's section lists their rules + (`field_range`), then, for `WIV1`, the keyframe kind (`keyframe_capability_mismatch`); +6. the caps of §7 on the sizes the header declares (`too_large`), in the order the + format's section lists them; +7. each later section in order: present (`stream_length`), then its rules + (`field_range`, `cdf_malformed`). A section of one table per channel (`WIA1`, + `WIS1`) is read table by table: each table present, then checked, before the next; +8. nothing after the last section (`trailing`, or `nested_length` for `WIV1`'s + nested keyframe). + +A decode then makes the checks of §7, and refuses a stream that runs out or does not +end canonically (§2). Two conforming sinks refuse the same bytes with the same code. + +## 3. `WIV1` — integer inter-frame video + +| off | size | field | rule | +|---|---|---|---| +| 0 | 4 | magic `WIV1` | | +| 4 | 1 | version | `1` | +| 5 | 1 | `mv_precision` | `1` (whole-pel) or `2` (half-pel); `1` when `n_frames` is 1 | +| 6 | 1 | `keyframe` | `0` or `1` | +| 7 | 1 | reserved | `0` | +| 8 | 4 | `n_frames` | ≥ 1 | +| 12 | 4 | `h` | ≥ 1 | +| 16 | 4 | `w` | ≥ 1 | +| 20 | 4 | `block` | ≥ 1; `1` when `n_frames` is 1 | +| 24 | 4 | `q` | 1..=65535 | +| 28 | 4 | `offset` | `i32` | +| 32 | 2 | `cdf_len` | 3..=4097 | +| 34 | 2 | reserved | `0` | +| 36 | `cdf_len`·4 | the residual table | §2 | +| … | (`n_frames` − 1)·`nb`·4 | motion vectors | `i16 dy, i16 dx` | +| … | 4 + `L` | the residual stream | §2 | +| … | 4 + `K` | keyframe 1 only: `u32 K`, then a `WIH1` payload | `K` is exactly the bytes that remain | + +The header's field rules are checked in the order `mv_precision`, `keyframe`, +`n_frames`, `h` and `w`, `block`, `q`, `cdf_len`, then, for a one-frame clip, +`mv_precision` 1 and `block` 1: such a clip codes no motion vector, so neither field +would change its decode, and they are fixed so that it has one header. `keyframe` 0 is +`wai.video.int_motion`'s and 1 is `wai.video.int_hyper`'s; the other is +`keyframe_capability_mismatch`. The frame `h·w·3`, the clip `n_frames·h·w·3`, the frame +count `n_frames` and the motion-vector field `2·nb·(n_frames − 1)` are capped (§7), in +that order. + +**Motion vectors.** `nb = ⌈h/block⌉·⌈w/block⌉` and `nbx = ⌈w/block⌉`. The vectors +are frames 1 to `n_frames − 1` in order, each its block grid row-major: the vector of +pixel `(y, x)` in frame `f` is entry `(f − 1)·nb + (y div block)·nbx + (x div block)`, +in units of `1/mv_precision` pel. + +**Decode.** One decoder reads the residual stream for the whole clip, against the +residual table at `offset`. For each frame in order, for each pixel in raster order +`(y, x)`, for each channel `c` of R, G, B: decode the symbol `r`, and the sample is +`clamp(pred + r·q, 0, 255)`. Since `|r| < 2^31` and `q < 2^16`, every step fits `i64`. + +- Frame 0 with keyframe 0: `pred = 128`. +- Frame 0 with keyframe 1: the frame is the nested `WIH1`'s decode (§4), which reads + no symbol of the residual stream. It must be `h × w` (`geometry_mismatch`). +- Frame `f ≥ 1`: `pred` reads the previous **decoded** frame `R` with the pixel's + vector `(dy, dx)`, where `p(Y, X)` is `R` at row `clamp(Y, 0, h − 1)`, column + `clamp(X, 0, w − 1)`, channel `c`: + - whole-pel: `pred = p(y + dy, x + dx)`; + - half-pel: `iy = y + (dy >> 1)`, `fy = dy & 1`, `ix = x + (dx >> 1)`, + `fx = dx & 1` (arithmetic shift); then `p(iy, ix)` when both are 0, + `(p(iy, ix) + p(iy, ix + 1) + 1) >> 1` when only `fx` is 1, + `(p(iy, ix) + p(iy + 1, ix) + 1) >> 1` when only `fy` is 1, and + `(p(iy, ix) + p(iy, ix + 1) + p(iy + 1, ix) + p(iy + 1, ix + 1) + 2) >> 2` + when both are. + +The output is `n_frames` RGB8 frames of `h × w` (rows top to bottom, pixels left to +right, R, G, B bytes): the buffers of the capability's frame-state-equivalence digests +(SPEC §7 *Digests*). + +## 4. `WIH1` — the integer conditional image codec + +| off | size | field | rule | +|---|---|---|---| +| 0 | 4 | magic `WIH1` | | +| 4 | 1 | version | `1` | +| 5 | 3 | reserved | `0` | +| 8 | 4 | `m` | ≥ 1: latent channels | +| 12 | 4 | `sh` | ≥ 1: latent height | +| 16 | 4 | `sw` | ≥ 1: latent width | +| 20 | 4 | `n_z` | ≥ 1: hyper-latent channels | +| 24 | 4 | `zh` | ≥ 1: hyper-latent height | +| 28 | 4 | `zw` | ≥ 1: hyper-latent width | +| 32 | 4 + `Lz` | the hyper-latent stream | §2 | +| … | 4 + `Ly` | the latent stream | §2 | + +The caps (§7) apply to `n_z·zh·zw` and to `2·m·sh·sw`. + +**Decode,** against the set of §9 (`g_s.whm` and `h_s.whm`, two integer networks at +one fixed-point scale `2^afrac`; `entropy.json`, the tables): + +1. For each hyper-latent channel `k` and each of its `zh·zw` elements, decode a + symbol `s` from the hyper-latent stream against channel `k`'s table and offset; + the element is `s·2^afrac + median_k`. +2. `h_s` on the hyper-latent `[n_z, zh, zw]` gives `[2·m, sh, sw]`: its first `m` + channels are the scales `σ`, the rest the means `μ`. +3. For each latent element `p` in order (channel, then row, then column), its table + is `b`, where, with the scale edges `e_0 … e_{N−1}` and `σ' = max(σ_p, e_0)`, + `b = N − 1 − #{ i < N − 1 : σ' ≤ e_i }`. Decode a symbol `r` from the latent + stream against table `b` and its offset; the element is `r·2^afrac + μ_p`. +4. `g_s` on the latent `[m, sh, sw]` gives three channels; each value `v` is clamped + to `[0, 2^afrac]` and becomes `(v·255 + 2^(afrac−1)) >> afrac`, as RGB8. + +Every value is formed in `i64` and must fit it (§7). + +## 5. `WIA1` — the integer learned audio codec + +| off | size | field | rule | +|---|---|---|---| +| 0 | 4 | magic `WIA1` | | +| 4 | 1 | version | `1` | +| 5 | 1 | `afrac` | 1..=30; equal to the model's (§7) | +| 6 | 2 | reserved | `0` | +| 8 | 4 | `sr` | ≥ 1: the sample rate the PCM is presented at, in hertz | +| 12 | 4 | `n_ch` | ≥ 1: latent channels | +| 16 | 4 | `lat_len` | ≥ 1: latent length | +| 20 | 4 | `t_out` | ≥ 1: samples presented | +| 24 | 4 | `offset` | `i32`: the symbol offset of every channel's table | +| 28 | 2 | `cdf_len` | 3..=4097 | +| 30 | 2 | reserved | `0` | +| 32 | `n_ch`·8 | medians | `i64` each, `|median| ≤ 2^47` | +| … | `n_ch`·`cdf_len`·4 | one table per channel | §2 | +| … | 4 + `L` | the latent stream | §2 | + +The header's field rules are checked in the order `afrac`, then `sr`, `n_ch`, +`lat_len` and `t_out`, then `cdf_len`. The caps (§7) apply to `n_ch·lat_len` and to +`t_out`. + +**Decode,** against the set of §9 (`model.wam`, the integer synthesis): + +1. For each channel `k` and each of its `lat_len` elements, decode a symbol `s` + against channel `k`'s table at `offset`; the element is `s·2^afrac + median_k`. +2. The synthesis (one-dimensional integer transposed convolutions and integer + inverse GDN) on `[n_ch, lat_len]` gives one channel of `t ≥ t_out` samples. +3. The first `t_out` samples `v` become `(v·32767 + 2^(afrac−1)) >> afrac`, clamped + to `[−32768, 32767]`. + +The output is `t_out` samples of `i16` PCM, one channel, at `sr` hertz: the buffer of +the capability's sample-equivalence digest (signed 16-bit little-endian samples). + +## 6. `WIS1` — the integer learned Gaussian-splat attribute codec + +| off | size | field | rule | +|---|---|---|---| +| 0 | 4 | magic `WIS1` | | +| 4 | 1 | version | `1` | +| 5 | 1 | `afrac` | 1..=30; equal to the model's (§7) | +| 6 | 2 | reserved | `0` | +| 8 | 4 | `c` | ≥ 1: latent channels | +| 12 | 4 | `sh` | ≥ 1: latent height | +| 16 | 4 | `sw` | ≥ 1: latent width | +| 20 | 4 | `s` | ≥ 1: the attribute planes' side | +| 24 | 4 | `attr` | ≥ 1: attribute planes | +| 28 | 4 | `n_gaussians` | 1..=`s·s` | +| 32 | `attr`·8 | ranges | per plane, `f32 lo` then `f32 hi` (IEEE 754 binary32): finite, `lo ≤ hi` | +| … | `c`·2 | table lengths | `u16` each, 3..=4097 | +| … | `c`·4 | symbol offsets | `i32` each | +| … | `c`·8 | medians | `i64` each, `|median| ≤ 2^47` | +| … | the lengths' sum ·4 | one table per channel | §2 | +| … | 4 + `L` | the latent stream | §2 | + +The header's field rules are checked in the order `afrac`, the dimensions, +`n_gaussians`. The caps (§7) apply to `c·sh·sw` and to `attr·s·s`. + +**Decode,** against the set of §9 (`g_s.whm`, the integer synthesis): + +1. For each channel `k` and each of its `sh·sw` elements, decode a symbol `s` + against channel `k`'s table and offset; the element is `s·2^afrac + median_k`. +2. `g_s` on `[c, sh, sw]` gives `[attr, s, s]` (`geometry_mismatch` otherwise). +3. Each value `v` is clamped to `[0, 2^afrac]` and becomes + `(v·255 + 2^(afrac−1)) >> afrac`, a byte. + +The output, plane after plane, is the buffer of the capability's splat-equivalence +digest. The first `n_gaussians` cells of the planes, in raster order, hold one +Gaussian each. The ranges are presentation data: a sink presents a plane's byte `u` +at about `lo + (hi − lo)·u/255`. They are outside the digest, and this extension does +not fix presentation. + +## 7. Caps and the checks a decode makes + +**Caps, the same on every target.** No tensor a payload sizes may exceed `2^26` +elements: a frame `h·w·3`, a `WIV1` motion-vector field `2·nb·(n_frames − 1)`, a +latent, a hyper-latent, a network's output, an attribute tensor, a clip's `t_out`. A +`WIV1` clip may not exceed `2^30` bytes, nor `2^20` frames. + +**Planning the networks.** Before a decode allocates a latent, it plans every network +on the payload's shapes from the layers' geometry and parameter lengths alone. The +network's input must be within the cap (`too_large`); then, layer by layer in order: + +1. the layer fits the running channel count (`model_mismatch`): a convolution's output + channels, kernel and stride are non-zero, it has `c·out·k·k` weights (`c·out·k` for + the audio synthesis) and `out` multipliers and biases, and its requantisation shift + is under 128; an inverse GDN has `c·c` and `c` parameters and shifts under 128; a + leaky ReLU's shift is under 64; +2. every size the layer forms is within `2^26` (`too_large`): a convolution's padded + input side `n + 2·pad`, a transposed convolution's uncropped side + `(n − 1)·stride + k + opad` and uncropped plane, and the output; +3. the layer leaves an output (`model_mismatch`): a kernel no larger than the padded + input, a crop `2·pad` smaller than the uncropped side. + +Every size is formed in 64-bit arithmetic with overflow checked, and a size past 64 +bits is past the cap, so a 32-bit sink refuses exactly what a 64-bit one does, with +the same code; once a network is planned, every size it forms fits a 32-bit index. + +**Memory and work.** A payload's header declares its decode's output and the work +that output takes, and a sink can read both before it decodes anything. A stream does +not bound them: a table can make one symbol cost a small fraction of a bit, so a few +kilobytes of stream can code a clip at the `2^30`-byte cap, as any codec can describe +a large output in a small file. The bound is the header's, within the caps: + +- the output: a `WIV1` clip's `n_frames·h·w·3` bytes (one per sample), a `WIH1` + image's `3·h·w` bytes at its synthesis network's size, a `WIA1` clip's `2·t_out` + bytes (`i16` samples), a `WIS1` attribute tensor's `attr·s·s` bytes; +- the entropy decode: one step per symbol, `n_frames·h·w·3` for a keyframe-0 `WIV1` + (`h·w·3` fewer, plus its keyframe's, for keyframe 1), `n_z·zh·zw + m·sh·sw` for a + `WIH1`, `n_ch·lat_len` for a `WIA1`, `c·sh·sw` for a `WIS1`; +- the buffers: latents and network activations of 8 bytes per element (`i64`); while + a network layer runs, its input, its output and, for a transposed convolution, the + uncropped plane it accumulates into, each at most `2^26` elements (*Planning the + networks*); a `WIV1` decode's frames (one byte per sample) and its motion vectors + (4 bytes each); +- each network's work, its multiply-accumulates layer by layer: a convolution's + every output times `c·k·k` (every tap of its padded input), a transposed + convolution's every input times `out·k·k` (`out·k` for the audio synthesis), an + inverse GDN's every element times `c`; a `WIV1` with keyframe 0 runs no network. A + kernel may skip a zero term, so it performs at most this many. + +A `WIA1`'s latent is the one its `t_out` needs and no longer: the synthesis of +`lat_len` samples gives at least `t_out` samples and that of `lat_len − 1` fewer +(`geometry_mismatch`), so a payload cannot declare a large latent to present a few +samples. + +A decode refuses a stream that runs out at its first missing symbol (§2), so a short +stream stops early, but a stream that carries its symbols is decoded in full: a sink +that will not spend what a header declares refuses before decoding, by its own size +policy, applied to the declared cost (`int_payload::declared_cost` and +`int_payload::check_cost`, with the set; `neural::declared_cost` over the set a +registry resolved; the C ABI's `wai_int_payload_declared`, given the set's path; +`wai-web`'s `intPayloadCost`; the header alone through `container::declared_size`), +`cost_over_limit` (§8). The reference neural sink applies one when it is given one +(`neural::ModelRegistry::with_cost_limit`); its C ABI does not decode integer video, +PCM or splat attributes, which it has no output kind for. This extension fixes the +caps; a sink's policy may be stricter. + +**The parameter set must fit the payload** (`model_mismatch`): + +- every network parses (the `WHM1` and `WAM1` forms, §9), and the networks of one set + share one fixed-point scale `afrac` in 1..=30; +- a `WIA1` or `WIS1` payload's `afrac` is the model's; +- the entropy tables of `entropy.json` are consistent (a table, length and offset per + scale edge, at least one edge; a table, length, offset and median per hyper-latent + channel) and every table is a CDF of §2 over its length; a payload has no more + hyper-latent channels than the tables; +- each layer's parameters have the lengths the running channel count needs, and its + geometry leaves an output (*Planning the networks*); +- the audio synthesis gives one channel; the conditional codec's `g_s` gives three. + +**The shapes the payload declares must be the ones the decode gives** +(`geometry_mismatch`): `h_s`'s output and `[2·m, sh, sw]`; a `WIV1` keyframe and +`h × w`; the audio synthesis's length and `t_out` (it may be longer, by less than one +latent sample's worth: the synthesis of `lat_len − 1` samples gives fewer than +`t_out`); `g_s`'s output and `[attr, s, s]`. + +**The order of a decode's checks.** After the parse (§2), a decode checks in this +order and refuses with the first that fails: + +1. the set's members, in the order of §9's list, each present (`set_incomplete`) and + parsed (`model_mismatch`); +2. the parameter set against the payload (`model_mismatch`): one fixed-point scale, + the payload's `afrac`, the entropy tables, the hyper-latent channels; +3. every network planned, in decode order (`h_s` then `g_s`; the audio synthesis; + the splat `g_s`), each layer by the three rules above (`model_mismatch`, + `too_large`): the caps come before any shape is compared; +4. what the networks give against the payload: for `WIH1`, `h_s`'s shape + (`geometry_mismatch`), then `g_s`'s three channels (`model_mismatch`); for `WIV1` + with keyframe 1, its keyframe's as for `WIH1`, then the keyframe's size against + `h × w` (`geometry_mismatch`); for `WIA1`, + one channel (`model_mismatch`), then at least `t_out` samples, then a latent no + longer than that needs (both `geometry_mismatch`); for `WIS1`, `g_s`'s shape + (`geometry_mismatch`); +5. each stream in decode order (`WIH1`: the hyper-latent's, then, after `h_s` runs, + the latent's; `WIV1` with keyframe 1: the keyframe's streams, then the residual + stream), symbol by symbol: a + symbol that runs its stream out (`stream_exhausted`) before its value's range + (`field_range`); after its last symbol, its canonical end (`stream_noncanonical`); +6. each network as it runs, layer by layer (`field_range`, below). + +So a run-out is reported before the value it would have given, and a size past the +caps before a shape that disagrees. A sink that applies a cost limit (*Memory and +work*) compares the declared cost with it after step 4, once everything the cost reads +has been checked, and before any symbol is decoded (`cost_over_limit`, §8). + +**Exactness.** A latent rebuilt as `s·2^afrac + base` must fit `i64`; before each +network layer runs, the decode checks on the values present that no product, partial +sum, requantised value or inverse-GDN term can leave the type its arithmetic is +formed in. A stream whose values fail either check is refused (`field_range`). So a +decode with overflow checks and one without compute the same integers, and neither +panics. + +## 8. Refusal codes + +The list is closed: a sink reports exactly one of these, the first that applies in the +order of §2 and §7. + +A refusal of the payload or of the parameter set against it, `bad_magic` through +`model_mismatch` in the table below (and `payload_malformed`), is final: the sink does +not decode the payload by another path, and SPEC §4's fallback is not tried, since it +would decode the same bytes. The other codes say the sink cannot decode through the +capability at all, and route as SPEC §4 routes such a capability: + +- `set_incomplete` is a pin that did not resolve (SPEC §3.1): a `WAI1` continues at + SPEC §4 step 4 (or reports it at step 6), and a `WAI2` moves on to its next entry + (SPEC §7.1); +- `not_dispatchable` and `unsupported_capability` name a capability the sink does not + decode: SPEC §4 does not take step 3 (or 5) with it, so a `WAI1` continues at step 4 + and a `WAI2` at its next entry, as for a capability the sink does not advertise; +- `cost_over_limit` is the sink's own policy, not a fault of the payload: a payload + whose declared cost exceeds the limit the sink applies (§7, *Memory and work*) is + refused before any symbol is decoded, and routes as `unsupported_capability` does, + so a `WAI1` continues at step 4 and a `WAI2` at its next entry. + +| code | when | +|---|---| +| `bad_magic` | the payload does not start with any of the four magics: a tool's JSON form lands here | +| `format_capability_mismatch` | the payload is another of the four formats than the capability's | +| `keyframe_capability_mismatch` | a `WIV1` keyframe kind that is not the capability's | +| `version` | a version this revision does not define | +| `reserved_nonzero` | a reserved field that is not zero | +| `field_range` | a field of any section outside its range (a header field, a `WIS1` range or table length, a median); or a value the stream decodes that leaves the exact integer range (§7) | +| `too_large` | a size past the caps of §7 | +| `cdf_malformed` | a table that breaks the rules of §2 | +| `stream_length` | the payload ends before a field or section it declares; or a stream length that is not a multiple of 4, or is under 8 | +| `nested_length` | a `WIV1` keyframe length `K` that is not the rest of the payload | +| `trailing` | bytes after the last section | +| `stream_exhausted` | a stream that ran out before its last symbol | +| `stream_noncanonical` | a stream that does not end canonically after its last symbol: a word left unread, or a state other than `2^31` (§2) | +| `geometry_mismatch` | a shape the payload declares that the decode does not give (§7) | +| `model_mismatch` | a parameter set that does not fit the payload (§7) | +| `set_incomplete` | a parameter set without a member the decode reads (§9); a pinned decode reports SPEC §3.1's `set_incomplete` | +| `not_dispatchable` | a capability SPEC §4 does not dispatch | +| `unsupported_capability` | a capability the sink has no integer decoder for | +| `payload_malformed` | a `wai.neural.int_synth` or `wai.neural.int_mlicpp` payload its decoder refuses | +| `cost_over_limit` | a payload whose declared cost (§7) exceeds the limit the sink applies before decoding | + +## 9. Parameter sets + +A capability that decodes against a parameter set reads these members, by the names a +set's listing gives them ([prior-carriage](prior-carriage.md) §7): + +| capability | shape | members | +|---|---|---| +| `wai.neural.int_hyper`, `wai.video.int_hyper` | several files | `g_s.whm` and `h_s.whm` (networks in `WHM1`), `entropy.json` (the tables) | +| `wai.audio.int_codec` | one file | `model.wam` (the synthesis in `WAM1`) | +| `wai.splat.int_codec` | one file | `g_s.whm` (the synthesis in `WHM1`) | +| `wai.video.int_motion` | none | — | + +A set of several files is pinned through its listing (`wai.prior.set`); a one-file set +MAY be pinned form-less, by its file's SHA-256 (SPEC §3.1). `wai.audio.int_codec` and +`wai.splat.int_codec` were registered with sets of several files, because no one-file +format had been named for them; this revision registers each as one file, the file its +decoder reads, which holds every parameter its output depends on. That widens their +registrations, as SPEC §8 allows a MINOR revision to; a set pinned through its listing +still verifies. + +**`WHM1`** (`int_hyper_synth::IntModel::from_bytes`): magic, then `afrac`, `gf`, +`shift` and the layer count (`u32` each), then per layer a tag byte: 0 a transposed +convolution and 1 a convolution (input and output channels, kernel, stride, padding and, +for 0, output padding as `u32`; weights as `i16`; per-output-channel multipliers and +biases as `i64`), 2 an inverse GDN (channels as `u32`; `gamma` as `i32`, `beta` as +`i64`), 3 a ReLU, 4 a leaky ReLU (slope `i64`, its shift `u32`). **`WAM1`** +(`int_audio::AudioModel::from_bytes`): the same header, then layers of tag 0, a +one-dimensional transposed convolution, and tag 2, an inverse GDN. In both forms +nothing follows the last layer: a file with bytes after it is refused +(`model_mismatch`), so one network has one file and one digest. The layers' integer +arithmetic is the reference decoders' (`int_hyper_synth.rs`, `int_audio.rs`, +`int_transform.rs`), which the conformance vectors pin. + +## 10. The tools' JSON forms + +The encode and export tools under `tools/` write a JSON bitstream. It is a converter input and a +debugging aid, never a payload. An encoder MUST NOT carry it as the payload of any +capability of §1, and a sink refuses it (`bad_magic`). The converters +(`int_payload::{wiv1,wih1,wia1,wis1}_from_json`, and the independent +`int-payload-conformance/int_payload_verify.py`) make explicit what the JSON leaves +implicit: + +- `WIV1`: an absent `mv_precision` is 1, and only 1 and 2 are converted; a one-frame + clip is written with `mv_precision` 1 and `block` 1, whatever the JSON states; `mvs` has one + list per frame, and the first is empty; a frame's list shorter than its block grid is + padded with `(0, 0)`, the vector the JSON decoder read for a missing block; a list + longer than the grid, or a vector outside `i16`, is refused; `rmax` is not carried. + The keyframe of `wai.video.int_hyper` is the `WIH1` converted from its own JSON. +- `WIH1`: the fields of `cond_bitstream.json`, the streams as they are. +- `WIA1`: `sr` is the export's `sample_rate` (in `audio.json`, which the bitstream + does not carry); `lo` is the symbol offset; every channel's table must have the first + table's length; `afrac` is carried and must be the model's. +- `WIS1`: the ranges `lo` and `hi` must each be a binary32 value, carried exactly (a + JSON integer included: `2^60 + 1` is refused, not rounded to `2^60`); + `afrac` is carried and must be the model's. + +A payload and the JSON it is converted from decode to the same bytes; the conformance +vectors check that for every committed JSON bitstream. + +## 11. Envelopes this revision makes non-conforming + +This revision completes these strings' registrations with a payload format they did +not have (SPEC §8, *Completing a registration*). It makes non-conforming, and they +still parse: + +- a `WAI1` envelope whose `capability` or `fallback` is a string of §1 and whose + payload is not that string's format: a tool's JSON form among them; +- a `WAI2` envelope with a component of a string of §1 whose bytes are not that + string's format. + +A sink refuses such a payload when it decodes it (§8), with no fallback attempt. An +encoder MUST NOT write one; the reference implementation's emit paths (`wai wrap`, the +C packers and the browser writers) parse the payload as a sink does and refuse to. An +encrypted envelope's payload (SPEC §6) is ciphertext until a sink decrypts it, so the +emit paths do not check it; the sink checks the plaintext when it decodes it (SPEC +§6.4). +No committed envelope or conformance corpus carried such a payload when this revision +was made. + +## 12. Reference implementation and conformance + +- `int_payload::wire` parses and writes the four formats, with no dependency on the + rest of the crate; `int_payload::decode_int` decodes every integer capability from + parameter files held in memory, with no ONNX runtime and no filesystem, for native + and `wasm32` sinks; `int_payload::resolve` verifies a pin over sets held in memory + (`pin::verify_bytes_with`) and runs SPEC §3.1's check 5, for a pinned decode and over + the files an unpinned decode reads (a listing the set carries must match them, and + the digests include the listing of the files read); it is the only way to build the + parameters a decode reads (`IntParams`). The neural sink (`neural::decode_payload`) + delegates every integer capability to `decode_int`, and its registry reads an + unpinned integer set once and runs check 5 over those bytes. +- `codecs::PINNED_INT_PAYLOADS` is the one table of strings and magics; + `container::check_int_payload_framing` parses a payload with `int_payload::wire` and + the capability's keyframe kind, which needs no integer decoder and no feature, so + every emit path runs it: `Wai::to_emit_bytes` and `WaiMulti::to_emit_bytes` (and so + the C packers), `wai-web`'s `pack` and `packMulti`, and `wai wrap`; `wai inspect` + reports it. +- `video_receipt::VideoReceipt::seal_decode_wiv1` seals a `wai.video.int_motion` + receipt whose identity is BLAKE3 over the `WIV1` payload. +- `wai-web` decodes the four formats in the browser (`decodeWiv1`, `decodeWih1`, + `decodeWia1`, `decodeWis1`). +- Conformance: the payloads beside their JSON sources in `wai-rs/tests/vectors/`, + and the refusal fixtures and index in + [`int-payload-conformance/`](../int-payload-conformance/), written by + `wai_int_payload_vectors gen` and checked by `wai_int_payload_vectors verify`; the + same corpus checked by `int_payload_verify.py`, which shares no code with wai-rs and + decodes the `WIV1` keyframe-0 payloads in full; and the cross-architecture goldens of + `byte-exact-conformance`, which parse and decode the `WIV1` and `WIH1` payloads on + every platform of its matrix. diff --git a/wai/extensions/jwp-receipts.md b/wai/extensions/jwp-receipts.md index 76e9d2c97..5dd2439e5 100644 --- a/wai/extensions/jwp-receipts.md +++ b/wai/extensions/jwp-receipts.md @@ -2602,10 +2602,14 @@ left the file inert because no fallback was declared. `key_log::module_check` in `wai-rs` implements these checks. The reference dispatcher runs SPEC §3.1's check 5 when the sink is configured with a key log (`key_log::SinkKeyLog`, given to -`pin::verify_with` or to `neural::ModelRegistry::with_key_log`, which -also runs it before an unpinned decode): the revocation lists for every -capability a pin applies to, and these checks for the capabilities a sink -holds to this section. A sink configured with no key log runs neither. The reference dispatcher runs no module obtained +`pin::verify_with`, to `int_payload::resolve`, which runs it for a pinned +decode and over the files an unpinned integer decode reads, or to +`neural::ModelRegistry::with_key_log`, which also runs it before an +unpinned decode, over the bytes that decode then reads, read once, by +every digest a pin over them could name): the revocation +lists for every capability a pin applies to, and these checks for the +capabilities a sink holds to this section. A sink configured with no key +log runs neither. The reference dispatcher runs no module obtained apart from an envelope, and `wai-web` and the C ABI take no key log ([SPEC](../SPEC.md) §4). diff --git a/wai/extensions/prior-carriage.md b/wai/extensions/prior-carriage.md index a89f0d009..abe3d98d8 100644 --- a/wai/extensions/prior-carriage.md +++ b/wai/extensions/prior-carriage.md @@ -287,7 +287,10 @@ WAI's binding sits beside the standard's, never in place of it. decode through. - The decoders read parameters only through `neural::ParamSource`, which hands a pinned decode the verified bytes; ONNX sessions are built from those bytes, and - a file the verified set lacks is refused with `set_incomplete`. + a file the verified set lacks is refused with `set_incomplete`. The integer + capabilities decode through `int_payload::decode_int`, which reads its set members + from memory; `int_payload::resolve` verifies a pin over sets held in memory and + runs check 5, pinned or unpinned ([integer-payloads](integer-payloads.md) §12). `wai.neural.bmshj2018` caches its CDF tables by content digest. - The C ABI (`ffi`) returns each manifest as its original bytes, carries a `WAI2` rendition's identity and pin in the rendition table, and reports a refused pin @@ -311,6 +314,9 @@ file the set lacks is refused with `set_incomplete`. |---|---|---| | `wai.neural.int_mlicpp` | one file | `model.wmb` | | `wai.neural.int_synth` | one file | `int_synth.json` | +| `wai.audio.int_codec` | one file | `model.wam` | +| `wai.splat.int_codec` | one file | `g_s.whm` | +| `wai.neural.int_hyper`, `wai.video.int_hyper` | several | `g_s.whm`, `h_s.whm`, `entropy.json` | | `wai.neural.encodec32`, `.dac`, `.mimi`, `.wavtokenizer`, `.snac`, `.video_bmshj2018` | one file | `decoder.onnx` | | `wai.neural.bmshj2018` | several | `decoder.onnx`, `cdfs_q.json` | | `wai.neural.mbt2018_mean` | several | `hyper_cdfs_q.json`, `hyper_synthesis.onnx`, `gauss_cdfs_q.json`, `synthesis.onnx` | @@ -319,7 +325,7 @@ file the set lacks is refused with `set_incomplete`. | `wai.neural.glc` | several | `hyper_cdfs_q.json`, `hyper_entropy.onnx`, `context_entropy.onnx`, `synthesis.onnx` | | `wai.neural.dcvc_rt`, `.dcvc_fm` | several | `hyper_cdfs_q.json`, `hyper_synthesis.onnx`, `gauss_cdfs_q.json`, `temporal_decoder.onnx` | | `wai.image.jpegai` | several | exact path: `norm.json`, `weights/{hs,hyp,synth}_{y,uv}.json`, `weights/mcm{0..3}_y.json`, `weights/icci.json`. Float path: `hyper_scale_decoder_int_weights.json`, `model_constants.json`, `jpegai_normative_tables.json`, `me-tANS/bounds.csv`, `jpegai_factorized_cdfs.json`, `jpegai_scaler_vec.json`, `jpegai_dequant_consts.json`, `model/CCS_SGMM/tools_0/model_{y,uv}/common_modules/hyper_decoder.onnx`, `model/CCS_SGMM/tools_0/model_y/common_modules/MCM/stage{0..3}.onnx`, `model16/CCS_SGMM/tools_0/model_{y,uv}/synthesis16.onnx`, `e2e/pfchain/icci_{y_m6,uv_m3}.onnx`. A set holding only the exact path's files decodes through that path; where that path refuses (a non-default floating-point environment), the float path needs files the set lacks, and the decode is refused with `set_incomplete` instead of running on unverified files. | -| `wai.neural.int_hyper`, `wai.video.int_hyper`, `wai.audio.int_codec`, `wai.splat.int_codec`, `wai.generate.spectral` | several, until an envelope decode path registers its files | — (no envelope decode path in the reference sink) | +| `wai.generate.spectral` | several, until an envelope decode path registers its files | — (no envelope decode path in the reference sink) | ## 8. Sources diff --git a/wai/int-payload-conformance/.gitattributes b/wai/int-payload-conformance/.gitattributes new file mode 100644 index 000000000..eff1df73a --- /dev/null +++ b/wai/int-payload-conformance/.gitattributes @@ -0,0 +1,2 @@ +# Hashed files: a line-ending conversion on checkout would change every digest. +* -text diff --git a/wai/int-payload-conformance/README.md b/wai/int-payload-conformance/README.md new file mode 100644 index 000000000..868405d07 --- /dev/null +++ b/wai/int-payload-conformance/README.md @@ -0,0 +1,80 @@ +# int-payload-conformance — the integer payloads + +Vectors for the payload formats SPEC §5 pins to the WAI-native integer codecs +([`extensions/integer-payloads.md`](../extensions/integer-payloads.md)): `WIV1` for +`wai.video.int_motion` (keyframe 0) and `wai.video.int_hyper` (keyframe 1), `WIH1` +for `wai.neural.int_hyper`, `WIA1` for `wai.audio.int_codec` and `WIS1` for +`wai.splat.int_codec`. + +**Code-equivalence.** A conforming sink decodes every payload to the output digests +its vector states, and refuses every fixture with the code it states, from the +extension's closed list (§8). + +- `vectors.json` — the index. For each payload: its capability, its path, its + SHA-256, the JSON it is converted from, the parameter set it decodes against (each + member's name in the set, and the committed file that holds it), and the + `sha256.json` sidecar that states its decoded output's SHA-256. For each refusal + fixture: its capability, its path, its code, and whether parsing alone refuses it + (`parse`) or the decode does (`decode`). Paths are relative to `wai/`. +- The payloads sit beside their JSON sources under + [`../wai-rs/tests/vectors/`](../wai-rs/tests/vectors/): `int_video/*.wiv1`, + `int_hyper/tiny_bitstream.wih1`, `int_audio/audio_bitstream.wia1` and + `int_splat/splat.wis1`. Their sidecars are the ones the JSON forms' decodes are + checked against: a payload and its JSON source decode to the same bytes. +- `refusals/*.bin` — each a committed payload changed in one place (a byte, a + field, a truncation, a word appended to a stream), so that it is refused with one + code, or, for `wiv1_end_state` and `wiv1_one_frame_end_state`, a small clip written + to end its stream in a state other than `2^31`. The fixtures named `*_order_*` + break two rules at once and pin which one a sink reports (the extension's §2 and + §7), and so do the `wiv1_geometry_flip_*` fixtures: `wiv1_geometry` with one byte + of its keyframe's last stream words changed, refused for the keyframe's size before + that stream is read. The fixtures for `bad_magic` are the JSON sources themselves: a tool's JSON + output is a converter input, never a payload. +- `sets/` — a parameter-set member a fixture decodes against in place of a committed + one: `entropy_median_max.json` is the tiny conditional codec's tables with one + median at `i64::MAX`. + +Files are compared as bytes; `.gitattributes` turns off line-ending conversion. + +**Generation.** Every file here, and every payload under `tests/vectors/`, is +written by one generator from the committed JSON alone, so it can be rebuilt and +compared: + + cd wai/wai-rs + cargo run --no-default-features --features video_int,audio_int,splat_int \ + --bin wai_int_payload_vectors -- gen ../int-payload-conformance + +**Verifiers.** + +- `wai_int_payload_vectors verify ../int-payload-conformance` (wai-rs) regenerates + every file and requires it to equal the committed one byte for byte, decodes every + payload through `int_payload::decode_int` to its sidecar's SHA-256, and requires + every fixture to be refused with its code at its layer, and by + `int_payload::declared_cost` with the same code unless the decode's refusal comes + from a stream or a network run. `tests/int_payload.rs` runs + the fixtures again in both build profiles, refuses every strict prefix of every + payload, and runs a sweep of single-byte changes in which nothing may panic; + `tests/int_payload_preflight.rs` checks that a header past the caps or past what the + parameter set can fit is refused before anything sized from it is allocated, and + that a payload whose streams cannot carry what its header declares is refused at + its first exhausted symbol, having allocated only what it decoded. +- `int_payload_verify.py`, Python standard library only and sharing no code with + wai-rs, written from the extension: it parses every payload with the extension's + rules and check order, converts every JSON source independently and requires the + committed bytes, decodes the `WIV1` payloads with keyframe 0 in full (the rANS + decoder with its bypass escape, its stop at the first exhausted symbol and its + canonical end, whole-pel and half-pel motion compensation) to their sidecars' + SHA-256, and checks every fixture: a `parse` fixture's code, and that a `decode` + fixture parses (a `WIV1` keyframe-0 one is decoded and must be refused with its + code). + + python3 int-payload-conformance/int_payload_verify.py int-payload-conformance + +**Not decoded independently.** The Python verifier does not decode `WIH1`, `WIV1` +with keyframe 1 (its keyframe is a `WIH1`), `WIA1` or `WIS1`: their decodes run +integer networks it does not implement. Their outputs are checked by wai-rs against +the sidecars above, and the `WIV1` and `WIH1` payloads also by the +cross-architecture goldens in +[`../byte-exact-conformance/`](../byte-exact-conformance/). For their `decode` +fixtures the Python verifier checks that they parse; their codes are checked by +wai-rs. diff --git a/wai/int-payload-conformance/int_payload_verify.py b/wai/int-payload-conformance/int_payload_verify.py new file mode 100644 index 000000000..8705e6316 --- /dev/null +++ b/wai/int-payload-conformance/int_payload_verify.py @@ -0,0 +1,544 @@ +#!/usr/bin/env python3 +"""Independent verifier for the integer payload corpus (extensions/integer-payloads.md). + +Python standard library only; it shares no code with wai-rs. Written from the +extension's text, it: + + - parses every committed payload (WIV1, WIH1, WIA1, WIS1) with the rules and in the + check order of the extension's §2-§6, and requires each to parse; + - converts each payload's JSON source independently (§10) and requires the result to + equal the committed payload byte for byte, and the payload's SHA-256 to be the one + the index states; + - decodes the WIV1 payloads with keyframe 0 in full - the rANS decoder with its + bypass escape, its stop at the first exhausted symbol and its canonical end, + whole-pel and half-pel motion compensation - and requires every frame's SHA-256 + and the clip's to be the ones the vector's sha256.json states; + - parses every refusal fixture and requires a parse-layer fixture to be refused with + its code, and a decode-layer fixture to parse; a decode-layer fixture of WIV1 + keyframe 0 is decoded and must be refused with its code (stream_exhausted, + stream_noncanonical). + +Not decoded independently: WIH1, WIV1 with keyframe 1 (its keyframe is a WIH1), WIA1 +and WIS1, whose decodes run integer networks this script does not implement. Their +decoded outputs are checked by wai-rs against the same sidecars, on every leg of the +cross-architecture matrix for WIV1 and WIH1. The decode-layer fixtures of those +formats are checked here to parse, and by wai-rs for their codes. + + python3 int-payload-conformance/int_payload_verify.py int-payload-conformance + +Exit status 1 on any failure. +""" +import base64 +import hashlib +import json +import math +import os +import struct +import sys + +MAGICS = (b"WIV1", b"WIH1", b"WIA1", b"WIS1") +MAX_TENSOR = 1 << 26 +MAX_CLIP = 1 << 30 +MAX_FRAMES = 1 << 20 +MAX_MEDIAN_Q = 1 << 47 +CAP_MAGIC = { + "wai.video.int_motion": b"WIV1", + "wai.video.int_hyper": b"WIV1", + "wai.neural.int_hyper": b"WIH1", + "wai.audio.int_codec": b"WIA1", + "wai.splat.int_codec": b"WIS1", +} + + +class Refused(Exception): + def __init__(self, code, why): + super().__init__(f"{code}: {why}") + self.code = code + + +class Reader: + def __init__(self, b, p): + self.b, self.p = b, p + + def left(self): + return len(self.b) - self.p + + def take(self, n): + if n > self.left(): + raise Refused("stream_length", "the payload ends inside a section") + s = self.b[self.p:self.p + n] + self.p += n + return s + + def u8(self): + return self.take(1)[0] + + def u16(self): + return struct.unpack("= b for a, b in zip(t, t[1:])): + raise Refused("cdf_malformed", "not strictly increasing") + + +def check_magic(p, want): + m = p[:4] + if len(m) == 4 and m == want: + return + if len(m) == 4 and m in MAGICS: + raise Refused("format_capability_mismatch", "another integer format") + raise Refused("bad_magic", "not the capability's magic") + + +def opening(p, magic, header): + check_magic(p, magic) + if len(p) < 5: + raise Refused("stream_length", "no version byte") + if p[4] != 1: + raise Refused("version", "a version this revision does not define") + if len(p) < header: + raise Refused("stream_length", "the payload ends inside its fixed header") + return Reader(p, 5) + + +def too_large(dims, cap): + n = 1 + for d in dims: + n *= d + if n > cap: + raise Refused("too_large", "over the cap") + + +def parse_wiv1(p, want_keyframe=None): + r = opening(p, b"WIV1", 36) + mvp, kf, res0 = r.u8(), r.u8(), r.u8() + n_frames, h, w, block, q = r.u32(), r.u32(), r.u32(), r.u32(), r.u32() + offset, cdf_len, res1 = r.i32(), r.u16(), r.u16() + if res0 or res1: + raise Refused("reserved_nonzero", "reserved") + if mvp not in (1, 2): + raise Refused("field_range", "mv_precision") + if kf > 1: + raise Refused("field_range", "keyframe") + if 0 in (n_frames, h, w): + raise Refused("field_range", "n_frames, h or w") + if block == 0: + raise Refused("field_range", "block") + if not 1 <= q <= 65535: + raise Refused("field_range", "q") + if not 3 <= cdf_len <= 4097: + raise Refused("field_range", "cdf_len") + if n_frames == 1 and (mvp != 1 or block != 1): + raise Refused("field_range", "a one-frame clip has mv_precision 1 and block 1") + if want_keyframe is not None and want_keyframe != kf: + raise Refused("keyframe_capability_mismatch", "keyframe") + too_large([h, w, 3], MAX_TENSOR) + too_large([n_frames, h, w, 3], MAX_CLIP) + too_large([n_frames], MAX_FRAMES) + nb = -(-h // block) * -(-w // block) + too_large([2, n_frames - 1, nb], MAX_TENSOR) + cdf = r.cdf(cdf_len) + raw = r.take((n_frames - 1) * nb * 4) + mvs = [struct.unpack_from(" MAX_MEDIAN_Q for x in med): + raise Refused("field_range", "median") + cdfs = [r.cdf(cdf_len) for _ in range(n_ch)] + rans = r.stream() + r.end() + return dict(afrac=afrac, sr=sr, n_ch=n_ch, lat_len=lat_len, t_out=t_out, offset=offset, + median_q=med, cdfs=cdfs, rans=rans) + + +def parse_wis1(p): + r = opening(p, b"WIS1", 32) + afrac, res = r.u8(), r.u16() + c, sh, sw, s, attr, n_g = (r.u32() for _ in range(6)) + if res: + raise Refused("reserved_nonzero", "reserved") + if not 1 <= afrac <= 30: + raise Refused("field_range", "afrac") + if 0 in (c, sh, sw, s, attr, n_g): + raise Refused("field_range", "a dimension") + if n_g > s * s: + raise Refused("field_range", "n_gaussians") + too_large([c, sh, sw], MAX_TENSOR) + too_large([attr, s, s], MAX_TENSOR) + raw = r.take(8 * attr) + ranges = [struct.unpack_from(" hi: + raise Refused("field_range", "a range") + lens = list(struct.unpack(f"<{c}H", r.take(2 * c))) + if any(not 3 <= n <= 4097 for n in lens): + raise Refused("field_range", "a CDF length") + offs = list(struct.unpack(f"<{c}i", r.take(4 * c))) + med = list(struct.unpack(f"<{c}q", r.take(8 * c))) + if any(abs(x) > MAX_MEDIAN_Q for x in med): + raise Refused("field_range", "median") + cdfs = [r.cdf(n) for n in lens] + rans = r.stream() + r.end() + return dict(afrac=afrac, c=c, sh=sh, sw=sw, s=s, attr=attr, n_gaussians=n_g, ranges=ranges, + cdfs=cdfs, offsets=offs, median_q=med, rans=rans) + + +def parse(cap, p): + m = CAP_MAGIC[cap] + if m == b"WIV1": + return parse_wiv1(p, 1 if cap == "wai.video.int_hyper" else 0) + return {b"WIH1": parse_wih1, b"WIA1": parse_wia1, b"WIS1": parse_wis1}[m](p) + + +# ---- writers: the JSON sources converted independently (extensions/integer-payloads.md §10) ---- + +def put_stream(b): + return struct.pack("= len(self.w): + self.exhausted = True + return + self.x = (self.x << 32) | self.w[self.i] + self.i += 1 + + def bits(self, n): + v = self.x & ((1 << n) - 1) + self.x >>= n + self.renorm() + return v + + def decode(self, cdf, offset): + n = len(cdf) + max_value = n - 2 + cum = self.x & 0xFFFF + lo, hi = 1, n - 1 + while lo < hi: + mid = (lo + hi) // 2 + if cdf[mid] > cum: + hi = mid + else: + lo = mid + 1 + s = lo - 1 + self.x = (cdf[s + 1] - cdf[s]) * (self.x >> 16) + (self.x & 0xFFFF) - cdf[s] + self.renorm() + value = s + if value == max_value: + val = self.bits(4) + count, groups = val, 0 + while val == 15 and groups < 8 and not self.exhausted: + val = self.bits(4) + count += val + groups += 1 + raw = 0 + for j in range(min(count, 8)): + raw |= self.bits(4) << (4 * j) + value = raw >> 1 + value = sat(-value - 1) if raw & 1 else sat(value + max_value) + return sat(value + offset) + + +def decode_wiv1_k0(v): + h, w, block, q, prec = v["h"], v["w"], v["block"], v["q"], v["mv_precision"] + nbx = -(-w // block) + nb = nbx * -(-h // block) + d = Rans(v["rans"]) + frames, prev = [], None + for f in range(v["n_frames"]): + mvs = v["mvs"][(f - 1) * nb:f * nb] if f else None + # The frame grows a sample at a time as its symbols decode: nothing is + # allocated ahead of the stream that fills it. + cur = bytearray() + for y in range(h): + for x in range(w): + dy, dx = mvs[(y // block) * nbx + x // block] if f else (0, 0) + for c in range(3): + if f == 0: + pred = 128 + else: + def p(yy, xx): + return prev[(min(max(yy, 0), h - 1) * w + min(max(xx, 0), w - 1)) * 3 + c] + if prec == 1: + pred = p(y + dy, x + dx) + else: + iy, fy, ix, fx = y + (dy >> 1), dy & 1, x + (dx >> 1), dx & 1 + if not fy and not fx: + pred = p(iy, ix) + elif not fy: + pred = (p(iy, ix) + p(iy, ix + 1) + 1) >> 1 + elif not fx: + pred = (p(iy, ix) + p(iy + 1, ix) + 1) >> 1 + else: + pred = (p(iy, ix) + p(iy, ix + 1) + p(iy + 1, ix) + p(iy + 1, ix + 1) + 2) >> 2 + r = d.decode(v["cdf"], v["offset"]) + if d.exhausted: + # §2: the first symbol during whose decode the stream runs out. + raise Refused("stream_exhausted", "the residual stream ran out") + cur.append(min(max(pred + r * q, 0), 255)) + frames.append(bytes(cur)) + prev = cur + if d.i != len(d.w) or d.x != 1 << 31: + # §2: after the last symbol every word has been read and the state is 2^31. + raise Refused("stream_noncanonical", "the residual stream does not end canonically") + return frames + + +def main(cdir): + wai = os.path.normpath(os.path.join(cdir, "..")) + + def rd(rel): + with open(os.path.join(wai, rel), "rb") as f: + return f.read() + + def jl(rel): + return json.loads(rd(rel)) + + idx = jl("int-payload-conformance/vectors.json") + fails, decoded, parsed_only = [], 0, [] + for v in idx["vectors"]: + name, cap = v["name"], v["capability"] + p = rd(v["payload"]) + try: + fields = parse(cap, p) + except Refused as e: + fails.append(f"{name}: does not parse: {e}") + continue + if hashlib.sha256(p).hexdigest() != v["payload_sha256"]: + fails.append(f"{name}: payload SHA-256 differs from the index") + src = v["converted_from"] + j = jl(src["json"]) + m = CAP_MAGIC[cap] + try: + if m == b"WIV1": + conv = wiv1_from_json(j, rd(src["keyframe"]) if "keyframe" in src else None) + elif m == b"WIH1": + conv = wih1_from_json(j) + elif m == b"WIA1": + conv = wia1_from_json(j, jl(src["sample_rate"])["sample_rate"]) + else: + conv = wis1_from_json(j) + except NotConvertible as e: + fails.append(f"{name}: {src['json']} is not convertible: {e}") + continue + if conv != p: + fails.append(f"{name}: the independent conversion of {src['json']} differs from the payload") + if m == b"WIV1" and fields["keyframe"] == 0: + frames = decode_wiv1_k0(fields) + side = jl(v["sidecar"])[name] + digests = [hashlib.sha256(f).digest() for f in frames] + if [d.hex() for d in digests] != side["frames_sha256"] or hashlib.sha256(b"".join(digests)).hexdigest() != side["clip_sha256"]: + fails.append(f"{name}: the independent decode differs from {v['sidecar']}") + else: + decoded += 1 + print(f"ok {name}: parsed, converted, decoded to its sidecar's SHA-256 ({len(frames)} frames)") + else: + parsed_only.append(name) + print(f"ok {name}: parsed and converted (not decoded independently)") + checked, decode_layer = 0, [] + for f in idx["refusals"]: + name, cap, code, layer = f["name"], f["capability"], f["code"], f["layer"] + p = rd(f["payload"]) + try: + fields = parse(cap, p) + got = None + except Refused as e: + got = e.code + if layer == "parse": + if got != code: + fails.append(f"{name}: parsed with {got}, want {code}") + continue + else: + if got is not None: + fails.append(f"{name}: refused at parse with {got}, want a decode-layer {code}") + continue + if CAP_MAGIC[cap] == b"WIV1" and fields["keyframe"] == 0: + try: + decode_wiv1_k0(fields) + fails.append(f"{name}: decoded, want {code}") + continue + except Refused as e: + if e.code != code: + fails.append(f"{name}: decode refused with {e.code}, want {code}") + continue + else: + decode_layer.append(name) + checked += 1 + print(f"ok {name}: {code}" + ("" if layer == "parse" or name not in decode_layer else " (parses; the code is the decode's)")) + # wiv1_end_state codes three symbols from the state 2^31 + 1, as the words [1, 4]; its + # canonical twin, the words [0, 4], decodes to one grey pixel. + twin = bytearray(rd("int-payload-conformance/refusals/wiv1_end_state.bin")) + twin[-8:-4] = struct.pack(" 1 else os.path.dirname(os.path.abspath(__file__)))) diff --git a/wai/int-payload-conformance/refusals/wia1_afrac_model.bin b/wai/int-payload-conformance/refusals/wia1_afrac_model.bin new file mode 100644 index 000000000..3c5391d10 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_afrac_model.bin differ diff --git a/wai/int-payload-conformance/refusals/wia1_afrac_zero.bin b/wai/int-payload-conformance/refusals/wia1_afrac_zero.bin new file mode 100644 index 000000000..d06bc2990 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_afrac_zero.bin differ diff --git a/wai/int-payload-conformance/refusals/wia1_cdf_malformed.bin b/wai/int-payload-conformance/refusals/wia1_cdf_malformed.bin new file mode 100644 index 000000000..5a2276262 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_cdf_malformed.bin differ diff --git a/wai/int-payload-conformance/refusals/wia1_exhausted.bin b/wai/int-payload-conformance/refusals/wia1_exhausted.bin new file mode 100644 index 000000000..758a5fa32 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_exhausted.bin differ diff --git a/wai/int-payload-conformance/refusals/wia1_latent_too_long.bin b/wai/int-payload-conformance/refusals/wia1_latent_too_long.bin new file mode 100644 index 000000000..a13648788 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_latent_too_long.bin differ diff --git a/wai/int-payload-conformance/refusals/wia1_median_range.bin b/wai/int-payload-conformance/refusals/wia1_median_range.bin new file mode 100644 index 000000000..d932c5d59 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_median_range.bin differ diff --git a/wai/int-payload-conformance/refusals/wia1_order_table_before_length.bin b/wai/int-payload-conformance/refusals/wia1_order_table_before_length.bin new file mode 100644 index 000000000..df51fa5f3 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_order_table_before_length.bin differ diff --git a/wai/int-payload-conformance/refusals/wia1_sr_zero.bin b/wai/int-payload-conformance/refusals/wia1_sr_zero.bin new file mode 100644 index 000000000..84a51220d Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_sr_zero.bin differ diff --git a/wai/int-payload-conformance/refusals/wia1_t_out.bin b/wai/int-payload-conformance/refusals/wia1_t_out.bin new file mode 100644 index 000000000..4e8394902 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_t_out.bin differ diff --git a/wai/int-payload-conformance/refusals/wia1_too_large.bin b/wai/int-payload-conformance/refusals/wia1_too_large.bin new file mode 100644 index 000000000..7556093a4 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_too_large.bin differ diff --git a/wai/int-payload-conformance/refusals/wia1_trailing.bin b/wai/int-payload-conformance/refusals/wia1_trailing.bin new file mode 100644 index 000000000..343085bba Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_trailing.bin differ diff --git a/wai/int-payload-conformance/refusals/wia1_under_splat.bin b/wai/int-payload-conformance/refusals/wia1_under_splat.bin new file mode 100644 index 000000000..05d5af267 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_under_splat.bin differ diff --git a/wai/int-payload-conformance/refusals/wia1_unread_word.bin b/wai/int-payload-conformance/refusals/wia1_unread_word.bin new file mode 100644 index 000000000..36cfb7639 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wia1_unread_word.bin differ diff --git a/wai/int-payload-conformance/refusals/wih1_exhausted.bin b/wai/int-payload-conformance/refusals/wih1_exhausted.bin new file mode 100644 index 000000000..524fef59d Binary files /dev/null and b/wai/int-payload-conformance/refusals/wih1_exhausted.bin differ diff --git a/wai/int-payload-conformance/refusals/wih1_geometry.bin b/wai/int-payload-conformance/refusals/wih1_geometry.bin new file mode 100644 index 000000000..b26b407ca Binary files /dev/null and b/wai/int-payload-conformance/refusals/wih1_geometry.bin differ diff --git a/wai/int-payload-conformance/refusals/wih1_model_mismatch.bin b/wai/int-payload-conformance/refusals/wih1_model_mismatch.bin new file mode 100644 index 000000000..99dccfd8f Binary files /dev/null and b/wai/int-payload-conformance/refusals/wih1_model_mismatch.bin differ diff --git a/wai/int-payload-conformance/refusals/wih1_order_caps_before_geometry.bin b/wai/int-payload-conformance/refusals/wih1_order_caps_before_geometry.bin new file mode 100644 index 000000000..96694b5c9 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wih1_order_caps_before_geometry.bin differ diff --git a/wai/int-payload-conformance/refusals/wih1_order_exhausted_before_value.bin b/wai/int-payload-conformance/refusals/wih1_order_exhausted_before_value.bin new file mode 100644 index 000000000..82e6396ee Binary files /dev/null and b/wai/int-payload-conformance/refusals/wih1_order_exhausted_before_value.bin differ diff --git a/wai/int-payload-conformance/refusals/wih1_reserved.bin b/wai/int-payload-conformance/refusals/wih1_reserved.bin new file mode 100644 index 000000000..fcf15645b Binary files /dev/null and b/wai/int-payload-conformance/refusals/wih1_reserved.bin differ diff --git a/wai/int-payload-conformance/refusals/wih1_stream_length.bin b/wai/int-payload-conformance/refusals/wih1_stream_length.bin new file mode 100644 index 000000000..dfc87e16a Binary files /dev/null and b/wai/int-payload-conformance/refusals/wih1_stream_length.bin differ diff --git a/wai/int-payload-conformance/refusals/wih1_too_large.bin b/wai/int-payload-conformance/refusals/wih1_too_large.bin new file mode 100644 index 000000000..bfd420f75 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wih1_too_large.bin differ diff --git a/wai/int-payload-conformance/refusals/wih1_trailing.bin b/wai/int-payload-conformance/refusals/wih1_trailing.bin new file mode 100644 index 000000000..78c6285ab Binary files /dev/null and b/wai/int-payload-conformance/refusals/wih1_trailing.bin differ diff --git a/wai/int-payload-conformance/refusals/wih1_under_video.bin b/wai/int-payload-conformance/refusals/wih1_under_video.bin new file mode 100644 index 000000000..f9004d337 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wih1_under_video.bin differ diff --git a/wai/int-payload-conformance/refusals/wih1_unread_word.bin b/wai/int-payload-conformance/refusals/wih1_unread_word.bin new file mode 100644 index 000000000..35debc43c Binary files /dev/null and b/wai/int-payload-conformance/refusals/wih1_unread_word.bin differ diff --git a/wai/int-payload-conformance/refusals/wih1_zero_dim.bin b/wai/int-payload-conformance/refusals/wih1_zero_dim.bin new file mode 100644 index 000000000..bbe93f49e Binary files /dev/null and b/wai/int-payload-conformance/refusals/wih1_zero_dim.bin differ diff --git a/wai/int-payload-conformance/refusals/wis1_afrac_model.bin b/wai/int-payload-conformance/refusals/wis1_afrac_model.bin new file mode 100644 index 000000000..d4453b279 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wis1_afrac_model.bin differ diff --git a/wai/int-payload-conformance/refusals/wis1_cdf_len.bin b/wai/int-payload-conformance/refusals/wis1_cdf_len.bin new file mode 100644 index 000000000..eb1795013 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wis1_cdf_len.bin differ diff --git a/wai/int-payload-conformance/refusals/wis1_exhausted.bin b/wai/int-payload-conformance/refusals/wis1_exhausted.bin new file mode 100644 index 000000000..8810add87 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wis1_exhausted.bin differ diff --git a/wai/int-payload-conformance/refusals/wis1_geometry.bin b/wai/int-payload-conformance/refusals/wis1_geometry.bin new file mode 100644 index 000000000..cf6c9bb4b Binary files /dev/null and b/wai/int-payload-conformance/refusals/wis1_geometry.bin differ diff --git a/wai/int-payload-conformance/refusals/wis1_n_gaussians.bin b/wai/int-payload-conformance/refusals/wis1_n_gaussians.bin new file mode 100644 index 000000000..89da80d06 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wis1_n_gaussians.bin differ diff --git a/wai/int-payload-conformance/refusals/wis1_range_nan.bin b/wai/int-payload-conformance/refusals/wis1_range_nan.bin new file mode 100644 index 000000000..06b4da320 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wis1_range_nan.bin differ diff --git a/wai/int-payload-conformance/refusals/wis1_range_order.bin b/wai/int-payload-conformance/refusals/wis1_range_order.bin new file mode 100644 index 000000000..bb9f866de Binary files /dev/null and b/wai/int-payload-conformance/refusals/wis1_range_order.bin differ diff --git a/wai/int-payload-conformance/refusals/wis1_trailing.bin b/wai/int-payload-conformance/refusals/wis1_trailing.bin new file mode 100644 index 000000000..befb707ce Binary files /dev/null and b/wai/int-payload-conformance/refusals/wis1_trailing.bin differ diff --git a/wai/int-payload-conformance/refusals/wis1_unread_word.bin b/wai/int-payload-conformance/refusals/wis1_unread_word.bin new file mode 100644 index 000000000..3d36170dd Binary files /dev/null and b/wai/int-payload-conformance/refusals/wis1_unread_word.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_cdf_malformed.bin b/wai/int-payload-conformance/refusals/wiv1_cdf_malformed.bin new file mode 100644 index 000000000..418fc2a89 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_cdf_malformed.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_end_state.bin b/wai/int-payload-conformance/refusals/wiv1_end_state.bin new file mode 100644 index 000000000..f9f435708 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_end_state.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_exhausted.bin b/wai/int-payload-conformance/refusals/wiv1_exhausted.bin new file mode 100644 index 000000000..d569a055a Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_exhausted.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_geometry.bin b/wai/int-payload-conformance/refusals/wiv1_geometry.bin new file mode 100644 index 000000000..14e73bf8a Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_geometry.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2133_fe.bin b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2133_fe.bin new file mode 100644 index 000000000..e398c3de4 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2133_fe.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2136_96.bin b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2136_96.bin new file mode 100644 index 000000000..7eb49444b Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2136_96.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2136_9b.bin b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2136_9b.bin new file mode 100644 index 000000000..64e335c0c Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2136_9b.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2137_8c.bin b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2137_8c.bin new file mode 100644 index 000000000..f8727cadb Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2137_8c.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2138_be.bin b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2138_be.bin new file mode 100644 index 000000000..2f8f05a97 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2138_be.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2139_03.bin b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2139_03.bin new file mode 100644 index 000000000..c5f5d2e24 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2139_03.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2146_4a.bin b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2146_4a.bin new file mode 100644 index 000000000..f713f62dc Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_geometry_flip_2146_4a.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_keyframe_mismatch.bin b/wai/int-payload-conformance/refusals/wiv1_keyframe_mismatch.bin new file mode 100644 index 000000000..f948456f9 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_keyframe_mismatch.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_mv_field.bin b/wai/int-payload-conformance/refusals/wiv1_mv_field.bin new file mode 100644 index 000000000..8350fedaa Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_mv_field.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_mv_precision.bin b/wai/int-payload-conformance/refusals/wiv1_mv_precision.bin new file mode 100644 index 000000000..c9f406468 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_mv_precision.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_nested_length.bin b/wai/int-payload-conformance/refusals/wiv1_nested_length.bin new file mode 100644 index 000000000..263af925f Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_nested_length.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_nested_version.bin b/wai/int-payload-conformance/refusals/wiv1_nested_version.bin new file mode 100644 index 000000000..af56afc0b Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_nested_version.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_one_frame_block.bin b/wai/int-payload-conformance/refusals/wiv1_one_frame_block.bin new file mode 100644 index 000000000..ae614adc4 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_one_frame_block.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_one_frame_end_state.bin b/wai/int-payload-conformance/refusals/wiv1_one_frame_end_state.bin new file mode 100644 index 000000000..6ba80e96b Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_one_frame_end_state.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_one_frame_mv_precision.bin b/wai/int-payload-conformance/refusals/wiv1_one_frame_mv_precision.bin new file mode 100644 index 000000000..f8260e7fb Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_one_frame_mv_precision.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_q_zero.bin b/wai/int-payload-conformance/refusals/wiv1_q_zero.bin new file mode 100644 index 000000000..3a8161b05 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_q_zero.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_reserved.bin b/wai/int-payload-conformance/refusals/wiv1_reserved.bin new file mode 100644 index 000000000..cd003d7a7 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_reserved.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_stream_length.bin b/wai/int-payload-conformance/refusals/wiv1_stream_length.bin new file mode 100644 index 000000000..754ef4b31 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_stream_length.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_too_large.bin b/wai/int-payload-conformance/refusals/wiv1_too_large.bin new file mode 100644 index 000000000..f83b0c107 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_too_large.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_too_many_frames.bin b/wai/int-payload-conformance/refusals/wiv1_too_many_frames.bin new file mode 100644 index 000000000..443565a62 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_too_many_frames.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_trailing.bin b/wai/int-payload-conformance/refusals/wiv1_trailing.bin new file mode 100644 index 000000000..695bc1d55 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_trailing.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_truncated.bin b/wai/int-payload-conformance/refusals/wiv1_truncated.bin new file mode 100644 index 000000000..c85f9cc30 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_truncated.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_under_audio.bin b/wai/int-payload-conformance/refusals/wiv1_under_audio.bin new file mode 100644 index 000000000..f948456f9 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_under_audio.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_unread_word.bin b/wai/int-payload-conformance/refusals/wiv1_unread_word.bin new file mode 100644 index 000000000..8b4a18499 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_unread_word.bin differ diff --git a/wai/int-payload-conformance/refusals/wiv1_version.bin b/wai/int-payload-conformance/refusals/wiv1_version.bin new file mode 100644 index 000000000..55f11d882 Binary files /dev/null and b/wai/int-payload-conformance/refusals/wiv1_version.bin differ diff --git a/wai/int-payload-conformance/sets/entropy_median_max.json b/wai/int-payload-conformance/sets/entropy_median_max.json new file mode 100644 index 000000000..ffb4e9fe9 --- /dev/null +++ b/wai/int-payload-conformance/sets/entropy_median_max.json @@ -0,0 +1 @@ +{"afrac":12,"frac":12,"scale_edges":[4096,8192,16384,32768],"cdfs":[[0,1,2,3,4,6,296,3834,19692,45837,61695,65233,65523,65532,65533,65534,65535,65536],[0,4,33,178,752,2521,6765,14695,26232,39306,50843,58773,63017,64786,65360,65505,65534,65536],[0,927,2410,4636,7775,11933,17108,23158,29803,36659,43304,49354,54529,58687,61826,64052,65535,65536],[0,2905,6173,9790,13731,17960,22426,27070,31824,36616,41370,46014,50480,54709,58650,62267,65535,65536]],"cdf_lengths":[18,18,18,18],"offsets":[-8,-8,-8,-8],"z_cdfs":[[0,4,33,178,752,2521,6765,14695,26232,39306,50843,58773,63017,64786,65360,65505,65534,65536],[0,4,33,178,752,2521,6765,14695,26232,39306,50843,58773,63017,64786,65360,65505,65534,65536]],"z_cdf_lengths":[18,18],"z_offsets":[1,-8],"z_median_q":[9223372036854775807,0]} diff --git a/wai/int-payload-conformance/vectors.json b/wai/int-payload-conformance/vectors.json new file mode 100644 index 000000000..a06965a83 --- /dev/null +++ b/wai/int-payload-conformance/vectors.json @@ -0,0 +1,806 @@ +{ + "format": "wai-int-payload-vectors/1", + "about": "Binary payload vectors of the WAI-native integer codecs and their refusal fixtures (extensions/integer-payloads.md §12). Paths are relative to wai/. Written by wai_int_payload_vectors gen.", + "vectors": [ + { + "name": "int_video", + "capability": "wai.video.int_motion", + "payload": "wai-rs/tests/vectors/int_video/int_video_bitstream.wiv1", + "payload_sha256": "3bbf0a864d9528e73aff036d1a636893f839d3f6c5f31dde7824a59e6c116565", + "converted_from": { + "json": "wai-rs/tests/vectors/int_video/int_video_bitstream.json" + }, + "set": {}, + "sidecar": "wai-rs/tests/vectors/int_video/sha256.json" + }, + { + "name": "video_real", + "capability": "wai.video.int_motion", + "payload": "wai-rs/tests/vectors/int_video/video_real_bitstream.wiv1", + "payload_sha256": "efce5ada256cb3443a673ee0c10aa4e0a8ce4a372102cd918627e5eec746880f", + "converted_from": { + "json": "wai-rs/tests/vectors/int_video/video_real_bitstream.json" + }, + "set": {}, + "sidecar": "wai-rs/tests/vectors/int_video/sha256.json" + }, + { + "name": "tiny", + "capability": "wai.neural.int_hyper", + "payload": "wai-rs/tests/vectors/int_hyper/tiny_bitstream.wih1", + "payload_sha256": "6d8a195a5a138452292b55d70a575d1fbb4372fd8614b3b821b676e5090c3c17", + "converted_from": { + "json": "wai-rs/tests/vectors/int_hyper/tiny_bitstream.json" + }, + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + }, + "sidecar": "wai-rs/tests/vectors/int_hyper/sha256.json" + }, + { + "name": "video_nk", + "capability": "wai.video.int_hyper", + "payload": "wai-rs/tests/vectors/int_video/video_nk_bitstream.wiv1", + "payload_sha256": "d2c62da643328bfbdf9eab129283322e9077a491c5ddd269464577f51a958bc8", + "converted_from": { + "json": "wai-rs/tests/vectors/int_video/video_nk_bitstream.json", + "keyframe": "wai-rs/tests/vectors/int_hyper/tiny_bitstream.wih1" + }, + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + }, + "sidecar": "wai-rs/tests/vectors/int_video/sha256.json" + }, + { + "name": "audio", + "capability": "wai.audio.int_codec", + "payload": "wai-rs/tests/vectors/int_audio/audio_bitstream.wia1", + "payload_sha256": "b45e727525f06ac1ac5a5a7121886c85a7c33613451acbb482c20e37f3c79055", + "converted_from": { + "json": "wai-rs/tests/vectors/int_audio/audio_bitstream.json", + "sample_rate": "wai-rs/tests/vectors/int_audio/audio.json" + }, + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + }, + "sidecar": "wai-rs/tests/vectors/int_audio/sha256.json" + }, + { + "name": "splat", + "capability": "wai.splat.int_codec", + "payload": "wai-rs/tests/vectors/int_splat/splat.wis1", + "payload_sha256": "473ea0223840758b26ff7d540d861e3b2c1565dbc2bec68b0235b3f50f19d422", + "converted_from": { + "json": "wai-rs/tests/vectors/int_splat/splat.json" + }, + "set": { + "g_s.whm": "wai-rs/tests/vectors/int_splat/splat_gs.bin" + }, + "sidecar": "wai-rs/tests/vectors/int_splat/sha256.json" + } + ], + "refusals": [ + { + "name": "wiv1_json_form", + "capability": "wai.video.int_motion", + "payload": "wai-rs/tests/vectors/int_video/int_video_bitstream.json", + "code": "bad_magic", + "layer": "parse", + "set": {} + }, + { + "name": "wiv1_under_audio", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wiv1_under_audio.bin", + "code": "format_capability_mismatch", + "layer": "parse", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wiv1_keyframe_mismatch", + "capability": "wai.video.int_motion", + "payload": "int-payload-conformance/refusals/wiv1_keyframe_mismatch.bin", + "code": "keyframe_capability_mismatch", + "layer": "parse", + "set": {} + }, + { + "name": "wiv1_version", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_version.bin", + "code": "version", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_reserved", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_reserved.bin", + "code": "reserved_nonzero", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_mv_precision", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_mv_precision.bin", + "code": "field_range", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_q_zero", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_q_zero.bin", + "code": "field_range", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_too_large", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_too_large.bin", + "code": "too_large", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_too_many_frames", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_too_many_frames.bin", + "code": "too_large", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_mv_field", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_mv_field.bin", + "code": "too_large", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_cdf_malformed", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_cdf_malformed.bin", + "code": "cdf_malformed", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_stream_length", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_stream_length.bin", + "code": "stream_length", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_truncated", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_truncated.bin", + "code": "stream_length", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_nested_length", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_nested_length.bin", + "code": "nested_length", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_nested_version", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_nested_version.bin", + "code": "version", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_trailing", + "capability": "wai.video.int_motion", + "payload": "int-payload-conformance/refusals/wiv1_trailing.bin", + "code": "trailing", + "layer": "parse", + "set": {} + }, + { + "name": "wiv1_exhausted", + "capability": "wai.video.int_motion", + "payload": "int-payload-conformance/refusals/wiv1_exhausted.bin", + "code": "stream_exhausted", + "layer": "decode", + "set": {} + }, + { + "name": "wiv1_unread_word", + "capability": "wai.video.int_motion", + "payload": "int-payload-conformance/refusals/wiv1_unread_word.bin", + "code": "stream_noncanonical", + "layer": "decode", + "set": {} + }, + { + "name": "wiv1_end_state", + "capability": "wai.video.int_motion", + "payload": "int-payload-conformance/refusals/wiv1_end_state.bin", + "code": "stream_noncanonical", + "layer": "decode", + "set": {} + }, + { + "name": "wiv1_one_frame_block", + "capability": "wai.video.int_motion", + "payload": "int-payload-conformance/refusals/wiv1_one_frame_block.bin", + "code": "field_range", + "layer": "parse", + "set": {} + }, + { + "name": "wiv1_one_frame_mv_precision", + "capability": "wai.video.int_motion", + "payload": "int-payload-conformance/refusals/wiv1_one_frame_mv_precision.bin", + "code": "field_range", + "layer": "parse", + "set": {} + }, + { + "name": "wiv1_one_frame_end_state", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_one_frame_end_state.bin", + "code": "stream_noncanonical", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_geometry", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_geometry.bin", + "code": "geometry_mismatch", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_geometry_flip_2146_4a", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_geometry_flip_2146_4a.bin", + "code": "geometry_mismatch", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_geometry_flip_2138_be", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_geometry_flip_2138_be.bin", + "code": "geometry_mismatch", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_geometry_flip_2136_96", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_geometry_flip_2136_96.bin", + "code": "geometry_mismatch", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_geometry_flip_2139_03", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_geometry_flip_2139_03.bin", + "code": "geometry_mismatch", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_geometry_flip_2133_fe", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_geometry_flip_2133_fe.bin", + "code": "geometry_mismatch", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_geometry_flip_2136_9b", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_geometry_flip_2136_9b.bin", + "code": "geometry_mismatch", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wiv1_geometry_flip_2137_8c", + "capability": "wai.video.int_hyper", + "payload": "int-payload-conformance/refusals/wiv1_geometry_flip_2137_8c.bin", + "code": "geometry_mismatch", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wih1_json_form", + "capability": "wai.neural.int_hyper", + "payload": "wai-rs/tests/vectors/int_hyper/tiny_bitstream.json", + "code": "bad_magic", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wih1_under_video", + "capability": "wai.video.int_motion", + "payload": "int-payload-conformance/refusals/wih1_under_video.bin", + "code": "format_capability_mismatch", + "layer": "parse", + "set": {} + }, + { + "name": "wih1_reserved", + "capability": "wai.neural.int_hyper", + "payload": "int-payload-conformance/refusals/wih1_reserved.bin", + "code": "reserved_nonzero", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wih1_zero_dim", + "capability": "wai.neural.int_hyper", + "payload": "int-payload-conformance/refusals/wih1_zero_dim.bin", + "code": "field_range", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wih1_too_large", + "capability": "wai.neural.int_hyper", + "payload": "int-payload-conformance/refusals/wih1_too_large.bin", + "code": "too_large", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wih1_stream_length", + "capability": "wai.neural.int_hyper", + "payload": "int-payload-conformance/refusals/wih1_stream_length.bin", + "code": "stream_length", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wih1_trailing", + "capability": "wai.neural.int_hyper", + "payload": "int-payload-conformance/refusals/wih1_trailing.bin", + "code": "trailing", + "layer": "parse", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wih1_model_mismatch", + "capability": "wai.neural.int_hyper", + "payload": "int-payload-conformance/refusals/wih1_model_mismatch.bin", + "code": "model_mismatch", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wih1_geometry", + "capability": "wai.neural.int_hyper", + "payload": "int-payload-conformance/refusals/wih1_geometry.bin", + "code": "geometry_mismatch", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wih1_exhausted", + "capability": "wai.neural.int_hyper", + "payload": "int-payload-conformance/refusals/wih1_exhausted.bin", + "code": "stream_exhausted", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wih1_order_caps_before_geometry", + "capability": "wai.neural.int_hyper", + "payload": "int-payload-conformance/refusals/wih1_order_caps_before_geometry.bin", + "code": "too_large", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wih1_order_exhausted_before_value", + "capability": "wai.neural.int_hyper", + "payload": "int-payload-conformance/refusals/wih1_order_exhausted_before_value.bin", + "code": "stream_exhausted", + "layer": "decode", + "set": { + "entropy.json": "int-payload-conformance/sets/entropy_median_max.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wih1_unread_word", + "capability": "wai.neural.int_hyper", + "payload": "int-payload-conformance/refusals/wih1_unread_word.bin", + "code": "stream_noncanonical", + "layer": "decode", + "set": { + "entropy.json": "wai-rs/tests/vectors/int_hyper/tiny_entropy.json", + "g_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_g_s.bin", + "h_s.whm": "wai-rs/tests/vectors/int_hyper/tiny_h_s.bin" + } + }, + { + "name": "wia1_json_form", + "capability": "wai.audio.int_codec", + "payload": "wai-rs/tests/vectors/int_audio/audio_bitstream.json", + "code": "bad_magic", + "layer": "parse", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wia1_under_splat", + "capability": "wai.splat.int_codec", + "payload": "int-payload-conformance/refusals/wia1_under_splat.bin", + "code": "format_capability_mismatch", + "layer": "parse", + "set": { + "g_s.whm": "wai-rs/tests/vectors/int_splat/splat_gs.bin" + } + }, + { + "name": "wia1_afrac_zero", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wia1_afrac_zero.bin", + "code": "field_range", + "layer": "parse", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wia1_sr_zero", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wia1_sr_zero.bin", + "code": "field_range", + "layer": "parse", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wia1_median_range", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wia1_median_range.bin", + "code": "field_range", + "layer": "parse", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wia1_cdf_malformed", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wia1_cdf_malformed.bin", + "code": "cdf_malformed", + "layer": "parse", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wia1_order_table_before_length", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wia1_order_table_before_length.bin", + "code": "cdf_malformed", + "layer": "parse", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wia1_too_large", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wia1_too_large.bin", + "code": "too_large", + "layer": "parse", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wia1_trailing", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wia1_trailing.bin", + "code": "trailing", + "layer": "parse", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wia1_afrac_model", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wia1_afrac_model.bin", + "code": "model_mismatch", + "layer": "decode", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wia1_t_out", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wia1_t_out.bin", + "code": "geometry_mismatch", + "layer": "decode", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wia1_latent_too_long", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wia1_latent_too_long.bin", + "code": "geometry_mismatch", + "layer": "decode", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wia1_exhausted", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wia1_exhausted.bin", + "code": "stream_exhausted", + "layer": "decode", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wia1_unread_word", + "capability": "wai.audio.int_codec", + "payload": "int-payload-conformance/refusals/wia1_unread_word.bin", + "code": "stream_noncanonical", + "layer": "decode", + "set": { + "model.wam": "wai-rs/tests/vectors/int_audio/audio_model.bin" + } + }, + { + "name": "wis1_json_form", + "capability": "wai.splat.int_codec", + "payload": "wai-rs/tests/vectors/int_splat/splat.json", + "code": "bad_magic", + "layer": "parse", + "set": { + "g_s.whm": "wai-rs/tests/vectors/int_splat/splat_gs.bin" + } + }, + { + "name": "wis1_range_order", + "capability": "wai.splat.int_codec", + "payload": "int-payload-conformance/refusals/wis1_range_order.bin", + "code": "field_range", + "layer": "parse", + "set": { + "g_s.whm": "wai-rs/tests/vectors/int_splat/splat_gs.bin" + } + }, + { + "name": "wis1_range_nan", + "capability": "wai.splat.int_codec", + "payload": "int-payload-conformance/refusals/wis1_range_nan.bin", + "code": "field_range", + "layer": "parse", + "set": { + "g_s.whm": "wai-rs/tests/vectors/int_splat/splat_gs.bin" + } + }, + { + "name": "wis1_n_gaussians", + "capability": "wai.splat.int_codec", + "payload": "int-payload-conformance/refusals/wis1_n_gaussians.bin", + "code": "field_range", + "layer": "parse", + "set": { + "g_s.whm": "wai-rs/tests/vectors/int_splat/splat_gs.bin" + } + }, + { + "name": "wis1_cdf_len", + "capability": "wai.splat.int_codec", + "payload": "int-payload-conformance/refusals/wis1_cdf_len.bin", + "code": "field_range", + "layer": "parse", + "set": { + "g_s.whm": "wai-rs/tests/vectors/int_splat/splat_gs.bin" + } + }, + { + "name": "wis1_trailing", + "capability": "wai.splat.int_codec", + "payload": "int-payload-conformance/refusals/wis1_trailing.bin", + "code": "trailing", + "layer": "parse", + "set": { + "g_s.whm": "wai-rs/tests/vectors/int_splat/splat_gs.bin" + } + }, + { + "name": "wis1_geometry", + "capability": "wai.splat.int_codec", + "payload": "int-payload-conformance/refusals/wis1_geometry.bin", + "code": "geometry_mismatch", + "layer": "decode", + "set": { + "g_s.whm": "wai-rs/tests/vectors/int_splat/splat_gs.bin" + } + }, + { + "name": "wis1_afrac_model", + "capability": "wai.splat.int_codec", + "payload": "int-payload-conformance/refusals/wis1_afrac_model.bin", + "code": "model_mismatch", + "layer": "decode", + "set": { + "g_s.whm": "wai-rs/tests/vectors/int_splat/splat_gs.bin" + } + }, + { + "name": "wis1_exhausted", + "capability": "wai.splat.int_codec", + "payload": "int-payload-conformance/refusals/wis1_exhausted.bin", + "code": "stream_exhausted", + "layer": "decode", + "set": { + "g_s.whm": "wai-rs/tests/vectors/int_splat/splat_gs.bin" + } + }, + { + "name": "wis1_unread_word", + "capability": "wai.splat.int_codec", + "payload": "int-payload-conformance/refusals/wis1_unread_word.bin", + "code": "stream_noncanonical", + "layer": "decode", + "set": { + "g_s.whm": "wai-rs/tests/vectors/int_splat/splat_gs.bin" + } + } + ] +} diff --git a/wai/wai-rs/Cargo.toml b/wai/wai-rs/Cargo.toml index 3cd82e8cf..cebff6519 100644 --- a/wai/wai-rs/Cargo.toml +++ b/wai/wai-rs/Cargo.toml @@ -142,7 +142,7 @@ world_int = ["neural_int"] # Integer-exact LEARNED Gaussian-splat ATTRIBUTE codec ("the int_hyper of 3D"): # learned transform over the [14,S,S] attribute image → byte-exact attribute tensor # (splat-equivalence, a portable cloud hash). Reuses int_hyper_synth (neural_int). -splat_int = ["neural_int"] +splat_int = ["neural_int", "serde_json/float_roundtrip"] neural = ["neural_int", "dep:ort", "dep:ndarray", "dep:zstd-safe"] # The perceptual / generative-semantic path: CLIP-similarity conformance, # regenerate-at-sink. Reuses the ONNX runtime for the reference CLIP embedder; @@ -176,13 +176,16 @@ provenance = ["world"] # content-address and chain it, and sign it. Conformance is reason-set equality with # the reference verifier and the portable C kernel over oer-conformance/. oer = ["dep:blake3", "dep:ed25519-dalek", "dep:num-bigint", "serde_json/float_roundtrip"] +# jwp-receipts §2.1's object content hash (src/jwp_hash.rs), on its own so any +# receipt family that names an object can compute it with BLAKE3 alone. +jwp_hash = ["dep:blake3"] # Typed receipt classes on the JWP chain (extensions/jwp-receipts.md §2.4.3–§2.4.7, # §2.6, §3.1–§3.2): `claim` receipts of class energy / lawful-basis / log-inclusion, # the soft-binding class, and recovery under the ancestor rule. Needs `provenance` # for the coverage vocabulary and `oer` for the evidential grades and the OER receipt # an energy claim may name, which it reads and verifies through `crate::oer`. Its one # direct dependency, curve25519-dalek, is already in the graph under ed25519-dalek. -receipt_claims = ["provenance", "oer", "dep:curve25519-dalek"] +receipt_claims = ["provenance", "oer", "jwp_hash", "dep:curve25519-dalek"] # Key discovery and signing authority (extensions/jwp-receipts.md §7): the origin key # log a `did:web` signer_id names, anchoring, the confirmation reported beside a receipt's # or claim's verdict, and the authorisation and revocation of decoder and reconstruct @@ -467,6 +470,14 @@ name = "wai_int_synth_vectors" path = "src/bin/wai_int_synth_vectors.rs" required-features = ["neural_int"] +# The binary payload vectors of the integer codecs and their refusal fixtures +# (extensions/integer-payloads.md §12): `gen` converts the committed JSON, `verify` +# requires byte-identical regeneration, the sidecars' SHA-256 and every fixture's code. +[[bin]] +name = "wai_int_payload_vectors" +path = "src/bin/wai_int_payload_vectors.rs" +required-features = ["video_int", "audio_int", "splat_int"] + [[bin]] name = "wai_film_vectors" path = "src/bin/wai_film_vectors.rs" @@ -627,6 +638,21 @@ name = "int_mlicpp_preflight" path = "tests/int_mlicpp_preflight.rs" required-features = ["neural_int"] +[[test]] +name = "int_payload" +path = "tests/int_payload.rs" +required-features = ["video_int", "audio_int", "splat_int"] + +[[test]] +name = "int_payload_preflight" +path = "tests/int_payload_preflight.rs" +required-features = ["video_int", "audio_int", "splat_int"] + +[[test]] +name = "int_payload_cost" +path = "tests/int_payload_cost.rs" +required-features = ["video_int", "audio_int", "splat_int"] + [[test]] name = "generate" path = "tests/generate.rs" diff --git a/wai/wai-rs/cbindgen.toml b/wai/wai-rs/cbindgen.toml index c44d98c9e..f54688798 100644 --- a/wai/wai-rs/cbindgen.toml +++ b/wai/wai-rs/cbindgen.toml @@ -41,6 +41,7 @@ prefix = "" [export.rename] "WaiBuffer" = "wai_buffer_t" "WaiDecoded" = "wai_decoded_t" +"WaiIntDeclared" = "wai_int_declared_t" [fn] prefix = "" diff --git a/wai/wai-rs/include/wai.h b/wai/wai-rs/include/wai.h index 52b10caff..e80552da8 100644 --- a/wai/wai-rs/include/wai.h +++ b/wai/wai-rs/include/wai.h @@ -345,6 +345,24 @@ typedef struct wai_buffer_t { uintptr_t cap; } wai_buffer_t; +/** + * What decoding an integer payload will cost (`wai_int_payload_declared`, + * `extensions/integer-payloads.md` §7, *Memory and work*): the decoded output's + * bytes (`output_known` 1 where they are filled), the symbols its streams code, and, + * where `cost_known` is 1, the most bytes the reference decode holds at once beyond + * the payload and the parsed set (`working_bytes`) and the networks' + * multiply-accumulates, `macs_hi·2^64 + macs_lo`. + */ +typedef struct wai_int_declared_t { + uint64_t output_bytes; + uint8_t output_known; + uint64_t symbols; + uint64_t working_bytes; + uint64_t macs_lo; + uint64_t macs_hi; + uint8_t cost_known; +} wai_int_declared_t; + /** * What `wai_neural_decode_ex` decoded. * @@ -397,13 +415,45 @@ extern "C" { */ const char *wai_last_error(void); +/** + * Read what decoding the payload of an integer capability will cost before decoding + * it, so a sink can refuse a payload past its own size policy first + * (`extensions/integer-payloads.md` §7, *Memory and work*). `capability` is the + * envelope's capability (or the `WAI2` component's). The payload is parsed as a sink + * parses it: one that is not the format SPEC §5 pins `capability` to is + * `WAI_ERR_INVALID_ARG`, with its code in `wai_last_error`, and so is a capability + * with no pinned integer format. + * + * With `set_path` null, only the header is read: a `wai.neural.int_hyper` image's + * size is its synthesis network's, so `output_known` is 0 for it, and `cost_known` is + * 1 only for a `WIV1` with keyframe 0, which runs no network. With `set_path` naming + * the capability's parameter set — a directory holding its members by the names §9 + * gives, or the file of a one-file set — every figure is filled + * (`int_payload::declared_cost`), and the set is checked against the payload as a + * decode checks it first (`set_incomplete`, `model_mismatch`, `geometry_mismatch`, + * …). The set is read as given, unverified: a pin is verified when the envelope is + * decoded. A build without `neural_int` reads no set. + * + * # Safety + * `capability` NUL-terminated; `payload` valid for `payload_len` bytes; `set_path` + * null or NUL-terminated UTF-8; `out` writable. + */ + +int wai_int_payload_declared(const char *capability, + const uint8_t *payload, + uintptr_t payload_len, + const char *set_path, + struct wai_int_declared_t *out); + /** * Builds a complete WAI file from a codec-produced payload + manifest * fields. `manifest_json` is a UTF-8 JSON object overriding/supplying * any fields; `payload` is the bytes from one of the codec encoders. * Refuses (`WAI_ERR_INVALID_ARG`) a manifest that names a candidate capability * as its capability or fallback, which an encoder MUST NOT emit (SPEC §5, - * *Candidate capabilities*): it emits through `Wai::to_emit_bytes`. + * *Candidate capabilities*), and a payload that is not the format SPEC §5 pins its + * integer capability or fallback to (*Payloads of the WAI-native integer codecs*): + * it emits through `Wai::to_emit_bytes`. */ int wai_envelope_pack(const char *manifest_json, @@ -426,7 +476,8 @@ int wai_envelope_pack(const char *manifest_json, * - The i-th payload pairs with `manifest.renditions[i]`. * * Refuses (`WAI_ERR_INVALID_ARG`) a table with a component, in either role, - * whose capability is a candidate (SPEC §5, *Candidate capabilities*): it emits + * whose capability is a candidate (SPEC §5, *Candidate capabilities*), or whose + * payload is not the format SPEC §5 pins its integer capability to: it emits * through `WaiMulti::to_emit_bytes`. * * # Safety @@ -680,7 +731,10 @@ int wai_neural_decode_video(const uint8_t *envelope, * array of paths. The first path is the registered artifact; the others are * further sets a pin may resolve against (a set and its successor). Every pin is * verified first (SPEC §3.1); a `WAI2` rendition is picked by SPEC §7.1. With - * `flags & WAI_DECODE_STRICT`, SPEC §7's replicate rule applies too. + * `flags & WAI_DECODE_STRICT`, SPEC §7's replicate rule applies too. A capability + * whose output has no `wai_decoded_t` kind (integer video, PCM, splat attributes) is + * not advertised, whatever `registry_json` says: dispatch moves on past it before + * decoding, and when nothing else decodes the error names it. * * `*out` is written on every call with a non-null `out`, before anything else is * read: on failure it holds `kind` 0 and an empty `data`, so the caller may free diff --git a/wai/wai-rs/src/bin/wai.rs b/wai/wai-rs/src/bin/wai.rs index fbaf3b9b4..ec7ce19e5 100644 --- a/wai/wai-rs/src/bin/wai.rs +++ b/wai/wai-rs/src/bin/wai.rs @@ -364,6 +364,15 @@ fn inspect(path: &str) -> Result<(), String> { if let Err(e) = w.check_candidates() { println!(" candidates: \u{2717} {e}"); } + let encrypted = !w.manifest.security.as_ref().is_none_or(wai::container::Security::is_plaintext); + if [Some(cap), fb].into_iter().flatten().any(|c| wai::codecs::pinned_int_payload(c).is_some()) && encrypted { + println!(" integer payload: encrypted; a sink checks it after decrypting (SPEC \u{a7}6.4)"); + } else if [Some(cap), fb].into_iter().flatten().any(|c| wai::codecs::pinned_int_payload(c).is_some()) { + match w.check_int_payload_framing() { + Ok(()) => println!(" integer payload: \u{2713} parses as the format SPEC \u{a7}5 pins (every parse rule; the decode checks the rest)"), + Err(e) => println!(" integer payload: \u{2717} {e}; a sink refuses it at decode"), + } + } } else { // WAI2 inspection let m = wai::container::WaiMulti::from_bytes(&bytes).map_err(|e| e.to_string())?; @@ -422,6 +431,9 @@ fn inspect(path: &str) -> Result<(), String> { if let Err(e) = m.check_candidates() { println!(" candidates: \u{2717} {e}"); } + if let Err((i, e)) = m.check_int_payload_framing() { + println!(" integer payload: \u{2717} [{i}] {e}; a sink refuses it at decode"); + } } println!(); @@ -486,7 +498,9 @@ fn license_str(cap: &str) -> &'static str { /// capability is refused as a `WAI1` capability or fallback, and a `WAI2` table /// that breaks the binding rules is refused. A candidate capability (SPEC §5, /// *Candidate capabilities*) is refused in every place — capability, fallback or -/// companion — and nothing is written. +/// companion — and nothing is written. So is a payload that is not the format SPEC §5 +/// pins its integer capability to (*Payloads of the WAI-native integer codecs*): the +/// JSON a tool writes is converted first, never wrapped. fn wrap(cap: &str, input: &str, out: &str, fallback: Option<&str>, intent: Option<&str>, companions: &[(String, String)]) -> Result<(), String> { if !wai::codecs::is_known_capability(cap) { @@ -526,9 +540,11 @@ fn wrap(cap: &str, input: &str, out: &str, fallback: Option<&str>, }; let manifest = Manifest::new(media_for(cap), intent, requirement, Conditioning::new("bytes")); let envelope = Wai::new(manifest, payload); - // The reference tool does not write an envelope that names a candidate, or - // one that breaks the role rules. + // The reference tool does not write an envelope that names a candidate, one that + // breaks the role rules, or one whose payload is not the format SPEC §5 pins its + // integer capability to (a tool's JSON output, say). envelope.check_candidates().map_err(|e| e.to_string())?; + envelope.check_int_payload_framing().map_err(|e| format!("not writing {out}: {e}"))?; envelope.check_component_roles().map_err(|e| match e { wai::container::RoleError::CompanionOnlyAsCapability { .. } => format!( "{e} (SPEC \u{a7}5): `wai wrap --companion {cap} `"), @@ -573,6 +589,7 @@ fn wrap_with_companions(cap: &str, payload: Vec, out: &str, fallback: Option let envelope = wai::container::WaiMulti::new( media_for(cap), intent, serde_json::Value::Null, components); envelope.check_candidates().map_err(|e| e.to_string())?; + envelope.check_int_payload_framing().map_err(|(i, e)| format!("not writing {out}: component {i}: {e}"))?; envelope.check_component_roles() .map_err(|e| format!("{e} (SPEC \u{a7}5, \u{a7}7.1)"))?; let n = envelope.write(out).map_err(|e| e.to_string())?; diff --git a/wai/wai-rs/src/bin/wai_int_payload_vectors.rs b/wai/wai-rs/src/bin/wai_int_payload_vectors.rs new file mode 100644 index 000000000..438162bfc --- /dev/null +++ b/wai/wai-rs/src/bin/wai_int_payload_vectors.rs @@ -0,0 +1,532 @@ +//! `wai_int_payload_vectors` — writes and verifies the binary payload vectors of the +//! WAI-native integer codecs (`extensions/integer-payloads.md` §12). +//! +//! wai_int_payload_vectors gen +//! wai_int_payload_vectors verify +//! +//! `gen` converts each committed JSON bitstream under `wai-rs/tests/vectors/` to its +//! pinned payload, beside the JSON (`*.wiv1`, `*.wih1`, `*.wia1`, `*.wis1`), derives the +//! refusal fixtures from those payloads (`refusals/*.bin`), and writes the index +//! `vectors.json`. Every file it writes is a function of the committed JSON alone, so +//! a second run writes the same bytes. +//! +//! `verify` regenerates every file in memory and requires it to equal the committed +//! one byte for byte; decodes each payload through `int_payload::decode_int` against +//! the parameter set the index names, and requires the output's SHA-256 to be the one +//! the vector's `sha256.json` sidecar states; and requires each refusal fixture to be +//! refused with the code the index states. Exit status 1 on any failure. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; + +use wai::int_payload::{declared_cost, decode_int, wire, IntDecoded, IntParams}; +use wai::pin::HeldBytes; + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn sha256(b: &[u8]) -> String { + hex(&Sha256::digest(b)) +} + +/// The tree below `wai/`, read from and written to by path relative to it. +struct Tree { + wai: PathBuf, + out: BTreeMap>, +} + +impl Tree { + fn read(&self, rel: &str) -> Vec { + if let Some(b) = self.out.get(rel) { + return b.clone(); + } + let p = self.wai.join(rel); + std::fs::read(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())) + } + fn json(&self, rel: &str) -> Value { + serde_json::from_slice(&self.read(rel)).unwrap_or_else(|e| panic!("parse {rel}: {e}")) + } + fn put(&mut self, rel: &str, bytes: Vec) { + self.out.insert(rel.to_string(), bytes); + } +} + +const V: &str = "wai-rs/tests/vectors"; +const R: &str = "int-payload-conformance/refusals"; + +/// One decodable vector: its name, capability, payload path, the JSON it is converted +/// from, the set members it decodes against (member name → path) and its sidecar entry. +struct Vector { + name: &'static str, + cap: &'static str, + payload: String, + from: Value, + set: Vec<(&'static str, String)>, + sidecar: String, +} + +fn int_hyper_set() -> Vec<(&'static str, String)> { + vec![ + ("entropy.json", format!("{V}/int_hyper/tiny_entropy.json")), + ("g_s.whm", format!("{V}/int_hyper/tiny_g_s.bin")), + ("h_s.whm", format!("{V}/int_hyper/tiny_h_s.bin")), + ] +} + +/// Convert every committed JSON bitstream to its payload. +fn vectors(t: &mut Tree) -> Vec { + use wai::int_payload::{wia1_from_json, wih1_from_json, wis1_from_json, wiv1_from_json}; + let mut out = Vec::new(); + for (name, json_rel, wiv1) in [ + ("int_video", "int_video/int_video_bitstream.json", "int_video/int_video_bitstream.wiv1"), + ("video_real", "int_video/video_real_bitstream.json", "int_video/video_real_bitstream.wiv1"), + ] { + let p = wiv1_from_json(&t.json(&format!("{V}/{json_rel}")), None).unwrap_or_else(|e| panic!("{name}: {e}")); + t.put(&format!("{V}/{wiv1}"), p); + out.push(Vector { + name, + cap: wai::codecs::CAP_VIDEO_INT_MOTION, + payload: format!("{V}/{wiv1}"), + from: json!({ "json": format!("{V}/{json_rel}") }), + set: vec![], + sidecar: format!("{V}/int_video/sha256.json"), + }); + } + let wih1 = wih1_from_json(&t.json(&format!("{V}/int_hyper/tiny_bitstream.json"))).expect("tiny WIH1"); + t.put(&format!("{V}/int_hyper/tiny_bitstream.wih1"), wih1.clone()); + out.push(Vector { + name: "tiny", + cap: wai::codecs::CAP_NEURAL_INT_HYPER, + payload: format!("{V}/int_hyper/tiny_bitstream.wih1"), + from: json!({ "json": format!("{V}/int_hyper/tiny_bitstream.json") }), + set: int_hyper_set(), + sidecar: format!("{V}/int_hyper/sha256.json"), + }); + let nk = wiv1_from_json(&t.json(&format!("{V}/int_video/video_nk_bitstream.json")), Some(&wih1)).expect("video_nk WIV1"); + t.put(&format!("{V}/int_video/video_nk_bitstream.wiv1"), nk); + out.push(Vector { + name: "video_nk", + cap: wai::codecs::CAP_VIDEO_INT_HYPER, + payload: format!("{V}/int_video/video_nk_bitstream.wiv1"), + from: json!({ "json": format!("{V}/int_video/video_nk_bitstream.json"), "keyframe": format!("{V}/int_hyper/tiny_bitstream.wih1") }), + set: int_hyper_set(), + sidecar: format!("{V}/int_video/sha256.json"), + }); + let sr = t.json(&format!("{V}/int_audio/audio.json"))["sample_rate"].as_u64().expect("sample_rate") as u32; + let wia1 = wia1_from_json(&t.json(&format!("{V}/int_audio/audio_bitstream.json")), sr).expect("audio WIA1"); + t.put(&format!("{V}/int_audio/audio_bitstream.wia1"), wia1); + out.push(Vector { + name: "audio", + cap: wai::codecs::CAP_AUDIO_INT_CODEC, + payload: format!("{V}/int_audio/audio_bitstream.wia1"), + from: json!({ "json": format!("{V}/int_audio/audio_bitstream.json"), "sample_rate": format!("{V}/int_audio/audio.json") }), + set: vec![("model.wam", format!("{V}/int_audio/audio_model.bin"))], + sidecar: format!("{V}/int_audio/sha256.json"), + }); + let wis1 = wis1_from_json(&t.json(&format!("{V}/int_splat/splat.json"))).expect("splat WIS1"); + t.put(&format!("{V}/int_splat/splat.wis1"), wis1); + out.push(Vector { + name: "splat", + cap: wai::codecs::CAP_SPLAT_INT_CODEC, + payload: format!("{V}/int_splat/splat.wis1"), + from: json!({ "json": format!("{V}/int_splat/splat.json") }), + set: vec![("g_s.whm", format!("{V}/int_splat/splat_gs.bin"))], + sidecar: format!("{V}/int_splat/sha256.json"), + }); + out +} + +/// One refusal fixture: its name, capability, payload path, the code it is refused +/// with, whether parsing alone refuses it (`parse`) or the decode does (`decode`), and +/// the set its decode reads. +struct Fixture { + name: &'static str, + cap: &'static str, + payload: String, + code: &'static str, + layer: &'static str, + set: Vec<(&'static str, String)>, +} + +fn set_u32(b: &mut [u8], off: usize, v: u32) { + b[off..off + 4].copy_from_slice(&v.to_le_bytes()); +} +fn get_u32(b: &[u8], off: usize) -> u32 { + u32::from_le_bytes(b[off..off + 4].try_into().unwrap()) +} + +/// The refusal fixtures, each the committed payload changed in one place. +fn fixtures(t: &mut Tree) -> Vec { + use wai::codecs::{CAP_AUDIO_INT_CODEC as AUD, CAP_NEURAL_INT_HYPER as HYP, CAP_SPLAT_INT_CODEC as SPL, + CAP_VIDEO_INT_HYPER as VNK, CAP_VIDEO_INT_MOTION as MOT}; + let iv = t.read(&format!("{V}/int_video/int_video_bitstream.wiv1")); + let nk = t.read(&format!("{V}/int_video/video_nk_bitstream.wiv1")); + let ih = t.read(&format!("{V}/int_hyper/tiny_bitstream.wih1")); + let au = t.read(&format!("{V}/int_audio/audio_bitstream.wia1")); + let sp = t.read(&format!("{V}/int_splat/splat.wis1")); + let mut out = Vec::new(); + let mut fx = |t: &mut Tree, name: &'static str, cap: &'static str, bytes: Option>, payload: Option, code: &'static str, layer: &'static str| { + let payload = match bytes { + Some(b) => { + let rel = format!("{R}/{name}.bin"); + t.put(&rel, b); + rel + } + None => payload.expect("a fixture names its payload"), + }; + let set = match cap { + c if c == HYP || c == VNK => int_hyper_set(), + c if c == AUD => vec![("model.wam", format!("{V}/int_audio/audio_model.bin"))], + c if c == SPL => vec![("g_s.whm", format!("{V}/int_splat/splat_gs.bin"))], + _ => vec![], + }; + out.push(Fixture { name, cap, payload, code, layer, set }); + }; + let edit = |base: &[u8], f: &dyn Fn(&mut Vec)| { + let mut b = base.to_vec(); + f(&mut b); + Some(b) + }; + + // ── WIV1 ── (keyframe 1 is video_nk: 16×16, the nested tiny WIH1) + let parsed = wire::parse_wiv1(&nk, Some(1)).expect("video_nk parses"); + let nk_l = at(&nk, parsed.rans) - 4; // offset of L + let nk_k = at(&nk, parsed.wih1.expect("keyframe 1")) - 4; // offset of K + fx(t, "wiv1_json_form", MOT, None, Some(format!("{V}/int_video/int_video_bitstream.json")), "bad_magic", "parse"); + fx(t, "wiv1_under_audio", AUD, Some(nk.clone()), None, "format_capability_mismatch", "parse"); + fx(t, "wiv1_keyframe_mismatch", MOT, Some(nk.clone()), None, "keyframe_capability_mismatch", "parse"); + fx(t, "wiv1_version", VNK, edit(&nk, &|b| b[4] = 2), None, "version", "parse"); + fx(t, "wiv1_reserved", VNK, edit(&nk, &|b| b[7] = 1), None, "reserved_nonzero", "parse"); + fx(t, "wiv1_mv_precision", VNK, edit(&nk, &|b| b[5] = 3), None, "field_range", "parse"); + fx(t, "wiv1_q_zero", VNK, edit(&nk, &|b| set_u32(b, 24, 0)), None, "field_range", "parse"); + fx(t, "wiv1_too_large", VNK, edit(&nk, &|b| { set_u32(b, 12, 8192); set_u32(b, 16, 8192) }), None, "too_large", "parse"); + // 2^20 + 1 frames of 16 × 16: within the clip's 2^30 bytes, past the frame count. + fx(t, "wiv1_too_many_frames", VNK, edit(&nk, &|b| set_u32(b, 8, (1 << 20) + 1)), None, "too_large", "parse"); + // Block 1 over 131074 frames: 2·256·131073 motion-vector elements, past 2^26. + fx(t, "wiv1_mv_field", VNK, edit(&nk, &|b| { set_u32(b, 8, 131_074); set_u32(b, 20, 1) }), None, "too_large", "parse"); + fx(t, "wiv1_cdf_malformed", VNK, edit(&nk, &|b| { let v = get_u32(b, 40); set_u32(b, 44, v) }), None, "cdf_malformed", "parse"); + fx(t, "wiv1_stream_length", VNK, edit(&nk, &|b| { let l = get_u32(b, nk_l); set_u32(b, nk_l, l - 2) }), None, "stream_length", "parse"); + fx(t, "wiv1_truncated", VNK, edit(&nk, &|b| b.truncate(nk_k + 2)), None, "stream_length", "parse"); + fx(t, "wiv1_nested_length", VNK, edit(&nk, &|b| { let k = get_u32(b, nk_k); set_u32(b, nk_k, k + 4) }), None, "nested_length", "parse"); + fx(t, "wiv1_nested_version", VNK, edit(&nk, &|b| b[nk_k + 4 + 4] = 2), None, "version", "parse"); + fx(t, "wiv1_trailing", MOT, edit(&iv, &|b| b.push(0)), None, "trailing", "parse"); + let iv_l = at(&iv, wire::parse_wiv1(&iv, Some(0)).expect("int_video parses").rans) - 4; + fx(t, "wiv1_exhausted", MOT, edit(&iv, &|b| { set_u32(b, iv_l, 8); b.truncate(iv_l + 4 + 8) }), None, "stream_exhausted", "decode"); + // A word appended to the residual stream (the last section), which the decode + // never reads. + fx(t, "wiv1_unread_word", MOT, edit(&iv, &|b| { let l = get_u32(b, iv_l); set_u32(b, iv_l, l + 4); b.extend_from_slice(&[0; 4]) }), None, "stream_noncanonical", "decode"); + // A one-pixel clip whose three symbols are 0 (frequency 2^15 of 2^16), encoded + // from the state 2^31 + 1: the stream is the words [1, 4], and the decode ends at + // 2^31 + 1. Encoded from 2^31, the canonical stream is [0, 4] (`grey_pixel`). + fx(t, "wiv1_end_state", MOT, Some(grey_pixel(1)), None, "stream_noncanonical", "decode"); + // A one-frame clip's header fixes mv_precision 1 and block 1 (§3): the one-pixel + // clip with block 4, or half-pel, is refused at parse. + fx(t, "wiv1_one_frame_block", MOT, edit(&grey_pixel(0), &|b| set_u32(b, 20, 4)), None, "field_range", "parse"); + fx(t, "wiv1_one_frame_mv_precision", MOT, edit(&grey_pixel(0), &|b| b[5] = 2), None, "field_range", "parse"); + // One frame, the nested keyframe: the residual stream codes no symbol, so it must be + // the state 2^31 alone, and 2^31 + 1 is refused. + let one = |state: u32| { + let mut v = wire::parse_wiv1(&nk, Some(1)).expect("video_nk parses"); + let rans: Vec = [state, 0].iter().flat_map(|w| w.to_le_bytes()).collect(); + (v.n_frames, v.mv_precision, v.block) = (1, 1, 1); + v.mvs.clear(); + wire::write_wiv1(&wire::Wiv1 { rans: &rans, ..v }) + }; + fx(t, "wiv1_one_frame_end_state", VNK, Some(one((1 << 31) + 1)), None, "stream_noncanonical", "decode"); + let geo = { + let mut v = wire::parse_wiv1(&nk, Some(1)).expect("video_nk parses"); + (v.h, v.w) = (12, 12); // the same 2 × 2 block grid; the keyframe decodes 16 × 16 + wire::write_wiv1(&v) + }; + fx(t, "wiv1_geometry", VNK, Some(geo.clone()), None, "geometry_mismatch", "decode"); + // The keyframe's size is checked against the clip's h × w before any symbol of the + // keyframe is decoded (§7): with one byte of the keyframe's last stream words + // changed, so that its stream no longer ends canonically, each is still + // geometry_mismatch. Named by the byte changed and the value XORed into it. + for (name, back, x) in [ + ("wiv1_geometry_flip_2146_4a", 2, 0x4a), ("wiv1_geometry_flip_2138_be", 10, 0xbe), + ("wiv1_geometry_flip_2136_96", 12, 0x96), ("wiv1_geometry_flip_2139_03", 9, 0x03), + ("wiv1_geometry_flip_2133_fe", 15, 0xfe), ("wiv1_geometry_flip_2136_9b", 12, 0x9b), + ("wiv1_geometry_flip_2137_8c", 11, 0x8c), + ] { + fx(t, name, VNK, edit(&geo, &|b| { let at = b.len() - back; b[at] ^= x }), None, "geometry_mismatch", "decode"); + } + + // ── WIH1 ── (the tiny conditional codec: m 4, 4×4; n_z 2, 2×2) + fx(t, "wih1_json_form", HYP, None, Some(format!("{V}/int_hyper/tiny_bitstream.json")), "bad_magic", "parse"); + fx(t, "wih1_under_video", MOT, Some(ih.clone()), None, "format_capability_mismatch", "parse"); + fx(t, "wih1_reserved", HYP, edit(&ih, &|b| b[6] = 1), None, "reserved_nonzero", "parse"); + fx(t, "wih1_zero_dim", HYP, edit(&ih, &|b| set_u32(b, 8, 0)), None, "field_range", "parse"); + fx(t, "wih1_too_large", HYP, edit(&ih, &|b| { set_u32(b, 24, 65_536); set_u32(b, 28, 65_536) }), None, "too_large", "parse"); + fx(t, "wih1_stream_length", HYP, edit(&ih, &|b| set_u32(b, 32, 6)), None, "stream_length", "parse"); + fx(t, "wih1_trailing", HYP, edit(&ih, &|b| b.push(0)), None, "trailing", "parse"); + fx(t, "wih1_model_mismatch", HYP, edit(&ih, &|b| set_u32(b, 20, 3)), None, "model_mismatch", "decode"); + fx(t, "wih1_geometry", HYP, edit(&ih, &|b| { set_u32(b, 12, 8); set_u32(b, 16, 8) }), None, "geometry_mismatch", "decode"); + let ih_ly = at(&ih, wire::parse_wih1(&ih).expect("tiny parses").rans) - 4; + fx(t, "wih1_exhausted", HYP, edit(&ih, &|b| { set_u32(b, ih_ly, 8); b.truncate(ih_ly + 4 + 8) }), None, "stream_exhausted", "decode"); + // The order of the decode's checks (§7): every network is planned, caps included, + // before a shape is compared, so a 2100 × 2100 latent, whose g_s would form a + // buffer past the cap, is too_large rather than h_s's geometry_mismatch; and a + // symbol that runs its stream out is refused before its value is rebuilt, so a z + // stream of 8 zero bytes, whose first symbol rebuilt against a median of i64::MAX + // would leave i64, is stream_exhausted (the set's entropy.json is + // sets/entropy_median_max.json). + fx(t, "wih1_order_caps_before_geometry", HYP, edit(&ih, &|b| { set_u32(b, 12, 2100); set_u32(b, 16, 2100) }), None, "too_large", "decode"); + let short_z = { + let mut k = wire::parse_wih1(&ih).expect("tiny parses"); + k.z_rans = &[0; 8]; + wire::write_wih1(&k) + }; + fx(t, "wih1_order_exhausted_before_value", HYP, Some(short_z), None, "stream_exhausted", "decode"); + fx(t, "wih1_unread_word", HYP, edit(&ih, &|b| { let l = get_u32(b, ih_ly); set_u32(b, ih_ly, l + 4); b.extend_from_slice(&[0; 4]) }), None, "stream_noncanonical", "decode"); + + // ── WIA1 ── (16 channels × 512, tables of 14) + let au_n = at(&au, wire::parse_wia1(&au).expect("audio parses").rans) - 4; // offset of L + fx(t, "wia1_json_form", AUD, None, Some(format!("{V}/int_audio/audio_bitstream.json")), "bad_magic", "parse"); + fx(t, "wia1_under_splat", SPL, Some(au.clone()), None, "format_capability_mismatch", "parse"); + fx(t, "wia1_afrac_zero", AUD, edit(&au, &|b| b[5] = 0), None, "field_range", "parse"); + fx(t, "wia1_sr_zero", AUD, edit(&au, &|b| set_u32(b, 8, 0)), None, "field_range", "parse"); + fx(t, "wia1_median_range", AUD, edit(&au, &|b| b[32..40].copy_from_slice(&((1i64 << 47) + 1).to_le_bytes())), None, "field_range", "parse"); + fx(t, "wia1_cdf_malformed", AUD, edit(&au, &|b| set_u32(b, 160 + 13 * 4, 65_535)), None, "cdf_malformed", "parse"); + // §2's order: the tables are read one at a time, each present and then checked, so + // a malformed first table is cdf_malformed though the payload ends inside the second. + fx(t, "wia1_order_table_before_length", AUD, edit(&au, &|b| { set_u32(b, 160 + 13 * 4, 65_535); b.truncate(160 + 14 * 4 + 10) }), None, "cdf_malformed", "parse"); + fx(t, "wia1_too_large", AUD, edit(&au, &|b| { set_u32(b, 12, 65_536); set_u32(b, 16, 2_048) }), None, "too_large", "parse"); + fx(t, "wia1_trailing", AUD, edit(&au, &|b| b.push(0)), None, "trailing", "parse"); + fx(t, "wia1_afrac_model", AUD, edit(&au, &|b| b[5] = 11), None, "model_mismatch", "decode"); + fx(t, "wia1_t_out", AUD, edit(&au, &|b| { let n = get_u32(b, 20); set_u32(b, 20, n + 1) }), None, "geometry_mismatch", "decode"); + // A latent twice as long as t_out needs: one sample fewer still gives t_out (§7). + fx(t, "wia1_latent_too_long", AUD, edit(&au, &|b| { let n = get_u32(b, 16); set_u32(b, 16, 2 * n) }), None, "geometry_mismatch", "decode"); + fx(t, "wia1_exhausted", AUD, edit(&au, &|b| { set_u32(b, au_n, 8); b.truncate(au_n + 4 + 8) }), None, "stream_exhausted", "decode"); + fx(t, "wia1_unread_word", AUD, edit(&au, &|b| { let l = get_u32(b, au_n); set_u32(b, au_n, l + 4); b.extend_from_slice(&[0; 4]) }), None, "stream_noncanonical", "decode"); + + // ── WIS1 ── (32 channels of 8×8 → 14 planes of 32×32) + let wis = |f: &dyn Fn(&mut wire::Wis1<'_>)| { + let mut v = wire::parse_wis1(&sp).expect("splat parses"); + f(&mut v); + Some(wire::write_wis1(&v)) + }; + fx(t, "wis1_json_form", SPL, None, Some(format!("{V}/int_splat/splat.json")), "bad_magic", "parse"); + fx(t, "wis1_range_order", SPL, wis(&|v| v.ranges[0] = (1.0, -1.0)), None, "field_range", "parse"); + fx(t, "wis1_range_nan", SPL, wis(&|v| v.ranges[0].0 = f32::NAN), None, "field_range", "parse"); + fx(t, "wis1_n_gaussians", SPL, wis(&|v| v.n_gaussians = v.s * v.s + 1), None, "field_range", "parse"); + fx(t, "wis1_cdf_len", SPL, edit(&sp, &|b| b[32 + 14 * 8..32 + 14 * 8 + 2].copy_from_slice(&2u16.to_le_bytes())), None, "field_range", "parse"); + fx(t, "wis1_trailing", SPL, edit(&sp, &|b| b.push(0)), None, "trailing", "parse"); + fx(t, "wis1_geometry", SPL, wis(&|v| { v.attr = 13; v.ranges.truncate(13) }), None, "geometry_mismatch", "decode"); + fx(t, "wis1_afrac_model", SPL, edit(&sp, &|b| b[5] = 11), None, "model_mismatch", "decode"); + let sp_l = at(&sp, wire::parse_wis1(&sp).expect("splat parses").rans) - 4; + fx(t, "wis1_exhausted", SPL, edit(&sp, &|b| { set_u32(b, sp_l, 8); b.truncate(sp_l + 4 + 8) }), None, "stream_exhausted", "decode"); + fx(t, "wis1_unread_word", SPL, edit(&sp, &|b| { let l = get_u32(b, sp_l); set_u32(b, sp_l, l + 4); b.extend_from_slice(&[0; 4]) }), None, "stream_noncanonical", "decode"); + let sets = "int-payload-conformance/sets"; + t.put(&format!("{sets}/entropy_median_max.json"), entropy_median_max(t)); + for f in &mut out { + if f.name == "wih1_order_exhausted_before_value" { + f.set = vec![ + ("entropy.json", format!("{sets}/entropy_median_max.json")), + ("g_s.whm", format!("{V}/int_hyper/tiny_g_s.bin")), + ("h_s.whm", format!("{V}/int_hyper/tiny_h_s.bin")), + ]; + } + } + out +} + +/// A `WIV1` keyframe-0 clip of one 1 × 1 frame whose three residual symbols are 0 +/// against the table `[0, 32768, 49152, 65536]`, with the stream `[w0, 4]`: `w0 = 0` is +/// the canonical coding of those symbols, which decodes to a grey pixel; `w0 = 1` +/// codes the same symbols from the state `2^31 + 1`. +fn grey_pixel(w0: u32) -> Vec { + let rans: Vec = [w0, 4].iter().flat_map(|w| w.to_le_bytes()).collect(); + wire::write_wiv1(&wire::Wiv1 { + mv_precision: 1, keyframe: 0, n_frames: 1, h: 1, w: 1, block: 1, q: 1, offset: 0, + cdf: vec![0, 32_768, 49_152, 65_536], mvs: vec![], rans: &rans, wih1: None, + }) +} + +/// The tiny conditional codec's entropy tables with the first hyper-latent channel's +/// median at `i64::MAX` and its symbol offset at 1, so that channel's first symbol, if +/// its value were used, would rebuild past `i64`. +fn entropy_median_max(t: &Tree) -> Vec { + let mut e = t.json(&format!("{V}/int_hyper/tiny_entropy.json")); + e["z_median_q"][0] = json!(i64::MAX); + e["z_offsets"][0] = json!(1); + let mut text = serde_json::to_string(&e).expect("entropy"); + text.push('\n'); + text.into_bytes() +} + +/// The offset in `base` at which `part`, a slice of it, starts. +fn at(base: &[u8], part: &[u8]) -> usize { + part.as_ptr() as usize - base.as_ptr() as usize +} + +/// Build every file: the payloads, the fixtures, the index. +fn build(wai: &Path) -> Tree { + let mut t = Tree { wai: wai.to_path_buf(), out: BTreeMap::new() }; + let vs = vectors(&mut t); + let fs = fixtures(&mut t); + let set = |s: &[(&str, String)]| -> Value { s.iter().map(|(k, v)| (k.to_string(), json!(v))).collect::>().into() }; + let index = json!({ + "format": "wai-int-payload-vectors/1", + "about": "Binary payload vectors of the WAI-native integer codecs and their refusal fixtures (extensions/integer-payloads.md §12). Paths are relative to wai/. Written by wai_int_payload_vectors gen.", + "vectors": vs.iter().map(|v| json!({ + "name": v.name, + "capability": v.cap, + "payload": v.payload, + "payload_sha256": sha256(&t.read(&v.payload)), + "converted_from": v.from, + "set": set(&v.set), + "sidecar": v.sidecar, + })).collect::>(), + "refusals": fs.iter().map(|f| json!({ + "name": f.name, + "capability": f.cap, + "payload": f.payload, + "code": f.code, + "layer": f.layer, + "set": set(&f.set), + })).collect::>(), + }); + let mut text = serde_json::to_string_pretty(&index).expect("index"); + text.push('\n'); + t.put("int-payload-conformance/vectors.json", text.into_bytes()); + t +} + +/// The SHA-256 digests of a decode's output, in the form its sidecar entry states. +fn digests(d: &IntDecoded) -> Value { + match d { + IntDecoded::Video { frames, .. } => { + let per: Vec<[u8; 32]> = frames.iter().map(|f| Sha256::digest(f).into()).collect(); + let clip = Sha256::digest(per.concat()); + json!({ "frames_sha256": per.iter().map(|d| hex(d)).collect::>(), "clip_sha256": hex(&clip) }) + } + IntDecoded::Image { rgb, .. } => json!({ "sha256": sha256(rgb) }), + IntDecoded::Pcm { samples, .. } => { + let le: Vec = samples.iter().flat_map(|s| s.to_le_bytes()).collect(); + json!({ "sha256": sha256(&le) }) + } + IntDecoded::Attr { bytes, .. } => json!({ "sha256": sha256(bytes) }), + _ => json!(null), + } +} + +/// `f` with the parameters of the set `set` names (member → path) for `cap`. +fn with_set(t: &Tree, cap: &str, set: &Value, f: impl FnOnce(&IntParams<'_>) -> R) -> R { + let owned: BTreeMap> = set.as_object().into_iter().flatten() + .map(|(k, v)| (k.clone(), t.read(v.as_str().expect("a set path")))).collect(); + let view: BTreeMap = owned.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect(); + let params = wai::int_payload::resolve(cap, None, &[HeldBytes::Set { listing: b"", files: &view }], None) + .unwrap_or_else(|_| IntParams::none()); + f(¶ms) +} + +/// Decode `payload` through `cap` against the set `set` names (member → path). +fn decode(t: &Tree, cap: &str, payload: &[u8], set: &Value) -> Result { + with_set(t, cap, set, |p| decode_int(cap, payload, p)) +} + +fn verify(wai: &Path) -> bool { + let t = build(wai); + let mut ok = true; + for (rel, bytes) in &t.out { + match std::fs::read(wai.join(rel)) { + Ok(have) if &have == bytes => {} + Ok(_) => { println!("FAIL {rel}: differs from its regeneration"); ok = false } + Err(e) => { println!("FAIL {rel}: {e}"); ok = false } + } + } + let index: Value = serde_json::from_slice(&t.out["int-payload-conformance/vectors.json"]).expect("index"); + for v in index["vectors"].as_array().expect("vectors") { + let (name, cap) = (v["name"].as_str().unwrap(), v["capability"].as_str().unwrap()); + let payload = t.read(v["payload"].as_str().unwrap()); + match decode(&t, cap, &payload, &v["set"]) { + Ok(d) => { + let want = &t.json(v["sidecar"].as_str().unwrap())[name]; + let got = digests(&d); + let same = got.as_object().unwrap().iter().all(|(k, g)| &want[k] == g); + if same { + println!("ok {name} ({cap}): decodes to its sidecar's SHA-256"); + } else { + println!("FAIL {name} ({cap}): decoded {got} ≠ sidecar"); + ok = false; + } + } + Err(e) => { println!("FAIL {name} ({cap}): {e}"); ok = false } + } + } + let mut seen = std::collections::BTreeSet::new(); + for f in index["refusals"].as_array().expect("refusals") { + let (name, cap, code) = (f["name"].as_str().unwrap(), f["capability"].as_str().unwrap(), f["code"].as_str().unwrap()); + let payload = t.read(f["payload"].as_str().unwrap()); + let parsed = match wai::codecs::pinned_int_payload(cap) { + Some(wire::MAGIC_WIV1) => wire::parse_wiv1(&payload, Some((cap == wai::codecs::CAP_VIDEO_INT_HYPER) as u8)).err(), + Some(wire::MAGIC_WIH1) => wire::parse_wih1(&payload).err(), + Some(wire::MAGIC_WIA1) => wire::parse_wia1(&payload).err(), + Some(wire::MAGIC_WIS1) => wire::parse_wis1(&payload).err(), + _ => None, + }; + let layer_ok = (f["layer"] == "parse") == parsed.is_some(); + // The declared cost reads what a decode reads before its first symbol, so it + // refuses with the decode's code, or admits the payload when the decode's + // refusal comes from a stream or a network run (§7, steps 5 and 6). + let declared = with_set(&t, cap, &f["set"], |p| declared_cost(cap, &payload, p).map(|_| ()).map_err(|e| e.code())); + let declared_ok = match declared { + Err(c) => c == code, + Ok(()) => matches!(code, "stream_exhausted" | "stream_noncanonical" | "field_range"), + }; + match decode(&t, cap, &payload, &f["set"]) { + Err(e) if e.code() == code && layer_ok && declared_ok => { seen.insert(code); println!("ok {name}: {code}") } + Err(e) if e.code() == code && layer_ok => { + println!("FAIL {name}: the decode gives {code}, the declared cost {declared:?}"); + ok = false + } + Err(e) => { println!("FAIL {name}: refused with {e}, want {code} at {}", f["layer"]); ok = false } + Ok(_) => { println!("FAIL {name}: decoded, want {code}"); ok = false } + } + } + println!("{} refusal codes covered by fixtures", seen.len()); + // The end-state fixture's canonical twin, the same symbols coded from 2^31, decodes. + match decode(&t, wai::codecs::CAP_VIDEO_INT_MOTION, &grey_pixel(0), &Value::Null) { + Ok(IntDecoded::Video { frames, .. }) if frames == [vec![128u8; 3]] => { + println!("ok grey_pixel: wiv1_end_state's canonical twin decodes to one grey pixel") + } + other => { println!("FAIL grey_pixel: {other:?}"); ok = false } + } + ok +} + +fn main() -> ExitCode { + let args: Vec = std::env::args().collect(); + let (Some(cmd), Some(dir)) = (args.get(1), args.get(2)) else { + eprintln!("usage: wai_int_payload_vectors gen|verify "); + return ExitCode::from(2); + }; + let wai = Path::new(dir).join(".."); + match cmd.as_str() { + "gen" => { + let t = build(&wai); + for (rel, bytes) in &t.out { + let p = wai.join(rel); + if let Some(d) = p.parent() { + std::fs::create_dir_all(d).expect("create directory"); + } + std::fs::write(&p, bytes).unwrap_or_else(|e| panic!("write {}: {e}", p.display())); + } + println!("wrote {} files", t.out.len()); + ExitCode::SUCCESS + } + "verify" => { + if verify(&wai) { ExitCode::SUCCESS } else { ExitCode::FAILURE } + } + _ => { + eprintln!("usage: wai_int_payload_vectors gen|verify "); + ExitCode::from(2) + } + } +} diff --git a/wai/wai-rs/src/bin/wai_video_meter.rs b/wai/wai-rs/src/bin/wai_video_meter.rs index 41a99d950..46ae5ff34 100644 --- a/wai/wai-rs/src/bin/wai_video_meter.rs +++ b/wai/wai-rs/src/bin/wai_video_meter.rs @@ -8,9 +8,14 @@ //! decode burst against an idle baseline, and reports the **marginal** CPU //! energy attributable to the decode. //! -//! wai_video_meter [receipt_out.json [report_out.json]] +//! wai_video_meter [receipt_out.json [report_out.json]] //! [--no-report] [--previous-report ] //! +//! The codestream is the `WIV1` payload `wai.video.int_motion` carries +//! (`extensions/integer-payloads.md` §3), and the receipt identifies it. Given a +//! tool's JSON bitstream instead, the meter converts it to `WIV1` first +//! (`int_payload::wiv1_from_json`) and meters and seals the converted payload. +//! //! Video was the last cargo class with no meter: haptics, 4D splats, scores and //! film are metered by `wai_meter`, worlds/feeds/avatars by `wai_world_meter`, //! quantum circuits by `wai_quantum_meter`. It also has the strongest claim of @@ -42,21 +47,28 @@ use ed25519_dalek::SigningKey; use wai::energy_burst::{measure_with_macmon, power_reader}; use wai::energy_figure::meter_report; use wai::energy_class::CounterFiltering; -use wai::int_video::decode; -use wai::video_receipt::{work_pixel_ops, Provenance, VideoReceipt}; +use wai::video_receipt::{decode_wiv1, work_pixel_ops, Provenance, VideoReceipt}; fn main() { let report_args = meter_report::parse_args_or_exit(&std::env::args().collect::>()); let args = &report_args.positional; let Some(path) = args.get(1) else { - eprintln!("usage: wai_video_meter [receipt_out.json [report_out.json]] [--no-report] [--previous-report ]"); + eprintln!("usage: wai_video_meter [receipt_out.json [report_out.json]] [--no-report] [--previous-report ]"); std::process::exit(2); }; - let bytes = std::fs::read(path).expect("read bitstream"); - let bs: serde_json::Value = serde_json::from_slice(&bytes).expect("parse bitstream JSON"); + let bytes = std::fs::read(path).expect("read the codestream"); + // The payload is WIV1; a JSON bitstream is the tools' form, converted first. + let payload = if bytes.starts_with(b"WIV1") { + bytes + } else { + let bs: serde_json::Value = serde_json::from_slice(&bytes).expect("neither a WIV1 payload nor a JSON bitstream"); + let p = wai::int_payload::wiv1_from_json(&bs, None).expect("the JSON bitstream converts to WIV1"); + eprintln!("{path} is a JSON bitstream: metering and sealing the WIV1 payload converted from it"); + p + }; // A decode that does not even run is not meterable. - let clip = decode(&bs).expect("codestream decodes"); + let clip = decode_wiv1(&payload).expect("codestream decodes"); let (n, h, w) = (clip.frames.len() as u64, clip.h as u64, clip.w as u64); let work = work_pixel_ops(n, h, w); @@ -78,8 +90,8 @@ fn main() { // as one. let mut values = None; let measured: Option<(u64, Provenance)> = - match measure_with_macmon("decode", &bs, |b| { - let _ = decode(b); + match measure_with_macmon("decode", &payload, |p| { + let _ = decode_wiv1(p); }) { None => { eprintln!( @@ -144,19 +156,11 @@ fn main() { // label it also did not earn. None => (0, None), }; - let receipt = VideoReceipt::seal_decode( - &signer, - signer_id, - &bytes, - &bs, - uj, - class, - None, - ) - .expect("seal receipt"); + let receipt = VideoReceipt::seal_decode_wiv1(&signer, signer_id, &payload, uj, class, None) + .expect("seal receipt"); assert!(receipt.verify(), "emitted receipt must verify"); assert!( - receipt.verify_reconstruction(&bytes, &bs), + receipt.verify_reconstruction_wiv1(&payload), "emitted receipt must re-decode to the same frames" ); let tag = if measured.is_some() { "MEASURED" } else { "UNMETERED" }; diff --git a/wai/wai-rs/src/claim_receipt.rs b/wai/wai-rs/src/claim_receipt.rs index d4c414f40..54ad62508 100644 --- a/wai/wai-rs/src/claim_receipt.rs +++ b/wai/wai-rs/src/claim_receipt.rs @@ -113,22 +113,15 @@ use std::fmt::Write as _; pub const DOMAIN_CLAIM: &[u8] = b"moq-jwp:claim\x01"; /// Domain of a claim receipt's link hash (its identity for `parent_receipt_hash`). pub const DOMAIN_CLAIM_LINK: &[u8] = b"moq-jwp:claim-link\x01"; -/// jwp-receipts §2.1: the per-object content hash domain. -pub const DOMAIN_OBJECT: &[u8] = b"moq-jwp:object\x01"; +// jwp-receipts §2.1: the per-object content hash and its domain, kept under their +// names here so every existing caller of this module's API stays valid. +pub use crate::jwp_hash::{content_hash as object_content_hash, DOMAIN_OBJECT}; /// The largest integer a claim may carry: I-JSON's (RFC 7493) exact-integer bound, /// which is what keeps RFC 8785 number serialisation trivial. Crate-private so the /// C header (cbindgen exports public integer constants) does not gain a generic name. pub(crate) const MAX_INTEGER: u64 = (1 << 53) - 1; -/// jwp-receipts §2.1 content hash of a WAI envelope — the hard binding. -pub fn object_content_hash(envelope_bytes: &[u8]) -> [u8; 32] { - let mut h = blake3::Hasher::new(); - h.update(DOMAIN_OBJECT); - h.update(envelope_bytes); - *h.finalize().as_bytes() -} - // ───────────────────────────── reasons ───────────────────────────── /// Why a claim, or a recovery, was rejected. The label is the conformance name. diff --git a/wai/wai-rs/src/codecs/mod.rs b/wai/wai-rs/src/codecs/mod.rs index 09c794592..506fb5211 100644 --- a/wai/wai-rs/src/codecs/mod.rs +++ b/wai/wai-rs/src/codecs/mod.rs @@ -47,8 +47,8 @@ pub const CAP_AUDIO_FLAC: &str = "wai.audio.flac"; // lossless // Video pub const CAP_VIDEO_AV1: &str = "wai.video.av1"; // lossy AV1 pub const CAP_VIDEO_AV1_LOSSLESS: &str = "wai.video.av1.lossless"; // lossless AV1 -pub const CAP_VIDEO_INT_MOTION: &str = "wai.video.int_motion"; // DETERMINISTIC INTEGER inter-frame codec (int_video.rs): I-frame + integer block motion-compensated P-frames + integer-quantized residual over rANS. Decoder IMPLEMENTED, pure i64, no float/ONNX — every frame is byte-identical on every machine (frame-state-equivalence; per-frame portable hash). The temporal extension of the integer contract. Bitstream from tools/wai_int_video_encode.py -pub const CAP_VIDEO_INT_HYPER: &str = "wai.video.int_hyper"; // NEURAL-KEYFRAME inter-frame codec: the I-frame is the LEARNED integer codec int_hyper (wai.neural.int_hyper) decoded byte-exact, the P-frames are integer block-MC residuals on top (int_video::decode_pframes). A learned keyframe you can put in a receipt + integer inter-frames — the whole decode integer, byte-identical everywhere. Composes int_hyper_synth + int_video; pure i64 +pub const CAP_VIDEO_INT_MOTION: &str = "wai.video.int_motion"; // DETERMINISTIC INTEGER inter-frame codec (int_video.rs): I-frame + integer block motion-compensated P-frames + integer-quantized residual over rANS. Decoder IMPLEMENTED, pure i64, no float/ONNX — every frame is byte-identical on every machine (frame-state-equivalence; per-frame portable hash). The temporal extension of the integer contract. Payload WIV1, keyframe 0 (extensions/integer-payloads.md); the encode tools write a JSON form a converter reads +pub const CAP_VIDEO_INT_HYPER: &str = "wai.video.int_hyper"; // NEURAL-KEYFRAME inter-frame codec: the I-frame is the LEARNED integer codec int_hyper (wai.neural.int_hyper) decoded byte-exact, the P-frames are integer block-MC residuals on top (int_video::decode_pframes_typed). A learned keyframe you can put in a receipt + integer inter-frames — the whole decode integer, byte-identical everywhere. Composes int_hyper_synth + int_video; pure i64. Payload WIV1, keyframe 1, a nested WIH1 (extensions/integer-payloads.md) // Text pub const CAP_TEXT_ZSTD: &str = "wai.text.zstd"; // general-purpose pub const CAP_TEXT_XZ: &str = "wai.text.xz"; // maximum classical ratio @@ -61,7 +61,7 @@ pub const CAP_NEURAL_WAVTOKENIZER: &str = "wai.neural.wavtokenizer"; // ultra-lo pub const CAP_NEURAL_BMSHJ2018: &str = "wai.neural.bmshj2018"; // bmshj2018-factorized image codec (2018 baseline) pub const CAP_NEURAL_MBT2018_MEAN: &str = "wai.neural.mbt2018_mean"; // mean-scale HYPERPRIOR image codec (Gaussian-conditional entropy) — decoder IMPLEMENTED in this crate (neural/hyperprior.rs); the conditional-entropy family: a mean-scale hyperprior conditions each latent's Gaussian; sink-supplied ONNX (hyper_synthesis + synthesis) pub const CAP_NEURAL_INT_SYNTH: &str = "wai.neural.int_synth"; // INTEGER fixed-point synthesis (neural/int_synth.rs): all-i64 transposed-conv decode, no float/ONNX — bit-identical on every machine BY CONSTRUCTION. The first capability that can soundly back intent=replicate cross-hardware (determinism IS integer-exact). Weights are sink-supplied (int_synth.json from a QAT model) -pub const CAP_NEURAL_INT_HYPER: &str = "wai.neural.int_hyper"; // INTEGER fixed-point CONDITIONAL codec (int_hyper_synth.rs): the full Gaussian-conditional decode integer end-to-end — factorized rANS-decode z (z_q = s·2^afrac + median_q) → integer hyper-decoder h_s → σ,μ → integer-σ bucket (int_entropy) → integer rANS-decode residual → y_hat = r·2^afrac + μ → integer synthesis g_s. Both bitstreams entropy-coded, no raw latents. No float/ONNX anywhere, so the WHOLE bitstream→image decode is bit-identical on every machine BY CONSTRUCTION — int_synth's sibling that adds the conditional entropy. Quality: PTQ of the pretrained mbt2018_mean (quality 6); CI (wai-sota-vectors.yml) requires the integer decode within 1 dB of the same model's float decode on a pinned test photograph — measured cost there 0.19 dB (35.985 vs 36.175 dB, arm64 macOS); image-dependent, see int_hyper_synth.rs. Tables sink-supplied +pub const CAP_NEURAL_INT_HYPER: &str = "wai.neural.int_hyper"; // INTEGER fixed-point CONDITIONAL codec (int_hyper_synth.rs): the full Gaussian-conditional decode integer end-to-end — factorized rANS-decode z (z_q = s·2^afrac + median_q) → integer hyper-decoder h_s → σ,μ → integer-σ bucket (int_entropy) → integer rANS-decode residual → y_hat = r·2^afrac + μ → integer synthesis g_s. Both bitstreams entropy-coded, no raw latents; payload WIH1 (extensions/integer-payloads.md). No float/ONNX anywhere, so the WHOLE bitstream→image decode is bit-identical on every machine BY CONSTRUCTION — int_synth's sibling that adds the conditional entropy. Quality: PTQ of the pretrained mbt2018_mean (quality 6); CI (wai-sota-vectors.yml) requires the integer decode within 1 dB of the same model's float decode on a pinned test photograph — measured cost there 0.19 dB (35.985 vs 36.175 dB, arm64 macOS); image-dependent, see int_hyper_synth.rs. Tables sink-supplied pub const CAP_NEURAL_INT_MLICPP: &str = "wai.neural.int_mlicpp"; // INTEGER MLIC++-topology codec (int_mlicpp.rs): the channel-autoregressive × checkerboard entropy model — K channel slices, each decoded as an anchor pass then a non-anchor pass, with an entropy network over concat[y decoded so far, hyper] giving σ‖μ before every pass — with every decode-side network integer (int_hyper_synth networks, int_entropy bucketing, integer rANS). The encoder and every decoder compute the same σ, so the same bucket, at any thread count: no float/ONNX anywhere, byte-identical on every machine BY CONSTRUCTION. Refuses any stream value outside the exact integer range. Payload (WIM1) and one-file model bundle (WMB1, so a prior pins every network and table) specified in int_mlicpp.rs. Conformance vector tests/vectors/int_mlicpp/ has SYNTHETIC weights (seeded, quantised like a trained model): it shows decode-equivalence, not compression quality. Model sink-supplied pub const CAP_NEURAL_ELIC: &str = "wai.neural.elic"; // checkerboard CONTEXT-MODEL image codec (neural/context.rs) — the JPEG-AI/MLIC++ entropy class: hyperprior + 2-pass spatial context (anchor → context net → non-anchor). Decoder IMPLEMENTED; sink-supplied ONNX (hyper_synthesis + context + synthesis). WAI-native, NOT the ISO JPEG AI bitstream pub const CAP_NEURAL_MLICPP: &str = "wai.neural.mlicpp"; // MLIC++: the multi-reference entropy model — channel-autoregressive slices + checkerboard + global attention. Decoder IMPLEMENTED (neural/mlicpp.rs): the slice/checkerboard decode orchestration; the attention nets are sink-supplied ONNX (hyper_synthesis + entropy + synthesis) @@ -136,8 +136,10 @@ pub const CAP_SPLAT_SPZ: &str = "wai.splat.spz"; // synthesis (int_hyper machinery, 14 channels) over a factorized-rANS latent → // byte-exact attribute tensor on every machine (splat-equivalence; a portable cloud // hash). The int_hyper of 3D — the decoded attributes you can put in a receipt. -// Decoder IMPLEMENTED, pure i64, no float/ONNX. Model + bitstream from -// tools/wai_int_splat_export.py. The float rasterizer stays sink-supplied. +// Decoder IMPLEMENTED, pure i64, no float/ONNX. Payload WIS1, with each attribute's +// presentation range (extensions/integer-payloads.md); its parameter set is one file, +// the WHM1 synthesis. tools/wai_int_splat_export.py writes the model and a JSON form a +// converter reads. The float rasterizer stays sink-supplied. pub const CAP_SPLAT_INT_CODEC: &str = "wai.splat.int_codec"; // Object-based spatial audio (extensions/spatial-audio, Phase 1). Ship @@ -161,10 +163,29 @@ pub const CAP_AUDIO_SCORE: &str = "wai.audio.score"; // decoding the per-channel factorized-rANS latent to int16 PCM byte-identically on // every machine (sample-equivalence; a portable PCM hash). The audio analog of // int_hyper — a neural audio codec whose reconstruction you can put in a receipt. -// Decoder IMPLEMENTED, pure i64, no float/ONNX. Model + bitstream from -// tools/wai_int_audio_export.py +// Decoder IMPLEMENTED, pure i64, no float/ONNX. Payload WIA1, with its sample rate +// (extensions/integer-payloads.md); its parameter set is one file, the WAM1 synthesis. +// tools/wai_int_audio_export.py writes the model and a JSON form a converter reads. pub const CAP_AUDIO_INT_CODEC: &str = "wai.audio.int_codec"; +/// SPEC §5, *Payloads of the WAI-native integer codecs*: each capability whose payload +/// this revision pins to a format of `extensions/integer-payloads.md`, with the +/// format's magic. The one table `int_payload`, `container::check_int_payload_framing` +/// (and so `wai wrap` and `wai inspect`) and the SPEC test read. `WIV1` is pinned twice: +/// keyframe 0 for `wai.video.int_motion`, keyframe 1 for `wai.video.int_hyper`. +pub const PINNED_INT_PAYLOADS: &[(&str, [u8; 4])] = &[ + (CAP_VIDEO_INT_MOTION, *b"WIV1"), + (CAP_VIDEO_INT_HYPER, *b"WIV1"), + (CAP_NEURAL_INT_HYPER, *b"WIH1"), + (CAP_AUDIO_INT_CODEC, *b"WIA1"), + (CAP_SPLAT_INT_CODEC, *b"WIS1"), +]; + +/// The magic of the format SPEC §5 pins `cap`'s payload to, or `None`. +pub fn pinned_int_payload(cap: &str) -> Option<[u8; 4]> { + PINNED_INT_PAYLOADS.iter().find(|(c, _)| *c == cap).map(|&(_, m)| m) +} + // Haptics (extensions/haptics): per-channel intensity/sharpness envelope, // reconstructed to an actuator signal (sample-equivalence). Engine // behind the `haptic` feature. @@ -1265,9 +1286,11 @@ pub fn parameter_set_shape(cap: &str) -> Option { } Some(match cap { // int_synth: its model specification, int_synth.json. int_mlicpp: its WMB1 - // bundle. The audio codecs and the per-frame video codec: one self-contained - // ONNX decoder (a model that keeps tensors in external files is several files). - CAP_NEURAL_INT_SYNTH | CAP_NEURAL_INT_MLICPP + // bundle. audio.int_codec: its WAM1 synthesis (model.wam). splat.int_codec: its + // WHM1 synthesis (g_s.whm). The audio codecs and the per-frame video codec: one + // self-contained ONNX decoder (a model that keeps tensors in external files is + // several files). + CAP_NEURAL_INT_SYNTH | CAP_NEURAL_INT_MLICPP | CAP_AUDIO_INT_CODEC | CAP_SPLAT_INT_CODEC | CAP_NEURAL_ENCODEC32 | CAP_NEURAL_DAC | CAP_NEURAL_MIMI | CAP_NEURAL_WAVTOKENIZER | CAP_NEURAL_SNAC | CAP_NEURAL_VIDEO_BMSHJ2018 => ParameterSetShape::OneFile, _ => ParameterSetShape::SeveralFiles, @@ -1993,14 +2016,15 @@ mod tests { } } one_file.sort_unstable(); - let mut want = vec![CAP_NEURAL_INT_SYNTH, CAP_NEURAL_INT_MLICPP, CAP_NEURAL_ENCODEC32, CAP_NEURAL_DAC, - CAP_NEURAL_MIMI, CAP_NEURAL_WAVTOKENIZER, CAP_NEURAL_SNAC, CAP_NEURAL_VIDEO_BMSHJ2018]; + let mut want = vec![CAP_NEURAL_INT_SYNTH, CAP_NEURAL_INT_MLICPP, CAP_AUDIO_INT_CODEC, CAP_SPLAT_INT_CODEC, + CAP_NEURAL_ENCODEC32, CAP_NEURAL_DAC, CAP_NEURAL_MIMI, CAP_NEURAL_WAVTOKENIZER, + CAP_NEURAL_SNAC, CAP_NEURAL_VIDEO_BMSHJ2018]; want.sort_unstable(); assert_eq!(one_file, want); for c in [CAP_IMAGE_PNG, CAP_SEMANTIC_CLIP, CAP_PRIOR_SET, CAP_WORLD_MODEL] { assert!(!decodes_against_parameter_set(c), "{c}"); } - for c in [CAP_IMAGE_JPEGAI, CAP_NEURAL_GLC, CAP_GENERATE_SPECTRAL, CAP_AUDIO_INT_CODEC] { + for c in [CAP_IMAGE_JPEGAI, CAP_NEURAL_GLC, CAP_GENERATE_SPECTRAL, CAP_NEURAL_INT_HYPER, CAP_VIDEO_INT_HYPER] { assert_eq!(parameter_set_shape(c), Some(ParameterSetShape::SeveralFiles), "{c}"); } // The listing form: lossless, royalty-free, StandardDefined. diff --git a/wai/wai-rs/src/codecs/registry.rs b/wai/wai-rs/src/codecs/registry.rs index be4ce30d5..29381fcff 100644 --- a/wai/wai-rs/src/codecs/registry.rs +++ b/wai/wai-rs/src/codecs/registry.rs @@ -1212,6 +1212,54 @@ mod tests { assert!(problems.is_empty(), "SPEC and the registry disagree:\n{}", problems.join("\n")); } + /// SPEC §5, *Payloads of the WAI-native integer codecs*, pins exactly the strings + /// and formats of `PINNED_INT_PAYLOADS`, and names each string's parameter-set + /// members as `int_payload` reads them; the extension defines each format in its + /// own section. + #[test] + fn spec_pins_the_integer_payloads_the_registry_lists() { + let spec = read("SPEC.md"); + let at = spec.find("### Payloads of the WAI-native integer codecs").expect("the SPEC §5 subsection"); + let section = &spec[at..]; + let section = §ion[..section[4..].find("\n### ").map_or(section.len(), |e| e + 4)]; + let mut rows: Vec<(String, String, String)> = Vec::new(); + for line in section.lines().filter(|l| l.starts_with("| `wai.")) { + let cells: Vec<&str> = line.split('|').map(str::trim).collect(); + let tick = |c: &str| c.split('`').nth(1).unwrap_or_default().to_string(); + rows.push((tick(cells[1]), tick(cells[2]), cells[3].to_string())); + } + let mut want: Vec<(String, String)> = super::super::PINNED_INT_PAYLOADS.iter() + .map(|(c, m)| (c.to_string(), String::from_utf8_lossy(m).into_owned())).collect(); + let mut got: Vec<(String, String)> = rows.iter().map(|(c, m, _)| (c.clone(), m.clone())).collect(); + want.sort(); + got.sort(); + assert_eq!(got, want, "SPEC's table and PINNED_INT_PAYLOADS"); + // Each row's set: its shape is the registry's, and its members are named. + let members: &[(&str, &[&str])] = &[ + (CAP_VIDEO_INT_MOTION, &[]), + (CAP_VIDEO_INT_HYPER, &["g_s.whm", "h_s.whm", "entropy.json"]), + (CAP_NEURAL_INT_HYPER, &["g_s.whm", "h_s.whm", "entropy.json"]), + (CAP_AUDIO_INT_CODEC, &["model.wam"]), + (CAP_SPLAT_INT_CODEC, &["g_s.whm"]), + ]; + for (cap, _, set) in &rows { + let (_, names) = members.iter().find(|(c, _)| c == cap).expect("a pinned string"); + let shape = match crate::codecs::parameter_set_shape(cap) { + None => "none", + Some(crate::codecs::ParameterSetShape::OneFile) => "one file", + Some(crate::codecs::ParameterSetShape::SeveralFiles) => "several files", + }; + assert!(set.starts_with(shape), "{cap}: SPEC says `{set}`, the registry {shape}"); + for n in *names { + assert!(set.contains(&format!("`{n}`")), "{cap}: SPEC names no `{n}`"); + } + } + let ext = read("extensions/integer-payloads.md"); + for (heading, magic) in [("## 3.", "WIV1"), ("## 4.", "WIH1"), ("## 5.", "WIA1"), ("## 6.", "WIS1")] { + assert!(ext.lines().any(|l| l.starts_with(heading) && l.contains(&format!("`{magic}`"))), "the extension's {heading} is {magic}"); + } + } + #[test] fn vocab_v2_names_every_registered_string_and_no_candidate_as_emittable() { let shapes = vocab_v2_shapes(); diff --git a/wai/wai-rs/src/container.rs b/wai/wai-rs/src/container.rs index 581a03fb5..1d5223d6d 100644 --- a/wai/wai-rs/src/container.rs +++ b/wai/wai-rs/src/container.rs @@ -733,6 +733,15 @@ impl Security { } } +/// SPEC §6.4 step 2 at a sink that implements no encryption scheme, as the reference +/// dispatchers are: a manifest whose `security` declares an encryption other than +/// `"none"` makes the envelope inert here, and no ciphertext reaches a codec. A sink +/// that opens a scheme decrypts first and dispatches the plaintext envelope (§6.4 +/// step 3), whose `security` is plaintext. `declared` is the encryption as written. +fn encryption_inert(declared: impl Into) -> Inert { + Inert::Encrypted { encryption: declared.into() } +} + /// A `WAI1` manifest (SPEC §3). Build one with [`Manifest::new`]. #[derive(Debug, Clone, Serialize, Deserialize)] #[non_exhaustive] @@ -809,14 +818,19 @@ pub enum Inert { /// The pin was refused for `capability` (the first refusal kept, step 3 before /// step 5). PinRefused { capability: String, refusal: PinRefusal }, + /// The manifest declares an encryption this sink cannot open (SPEC §6.4 step 2, + /// `unsupported_encryption`): the reference dispatchers implement no scheme, so an + /// encrypted envelope is inert at them before any capability is considered. + Encrypted { encryption: String }, } impl Inert { - /// `missing_capability`, or the pin's reason. + /// `missing_capability`, `unsupported_encryption`, or the pin's reason. pub fn code(&self) -> &'static str { match self { Self::MissingCapability => "missing_capability", Self::PinRefused { refusal, .. } => refusal.code(), + Self::Encrypted { .. } => "unsupported_encryption", } } } @@ -826,6 +840,11 @@ impl std::fmt::Display for Inert { match self { Self::MissingCapability => write!(f, "missing_capability: the sink advertises neither the capability nor the fallback"), Self::PinRefused { capability, refusal } => write!(f, "{}: {capability}: {refusal}", refusal.code()), + Self::Encrypted { encryption } => write!( + f, + "unsupported_encryption: the manifest declares the encryption {encryption}, which this sink cannot \ + open (SPEC §6.4): the file is inert here, and its payload is not decoded" + ), } } } @@ -904,7 +923,9 @@ impl Wai { } /// SPEC §4 dispatch at a sink that advertises the capabilities `advertises` - /// accepts and verifies pins with `verify` (SPEC §3.1). Step 3 takes the + /// accepts and verifies pins with `verify` (SPEC §3.1). First, SPEC §6.4 step 2: + /// a manifest that declares an encryption other than `"none"` is inert here + /// ([`Inert::Encrypted`]), since this dispatcher opens no scheme. Step 3 takes the /// capability when the sink advertises it and, where the pin applies to it, /// `verify` accepts the pin; step 5 does the same for the fallback. A refused /// pin makes that capability unsupported for this envelope: dispatch moves on, @@ -935,6 +956,11 @@ impl Wai { advertises: impl Fn(&str) -> bool, mut verify: impl FnMut(&str, &Prior) -> Result, ) -> Result, Inert> { + // SPEC §6.4 step 2, before §4: an encrypted payload is inert at a sink that + // opens no scheme. + if let Some(sec) = self.manifest.security.as_ref().filter(|s| !s.is_plaintext()) { + return Err(encryption_inert(format!("{:?}", sec.encryption.as_deref().unwrap_or_default()))); + } let primary = self.capability(); if crate::codecs::is_companion_only(primary) { return Err(Inert::MissingCapability); @@ -1036,11 +1062,31 @@ impl Wai { check_candidates_v1(self.capability(), self.fallback()) } - /// The envelope's bytes, for an encoder to emit: [`Self::check_candidates`], - /// then [`Self::to_bytes`]. The C packer `wai_envelope_pack` emits through - /// this, so an encoder that uses it cannot emit a candidate (SPEC §5). + /// Check that the payload is the format SPEC §5 pins for the `capability`, and for + /// the `fallback` (step 5 decodes the same payload through it), where either is an + /// integer capability with a pinned format ([`check_int_payload_framing`]). An + /// encrypted payload (`security.encryption` other than `"none"`) is ciphertext, so + /// it is not checked here: a sink decrypts it first (SPEC §6.4) and checks the + /// plaintext when it decodes it. + pub fn check_int_payload_framing(&self) -> Result<(), FramingError> { + if !self.manifest.security.as_ref().is_none_or(Security::is_plaintext) { + return Ok(()); + } + check_int_payload_framing(self.capability(), &self.payload)?; + match self.fallback() { + Some(fb) => check_int_payload_framing(fb, &self.payload), + None => Ok(()), + } + } + + /// The envelope's bytes, for an encoder to emit: [`Self::check_candidates`], then + /// [`Self::check_int_payload_framing`], then [`Self::to_bytes`]. The C packer + /// `wai_envelope_pack` emits through this, so an encoder that uses it cannot emit a + /// candidate, or an integer payload that is not its capability's pinned format + /// (SPEC §5). pub fn to_emit_bytes(&self) -> Result, EmitError> { self.check_candidates().map_err(EmitError::Candidate)?; + self.check_int_payload_framing().map_err(|error| EmitError::IntPayload { component: None, error })?; self.to_bytes().map_err(EmitError::Serialize) } @@ -1330,6 +1376,131 @@ pub fn check_component_candidates<'a>(capabilities: impl IntoIterator bool { + matches!(manifest_v2_security(m), Ok(s) if s.as_ref().is_none_or(Security::is_plaintext)) +} + +/// A `WAI2` manifest's `security` member (SPEC §6.1, §7.1): `None` when absent or +/// `null`; the object; or why the member is not a security object (`true`, a string, +/// an array, an `encryption` that is not a string, …), which an encoder does not write +/// ([`WaiMulti::to_emit_bytes`]). +pub fn manifest_v2_security(m: &ManifestV2) -> Result, String> { + match m.extra.get("security") { + None | Some(Value::Null) => Ok(None), + Some(v) => serde_json::from_value::(v.clone()).map(Some) + .map_err(|e| format!("the manifest's `security` member {v} is not a security object (SPEC §6.1): {e}")), + } +} + +/// Why a payload is not the format SPEC §5 pins its capability to (*Payloads of the +/// WAI-native integer codecs*, `extensions/integer-payloads.md`): the capability, the +/// code the extension's §8 names and what failed. The payload is parsed as a sink parses +/// it (§2–§6), so every code a parse can give is possible here: `bad_magic` (none of the +/// four formats; a JSON form lands here), `format_capability_mismatch`, +/// `keyframe_capability_mismatch`, `version`, `reserved_nonzero`, `field_range`, +/// `too_large`, `cdf_malformed`, `stream_length`, `nested_length` and `trailing`. What +/// only a decode checks, against a parameter set and through the streams (§7), is not. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FramingError { + pub capability: String, + pub code: &'static str, + pub detail: &'static str, +} + +impl std::fmt::Display for FramingError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let want = crate::codecs::pinned_int_payload(&self.capability).map(|m| String::from_utf8_lossy(&m).into_owned()); + write!(f, "the payload is not the {} payload SPEC \u{a7}5 pins {} to ({}: {}; extensions/integer-payloads.md)", + want.as_deref().unwrap_or("?"), self.capability, self.code, self.detail) + } +} + +impl std::error::Error for FramingError {} + +/// Check that `payload` is the format SPEC §5 pins `capability`'s payload to +/// ([`crate::codecs::PINNED_INT_PAYLOADS`]): it is parsed with the wire parser a sink +/// runs (`int_payload::wire`, with the capability's keyframe kind for `WIV1`), which +/// checks every rule of the extension's §2–§6 in its order and reports the first that +/// fails ([`FramingError`]). `Ok` for a capability with no pinned integer format. It +/// needs no parameter set and no integer decoder, so every emit path runs it +/// ([`Wai::to_emit_bytes`], [`WaiMulti::to_emit_bytes`], `wai wrap`, the browser's +/// writers). `payload` is plaintext: an encrypted envelope's payload is checked after +/// it is decrypted (SPEC §6.4), and its emit paths skip the check. +pub fn check_int_payload_framing(capability: &str, payload: &[u8]) -> Result<(), FramingError> { + use crate::int_wire as w; + let parsed = match crate::codecs::pinned_int_payload(capability) { + None => return Ok(()), + Some(w::MAGIC_WIV1) => w::parse_wiv1(payload, Some((capability == crate::codecs::CAP_VIDEO_INT_HYPER) as u8)).map(|_| ()), + Some(w::MAGIC_WIH1) => w::parse_wih1(payload).map(|_| ()), + Some(w::MAGIC_WIA1) => w::parse_wia1(payload).map(|_| ()), + Some(_) => w::parse_wis1(payload).map(|_| ()), + }; + parsed.map_err(|e| FramingError { capability: capability.to_string(), code: e.code(), detail: e.detail }) +} + +/// What an integer payload's header declares about its decode, read before any +/// symbol is decoded and with no parameter set (`extensions/integer-payloads.md` §7). +/// A sink applies its own size policy to it and refuses a payload past that policy +/// before decoding; `int_payload::declared_cost` adds what the parameter set fixes +/// (the networks' buffers, a `WIH1` image's size). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct DeclaredSize { + /// The decoded output's bytes, where the header alone fixes it: the frames of a + /// `WIV1` (`n_frames·h·w·3`), the PCM of a `WIA1` (`2·t_out`), the attribute tensor + /// of a `WIS1` (`attr·s·s`). `None` for a `WIH1`, whose image size is its + /// synthesis network's. + pub output_bytes: Option, + /// The symbols the payload's streams code: one entropy-decode step each. + pub symbols: u64, + /// The most bytes the reference decode holds at once beyond the payload, where the + /// header alone fixes it: a `WIV1` with keyframe 0 + /// ([`crate::int_wire::Wiv1::keyframe0_working_bytes`]). `None` where a network + /// runs, whose buffers the parameter set sizes. + pub working_bytes: Option, + /// The networks' multiply-accumulates, where the header alone fixes them: 0 for a + /// `WIV1` with keyframe 0, which runs no network; `None` otherwise. + pub macs: Option, +} + +/// [`DeclaredSize`] for `payload` under `capability`, after the parse +/// [`check_int_payload_framing`] makes. `Ok(None)` for a capability with no pinned +/// integer format. +pub fn declared_size(capability: &str, payload: &[u8]) -> Result, FramingError> { + use crate::int_wire as w; + let fail = |e: w::WireError| FramingError { capability: capability.to_string(), code: e.code(), detail: e.detail }; + let wih1 = |k: &w::Wih1<'_>| k.n_z as u64 * k.zh as u64 * k.zw as u64 + k.m as u64 * k.sh as u64 * k.sw as u64; + Ok(Some(match crate::codecs::pinned_int_payload(capability) { + None => return Ok(None), + Some(w::MAGIC_WIV1) => { + let v = w::parse_wiv1(payload, Some((capability == crate::codecs::CAP_VIDEO_INT_HYPER) as u8)).map_err(fail)?; + let frame = v.h as u64 * v.w as u64 * 3; + let output_bytes = Some(frame * v.n_frames as u64); + match v.wih1 { + None => DeclaredSize { output_bytes, symbols: frame * v.n_frames as u64, working_bytes: Some(v.keyframe0_working_bytes()), macs: Some(0) }, + Some(k) => { + let symbols = frame * (v.n_frames as u64 - 1) + wih1(&w::parse_wih1(k).map_err(fail)?); + DeclaredSize { output_bytes, symbols, working_bytes: None, macs: None } + } + } + } + Some(w::MAGIC_WIH1) => DeclaredSize { output_bytes: None, symbols: wih1(&w::parse_wih1(payload).map_err(fail)?), working_bytes: None, macs: None }, + Some(w::MAGIC_WIA1) => { + let a = w::parse_wia1(payload).map_err(fail)?; + DeclaredSize { output_bytes: Some(2 * a.t_out as u64), symbols: a.n_ch as u64 * a.lat_len as u64, working_bytes: None, macs: None } + } + Some(_) => { + let v = w::parse_wis1(payload).map_err(fail)?; + DeclaredSize { output_bytes: Some(v.attr as u64 * v.s as u64 * v.s as u64), symbols: v.c as u64 * v.sh as u64 * v.sw as u64, working_bytes: None, macs: None } + } + })) +} + /// Why [`Wai::to_emit_bytes`] or [`WaiMulti::to_emit_bytes`] wrote nothing. #[derive(Debug)] #[non_exhaustive] @@ -1338,6 +1509,14 @@ pub enum EmitError { Candidate(CandidateError), /// The envelope did not serialise. Serialize(std::io::Error), + /// A payload is not the format SPEC §5 pins its integer capability to + /// ([`check_int_payload_framing`]): the `WAI2` component's index, or `None` for a + /// `WAI1`'s payload. + IntPayload { component: Option, error: FramingError }, + /// A `WAI2` manifest's `security` member is not a security object + /// ([`manifest_v2_security`]): an encoder that writes one could claim an + /// encryption no sink can read, or none. + Security(String), } impl std::fmt::Display for EmitError { @@ -1345,6 +1524,9 @@ impl std::fmt::Display for EmitError { match self { EmitError::Candidate(e) => e.fmt(f), EmitError::Serialize(e) => write!(f, "envelope serialize: {e}"), + EmitError::IntPayload { component: Some(i), error } => write!(f, "component {i}: {error}"), + EmitError::IntPayload { component: None, error } => error.fmt(f), + EmitError::Security(why) => f.write_str(why), } } } @@ -1456,11 +1638,15 @@ pub enum Skip { /// A policy that serves only sound `replicate` claims left the entry out: /// SPEC §7's rule refuses `replicate` through it (`replicate_*`). Replicate(crate::codecs::ReplicateRefusal), + /// The manifest declares an encryption the sink cannot open (SPEC §6.4 step 2, + /// `unsupported_encryption`): every entry's payload is ciphertext here. + Encrypted, } impl Skip { /// The reason: `not_selectable`, `missing_capability`, - /// `component_digest_mismatch`, the pin's reason, or the `replicate_*` reason. + /// `component_digest_mismatch`, the pin's reason, the `replicate_*` reason, or + /// `unsupported_encryption`. pub fn code(&self) -> &'static str { match self { Self::NotSelectable => "not_selectable", @@ -1468,6 +1654,7 @@ impl Skip { Self::ComponentDigest => "component_digest_mismatch", Self::Pin(r) => r.code(), Self::Replicate(r) => r.code(), + Self::Encrypted => "unsupported_encryption", } } } @@ -1645,7 +1832,10 @@ impl WaiMulti { /// pin applies ([`pin_applies_to`]), `verify` accepts. An entry whose pin is /// refused is skipped as if its capability were not advertised, and its reason /// kept. When nothing is picked the envelope is inert at this sink, and the - /// error gives every entry's index and reason. + /// error gives every entry's index and reason. An envelope whose manifest declares + /// an encryption other than `"none"` is inert before any entry is considered (SPEC + /// §6.4 step 2; this dispatcher opens no scheme): every entry is + /// [`Skip::Encrypted`]. pub fn pick( &self, advertises: impl Fn(&str) -> bool, @@ -1672,6 +1862,11 @@ impl WaiMulti { eligible: impl Fn(usize, &RenditionMeta) -> Result<(), Skip>, mut verify: impl FnMut(&str, &Prior) -> Result, ) -> Result, Vec<(usize, Skip)>> { + // SPEC §6.4 step 2, before selection: an encrypted envelope is inert at a sink + // that opens no scheme, and every entry is reported so. + if !manifest_v2_plaintext(&self.manifest) { + return Err((0..self.manifest.renditions.len()).map(|i| (i, Skip::Encrypted)).collect()); + } let mut skipped = Vec::new(); for (index, (meta, payload)) in self.manifest.renditions.iter().zip(self.renditions.iter()).enumerate() { if !meta.is_selectable() { @@ -1708,11 +1903,31 @@ impl WaiMulti { check_component_candidates(self.manifest.renditions.iter().map(|m| m.capability.as_str())) } - /// The envelope's bytes, for an encoder to emit: [`Self::check_candidates`], - /// then [`Self::to_bytes`]. The C packer `wai_envelope_pack_multi` emits - /// through this, so an encoder that uses it cannot emit a candidate (SPEC §5). + /// [`check_int_payload_framing`] over every component, in table order: the + /// first component whose payload is not its capability's pinned format, with its + /// index. Skipped for an encrypted envelope (a `security` member whose + /// `encryption` is not `"none"`), whose payloads are ciphertext until a sink + /// decrypts them (SPEC §6.4). + pub fn check_int_payload_framing(&self) -> Result<(), (usize, FramingError)> { + if !manifest_v2_plaintext(&self.manifest) { + return Ok(()); + } + for (i, (m, p)) in self.manifest.renditions.iter().zip(&self.renditions).enumerate() { + check_int_payload_framing(&m.capability, p).map_err(|e| (i, e))?; + } + Ok(()) + } + + /// The envelope's bytes, for an encoder to emit: [`Self::check_candidates`], then + /// [`manifest_v2_security`], then [`Self::check_int_payload_framing`], then + /// [`Self::to_bytes`]. The C packer `wai_envelope_pack_multi` emits through this, so + /// an encoder that uses it cannot emit a candidate, a `security` member that is not + /// a security object (SPEC §6.1), or a component whose integer payload is not its + /// capability's pinned format (SPEC §5). pub fn to_emit_bytes(&self) -> Result, EmitError> { self.check_candidates().map_err(EmitError::Candidate)?; + manifest_v2_security(&self.manifest).map_err(EmitError::Security)?; + self.check_int_payload_framing().map_err(|(i, error)| EmitError::IntPayload { component: Some(i), error })?; self.to_bytes().map_err(EmitError::Serialize) } @@ -2676,6 +2891,112 @@ mod tests { assert_eq!(m.pick(|c| c == "wai.image.nope", no_parameter_sets()).unwrap().meta.capability, "wai.image.nope"); } + /// SPEC §5, *Payloads of the WAI-native integer codecs*: `to_emit_bytes` (the C + /// packers' emit path) parses an integer payload as a sink does and writes nothing + /// when it is not its capability's pinned format, as the capability, the fallback + /// or a `WAI2` component: a keyframe-0 `WIV1` under `wai.video.int_hyper`, a `WIV1` + /// cut inside its tables, a tool's JSON form. The committed payloads are written. + #[test] + fn the_emit_path_refuses_an_integer_payload_that_is_not_its_format() { + let read = |rel: &str| std::fs::read(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors").join(rel)).unwrap(); + let (k0, k1) = (read("int_video/int_video_bitstream.wiv1"), read("int_video/video_nk_bitstream.wiv1")); + let (mot, hyp) = (crate::codecs::CAP_VIDEO_INT_MOTION, crate::codecs::CAP_VIDEO_INT_HYPER); + let v1 = |cap: &str, fb: Option<&str>, p: &[u8]| { + let mut req = ModelRequirement::new(cap); + if let Some(f) = fb { + req = req.with_fallback(f); + } + Wai::new(Manifest::new("video", "improve", req, Conditioning::new("bytes")), p.to_vec()).to_emit_bytes() + }; + let code = |r: Result, EmitError>| match r { + Err(EmitError::IntPayload { component, error }) => Some((component, error.code)), + Err(e) => panic!("{e}"), + Ok(_) => None, + }; + assert_eq!(code(v1(hyp, None, &k0)), Some((None, "keyframe_capability_mismatch"))); + assert_eq!(code(v1(mot, None, &k0[..40])), Some((None, "stream_length"))); + assert_eq!(code(v1(mot, None, &read("int_video/int_video_bitstream.json"))), Some((None, "bad_magic"))); + assert_eq!(code(v1(crate::codecs::CAP_VIDEO_AV1, Some(mot), &k1)), Some((None, "keyframe_capability_mismatch"))); + assert_eq!(code(v1(mot, None, &k0)), None); + assert_eq!(code(v1(hyp, None, &k1)), None); + let v2 = |a: &[u8], b: &[u8]| WaiMulti::new("video", "improve", serde_json::Value::Null, vec![ + Rendition::media(mot, "bytes", a.to_vec()), Rendition::media(hyp, "bytes", b.to_vec()), + ]).to_emit_bytes(); + assert_eq!(code(v2(&k0, &k0)), Some((Some(1), "keyframe_capability_mismatch"))); + assert_eq!(code(v2(&k0, &k1)), None); + } + + /// An encrypted envelope's payload is ciphertext (SPEC §6): the emit paths do not + /// check it against the integer format, since a sink decrypts it first (§6.4) and + /// checks the plaintext when it decodes it. A `WAI1` and a `WAI2` that declare an + /// encryption are written; the same bytes in plaintext are refused. + #[test] + fn an_encrypted_integer_payload_is_emitted() { + let read = |rel: &str| std::fs::read(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors").join(rel)).unwrap(); + let plain = read("int_video/int_video_bitstream.wiv1"); + let cipher: Vec = plain.iter().enumerate().map(|(i, b)| b ^ (i as u8).wrapping_mul(151).wrapping_add(7)).collect(); + let mot = crate::codecs::CAP_VIDEO_INT_MOTION; + let v1 = |security: Option| { + let mut m = Manifest::new("video", "improve", ModelRequirement::new(mot), Conditioning::new("bytes")); + m.security = security; + Wai::new(m, cipher.clone()).to_emit_bytes() + }; + assert!(v1(Some(Security::new("aes-256-gcm", 7))).is_ok(), "an encrypted WAI1 is written"); + assert!(matches!(v1(None), Err(EmitError::IntPayload { .. })), "the ciphertext in plaintext is refused"); + assert!(matches!(v1(Some(Security::new("none", 0))), Err(EmitError::IntPayload { .. })), "`none` is plaintext"); + let mut multi = WaiMulti::new("video", "improve", serde_json::Value::Null, vec![Rendition::media(mot, "bytes", cipher.clone())]); + assert!(matches!(multi.to_emit_bytes(), Err(EmitError::IntPayload { .. }))); + multi.manifest.extra.insert("security".into(), serde_json::json!({"encryption": "mls-sframe", "kid": 0})); + assert!(multi.to_emit_bytes().is_ok(), "an encrypted WAI2 is written"); + } + + /// SPEC §6.4 step 2 at the reference dispatchers, which open no scheme: a manifest + /// that declares an encryption other than `"none"` is inert before any capability + /// is considered (`unsupported_encryption`), so a payload sent in the clear under a + /// false claim of encryption is not decoded as plaintext; `"none"`, `null` and an + /// absent member are plaintext. A `WAI2` `security` member that is not a security + /// object is inert too, and is never written. + #[test] + fn an_encrypted_envelope_is_inert_at_the_reference_dispatchers() { + let png = "wai.image.png"; + let all = |_: &str| true; + let v1 = |security: Option| { + let mut m = Manifest::new("image", "improve", ModelRequirement::new(png), Conditioning::new("bytes")); + m.security = security; + Wai::new(m, b"bytes".to_vec()) + }; + assert_eq!(v1(None).dispatch_verified(all, no_parameter_sets()).unwrap().capability, png); + assert_eq!(v1(Some(Security::new("none", 0))).dispatch_verified(all, no_parameter_sets()).unwrap().capability, png); + let e = v1(Some(Security::new("aes-256-gcm", 7))).dispatch_verified(all, no_parameter_sets()).unwrap_err(); + assert_eq!(e.code(), "unsupported_encryption"); + assert!(e.to_string().contains("\"aes-256-gcm\""), "{e}"); + + let multi = |security: Option| { + let mut m = WaiMulti::new("image", "improve", Value::Null, + vec![Rendition::media(png, "bytes", b"a".to_vec()), Rendition::media(png, "bytes", b"b".to_vec())]); + if let Some(v) = security { + m.manifest.extra.insert("security".into(), v); + } + m + }; + for plain in [None, Some(Value::Null), Some(serde_json::json!({"encryption": "none"})), Some(serde_json::json!({}))] { + let m = multi(plain.clone()); + assert_eq!(m.pick(all, no_parameter_sets()).unwrap().index, 0, "{plain:?}"); + assert!(m.to_emit_bytes().is_ok(), "{plain:?}"); + } + let encrypted = multi(Some(serde_json::json!({"encryption": "mls-sframe", "kid": 0}))); + assert!(encrypted.to_emit_bytes().is_ok(), "an encrypted WAI2 is written"); + let skips = encrypted.pick(all, no_parameter_sets()).unwrap_err(); + assert_eq!(skips, vec![(0, Skip::Encrypted), (1, Skip::Encrypted)]); + assert_eq!(skips[0].1.code(), "unsupported_encryption"); + for bad in [serde_json::json!(true), serde_json::json!("aes"), serde_json::json!([]), serde_json::json!({"encryption": 5})] { + let m = multi(Some(bad.clone())); + assert!(matches!(m.to_emit_bytes(), Err(EmitError::Security(ref why)) if why.contains("is not a security object")), "{bad}"); + let back = WaiMulti::from_bytes(&m.to_bytes().unwrap()).unwrap(); + assert_eq!(back.pick(all, no_parameter_sets()).unwrap_err()[0].1, Skip::Encrypted, "{bad}: read, it is inert"); + } + } + /// The emit path of the C packers (`wai_envelope_pack`, /// `wai_envelope_pack_multi`), driven exactly as they drive it: the manifest /// JSON parsed into the manifest type, the envelope built, and the bytes taken diff --git a/wai/wai-rs/src/energy_figure/tests.rs b/wai/wai-rs/src/energy_figure/tests.rs index 0c2f2b2a2..901c53838 100644 --- a/wai/wai-rs/src/energy_figure/tests.rs +++ b/wai/wai-rs/src/energy_figure/tests.rs @@ -96,8 +96,8 @@ fn a_delivered_object_is_viewed_at_its_index_within_its_group() { use crate::jwp_delivery::{GroupBuilder, ObjectLabel}; let env = |i: u8| vec![i; 9]; let mut b = GroupBuilder::default(); - b.add(crate::claim_receipt::object_content_hash(&env(0)), 300, "o", Some(ObjectLabel::Class(chip(20_000)))); - b.add(crate::claim_receipt::object_content_hash(&env(1)), 200, "o", None); + b.add(crate::jwp_hash::content_hash(&env(0)), 300, "o", Some(ObjectLabel::Class(chip(20_000)))); + b.add(crate::jwp_hash::content_hash(&env(1)), 200, "o", None); let (g, objs) = b.finish(&key(0x11), "p", 1, 2, "2026-09-30T12:00:00Z", "2026-09-30T12:00:01Z", None, false).unwrap(); let v = DeliveredObject { object: &objs[1], group: &g, envelope: &env(1) }.figure_view().unwrap(); assert_eq!(v.profile, "moq-jwp:object"); @@ -143,6 +143,7 @@ fn a_provenance_receipt_is_viewed_step_by_step() { #[cfg(feature = "video_receipt")] #[test] +#[allow(deprecated)] // the receipt sealed over the JSON form, as its legacy pins are fn the_video_receipt_is_viewed_by_its_receipt_hash() { let p = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/vectors/int_video/int_video_bitstream.json"); let bytes = std::fs::read(p).unwrap(); diff --git a/wai/wai-rs/src/ffi.rs b/wai/wai-rs/src/ffi.rs index 0a8699a98..dcc85210c 100644 --- a/wai/wai-rs/src/ffi.rs +++ b/wai/wai-rs/src/ffi.rs @@ -162,17 +162,149 @@ where F: FnOnce() -> std::result::Result, (c_int, String)> + panic::Unwi /// container's (`WAI_ERR_CONTAINER`). fn emit_err(e: crate::container::EmitError) -> (c_int, String) { match e { - crate::container::EmitError::Candidate(_) => (WAI_ERR_INVALID_ARG, e.to_string()), + crate::container::EmitError::Candidate(_) | crate::container::EmitError::IntPayload { .. } => (WAI_ERR_INVALID_ARG, e.to_string()), _ => (WAI_ERR_CONTAINER, e.to_string()), } } +/// What decoding an integer payload will cost (`wai_int_payload_declared`, +/// `extensions/integer-payloads.md` §7, *Memory and work*): the decoded output's +/// bytes (`output_known` 1 where they are filled), the symbols its streams code, and, +/// where `cost_known` is 1, the most bytes the reference decode holds at once beyond +/// the payload and the parsed set (`working_bytes`) and the networks' +/// multiply-accumulates, `macs_hi·2^64 + macs_lo`. +#[repr(C)] +#[derive(Debug, Clone, Copy, Default)] +pub struct WaiIntDeclared { + pub output_bytes: u64, + pub output_known: u8, + pub symbols: u64, + pub working_bytes: u64, + pub macs_lo: u64, + pub macs_hi: u64, + pub cost_known: u8, +} + +impl WaiIntDeclared { + /// The figures `container::declared_size` reads from the header alone. + fn from_size(d: crate::container::DeclaredSize) -> Self { + let macs = d.macs.unwrap_or(0); + WaiIntDeclared { + output_bytes: d.output_bytes.unwrap_or(0), + output_known: d.output_bytes.is_some() as u8, + symbols: d.symbols, + working_bytes: d.working_bytes.unwrap_or(0), + macs_lo: macs as u64, + macs_hi: (macs >> 64) as u64, + cost_known: (d.working_bytes.is_some() && d.macs.is_some()) as u8, + } + } +} + +/// The members of `capability`'s parameter set read from `set_path`: inside it where +/// it is a directory, by the names the set lists them under; the file itself for a set +/// of one file; else the file named for the member beside it. A member that cannot be +/// read is absent, and the cost then refuses with `set_incomplete`. +#[cfg(feature = "neural_int")] +fn read_set(capability: &str, set_path: &std::path::Path) -> std::collections::BTreeMap> { + let members = crate::int_payload::set_members(capability); + members.iter() + .filter_map(|m| { + let f = if set_path.is_dir() { + set_path.join(m) + } else if members.len() == 1 { + set_path.to_path_buf() + } else { + set_path.parent().map(|d| d.join(m)).unwrap_or_else(|| std::path::PathBuf::from(m)) + }; + std::fs::read(f).ok().map(|b| (m.to_string(), b)) + }) + .collect() +} + +/// Every figure of the declared cost, over the set read from `set_path`. +#[cfg(feature = "neural_int")] +fn cost_with_set(capability: &str, payload: &[u8], set_path: &str) -> std::result::Result { + use crate::int_payload::{declared_cost, resolve}; + let files = read_set(capability, std::path::Path::new(set_path)); + let view: std::collections::BTreeMap = files.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect(); + let params = resolve(capability, None, &[crate::pin::HeldBytes::Set { listing: b"", files: &view }], None) + .map_err(|r| (WAI_ERR_INVALID_ARG, format!("{}: {r}", r.code())))?; + let c = declared_cost(capability, payload, ¶ms).map_err(|e| (WAI_ERR_INVALID_ARG, e.to_string()))?; + Ok(WaiIntDeclared { + output_bytes: c.output_bytes, output_known: 1, symbols: c.symbols, working_bytes: c.working_bytes, + macs_lo: c.macs as u64, macs_hi: (c.macs >> 64) as u64, cost_known: 1, + }) +} + +#[cfg(not(feature = "neural_int"))] +fn cost_with_set(capability: &str, _: &[u8], _: &str) -> std::result::Result { + Err((WAI_ERR_INVALID_ARG, format!("{capability}: this build reads no parameter set (feature neural_int)"))) +} + +/// Read what decoding the payload of an integer capability will cost before decoding +/// it, so a sink can refuse a payload past its own size policy first +/// (`extensions/integer-payloads.md` §7, *Memory and work*). `capability` is the +/// envelope's capability (or the `WAI2` component's). The payload is parsed as a sink +/// parses it: one that is not the format SPEC §5 pins `capability` to is +/// `WAI_ERR_INVALID_ARG`, with its code in `wai_last_error`, and so is a capability +/// with no pinned integer format. +/// +/// With `set_path` null, only the header is read: a `wai.neural.int_hyper` image's +/// size is its synthesis network's, so `output_known` is 0 for it, and `cost_known` is +/// 1 only for a `WIV1` with keyframe 0, which runs no network. With `set_path` naming +/// the capability's parameter set — a directory holding its members by the names §9 +/// gives, or the file of a one-file set — every figure is filled +/// (`int_payload::declared_cost`), and the set is checked against the payload as a +/// decode checks it first (`set_incomplete`, `model_mismatch`, `geometry_mismatch`, +/// …). The set is read as given, unverified: a pin is verified when the envelope is +/// decoded. A build without `neural_int` reads no set. +/// +/// # Safety +/// `capability` NUL-terminated; `payload` valid for `payload_len` bytes; `set_path` +/// null or NUL-terminated UTF-8; `out` writable. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn wai_int_payload_declared( + capability: *const c_char, + payload: *const u8, payload_len: usize, + set_path: *const c_char, + out: *mut WaiIntDeclared, +) -> c_int { + if capability.is_null() || out.is_null() { + set_err("int_payload_declared: capability/out null"); + return WAI_ERR_INVALID_ARG; + } + let r = panic::catch_unwind(|| -> std::result::Result { + let cap = unsafe { std::ffi::CStr::from_ptr(capability) }.to_string_lossy().to_string(); + let p = unsafe { input_slice(payload, payload_len) } + .ok_or((WAI_ERR_INVALID_ARG, "payload null + len>0".into()))?; + let header = match crate::container::declared_size(&cap, p) { + Ok(Some(d)) => WaiIntDeclared::from_size(d), + Ok(None) => return Err((WAI_ERR_INVALID_ARG, format!("{cap} has no pinned integer payload format"))), + Err(e) => return Err((WAI_ERR_INVALID_ARG, e.to_string())), + }; + if set_path.is_null() { + return Ok(header); + } + let set = unsafe { std::ffi::CStr::from_ptr(set_path) }.to_str() + .map_err(|_| (WAI_ERR_INVALID_ARG, "set_path is not UTF-8".to_string()))?; + cost_with_set(&cap, p, set) + }); + match r { + Ok(Ok(d)) => { unsafe { *out = d; } WAI_OK } + Ok(Err((code, msg))) => { set_err(msg); code } + Err(_) => { set_err("internal panic in int_payload_declared"); WAI_ERR_PANIC } + } +} + /// Builds a complete WAI file from a codec-produced payload + manifest /// fields. `manifest_json` is a UTF-8 JSON object overriding/supplying /// any fields; `payload` is the bytes from one of the codec encoders. /// Refuses (`WAI_ERR_INVALID_ARG`) a manifest that names a candidate capability /// as its capability or fallback, which an encoder MUST NOT emit (SPEC §5, -/// *Candidate capabilities*): it emits through `Wai::to_emit_bytes`. +/// *Candidate capabilities*), and a payload that is not the format SPEC §5 pins its +/// integer capability or fallback to (*Payloads of the WAI-native integer codecs*): +/// it emits through `Wai::to_emit_bytes`. #[unsafe(no_mangle)] pub unsafe extern "C" fn wai_envelope_pack( manifest_json: *const c_char, @@ -226,7 +358,8 @@ pub unsafe extern "C" fn wai_envelope_detect_version( /// - The i-th payload pairs with `manifest.renditions[i]`. /// /// Refuses (`WAI_ERR_INVALID_ARG`) a table with a component, in either role, -/// whose capability is a candidate (SPEC §5, *Candidate capabilities*): it emits +/// whose capability is a candidate (SPEC §5, *Candidate capabilities*), or whose +/// payload is not the format SPEC §5 pins its integer capability to: it emits /// through `WaiMulti::to_emit_bytes`. /// /// # Safety diff --git a/wai/wai-rs/src/ffi_neural.rs b/wai/wai-rs/src/ffi_neural.rs index 6b363fb69..2a181733b 100644 --- a/wai/wai-rs/src/ffi_neural.rs +++ b/wai/wai-rs/src/ffi_neural.rs @@ -19,6 +19,12 @@ //! `WAI_ERR_UNSOUND_REPLICATE`, and a `WAI2` envelope with nothing to pick is //! `WAI_ERR_NO_RENDITION`. Output buffers are heap-allocated here; free them with //! `wai_buffer_free`. +//! +//! Integer video, PCM and splat attributes (`neural::decodes_to_int`) have no +//! `wai_decoded_t` kind; the Rust API returns them (`int_payload::decode_int`). These +//! entry points do not advertise those capabilities, so such a payload is refused +//! before anything is decoded, and dispatch moves on to a fallback this ABI can +//! return. use std::ffi::{c_char, c_int, CStr}; use std::panic; @@ -28,8 +34,8 @@ use std::slice; use crate::container::{Wai, WaiMulti}; use crate::ffi::{set_err, WaiBuffer, WAI_ERR_CODEC, WAI_ERR_CONTAINER, WAI_ERR_INVALID_ARG, WAI_ERR_NO_RENDITION, WAI_ERR_PANIC, WAI_ERR_PRIOR, WAI_ERR_UNSOUND_REPLICATE, WAI_OK}; -use crate::neural::{decode_envelope, decode_envelope_strict, decode_multi, decode_multi_strict, DecodeError, - Decoded, ModelRegistry}; +use crate::neural::{decode_envelope, decode_envelope_strict, decode_multi, decode_multi_strict, decodes_to_int, + DecodeError, Decoded, ModelRegistry}; unsafe fn input_envelope<'a>(bytes: *const u8, len: usize) -> Option<&'a [u8]> { if bytes.is_null() && len > 0 { return None; } @@ -48,6 +54,7 @@ fn decode_status(e: DecodeError) -> (c_int, String) { DecodeError::PriorMismatch(r) => (WAI_ERR_PRIOR, format!("{}: {r}", r.code())), DecodeError::UnsoundReplicate(why) => (WAI_ERR_UNSOUND_REPLICATE, why), DecodeError::NoRendition(_) => (WAI_ERR_NO_RENDITION, e.to_string()), + DecodeError::Encrypted(why) => (WAI_ERR_CONTAINER, why), other => (WAI_ERR_CODEC, other.to_string()), } } @@ -66,14 +73,25 @@ fn run Result<(), (c_int, String)> + panic::UnwindSafe>(f: F) -> } } +/// The refusal of a capability whose output this ABI has no kind for. +fn no_kind(caps: &[&str]) -> (c_int, String) { + (WAI_ERR_CODEC, format!( + "{} decodes to integer video, PCM or splat attributes, which have no wai_decoded_t kind: \ + not decoded here; decode it with the Rust API (int_payload::decode_int)", caps.join(", "))) +} + /// Parse a `WAI1` envelope and decode it with `model_path` registered for its -/// capability. +/// capability. A capability whose output has no `wai_decoded_t` kind is refused before +/// anything is decoded. fn decode_one(envelope: *const u8, envelope_len: usize, model_path: *const c_char) -> Result { let bytes = unsafe { input_envelope(envelope, envelope_len) } .ok_or((WAI_ERR_INVALID_ARG, "envelope null with len>0".to_string()))?; let path = unsafe { input_cstr(model_path) } .ok_or((WAI_ERR_INVALID_ARG, "model_path null or not UTF-8".to_string()))?; let env = Wai::from_bytes(bytes).map_err(|e| (WAI_ERR_CONTAINER, format!("envelope parse: {e}")))?; + if decodes_to_int(env.capability()) { + return Err(no_kind(&[env.capability()])); + } let reg = ModelRegistry::new().register(env.capability().to_string(), Path::new(path)); decode_envelope(&env, ®).map_err(decode_status) } @@ -211,7 +229,10 @@ impl WaiDecoded { /// array of paths. The first path is the registered artifact; the others are /// further sets a pin may resolve against (a set and its successor). Every pin is /// verified first (SPEC §3.1); a `WAI2` rendition is picked by SPEC §7.1. With -/// `flags & WAI_DECODE_STRICT`, SPEC §7's replicate rule applies too. +/// `flags & WAI_DECODE_STRICT`, SPEC §7's replicate rule applies too. A capability +/// whose output has no `wai_decoded_t` kind (integer video, PCM, splat attributes) is +/// not advertised, whatever `registry_json` says: dispatch moves on past it before +/// decoding, and when nothing else decodes the error names it. /// /// `*out` is written on every call with a non-null `out`, before anything else is /// read: on failure it holds `kind` 0 and an empty `data`, so the caller may free @@ -241,7 +262,12 @@ pub unsafe extern "C" fn wai_neural_decode_ex( let map: std::collections::BTreeMap> = serde_json::from_str(rj) .map_err(|e| (WAI_ERR_INVALID_ARG, format!("registry_json: {e}")))?; let mut reg = ModelRegistry::new(); + let mut kindless: Vec = Vec::new(); for (cap, paths) in map { + if decodes_to_int(&cap) { + kindless.push(cap); + continue; + } let mut it = paths.into_iter(); if let Some(first) = it.next() { reg = reg.register(cap.clone(), first); @@ -253,14 +279,22 @@ pub unsafe extern "C" fn wai_neural_decode_ex( let strict = flags & WAI_DECODE_STRICT != 0; let version = crate::container::detect_version(bytes) .map_err(|e| (WAI_ERR_CONTAINER, e.to_string()))?; + // A refusal at an envelope that names a capability left unadvertised above says so. + let explain = |named: Vec<&str>, e: DecodeError| { + let left: Vec<&str> = named.into_iter().filter(|c| kindless.iter().any(|k| k == c)).collect(); + let (code, msg) = decode_status(e); + if left.is_empty() { (code, msg) } else { (code, format!("{msg}; {}", no_kind(&left).1)) } + }; let (index, decoded) = if version == 1 { let env = Wai::from_bytes(bytes).map_err(|e| (WAI_ERR_CONTAINER, format!("envelope parse: {e}")))?; let d = if strict { decode_envelope_strict(&env, ®) } else { decode_envelope(&env, ®) }; - (0, d.map_err(decode_status)?) + let named = [Some(env.capability()), env.fallback()].into_iter().flatten().collect(); + (0, d.map_err(|e| explain(named, e))?) } else { let m = WaiMulti::from_bytes(bytes).map_err(|e| (WAI_ERR_CONTAINER, format!("envelope parse: {e}")))?; let r = if strict { decode_multi_strict(&m, ®) } else { decode_multi(&m, ®) }; - r.map_err(decode_status)? + let named = m.manifest.renditions.iter().map(|r| r.capability.as_str()).collect(); + r.map_err(|e| explain(named, e))? }; let mut o = WaiDecoded { index: index as u32, ..WaiDecoded::empty() }; match decoded { @@ -283,6 +317,11 @@ pub unsafe extern "C" fn wai_neural_decode_ex( o.fps_x_1000 = (v.fps * 1000.0).round() as u32; o.data = WaiBuffer::from_vec(v.frames_rgb.concat()); } + // Not reached: such a capability is not advertised above. Refused rather + // than converted, so no caller could read one as another kind. + Decoded::Int(_) => { + return Err((WAI_ERR_CODEC, "this integer output (video, PCM or splat attributes) has no wai_decoded_t kind; decode it with the Rust API".to_string())); + } } unsafe { out.write(o) }; Ok(()) diff --git a/wai/wai-rs/src/int_audio.rs b/wai/wai-rs/src/int_audio.rs index 64b2ac9ed..3b4a118f6 100644 --- a/wai/wai-rs/src/int_audio.rs +++ b/wai/wai-rs/src/int_audio.rs @@ -35,47 +35,48 @@ pub struct AudioModel { pub layers: Vec, } -/// Little-endian, bounds-checked cursor over the binary model. +/// Little-endian cursor over the binary model, bounds-checked: every read is of bytes +/// that are present, with its end formed in checked arithmetic, so a damaged model is +/// refused and never makes the parser allocate more than the bytes it holds. struct Cur<'a> { b: &'a [u8], p: usize, } -impl Cur<'_> { - fn u32(&mut self) -> Result { - let e = self.p + 4; +impl<'a> Cur<'a> { + /// The next `n` elements of `size` bytes each. + fn take(&mut self, n: usize, size: usize) -> Result<&'a [u8], &'static str> { + let e = n.checked_mul(size).and_then(|len| self.p.checked_add(len)).ok_or("model truncated")?; let s = self.b.get(self.p..e).ok_or("model truncated")?; self.p = e; - Ok(u32::from_le_bytes(s.try_into().unwrap()) as usize) + Ok(s) + } + fn u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.take(1, 4)?.try_into().unwrap()) as usize) } fn u8(&mut self) -> Result { - let v = *self.b.get(self.p).ok_or("model truncated")?; - self.p += 1; - Ok(v) + Ok(self.take(1, 1)?[0]) } fn i16v(&mut self, n: usize) -> Result, &'static str> { - let e = self.p + 2 * n; - let s = self.b.get(self.p..e).ok_or("model truncated")?; - self.p = e; - Ok(s.chunks_exact(2).map(|c| i16::from_le_bytes([c[0], c[1]]) as i64).collect()) + Ok(self.take(n, 2)?.chunks_exact(2).map(|c| i16::from_le_bytes([c[0], c[1]]) as i64).collect()) } fn i32v(&mut self, n: usize) -> Result, &'static str> { - let e = self.p + 4 * n; - let s = self.b.get(self.p..e).ok_or("model truncated")?; - self.p = e; - Ok(s.chunks_exact(4).map(|c| i32::from_le_bytes([c[0], c[1], c[2], c[3]]) as i64).collect()) + Ok(self.take(n, 4)?.chunks_exact(4).map(|c| i32::from_le_bytes([c[0], c[1], c[2], c[3]]) as i64).collect()) } fn i64v(&mut self, n: usize) -> Result, &'static str> { - let e = self.p + 8 * n; - let s = self.b.get(self.p..e).ok_or("model truncated")?; - self.p = e; - Ok(s.chunks_exact(8).map(|c| i64::from_le_bytes(c.try_into().unwrap())).collect()) + Ok(self.take(n, 8)?.chunks_exact(8).map(|c| i64::from_le_bytes(c.try_into().unwrap())).collect()) } } +/// The product of a layer's dimensions, refused where it overflows `usize`. +fn count(dims: &[usize]) -> Result { + dims.iter().try_fold(1usize, |a, &d| a.checked_mul(d)).ok_or("layer size overflows") +} + impl AudioModel { /// Parse the export tool's compact binary model: magic `WAM1`, then /// `afrac/gf/shift/n_layers` (u32) and per layer a `u8` tag — 0 convT1d, 2 igdn - /// — with int12 weights as `i16`, IGDN `gamma` as `i32`, the rest `i64`. + /// — with int12 weights as `i16`, IGDN `gamma` as `i32`, the rest `i64`, and + /// nothing after the last layer. pub fn from_bytes(b: &[u8]) -> Result { if b.get(0..4) != Some(b"WAM1") { return Err("bad model magic (want WAM1)"); @@ -83,7 +84,10 @@ impl AudioModel { let mut c = Cur { b, p: 4 }; let (afrac, gf, shift) = (c.u32()? as u32, c.u32()? as u32, c.u32()? as u32); let n = c.u32()?; - let mut layers = Vec::with_capacity(n); + // Each layer is at least its tag byte, so a count beyond the bytes left is a + // truncated model, and reserving for it would let four damaged bytes ask for a + // large allocation. + let mut layers = Vec::with_capacity(n.min(b.len() - c.p)); for _ in 0..n { layers.push(match c.u8()? { 0 => { @@ -91,15 +95,19 @@ impl AudioModel { let out = c.u32()?; let (k, stride, pad, opad) = (c.u32()?, c.u32()?, c.u32()?, c.u32()?); ALayer::ConvT1d { out_ch: out, k, stride, pad, opad, - weight: c.i16v(inc * out * k)?, mult: c.i64v(out)?, bias: c.i64v(out)? } + weight: c.i16v(count(&[inc, out, k])?)?, mult: c.i64v(out)?, bias: c.i64v(out)? } } 2 => { let cc = c.u32()?; - ALayer::Igdn { c: cc, gamma: c.i32v(cc * cc)?, beta: c.i64v(cc)? } + ALayer::Igdn { c: cc, gamma: c.i32v(count(&[cc, cc])?)?, beta: c.i64v(cc)? } } _ => return Err("bad layer tag"), }); } + // Nothing follows the last layer (`extensions/integer-payloads.md` §9). + if c.p != b.len() { + return Err("bytes follow the last layer"); + } Ok(Self { afrac, gf, shift, layers }) } } @@ -167,81 +175,288 @@ pub mod reference { } } -/// Run the integer synthesis `g_s` on the fixed-point latent `x` (`[c, t]` at -/// `2^afrac`) → `([1, to], to)` waveform at `2^afrac`. Pure i64. -fn run_g_s(model: &AudioModel, x: &[i64], mut c: usize, mut t: usize) -> (Vec, usize) { - let (afrac, gf, shift) = (model.afrac, model.gf, model.shift); - let mut x = x.to_vec(); - for layer in &model.layers { - match layer { - ALayer::ConvT1d { out_ch, k, stride, pad, opad, weight, mult, bias } => { - let (nx, to) = conv_t_1d(&x, c, t, weight, *out_ch, *k, *stride, *pad, *opad, mult, bias, shift); - x = nx; - c = *out_ch; - t = to; - } - ALayer::Igdn { c: _, gamma, beta } => { - x = igdn(&x, c, t, afrac, gamma, beta, gf); - } - } - } - (x, t) -} - fn i64s(v: &Value) -> Vec { v.as_array().map(|a| a.iter().filter_map(|n| n.as_i64()).collect()).unwrap_or_default() } -/// Decode a full audio **bitstream → int16 PCM**, end to end and fully integer: -/// -/// 1. per channel, factorized rANS-decode the symbol `s`; rebuild the fixed-point -/// latent `y_q = s·2^afrac + median_q` (no float); -/// 2. integer synthesis `g_s` (transposed convs + IGDN) → waveform at `2^afrac`; -/// 3. quantize → int16 PCM (round-half-up, clamp). -/// -/// `bs` is the export tool's `audio_bitstream.json`. Byte-identical on every machine. -pub fn decode(model: &AudioModel, bs: &Value) -> Result, &'static str> { - let c = bs["n_ch"].as_u64().ok_or("n_ch")? as usize; - let l = bs["lat_len"].as_u64().ok_or("lat_len")? as usize; - let t_out = bs["t_out"].as_u64().ok_or("t_out")? as usize; - let afrac = model.afrac; - let one = 1i64 << afrac; - let lo = bs["lo"].as_i64().ok_or("lo")? as i32; - let median_q = i64s(&bs["median_q"]); - if median_q.len() != c { - return Err("median_q length disagrees with n_ch"); - } - let cdfs: Vec> = bs["cdfs"] - .as_array() - .ok_or("cdfs")? - .iter() - .map(|row| row.as_array().unwrap().iter().map(|n| n.as_i64().unwrap() as i32).collect()) - .collect(); - let cdf_len = cdfs.first().ok_or("empty cdfs")?.len(); - let rans = b64_decode(bs["rans_b64"].as_str().ok_or("rans_b64")?)?; +use crate::int_hyper_synth::{refused, Refused, RefusalKind, MAX_AFRAC, MAX_TENSOR}; - // 1. factorized rANS-decode the per-channel latent (channel-major), rebuild y_q. - let mut y = vec![0i64; c * l]; - let mut dec = RansDecoder::new(&rans)?; - for ch in 0..c { - for p in 0..l { - let s = dec.decode(&cdfs[ch], cdf_len, lo); - y[ch * l + p] = s as i64 * one + median_q[ch]; +/// A clip's coded form, typed: what a `WIA1` payload carries +/// (`extensions/integer-payloads.md` §5), and what the tool's JSON form carries, read +/// by [`AudioJson`]. `cdfs` holds one table per latent channel, all of one length. +#[derive(Debug, Clone, Copy)] +pub struct AudioStream<'a> { + /// The fixed-point scale the stream was coded at; it must equal the model's. + pub afrac: u32, + pub n_ch: usize, + pub lat_len: usize, + /// Samples presented: the first `t_out` of the synthesis. + pub t_out: usize, + /// The symbol offset every channel's table is coded at (the JSON form's `lo`). + pub offset: i32, + pub median_q: &'a [i64], + pub cdfs: &'a [Vec], + pub rans: &'a [u8], +} + +/// The tool's JSON bitstream (`tools/wai_int_audio_export.py`'s +/// `audio_bitstream.json`), read into the typed form. A converter input and a +/// debugging aid; the payload is `WIA1`. What the old JSON decoder left implicit is +/// explicit here: its `afrac` is read (the old decoder ignored it) and must be the +/// model's; every channel's table must have the first table's length (the old decoder +/// read every table at that length). +#[derive(Debug, Clone)] +pub struct AudioJson { + pub afrac: u32, + pub n_ch: usize, + pub lat_len: usize, + pub t_out: usize, + pub offset: i32, + pub median_q: Vec, + pub cdfs: Vec>, + pub rans: Vec, +} + +impl AudioJson { + /// Read the tool's JSON bitstream. + pub fn from_value(bs: &Value) -> Result { + let uz = |k: &'static str| bs[k].as_u64().and_then(|v| usize::try_from(v).ok()).ok_or(k); + let (n_ch, lat_len, t_out) = (uz("n_ch")?, uz("lat_len")?, uz("t_out")?); + let afrac = bs["afrac"].as_u64().and_then(|v| u32::try_from(v).ok()).ok_or("afrac")?; + let offset = bs["lo"].as_i64().and_then(|v| i32::try_from(v).ok()).ok_or("lo")?; + let median_q = i64s(&bs["median_q"]); + let cdfs = bs["cdfs"].as_array().ok_or("cdfs")?.iter() + .map(|row| row.as_array().and_then(|r| r.iter().map(|n| n.as_i64().and_then(|v| i32::try_from(v).ok())).collect::>>())) + .collect::>>().ok_or("cdfs holds a value outside i32")?; + let first = cdfs.first().ok_or("empty cdfs")?.len(); + if cdfs.iter().any(|c| c.len() != first) { + return Err("the channels' tables are not of one length"); + } + let rans = b64_decode(bs["rans_b64"].as_str().ok_or("rans_b64")?)?; + Ok(AudioJson { afrac, n_ch, lat_len, t_out, offset, median_q, cdfs, rans }) + } + + /// The typed form, borrowing this one's tables. + pub fn stream(&self) -> AudioStream<'_> { + AudioStream { + afrac: self.afrac, n_ch: self.n_ch, lat_len: self.lat_len, t_out: self.t_out, offset: self.offset, + median_q: &self.median_q, cdfs: &self.cdfs, rans: &self.rans, } } - // 2. integer synthesis → waveform. - let (x, t) = run_g_s(model, &y, c, l); - // 3. waveform → int16 PCM (round-half-up, clamp). Take t_out samples. +} + +/// The model's output `[c, t]` on a `[c, t]` latent, planned from its layers' geometry +/// and parameter lengths alone, before anything is allocated +/// (`extensions/integer-payloads.md` §7). Layer by layer in order: the layer fits the +/// running channel count (`ModelMismatch`: output channels, kernel and stride +/// non-zero, `c·out·k` weights, `out` multipliers and biases, a requantisation shift +/// under 128; an inverse GDN of the running channel count, with `c·c` and `c` +/// parameters and shifts under 128); the uncropped length `(t − 1)·stride + k + opad` +/// and the buffer of `out` channels of it are within [`MAX_TENSOR`] (`TooLarge`); and +/// the crop `2·pad` leaves an output (`ModelMismatch`). Every size is formed in `u64` +/// with checked arithmetic, a size past `u64` being over the cap, so every target +/// refuses alike. Returns the output `[c, t]`, the most elements one layer holds, and +/// the multiply-accumulates the layers comprise (`extensions/integer-payloads.md` §7): +/// a transposed convolution's every input times `out·k`, an inverse GDN's every +/// element times `c`, summed in `u128`. +fn plan(model: &AudioModel, c: usize, t: usize) -> Result<(usize, usize, u64, u128), Refused> { + use RefusalKind::*; + const CAP: u64 = MAX_TENSOR as u64; + let (mut c, mut t) = (c as u64, t as u64); + // The most elements the reference kernels hold while one layer runs, its input + // included: a transposed convolution's input, uncropped buffer and output; an + // inverse GDN's input, output and one position's squares (`c` values of 16 bytes). + let mut live = c * t; + let mut macs: u128 = 0; + for layer in &model.layers { + match layer { + ALayer::ConvT1d { out_ch, k, stride, pad, opad, weight, mult, bias } => { + let (cout, k, s, p, op) = (*out_ch as u64, *k as u64, *stride as u64, *pad as u64, *opad as u64); + if cout == 0 || k == 0 || s == 0 { + return Err(refused(ModelMismatch, "a layer has zero output channels, kernel or stride")); + } + if c.checked_mul(cout).and_then(|v| v.checked_mul(k)) != Some(weight.len() as u64) + || mult.len() as u64 != cout || bias.len() as u64 != cout || model.shift > 127 + { + return Err(refused(ModelMismatch, "a layer's parameters do not match the running channel count")); + } + let full = (t - 1).checked_mul(s).and_then(|v| v.checked_add(k)).and_then(|v| v.checked_add(op)); + let full = full.filter(|&n| n <= CAP).ok_or(refused(TooLarge, "a layer's uncropped length exceeds 2^26"))?; + if cout.checked_mul(full).is_none_or(|n| n > CAP) { + return Err(refused(TooLarge, "a layer would form a buffer over 2^26 elements")); + } + let crop = p.checked_mul(2).unwrap_or(u64::MAX); + if full <= crop { + return Err(refused(ModelMismatch, "a layer's padding leaves no output")); + } + live = live.max(c * t + cout * full + cout * (full - crop)); + macs = macs.saturating_add((c * t) as u128 * (cout * k) as u128); + (c, t) = (cout, full - crop); + } + ALayer::Igdn { c: cc, gamma, beta } => { + if *cc as u64 != c || c.checked_mul(c) != Some(gamma.len() as u64) || beta.len() as u64 != c + || model.gf > 127 || model.afrac > 127 + { + return Err(refused(ModelMismatch, "an IGDN layer does not match the running channel count")); + } + live = live.max(2 * c * t + 2 * c); + macs = macs.saturating_add((c * t) as u128 * c as u128); + } + } + } + // Every running size is at most 2^26 here, so it fits usize on every target. + Ok((c as usize, t as usize, live, macs)) +} + +/// What [`decode_typed`] checks before it decodes a symbol, and the buffers it then +/// forms. Returns the synthesis's length, the most bytes the decode holds at once +/// on the reference kernels beyond the payload and the parsed model (the latent and +/// each layer's buffers, `i64` each, then the synthesis's output, `i64`, with the +/// PCM, `i16`), and the synthesis's multiply-accumulates on the latent. +/// +/// The latent's length is the one `t_out` needs and no longer: the synthesis of +/// `lat_len` latent samples gives at least `t_out` samples (`GeometryMismatch` +/// otherwise), and that of `lat_len − 1` gives fewer (`GeometryMismatch`), so a +/// payload cannot declare a large latent to present a short clip. A length for which +/// the model leaves no output gives none. +pub(crate) fn audio_plan(model: &AudioModel, s: &AudioStream<'_>) -> Result<(usize, u64, u128), Refused> { + use RefusalKind::*; + let afrac = model.afrac; + if !(1..=MAX_AFRAC).contains(&afrac) { + return Err(refused(ModelMismatch, "the model's fixed-point scale is outside 1..=30")); + } + if s.afrac != afrac { + return Err(refused(ModelMismatch, "the stream's afrac is not the model's")); + } + let (c, l) = (s.n_ch, s.lat_len); + if c == 0 || l == 0 || s.t_out == 0 || s.median_q.len() != c || s.cdfs.len() != c { + return Err(refused(FieldRange, "a dimension is zero, or the medians or tables are not one per channel")); + } + if s.median_q.iter().any(|m| m.unsigned_abs() > 1 << 47) { + return Err(refused(FieldRange, "a median's magnitude exceeds 2^47")); + } + let len = s.cdfs[0].len(); + let ok = |t: &Vec| t.len() == len && (3..=4097).contains(&len) && t[0] == 0 && t[len - 1] == 1 << 16 + && t.windows(2).all(|p| p[0] < p[1]); + if !s.cdfs.iter().all(ok) { + return Err(refused(CdfMalformed, "a table is not a CDF from 0 to 65536 of the channels' one length")); + } + if c.checked_mul(l).is_none_or(|n| n > MAX_TENSOR) || s.t_out > MAX_TENSOR { + return Err(refused(TooLarge, "the latent or the clip exceeds 2^26 elements")); + } + let (oc, t, live, macs) = plan(model, c, l)?; + if oc != 1 { + return Err(refused(ModelMismatch, "the model does not give one channel")); + } + if t < s.t_out { + return Err(refused(GeometryMismatch, "the synthesis gives fewer samples than t_out")); + } + let shorter = if l > 1 { plan(model, c, l - 1).map_or(0, |(_, t, ..)| t) } else { 0 }; + if shorter >= s.t_out { + return Err(refused(GeometryMismatch, "the latent is longer than t_out needs: one sample fewer gives t_out samples")); + } + let working = (8 * live).max(8 * t as u64 + 2 * s.t_out as u64); + Ok((t, working, macs)) +} + +/// One layer's arithmetic on the values `x` holds, checked before it runs, as +/// [`crate::int_hyper_synth::run_net_checked`] checks a 2-D layer: every product and +/// partial sum and the requantised value of a convolution fit `i64`; IGDN's terms, +/// norm and root fit their types. +fn layer_in_range(model: &AudioModel, layer: &ALayer, x: &[i64], c: usize) -> Result<(), Refused> { + use crate::int_kernels::max_abs; + const I64: u128 = i64::MAX as u128; + match layer { + ALayer::ConvT1d { k, weight, mult, bias, .. } => { + if model.shift > 127 { + return Err(refused(RefusalKind::ModelMismatch, "requantisation shift of 128 or more")); + } + let terms = (c as u128).saturating_mul(*k as u128); + let acc = terms.saturating_mul(max_abs(weight) as u128).saturating_mul(max_abs(x) as u128); + if acc > I64 { + return Err(refused(RefusalKind::FieldRange, "a latent could take an accumulation outside i64")); + } + let q = ((acc * max_abs(mult) as u128) >> model.shift) + 1; + if q.saturating_add(max_abs(bias) as u128) > I64 { + return Err(refused(RefusalKind::FieldRange, "a latent could take a requantised value outside i64")); + } + Ok(()) + } + ALayer::Igdn { gamma, beta, .. } => crate::int_hyper_synth::igdn_check(x, c, gamma, beta, model.gf, model.afrac) + .map_err(|(why, values)| refused(if values { RefusalKind::FieldRange } else { RefusalKind::ModelMismatch }, why)), + } +} + +/// Decode a typed clip → int16 PCM, mono, byte-exact on every machine, refusing +/// rather than panicking or overflowing on any input: +/// +/// 1. the stream's `afrac` is the model's, in `1..=30`; every table is well formed; +/// the model's layers fit the latent's shape, give one channel, and give at least +/// `t_out` samples from a latent no longer than that needs, with no buffer over +/// [`MAX_TENSOR`] elements — all before anything is allocated ([`audio_plan`]); +/// 2. per channel, rANS-decode the symbol `s`; the latent is `s·2^afrac + median_q` +/// ([`crate::int_entropy::dequantize`]), and a stream that runs out before its last +/// symbol is refused; +/// 3. integer synthesis `g_s`, each layer's arithmetic checked before it runs; +/// 4. the first `t_out` samples, `(x·32767 + 2^(afrac−1)) >> afrac` clamped to `i16`. +pub fn decode_typed(model: &AudioModel, s: &AudioStream<'_>) -> Result, Refused> { + use crate::int_entropy::dequantize; + use RefusalKind::*; + audio_plan(model, s)?; + let (afrac, c, l, len) = (model.afrac, s.n_ch, s.lat_len, s.cdfs[0].len()); + + // 1. factorized rANS-decode the per-channel latent (channel-major), rebuild y_q. + // The latent grows a symbol at a time, and the first symbol that runs the stream + // out refuses the payload before its value is used. + let mut y = Vec::new(); + let mut dec = RansDecoder::new(s.rans).map_err(|_| refused(FieldRange, "the latent stream is not a rANS stream"))?; + for ch in 0..c { + for _ in 0..l { + let sym = dec.decode(&s.cdfs[ch], len, s.offset); + if dec.is_exhausted() { + return Err(refused(StreamExhausted, "the latent stream ended before its last symbol")); + } + crate::int_hyper_synth::push_capped(&mut y, dequantize(sym, afrac, s.median_q[ch]).ok_or(refused(FieldRange, "a latent value leaves i64"))?, c * l); + } + } + if !dec.is_canonical_end() { + return Err(refused(StreamNoncanonical, "the latent stream does not end canonically after its last symbol")); + } + // 2. integer synthesis → waveform, each layer checked on the values it is given. + let (mut x, mut cc, mut tt) = (y, c, l); + for layer in &model.layers { + layer_in_range(model, layer, &x, cc)?; + (x, cc, tt) = run_layer(model, layer, x, cc, tt); + } + // 3. waveform → int16 PCM (round-half-up, clamp): the first t_out samples. let half = 1i64 << (afrac - 1); - let n = t_out.min(t); - let mut pcm = vec![0i16; n]; - for (i, s) in pcm.iter_mut().enumerate() { - let num = x[i] * 32767 + half; - *s = (num >> afrac).clamp(-32768, 32767) as i16; + let mut pcm = Vec::with_capacity(s.t_out); + for &v in &x[..s.t_out] { + let num = v.checked_mul(32767).and_then(|n| n.checked_add(half)) + .ok_or(refused(FieldRange, "a synthesised sample leaves i64 when scaled"))?; + pcm.push((num >> afrac).clamp(-32768, 32767) as i16); } Ok(pcm) } +/// One layer of the synthesis on `x` (`[c, t]`): the new activations and shape. +fn run_layer(model: &AudioModel, layer: &ALayer, x: Vec, c: usize, t: usize) -> (Vec, usize, usize) { + match layer { + ALayer::ConvT1d { out_ch, k, stride, pad, opad, weight, mult, bias } => { + let (nx, to) = conv_t_1d(&x, c, t, weight, *out_ch, *k, *stride, *pad, *opad, mult, bias, model.shift); + (nx, *out_ch, to) + } + ALayer::Igdn { gamma, beta, .. } => (igdn(&x, c, t, model.afrac, gamma, beta, model.gf), c, t), + } +} + +/// Decode the tool's JSON bitstream (`audio_bitstream.json`) → int16 PCM: +/// [`AudioJson`] then [`decode_typed`]. A debugging and converter path; the payload +/// is `WIA1`. Byte-identical on every machine. +pub fn decode(model: &AudioModel, bs: &Value) -> Result, &'static str> { + decode_typed(model, &AudioJson::from_value(bs)?.stream()).map_err(|r| r.detail) +} + /// FNV-1a 64-bit over the little-endian int16 PCM bytes — the dependency-free /// checksum the export tool and this decoder share to prove byte-for-byte agreement /// across languages (sample-equivalence). @@ -286,6 +501,38 @@ fn b64_decode(s: &str) -> Result, &'static str> { } /// The fast 1-D transposed conv against the scalar reference, element by element. +#[cfg(test)] +mod plan_tests { + use super::*; + + /// The committed synthesis's planned multiply-accumulates on the committed latent are + /// its layers' terms at the lengths the reference kernel gives (run here on a zero + /// latent): a transposed convolution's every input times `out·k`, an inverse GDN's + /// every element times `c`. + #[test] + fn the_planned_macs_are_every_layers_terms() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors/int_audio"); + let model = AudioModel::from_bytes(&std::fs::read(dir.join("audio_model.bin")).unwrap()).unwrap(); + let wia1 = std::fs::read(dir.join("audio_bitstream.wia1")).unwrap(); + let a = crate::int_wire::parse_wia1(&wia1).unwrap(); + let (c0, l) = (a.n_ch as usize, a.lat_len as usize); + let (.., macs) = plan(&model, c0, l).unwrap(); + let (mut c, mut t, mut terms) = (c0, l, 0u128); + for layer in &model.layers { + match layer { + ALayer::ConvT1d { out_ch, k, stride, pad, opad, weight, mult, bias } => { + let (_, to) = reference::conv_t_1d(&vec![0; c * t], c, t, weight, *out_ch, *k, *stride, *pad, *opad, mult, bias, model.shift); + terms += (c * t) as u128 * (out_ch * k) as u128; + (c, t) = (*out_ch, to); + } + ALayer::Igdn { .. } => terms += (c * t) as u128 * c as u128, + } + } + assert!(terms > 0); + assert_eq!(macs, terms); + } +} + #[cfg(test)] mod kernel_equivalence { use super::*; diff --git a/wai/wai-rs/src/int_hyper_synth.rs b/wai/wai-rs/src/int_hyper_synth.rs index 0112cc7a3..c1acd61c7 100644 --- a/wai/wai-rs/src/int_hyper_synth.rs +++ b/wai/wai-rs/src/int_hyper_synth.rs @@ -264,7 +264,52 @@ fn count(dims: &[usize]) -> Result { } impl IntModel { + /// [`Self::from_json`], refusing rather than panicking on any JSON. It reads what + /// `from_json` reads, an absent number as 0 and an absent list as empty, as that + /// does, but refuses a member of the wrong type, a size that does not fit (`u32` + /// for `afrac`, `gf`, `shift` and a shift, `usize` for the rest), and a layer type + /// it does not know. A layer's parameters are checked against the running shapes + /// when the network is planned ([`decode_conditional_checked`]). + pub fn try_from_json(v: &Value) -> Result { + let word = |o: &Value, k: &'static str| match o.get(k) { + None => Ok(0), + Some(n) => n.as_u64().and_then(|n| u32::try_from(n).ok()).ok_or(k), + }; + let size = |o: &Value, k: &'static str| match o.get(k) { + None => Ok(0), + Some(n) => n.as_u64().and_then(|n| usize::try_from(n).ok()).ok_or(k), + }; + let ints = |o: &Value, k: &'static str| -> Result, &'static str> { + match o.get(k) { + None => Ok(Vec::new()), + Some(a) => a.as_array().ok_or(k)?.iter().map(Value::as_i64).collect::>().ok_or(k), + } + }; + let layers = v.get("layers").and_then(Value::as_array).ok_or("layers")?.iter().map(|l| { + Ok(match l.get("type").and_then(Value::as_str).ok_or("a layer's type")? { + "convT" => IntLayer::ConvT { + out_ch: size(l, "out_ch")?, k: size(l, "k")?, stride: size(l, "stride")?, pad: size(l, "pad")?, + opad: size(l, "opad")?, weight: ints(l, "weight")?, mult: ints(l, "mult")?, bias: ints(l, "bias")?, + }, + "conv" => IntLayer::Conv { + out_ch: size(l, "out_ch")?, k: size(l, "k")?, stride: size(l, "stride")?, pad: size(l, "pad")?, + weight: ints(l, "weight")?, mult: ints(l, "mult")?, bias: ints(l, "bias")?, + }, + "igdn" => IntLayer::Igdn { gamma: ints(l, "gamma")?, beta: ints(l, "beta")? }, + "relu" => IntLayer::Relu, + "leaky" => IntLayer::Leaky { + slope_q: match l.get("slope_q") { None => 655, Some(q) => q.as_i64().ok_or("slope_q")? }, + slope_frac: word(l, "slope_frac")?, + }, + _ => return Err("an unknown layer type"), + }) + }).collect::, &'static str>>()?; + Ok(Self { afrac: word(v, "afrac")?, gf: word(v, "gf")?, shift: word(v, "shift")?, layers }) + } + /// Parse the export tool's JSON model (`int_synth_sota_model.json` / `int_hs_model.json`). + /// Panics on a JSON the tool would not write; [`Self::try_from_json`] refuses + /// instead. pub fn from_json(v: &Value) -> Self { let layers = v["layers"] .as_array() @@ -291,7 +336,8 @@ impl IntModel { /// Parse the export tool's **compact binary** model (`*_model.bin`): magic /// `WHM1`, then `afrac/gf/shift/n_layers` (u32) and per layer a `u8` tag — /// 0 convT, 1 conv, 2 igdn, 3 relu, 4 leaky — with int12 weights as `i16`, - /// IGDN `gamma` as `i32`, the rest `i64`. Decodes identically to [`from_json`]. + /// IGDN `gamma` as `i32`, the rest `i64`, and nothing after the last layer. + /// Decodes identically to [`from_json`]. pub fn from_bytes(b: &[u8]) -> Result { if b.get(0..4) != Some(b"WHM1") { return Err("bad model magic (want WHM1)"); @@ -328,6 +374,11 @@ impl IntModel { _ => return Err("bad layer tag"), }); } + // Nothing follows the last layer (`extensions/integer-payloads.md` §9), so one + // network has one file. + if c.p != b.len() { + return Err("bytes follow the last layer"); + } Ok(Self { afrac, gf, shift, layers }) } } @@ -449,7 +500,7 @@ pub fn run_net_checked( /// A check [`check_layer`] failed: its reason, and whether it concerned the values /// presented (`true`) or the layer itself (`false`). -type Failed = (&'static str, bool); +pub(crate) type Failed = (&'static str, bool); const fn shape(reason: &'static str) -> Failed { (reason, false) } @@ -548,7 +599,6 @@ fn check_layer(model: &IntModel, layer: &IntLayer, x: &[i64], c: usize, h: usize use crate::int_kernels::max_abs; let (out, _) = layer_geometry(layer, c, h, w)?; const I64: u128 = i64::MAX as u128; - const I128: u128 = i128::MAX as u128; let xm = max_abs(x) as u128; // One (transposed) convolution's arithmetic: every product and partial sum, then // the requantised value, must fit i64 (the reference's accumulator and result). @@ -579,28 +629,7 @@ fn check_layer(model: &IntModel, layer: &IntLayer, x: &[i64], c: usize, h: usize Ok(out) } IntLayer::Igdn { gamma, beta } => { - if c.checked_mul(c) != Some(gamma.len()) || beta.len() != c { - return Err(shape("IGDN parameter lengths do not match the channel count")); - } - if model.gf > 127 || model.afrac > 127 { - return Err(shape("IGDN shift of 128 or more")); - } - // x² ≤ 2^126 always fits; each term gamma·x², and the norm, must fit i128. - let x2 = xm * xm; - let term = (max_abs(gamma) as u128).checked_mul(x2).filter(|&t| t <= I128).ok_or(range("IGDN term could leave i128"))?; - let norm = (max_abs(beta) as u128).saturating_add((c as u128).saturating_mul((term >> model.gf) + 1)); - if norm > I128 { - return Err(range("IGDN norm could leave i128")); - } - // The reference narrows the root to i64 (`isqrt(norm) as i64`), so it must - // fit; then |x · root| < 2^63 · 2^63 fits u128, and the result must fit i64. - let root = norm.isqrt(); - if root > I64 { - return Err(range("IGDN root could leave i64")); - } - if ((xm * root) >> model.afrac) + 1 > I64 { - return Err(range("IGDN output could leave i64")); - } + igdn_check(x, c, gamma, beta, model.gf, model.afrac)?; Ok((c, h, w)) } IntLayer::Relu => Ok((c, h, w)), @@ -616,6 +645,39 @@ fn check_layer(model: &IntModel, layer: &IntLayer, x: &[i64], c: usize, h: usize } } +/// The IGDN checks [`run_net_checked`] documents, for activations `x` of `c` channels +/// at `2^afrac` against `gamma` (`c × c`, at `2^gf`) and `beta` (`c`). Shared with the +/// 1-D synthesis of `int_audio`, whose IGDN is the same [`igdn`]. +pub(crate) fn igdn_check(x: &[i64], c: usize, gamma: &[i64], beta: &[i64], gf: u32, afrac: u32) -> Result<(), Failed> { + use crate::int_kernels::max_abs; + const I64: u128 = i64::MAX as u128; + const I128: u128 = i128::MAX as u128; + if c.checked_mul(c) != Some(gamma.len()) || beta.len() != c { + return Err(shape("IGDN parameter lengths do not match the channel count")); + } + if gf > 127 || afrac > 127 { + return Err(shape("IGDN shift of 128 or more")); + } + let xm = max_abs(x) as u128; + // x² ≤ 2^126 always fits; each term gamma·x², and the norm, must fit i128. + let x2 = xm * xm; + let term = (max_abs(gamma) as u128).checked_mul(x2).filter(|&t| t <= I128).ok_or(range("IGDN term could leave i128"))?; + let norm = (max_abs(beta) as u128).saturating_add((c as u128).saturating_mul((term >> gf) + 1)); + if norm > I128 { + return Err(range("IGDN norm could leave i128")); + } + // The reference narrows the root to i64 (`isqrt(norm) as i64`), so it must + // fit; then |x · root| < 2^63 · 2^63 fits u128, and the result must fit i64. + let root = norm.isqrt(); + if root > I64 { + return Err(range("IGDN root could leave i64")); + } + if ((xm * root) >> afrac) + 1 > I64 { + return Err(range("IGDN output could leave i64")); + } + Ok(()) +} + /// Decode the integer synthesis `model` on the latent `input` (`[c, h, w]` at /// `2^afrac`) → RGB, row-major **HWC** `u8`. Byte-identical on every machine. pub fn decode(model: &IntModel, input: &[i64], c: usize, h: usize, w: usize) -> (Vec, usize, usize) { @@ -719,7 +781,24 @@ fn bytes(v: &Value) -> Vec { } impl CondBitstream { - /// Parse the bitstream the export tool emits (`cond_bitstream.json`). + /// [`Self::from_value`], refusing rather than panicking on any JSON: every member + /// present, a size an integer that fits `usize`, a stream an array of bytes. A + /// reader of JSON it did not write (the browser's debug decode) uses this, and + /// decodes with [`decode_conditional_checked`]. + pub fn try_from_value(v: &Value) -> Result { + let dim = |k: &'static str| v.get(k).and_then(Value::as_u64).and_then(|n| usize::try_from(n).ok()).ok_or(k); + let bytes = |k: &'static str| -> Result, &'static str> { + v.get(k).and_then(Value::as_array).ok_or(k)?.iter() + .map(|b| b.as_u64().and_then(|b| u8::try_from(b).ok())).collect::>().ok_or(k) + }; + Ok(Self { + z_rans: bytes("z_rans")?, n_z: dim("n_z")?, zh: dim("zh")?, zw: dim("zw")?, + m: dim("m")?, sh: dim("sh")?, sw: dim("sw")?, rans: bytes("rans")?, + }) + } + + /// Parse the bitstream the export tool emits (`cond_bitstream.json`). Panics on a + /// JSON the tool would not write; [`Self::try_from_value`] refuses instead. pub fn from_value(v: &Value) -> Self { Self { z_rans: bytes(&v["z_rans"]), @@ -780,6 +859,411 @@ pub fn decode_conditional( Ok(decode(g_s, &y_hat, bs.m, sh, sw)) } +/// The kind of rule a checked decode refused on. The integer-payload decoders +/// (`int_payload`, and `int_audio` and `int_splat`'s typed decoders) report these as +/// the refusal codes of `extensions/integer-payloads.md` §8. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RefusalKind { + /// A value the payload carries, or a latent its stream decodes, is outside the + /// range the decode is exact over. + FieldRange, + /// A tensor or an activation would exceed [`MAX_TENSOR`] elements. + TooLarge, + /// A CDF table breaks the table rules. + CdfMalformed, + /// A rANS stream ran out before its last symbol. + StreamExhausted, + /// A rANS stream does not end canonically after its last symbol. + StreamNoncanonical, + /// A shape the payload declares disagrees with the one the decode produces. + GeometryMismatch, + /// The parameter set does not fit the payload. + ModelMismatch, +} + +/// A checked decode's refusal: the kind of rule and which. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Refused { + pub kind: RefusalKind, + pub detail: &'static str, +} + +impl std::fmt::Display for Refused { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}: {}", self.kind, self.detail) + } +} + +pub(crate) const fn refused(kind: RefusalKind, detail: &'static str) -> Refused { + Refused { kind, detail } +} + +/// The most elements any one buffer a checked decode forms may hold, the same on +/// every target (`extensions/integer-payloads.md` §7). +/// +/// cbindgen:ignore +pub const MAX_TENSOR: usize = 1 << 26; +/// The largest fixed-point scale a checked decode accepts. +/// +/// cbindgen:ignore +pub const MAX_AFRAC: u32 = 30; + +/// A table of `extensions/integer-payloads.md` §2 over its first `len` entries: +/// `3 ≤ len ≤ 4097` and `len` within the row, `cdf[0] = 0`, `cdf[len − 1] = 2^16`, +/// strictly increasing over `len`. +fn table_ok(cdf: &[i32], len: usize) -> bool { + (3..=4097).contains(&len) && len <= cdf.len() && cdf[0] == 0 && cdf[len - 1] == 1 << 16 + && cdf[..len].windows(2).all(|p| p[0] < p[1]) +} + +/// The entropy model's consistency: a non-empty scale table with one Gaussian table, +/// length and offset per edge; one `z` table, length, offset and median per channel; +/// every table one the rANS decoder is total over. +fn check_entropy(t: &CondEntropy) -> Result<(), &'static str> { + let nb = t.edges.len(); + if nb == 0 || t.cdfs.len() != nb || t.cdf_lengths.len() != nb || t.offsets.len() != nb { + return Err("one Gaussian table, length and offset per scale edge, and at least one edge"); + } + let nz = t.z_cdfs.len(); + if nz == 0 || t.z_cdf_lengths.len() != nz || t.z_offsets.len() != nz || t.z_median_q.len() != nz { + return Err("one z table, length, offset and median per channel"); + } + let all = t.cdfs.iter().zip(&t.cdf_lengths).chain(t.z_cdfs.iter().zip(&t.z_cdf_lengths)); + if all.into_iter().any(|(c, &l)| !table_ok(c, l)) { + return Err("a table is not a CDF from 0 to 65536, strictly increasing over its length"); + } + Ok(()) +} + +impl CondEntropy { + /// Parse and check the entropy tables (`cond_entropy.json`, read from a set as + /// `entropy.json`): every member present with integers in range, the lengths + /// consistent and every table well formed. Refuses rather than panics. + pub fn try_from_value(v: &Value) -> Result { + fn arr<'a>(v: &'a Value, k: &str) -> Result<&'a Vec, &'static str> { + v.get(k).and_then(Value::as_array).ok_or("an entropy member is missing or not an array") + } + fn ints>(v: &Value, k: &str) -> Result, &'static str> { + arr(v, k)?.iter().map(|x| x.as_i64().and_then(|i| T::try_from(i).ok())).collect::>() + .ok_or("an entropy member holds a value out of range") + } + fn rows(v: &Value, k: &str) -> Result>, &'static str> { + arr(v, k)?.iter().map(|r| r.as_array().ok_or("an entropy table is not an array") + .and_then(|r| r.iter().map(|x| x.as_i64().and_then(|i| i32::try_from(i).ok())).collect::>() + .ok_or("an entropy table holds a value out of range"))).collect() + } + let t = CondEntropy { + edges: ints(v, "scale_edges")?, + cdfs: rows(v, "cdfs")?, + cdf_lengths: ints(v, "cdf_lengths")?, + offsets: ints(v, "offsets")?, + z_cdfs: rows(v, "z_cdfs")?, + z_cdf_lengths: ints(v, "z_cdf_lengths")?, + z_offsets: ints(v, "z_offsets")?, + z_median_q: ints(v, "z_median_q")?, + }; + check_entropy(&t)?; + Ok(t) + } +} + +/// A conditional bitstream's fields, borrowed: what a `WIH1` payload carries. +#[derive(Debug, Clone, Copy)] +pub struct CondStream<'a> { + pub m: usize, + pub sh: usize, + pub sw: usize, + pub n_z: usize, + pub zh: usize, + pub zw: usize, + pub z_rans: &'a [u8], + pub rans: &'a [u8], +} + +impl CondBitstream { + /// The borrowed form [`decode_conditional_checked`] reads. + pub fn stream(&self) -> CondStream<'_> { + CondStream { m: self.m, sh: self.sh, sw: self.sw, n_z: self.n_z, zh: self.zh, zw: self.zw, z_rans: &self.z_rans, rans: &self.rans } + } +} + +/// A network's output on a `[c, h, w]` input, planned from its layers' geometry and +/// parameter lengths alone, before anything is allocated or any value read +/// (`extensions/integer-payloads.md` §7). The input must be within [`MAX_TENSOR`] +/// (`TooLarge`). Then, layer by layer in order: +/// +/// 1. the layer fits the running channel count (`ModelMismatch`): a convolution's +/// output channels, kernel and stride are non-zero, its weights number +/// `c·out·k·k` and its multipliers and biases `out`, and its requantisation shift is +/// under 128; an inverse GDN's `gamma` numbers `c·c` and its `beta` `c`, and its +/// shifts are under 128; a leaky ReLU's shift is under 64; +/// 2. every size the layer forms is within [`MAX_TENSOR`] (`TooLarge`): a +/// convolution's padded input side `n + 2·pad`, a transposed convolution's uncropped +/// side `(n − 1)·stride + k + opad` and uncropped plane, and the output; +/// 3. the layer leaves an output (`ModelMismatch`): a kernel no larger than the padded +/// input, a crop `2·pad` smaller than the uncropped side. +/// +/// Every size is formed in `u64` with checked arithmetic, a size past `u64` being over +/// the cap, so a 32-bit sink refuses exactly what a 64-bit one does, with the same +/// kind; and once a network is planned, every size [`run_net_checked`] forms fits +/// `usize` on every target. The first rule that fails is the refusal. +pub(crate) fn plan_net(net: &IntModel, c: usize, h: usize, w: usize) -> Result { + plan_layers(net, c, h, w) +} + +/// What [`plan_net`] finds about a network on a `[c, h, w]` input, in elements. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct NetPlan { + /// The output `[c, h, w]`. + pub out: (usize, usize, usize), + /// The largest single buffer the network forms: its input, a layer's output, a + /// transposed convolution's uncropped plane. + pub largest: u64, + /// The most elements the reference kernels hold at once while one layer runs, its + /// input included: a transposed convolution's input, uncropped plane and output; a + /// convolution's input and output; an inverse GDN's input, output and one + /// position's squares (`c` values of 16 bytes, counted as `2·c`); a ReLU's input, + /// rewritten in place. Each element is an `i64`. + pub live: u64, + /// The multiply-accumulates the layers comprise (`extensions/integer-payloads.md` + /// §7): a convolution's every output times `c·k·k` (every tap of its padded + /// input), a transposed convolution's every input times `out·k·k`, an inverse + /// GDN's every element times `c`. A kernel may skip a zero term, so it performs + /// at most this many. Summed in `u128`: a layer's terms are its weights times at + /// most `2^26` positions, so no set a sink can hold reaches the type's limit. + pub macs: u128, +} + +fn plan_layers(net: &IntModel, c: usize, h: usize, w: usize) -> Result { + use RefusalKind::{ModelMismatch, TooLarge}; + const CAP: u64 = MAX_TENSOR as u64; + let big = |what: &'static str| refused(TooLarge, what); + let within = |v: Option, what: &'static str| v.filter(|&n| n <= CAP).ok_or(big(what)); + let (mut c, mut h, mut w) = (c as u64, h as u64, w as u64); + if c == 0 || h == 0 || w == 0 { + return Err(refused(ModelMismatch, "a network's input is empty")); + } + let input = within(c.checked_mul(h).and_then(|v| v.checked_mul(w)), "a network's input exceeds 2^26 elements")?; + let (mut largest, mut live) = (input, input); + let mut macs: u128 = 0; + let params = |cout: u64, k: u64, weight: usize, mult: usize, bias: usize, c: u64| { + let n = c.checked_mul(cout).and_then(|v| v.checked_mul(k)).and_then(|v| v.checked_mul(k)); + n == Some(weight as u64) && mult as u64 == cout && bias as u64 == cout + }; + for layer in &net.layers { + match layer { + IntLayer::ConvT { out_ch, k, stride, pad, opad, weight, mult, bias } => { + let (cout, k, s, p, op) = (*out_ch as u64, *k as u64, *stride as u64, *pad as u64, *opad as u64); + if cout == 0 || k == 0 || s == 0 || !params(cout, k, weight.len(), mult.len(), bias.len(), c) || net.shift > 127 { + return Err(refused(ModelMismatch, "a transposed convolution does not fit the running channel count")); + } + let full = |n: u64| (n - 1).checked_mul(s).and_then(|v| v.checked_add(k)).and_then(|v| v.checked_add(op)); + let (fh, fw) = (within(full(h), "a layer's uncropped side exceeds 2^26")?, within(full(w), "a layer's uncropped side exceeds 2^26")?); + let plane = within(cout.checked_mul(fh).and_then(|v| v.checked_mul(fw)), "a layer's uncropped plane exceeds 2^26 elements")?; + let crop = p.checked_mul(2).unwrap_or(u64::MAX); + if fh <= crop || fw <= crop { + return Err(refused(ModelMismatch, "a transposed convolution's padding leaves no output")); + } + let a = c * h * w; + macs = macs.saturating_add(a as u128 * (cout * k * k) as u128); + (c, h, w) = (cout, fh - crop, fw - crop); + let b = c * h * w; + largest = largest.max(plane).max(b); + live = live.max(a + plane + b); + } + IntLayer::Conv { out_ch, k, stride, pad, weight, mult, bias } => { + let (cout, k, s, p) = (*out_ch as u64, *k as u64, *stride as u64, *pad as u64); + if cout == 0 || k == 0 || s == 0 || !params(cout, k, weight.len(), mult.len(), bias.len(), c) || net.shift > 127 { + return Err(refused(ModelMismatch, "a convolution does not fit the running channel count")); + } + let padded = |n: u64| p.checked_mul(2).and_then(|v| v.checked_add(n)); + let (hp, wp) = (within(padded(h), "a layer's padded side exceeds 2^26")?, within(padded(w), "a layer's padded side exceeds 2^26")?); + if hp < k || wp < k { + return Err(refused(ModelMismatch, "a convolution's kernel is larger than its padded input")); + } + let (ho, wo) = ((hp - k) / s + 1, (wp - k) / s + 1); + let b = within(cout.checked_mul(ho).and_then(|v| v.checked_mul(wo)), "a layer's output exceeds 2^26 elements")?; + live = live.max(c * h * w + b); + largest = largest.max(b); + macs = macs.saturating_add(b as u128 * (c * k * k) as u128); + (c, h, w) = (cout, ho, wo); + } + IntLayer::Igdn { gamma, beta } => { + if c.checked_mul(c) != Some(gamma.len() as u64) || beta.len() as u64 != c || net.gf > 127 || net.afrac > 127 { + return Err(refused(ModelMismatch, "an inverse GDN does not fit the running channel count")); + } + live = live.max(2 * c * h * w + 2 * c); + macs = macs.saturating_add((c * h * w) as u128 * c as u128); + } + IntLayer::Relu => {} + IntLayer::Leaky { slope_frac, .. } => { + if *slope_frac > 63 { + return Err(refused(ModelMismatch, "a leaky ReLU's shift is 64 or more")); + } + } + } + } + // Every running size is at most 2^26 here, so it fits usize on every target. + Ok(NetPlan { out: (c as usize, h as usize, w as usize), largest, live, macs }) +} + +/// [`run_net_checked`] with its refusal as a [`Refused`]: values that could overflow a +/// layer are the stream's (`FieldRange`); a layer that does not fit is the model's. +fn run_checked(net: &IntModel, x: &[i64], c: usize, h: usize, w: usize) -> Result, Refused> { + run_net_checked(net, x, c, h, w).map(|(o, ..)| o).map_err(|r| { + if r.out_of_range { + refused(RefusalKind::FieldRange, "a decoded latent could take a layer outside the exact integer range") + } else { + refused(RefusalKind::ModelMismatch, "a network's parameters do not fit the running shapes") + } + }) +} + +/// [`decode_conditional`], refusing instead of panicking or overflowing on any input: +/// the decode the `WIH1` payload (`wai.neural.int_hyper`, and the keyframe of +/// `wai.video.int_hyper`) runs. On a stream [`decode_conditional`] decodes, both give +/// the same image. +/// +/// Before anything is allocated, in this order: the networks share one fixed-point +/// scale in `1..=`[`MAX_AFRAC`]; the entropy model is consistent and has a `z` table +/// for each of the payload's channels; the latents are within [`MAX_TENSOR`]; `h_s` on +/// the hyper-latent's shape, then `g_s` on `[m, sh, sw]`, are planned ([`plan_net`]: each +/// layer fits, forms nothing over the cap, and leaves an output); then `h_s` must give +/// `[2·m, sh, sw]` (`GeometryMismatch`) and `g_s` three channels. Then the hyper-latent +/// is decoded symbol by symbol (a symbol that runs the stream out is refused before +/// its value is rebuilt with [`dequantize`]), its stream must end canonically, `h_s` +/// runs through [`run_net_checked`], the latent is decoded the same way, and `g_s` +/// runs. Returns RGB HWC with its height and width. +pub fn decode_conditional_checked( + g_s: &IntModel, h_s: &IntModel, ent: &CondEntropy, bs: &CondStream<'_>, +) -> Result<(Vec, usize, usize), Refused> { + use crate::int_entropy::dequantize; + use RefusalKind::*; + let p = cond_plan(g_s, h_s, ent, bs)?; + let (afrac, n_z, zh, zw, m, sh, sw) = (g_s.afrac, bs.n_z, bs.zh, bs.zw, bs.m, bs.sh, bs.sw); + let (zplane, n, (oc, oh, ow)) = (zh * zw, m * sh * sw, p.gs.out); + // 1. factorized-decode z, rebuild z_q = s·2^afrac + median_q. The latent grows a + // symbol at a time, to its declared length and no further, and the first symbol + // that runs its stream out refuses the payload before its value is used. + let mut z = Vec::new(); + let mut dz = RansDecoder::new(bs.z_rans).map_err(|_| refused(FieldRange, "the z stream is not a rANS stream"))?; + for ch in 0..n_z { + for _ in 0..zplane { + let s = dz.decode(&ent.z_cdfs[ch], ent.z_cdf_lengths[ch], ent.z_offsets[ch]); + if dz.is_exhausted() { + return Err(refused(StreamExhausted, "the z stream ended before its last symbol")); + } + push_capped(&mut z, dequantize(s, afrac, ent.z_median_q[ch]).ok_or(refused(FieldRange, "a hyper-latent value leaves i64"))?, n_z * zplane); + } + } + if !dz.is_canonical_end() { + return Err(refused(StreamNoncanonical, "the z stream does not end canonically after its last symbol")); + } + let sm = run_checked(h_s, &z, n_z, zh, zw)?; + drop(z); + let (sigma, mu) = sm.split_at(n); + // 2. y, conditioned on the integer σ: y_hat = r·2^afrac + μ. + let mut dec = RansDecoder::new(bs.rans).map_err(|_| refused(FieldRange, "the y stream is not a rANS stream"))?; + let mut y_hat = Vec::new(); + for p in 0..n { + let b = det_build_index(sigma[p], &ent.edges); + let r = dec.decode(&ent.cdfs[b], ent.cdf_lengths[b], ent.offsets[b]); + if dec.is_exhausted() { + return Err(refused(StreamExhausted, "the y stream ended before its last symbol")); + } + push_capped(&mut y_hat, dequantize(r, afrac, mu[p]).ok_or(refused(FieldRange, "a latent value leaves i64"))?, n); + } + if !dec.is_canonical_end() { + return Err(refused(StreamNoncanonical, "the y stream does not end canonically after its last symbol")); + } + drop(sm); + // 3. integer synthesis g_s → pixels. + let x = run_checked(g_s, &y_hat, m, sh, sw)?; + drop(y_hat); + Ok((to_rgb8(&x, oc, oh, ow, afrac), oh, ow)) +} + +/// Append `x` to `v`, a buffer that will hold `total` elements, growing it by doubling +/// but never past `total`: a decode holds no more than it declared, and no buffer is +/// reserved ahead of the symbols that fill it. +pub(crate) fn push_capped(v: &mut Vec, x: T, total: usize) { + if v.len() == v.capacity() { + let want = v.capacity().saturating_mul(2).max(64).min(total).max(v.len() + 1); + v.reserve_exact(want - v.len()); + } + v.push(x); +} + +/// What [`decode_conditional_checked`] checks before it decodes a symbol, in its +/// order, and the buffers it then forms: the networks share one fixed-point scale in +/// `1..=`[`MAX_AFRAC`]; the entropy model is consistent and has a `z` table for each +/// of the payload's channels; the latents are within [`MAX_TENSOR`]; `h_s` on the +/// hyper-latent's shape, then `g_s` on `[m, sh, sw]`, are planned ([`plan_net`]); then `h_s` +/// must give `[2·m, sh, sw]` (`GeometryMismatch`) and `g_s` three channels. +pub(crate) fn cond_plan(g_s: &IntModel, h_s: &IntModel, ent: &CondEntropy, bs: &CondStream<'_>) -> Result { + use RefusalKind::*; + let afrac = g_s.afrac; + if h_s.afrac != afrac || !(1..=MAX_AFRAC).contains(&afrac) { + return Err(refused(ModelMismatch, "g_s and h_s do not share one fixed-point scale in 1..=30")); + } + check_entropy(ent).map_err(|e| refused(ModelMismatch, e))?; + let (n_z, zh, zw, m, sh, sw) = (bs.n_z, bs.zh, bs.zw, bs.m, bs.sh, bs.sw); + if [n_z, zh, zw, m, sh, sw].contains(&0) { + return Err(refused(FieldRange, "a latent dimension is zero")); + } + if n_z > ent.z_cdfs.len() { + return Err(refused(ModelMismatch, "the hyper-latent has more channels than the entropy model has z tables")); + } + let size = |d: &[usize]| d.iter().try_fold(1usize, |a, &x| a.checked_mul(x)).filter(|&n| n <= MAX_TENSOR); + if size(&[n_z, zh, zw]).is_none() || size(&[2, m, sh, sw]).is_none() { + return Err(refused(TooLarge, "a latent exceeds 2^26 elements")); + } + // Both networks are planned, caps included, before either output shape is + // compared with the payload's. + let hs = plan_net(h_s, n_z, zh, zw)?; + let gs = plan_net(g_s, m, sh, sw)?; + if hs.out != (2 * m, sh, sw) { + return Err(refused(GeometryMismatch, "h_s does not give the latent shape the payload declares")); + } + if gs.out.0 != 3 { + return Err(refused(ModelMismatch, "g_s does not give three channels")); + } + Ok(CondPlan { hs, gs, z: (n_z * zh * zw) as u64, y: (m * sh * sw) as u64 }) +} + +/// The buffers [`decode_conditional_checked`] forms on a payload [`cond_plan`] admits. +#[derive(Debug, Clone, Copy)] +pub(crate) struct CondPlan { + pub hs: NetPlan, + pub gs: NetPlan, + /// The hyper-latent's and the latent's elements: the symbols each stream codes. + pub z: u64, + pub y: u64, +} + +impl CondPlan { + /// The image's bytes (RGB8). + pub fn output_bytes(&self) -> u64 { + let (c, h, w) = self.gs.out; + (c * h * w) as u64 + } + + /// The most bytes the decode holds at once on the reference kernels, beyond the + /// payload and the parsed set: the hyper-latent while `h_s` runs; `h_s`'s output + /// (`[2·m, sh, sw]`) while the latent is decoded; the latent while `g_s` runs; and + /// `g_s`'s output while it becomes RGB8. Every value is an `i64`. + pub fn working_bytes(&self) -> u64 { + let gs_out = self.output_bytes(); + [8 * (self.z + self.hs.live), 8 * (2 * self.y + self.y), 8 * (self.y + self.gs.live), 8 * gs_out + gs_out] + .into_iter().max().unwrap_or(0) + } + + /// The networks' multiply-accumulates: `h_s`'s, then `g_s`'s ([`NetPlan::macs`]). + pub fn macs(&self) -> u128 { + self.hs.macs.saturating_add(self.gs.macs) + } +} + /// The fast kernels against the scalar reference, element by element. Operand /// ranges cover the `i32` fast path, the `i64` path, and sparse (zero-heavy) /// inputs; geometries cover strides, paddings, output paddings and kernels smaller @@ -1118,6 +1602,60 @@ mod checked { assert!(ran > 150, "too few nets passed the checks ({ran})"); } + /// `NetPlan::macs` is every layer's terms at the shapes the network runs at: + /// counted here one term at a time — a convolution's every output and every tap of + /// every input channel, a transposed convolution's every input and every tap of + /// every output channel, an inverse GDN's every element and every channel — over + /// the shapes `apply_layer` forms, not the plan's. + #[test] + fn the_planned_macs_are_every_layers_terms() { + let mut g = Rng(0x5EED_0003); + let mut ran = 0; + for _ in 0..200 { + let cin = 1 + g.below(4); + let (model, _) = random_net(&mut g, cin); + let (h, w) = (1 + g.below(6), 1 + g.below(6)); + let x = g.vec(cin * h * w, 1 << 16, 20); + if run_net_checked(&model, &x, cin, h, w).is_err() { + continue; + } + let plan = plan_net(&model, cin, h, w).expect("the checked run ran, so the plan admits it"); + let (mut a, mut c, mut hh, mut ww) = (x, cin, h, w); + let mut terms: u128 = 0; + for layer in &model.layers { + let (na, nc, nh, nw) = apply_layer(&model, layer, a, c, hh, ww); + match layer { + IntLayer::Conv { k, .. } => { + for _output in 0..nc * nh * nw { + for _tap in 0..c * k * k { + terms += 1; + } + } + } + IntLayer::ConvT { out_ch, k, .. } => { + for _input in 0..c * hh * ww { + for _tap in 0..out_ch * k * k { + terms += 1; + } + } + } + IntLayer::Igdn { .. } => { + for _element in 0..c * hh * ww { + for _channel in 0..c { + terms += 1; + } + } + } + IntLayer::Relu | IntLayer::Leaky { .. } => {} + } + (a, c, hh, ww) = (na, nc, nh, nw); + } + assert_eq!(plan.macs, terms); + ran += 1; + } + assert!(ran > 150, "too few nets passed the checks ({ran})"); + } + #[test] fn a_convolution_runs_up_to_the_accumulation_bound_and_no_further() { // One term: the bound is |w|·|x|. Exactly i64::MAX runs; one past refuses. diff --git a/wai/wai-rs/src/int_mlicpp.rs b/wai/wai-rs/src/int_mlicpp.rs index a6a457e0d..dacc63a9b 100644 --- a/wai/wai-rs/src/int_mlicpp.rs +++ b/wai/wai-rs/src/int_mlicpp.rs @@ -361,6 +361,14 @@ pub struct Trace { pub symbols: Vec, } +/// Parse the `WMB1` bundle `bundle` and decode the `WIM1` payload with it: the +/// byte-input entry of the integer decoders (`int_payload`, the browser crate). A pin +/// over the bundle is verified before this is called (`pin::verify_bytes`, +/// `int_payload::resolve`), so it takes no pin. +pub fn decode_bytes(payload: &[u8], bundle: &[u8]) -> Result { + decode(&IntMlicModel::from_bundle(bundle)?, payload) +} + /// Decode a `WIM1` payload with `model` → RGB. Byte-identical on every machine. /// /// Before it allocates anything, the decode derives from the model's layer geometry diff --git a/wai/wai-rs/src/int_payload/mod.rs b/wai/wai-rs/src/int_payload/mod.rs new file mode 100644 index 000000000..d8905d011 --- /dev/null +++ b/wai/wai-rs/src/int_payload/mod.rs @@ -0,0 +1,829 @@ +//! Payloads of the WAI-native integer codecs +//! ([`extensions/integer-payloads.md`](../../../extensions/integer-payloads.md)) and +//! the one integer decode path every sink shares. +//! +//! SPEC §5 pins each of these capabilities' payload to a binary format +//! ([`crate::codecs::PINNED_INT_PAYLOADS`]): `WIV1` for `wai.video.int_motion` +//! (keyframe 0) and `wai.video.int_hyper` (keyframe 1, a nested `WIH1`), `WIH1` for +//! `wai.neural.int_hyper`, `WIA1` for `wai.audio.int_codec` and `WIS1` for +//! `wai.splat.int_codec`. [`wire`] parses and writes them. The JSON the export and +//! encode tools write is a converter input ([`wiv1_from_json`], [`wih1_from_json`], +//! [`wia1_from_json`], [`wis1_from_json`]) and a debugging aid, never a payload: a +//! payload that is not the pinned format is refused (`bad_magic`). +//! +//! [`decode_int`] decodes every integer capability, these five and +//! `wai.neural.int_synth` and `wai.neural.int_mlicpp`, from the parameter files a +//! [`IntParams`] holds. It needs no ONNX runtime and no filesystem, so it builds under +//! `neural_int` and its media features alone, natively and for `wasm32`; the neural +//! sink (`neural::decode_payload`, feature `neural`) delegates these capabilities to +//! it. [`resolve`] verifies a pin over sets held in memory, running SPEC §3.1's check +//! 5 when the sink holds a key log, and runs the same check over the set an unpinned +//! decode reads (SPEC §4, *Module authorisation*). +//! +//! Every refusal is a code from the closed list of the extension's §8 +//! ([`Refusal`]). A refusal of the payload or of the parameter set against it is final: +//! nothing is decoded in its place, and SPEC §4's fallback, which would decode the same +//! bytes, is not tried. `set_incomplete`, `not_dispatchable` and +//! `unsupported_capability` say the sink cannot decode through the capability, and +//! dispatch moves on as SPEC §4 says (the extension's §8). + +pub use crate::int_wire as wire; + +#[cfg(test)] +mod tests; + +pub use wire::{Refusal, WireError}; + +use crate::codecs::{ + CAP_AUDIO_INT_CODEC, CAP_NEURAL_INT_HYPER, CAP_NEURAL_INT_MLICPP, CAP_NEURAL_INT_SYNTH, CAP_SPLAT_INT_CODEC, + CAP_VIDEO_INT_HYPER, +}; +use crate::container::{PinRefusal, Prior}; +use crate::pin::{HeldBytes, PinAuthority, Verified, VerifiedRef}; + +/// Why [`decode_int`] or a converter refused: a code of the extension's closed list, +/// and what failed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IntPayloadError { + refusal: Refusal, + detail: String, +} + +impl IntPayloadError { + pub fn new(refusal: Refusal, detail: impl Into) -> Self { + IntPayloadError { refusal, detail: detail.into() } + } + /// The refusal. + pub fn refusal(&self) -> Refusal { + self.refusal + } + /// The code the extension's §8 names. + pub fn code(&self) -> &'static str { + self.refusal.code() + } + /// What failed. For [`Refusal::SetIncomplete`], the path of the file the set lacks. + pub fn detail(&self) -> &str { + &self.detail + } +} + +impl std::fmt::Display for IntPayloadError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: {}", self.refusal.code(), self.detail) + } +} + +impl std::error::Error for IntPayloadError {} + +impl From for IntPayloadError { + fn from(e: WireError) -> Self { + IntPayloadError::new(e.refusal, e.detail) + } +} + +impl From for IntPayloadError { + fn from(r: crate::int_hyper_synth::Refused) -> Self { + use crate::int_hyper_synth::RefusalKind as K; + let refusal = match r.kind { + K::FieldRange => Refusal::FieldRange, + K::TooLarge => Refusal::TooLarge, + K::CdfMalformed => Refusal::CdfMalformed, + K::StreamExhausted => Refusal::StreamExhausted, + K::StreamNoncanonical => Refusal::StreamNoncanonical, + K::GeometryMismatch => Refusal::GeometryMismatch, + K::ModelMismatch => Refusal::ModelMismatch, + }; + IntPayloadError::new(refusal, r.detail) + } +} + +fn model(detail: impl Into) -> IntPayloadError { + IntPayloadError::new(Refusal::ModelMismatch, detail) +} + +/// The files a capability's decode reads from its parameter set, by the names a set +/// lists them under (SPEC §5; `extensions/prior-carriage.md` §7). Empty for a +/// capability that decodes against no set, or that this module does not decode. +pub fn set_members(cap: &str) -> &'static [&'static str] { + match cap { + CAP_NEURAL_INT_HYPER | CAP_VIDEO_INT_HYPER => &["entropy.json", "g_s.whm", "h_s.whm"], + CAP_AUDIO_INT_CODEC => &["model.wam"], + CAP_SPLAT_INT_CODEC => &["g_s.whm"], + CAP_NEURAL_INT_SYNTH => &["int_synth.json"], + CAP_NEURAL_INT_MLICPP => &["model.wmb"], + _ => &[], + } +} + +/// The parameter set one integer decode reads, in memory. +/// +/// A pinned decode reads only bytes its pin verified: what `pin::verify` returned (a +/// native sink), or the held set `pin::verify_bytes` matched and, for the form +/// `wai.prior.set`, the paths its listing names (a sink with no filesystem). An +/// unpinned decode reads the set the sink registers for the capability: only the +/// files check 5 was run over ([`resolve`]). +/// +/// Only [`resolve`] builds one that holds parameters, so a decode never reads a set +/// that no pin verified while reporting it pinned, nor an unpinned set other than the +/// one check 5 identified. [`IntParams::none`] holds none. +#[derive(Debug, Clone)] +pub struct IntParams<'a>(Params<'a>); + +#[derive(Debug, Clone)] +enum Params<'a> { + /// No parameter set: a capability that decodes against none (`wai.video.int_motion`). + None, + /// A set `pin::verify` verified, as the bytes that were hashed. + #[cfg_attr(not(feature = "neural"), allow(dead_code))] + Verified(&'a Verified), + /// A held set: one file, or a set whose readable files are `paths` (`None`: a + /// form-less pin's one file). `pinned` when a pin verified it. + Held { set: HeldBytes<'a>, paths: Option>, pinned: bool }, +} + +impl<'a> IntParams<'a> { + /// No parameter set. A decode through a capability that reads one refuses with + /// `set_incomplete`. + pub fn none() -> IntParams<'static> { + IntParams(Params::None) + } + + /// The set `pin::verify` returned for a pin over `cap`: the neural sink's pinned + /// decode, whose registry ran SPEC §3.1's checks. + #[cfg(feature = "neural")] + pub(crate) fn verified(v: &'a Verified) -> Self { + IntParams(Params::Verified(v)) + } + + /// The parameters a pin verified with `pin::verify_bytes` over `held`. + fn from_verified_ref(cap: &str, held: &[HeldBytes<'a>], r: &VerifiedRef) -> Result { + let (index, paths) = match r { + VerifiedRef::File { index } => (*index, None), + VerifiedRef::Set { index, paths } => (*index, Some(paths.clone())), + }; + let set = *held.get(index).ok_or_else(|| PinRefusal::NotHeld { capability: cap.to_string() })?; + Ok(IntParams(Params::Held { set, paths, pinned: true })) + } + + /// Whether a pin verified these parameters. + pub fn is_pinned(&self) -> bool { + matches!(self.0, Params::Verified(_) | Params::Held { pinned: true, .. }) + } + + /// The members of `cap`'s set these parameters hold, by name: the bytes a decode + /// reads. + pub fn read(&self, cap: &str) -> std::collections::BTreeMap { + set_members(cap).iter().filter_map(|m| self.member(cap, m).ok().map(|b| (m.to_string(), b))).collect() + } + + /// Whether these parameters hold no set ([`IntParams::none`], or [`resolve`] for a + /// capability that decodes against none). + pub fn is_none(&self) -> bool { + matches!(self.0, Params::None) + } + + /// The set member `name` that `cap`'s decode reads. A one-file set (a form-less + /// pin's, or a one-file capability's unpinned file) answers for the capability's + /// one member; a listed set answers for a path it lists. A file the set does not + /// hold is `set_incomplete`, naming it. + pub fn member(&self, cap: &str, name: &str) -> Result<&'a [u8], IntPayloadError> { + let one = set_members(cap) == [name]; + let found = match &self.0 { + Params::None => None, + Params::Verified(Verified::File(b)) => one.then_some(b.as_slice()), + Params::Verified(v @ Verified::Set { .. }) => v.file(name).ok(), + Params::Held { set: HeldBytes::File(b), .. } => one.then_some(*b), + Params::Held { set: HeldBytes::Set { files, .. }, paths, .. } => { + if paths.as_ref().is_none_or(|p| p.iter().any(|x| x == name)) { files.get(name).copied() } else { None } + } + }; + found.ok_or_else(|| IntPayloadError::new(Refusal::SetIncomplete, name)) + } +} + +/// What an unpinned decode of `cap` reads from `set`, and every digest a pin over +/// those bytes could name, for check 5 (SPEC §4, *Module authorisation*). +/// +/// The decode reads the capability's members (`set_members`) and nothing else. Where +/// the set carries a listing, it must be well formed (`set_malformed`) and every file +/// it names must be held with its digest (`part_mismatch`), as for a pinned set (SPEC +/// §3.1 checks 7 and 8), and the decode reads only the members it lists. The digests: +/// the listing's, where there is one; the listing of the members the decode reads +/// (`pin::listing_from`), which is what a pin in the form `wai.prior.set` over exactly +/// those files names; and, for a one-file capability, its one file's. So an empty +/// listing does not hide the set: its digest is the one derived from the files read. +/// A set whose listing is malformed or does not match its files has no one identity, +/// so it is refused before check 5 runs. +pub(crate) fn unpinned_read(cap: &str, set: &HeldBytes<'_>) -> Result<(Vec, Option>), PinRefusal> { + use crate::pin::{parse_listing, sha256, sha256_hex}; + let one_file = crate::container::pin_shape(cap) == crate::codecs::ParameterSetShape::OneFile; + let members = set_members(cap); + let mut digests = Vec::new(); + let mut read: std::collections::BTreeMap = std::collections::BTreeMap::new(); + let paths = match set { + HeldBytes::File(b) => { + if one_file { + if let Some(m) = members.first() { + read.insert(m.to_string(), *b); + } + } + None + } + HeldBytes::Set { listing, files } => { + let listed: Option> = if listing.is_empty() { + None + } else { + digests.push(sha256_hex(listing)); + let lines = parse_listing(listing)?; + if let Some((path, _)) = lines.iter().find(|(path, d)| files.get(path).is_none_or(|b| sha256(b) != *d)) { + return Err(PinRefusal::PartMismatch { path: path.clone() }); + } + Some(lines.into_iter().map(|(p, _)| p).collect()) + }; + for m in members { + if listed.as_ref().is_none_or(|l| l.iter().any(|p| p == m)) { + if let Some(b) = files.get(*m) { + read.insert(m.to_string(), *b); + } + } + } + Some(read.keys().cloned().collect()) + } + }; + digests.extend(read_identities(cap, &read)); + digests.sort_unstable(); + digests.dedup(); + Ok((digests, paths)) +} + +/// Every digest a pin over the bytes a decode of `cap` reads, `read` (member name → +/// bytes), could name besides its own: the listing of those members +/// (`pin::listing_from`, what a pin in the form `wai.prior.set` over exactly them +/// names), and, for a one-file capability, its one file's SHA-256 (what a form-less +/// pin names). Check 5 runs over these, pinned or unpinned, so a set is refused under +/// any identity of the bytes the decode reads (SPEC §3.1 check 5). +pub fn read_identities(cap: &str, read: &std::collections::BTreeMap) -> Vec { + use crate::pin::{listing_from, sha256_hex}; + let mut out = Vec::new(); + if crate::container::pin_shape(cap) == crate::codecs::ParameterSetShape::OneFile { + if let Some(b) = set_members(cap).first().and_then(|m| read.get(*m)) { + out.push(sha256_hex(b)); + } + } + if let Ok(l) = listing_from(read) { + out.push(sha256_hex(&l)); + } + out +} + +/// The parameters `cap`'s decode reads, at a sink that holds the sets `held` in memory +/// and, where it has one, the key log `authority` (SPEC §3.1, §4). The one way to build +/// [`IntParams`] that hold a set. +/// +/// - A capability that decodes against no parameter set reads none. +/// - Where a pin applies and the envelope carries one, `pin::verify_bytes_with` runs +/// SPEC §3.1's checks in order, check 5 included where `authority` is given, and the +/// decode reads the bytes it verified. Where `authority` is given, check 5 then runs +/// over every identity of those bytes as well ([`read_identities`]) and the pin's +/// digest. +/// - Otherwise the decode is unpinned and reads `held[0]`, the set the sink registers +/// for the capability, as [`unpinned_read`] states: a listing the set carries must +/// match its files, and the decode reads only the capability's members (only those +/// the listing names, where there is one). Where `authority` is given, check 5 runs +/// over every digest a pin over the bytes read could name, and refuses a revoked or +/// unauthorised set. +pub fn resolve<'a>( + cap: &str, + prior: Option<&Prior>, + held: &[HeldBytes<'a>], + authority: Option<&dyn PinAuthority>, +) -> Result, PinRefusal> { + if crate::codecs::is_known_capability(cap) && !crate::codecs::decodes_against_parameter_set(cap) { + return Ok(IntParams(Params::None)); + } + match prior { + Some(p) if crate::container::pin_applies_to(cap) => { + let r = crate::pin::verify_bytes_with(cap, p, held, crate::pin::READABLE_FORMS, authority)?; + let params = IntParams::from_verified_ref(cap, held, &r)?; + // Check 5 ran over the pin's digest; it runs again over every identity of the + // bytes the decode will read, so a set revoked (or not granted) under its + // file's digest or under the listing of its members is not reached through + // a pin that names it another way. + if let Some(a) = authority { + let mut ids = read_identities(cap, ¶ms.read(cap)); + ids.push(p.sha256.clone()); + a.check_unpinned(cap, &ids)?; + } + Ok(params) + } + _ => { + let set = *held.first().ok_or_else(|| PinRefusal::NotHeld { capability: cap.to_string() })?; + let (digests, paths) = unpinned_read(cap, &set)?; + if let Some(a) = authority { + a.check_unpinned(cap, &digests)?; + } + Ok(IntParams(Params::Held { set, paths, pinned: false })) + } + } +} + +/// What an integer decode gives. Pixels are RGB8, rows top to bottom, pixels left to +/// right; PCM is `i16`, channels interleaved. +#[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] +pub enum IntDecoded { + /// `wai.neural.int_hyper`, `wai.neural.int_synth`, `wai.neural.int_mlicpp`. + Image { rgb: Vec, width: u32, height: u32 }, + /// `wai.video.int_motion`, `wai.video.int_hyper`: every frame, in order. + Video { frames: Vec>, width: u32, height: u32 }, + /// `wai.audio.int_codec`: the PCM, its sample rate and channel count. + Pcm { samples: Vec, sample_rate: u32, channels: u32 }, + /// `wai.splat.int_codec`: the `[n_attr, s, s]` attribute tensor (the + /// splat-equivalence digest's buffer), the cloud's Gaussian count, and each + /// attribute's presentation range `(lo, hi)`, which the digest does not cover. + Attr { bytes: Vec, n_attr: u32, s: u32, n_gaussians: u32, ranges: Vec<(f32, f32)> }, +} + +/// Decode `payload` through the integer capability `cap` against `params`. +/// +/// The capability must be dispatchable (SPEC §4, `codecs::is_dispatchable`). For the +/// five capabilities SPEC §5 pins a format to, the payload must be that format +/// (`bad_magic` otherwise; a JSON form is refused here, before any parameter is +/// read) and is checked as the extension's §2–§7 give, in that order; the decode then +/// reads its set members, checks that they fit the payload, and runs the checked +/// integer decoder. `wai.neural.int_synth` and `wai.neural.int_mlicpp` decode through +/// their own decoders. A capability this build has no decoder for is +/// `unsupported_capability`. +pub fn decode_int(cap: &str, payload: &[u8], params: &IntParams<'_>) -> Result { + if !crate::codecs::is_dispatchable(cap) { + return Err(IntPayloadError::new(Refusal::NotDispatchable, format!("{cap} is not dispatchable (SPEC \u{a7}4)"))); + } + match crate::codecs::pinned_int_payload(cap) { + Some(wire::MAGIC_WIV1) => decode_wiv1(cap, payload, params), + Some(wire::MAGIC_WIH1) => decode_wih1(cap, payload, params), + Some(wire::MAGIC_WIA1) => decode_wia1(cap, payload, params), + Some(wire::MAGIC_WIS1) => decode_wis1(cap, payload, params), + _ => match cap { + CAP_NEURAL_INT_SYNTH => { + let spec = params.member(cap, "int_synth.json")?; + let img = crate::int_synth::decode_from_spec(payload, spec).map_err(|e| { + let crate::int_synth::IntSynthError::InvalidPayload(why) = e; + IntPayloadError::new(Refusal::PayloadMalformed, why) + })?; + Ok(IntDecoded::Image { rgb: img.rgb, width: img.width, height: img.height }) + } + CAP_NEURAL_INT_MLICPP => { + use crate::int_mlicpp::IntMlicError; + let mlic = |e: IntMlicError| match e { + IntMlicError::Payload(why) => IntPayloadError::new(Refusal::PayloadMalformed, why), + IntMlicError::Model(why) => model(why), + }; + let m = crate::int_mlicpp::IntMlicModel::from_bundle(params.member(cap, "model.wmb")?).map_err(mlic)?; + let img = crate::int_mlicpp::decode(&m, payload).map_err(mlic)?; + Ok(IntDecoded::Image { rgb: img.rgb, width: img.width as u32, height: img.height as u32 }) + } + _ => Err(unsupported(cap, "no integer decoder for this capability")), + }, + } +} + +/// What decoding a payload will cost, read before any symbol is decoded, from the +/// header and the parameter set's layer geometry alone (`extensions/integer-payloads.md` +/// §7, *Memory and work*). The header declares the output, and the caps bound it; a +/// stream can make a large output from few bytes (a table can make one symbol nearly +/// free), so it is the header, not the stream's length, that bounds the work. A sink +/// applies its own size policy to this and refuses a payload past it before +/// decoding. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct DeclaredCost { + /// The decoded output's bytes: frames (`n_frames·h·w·3`), an image (`h·w·3`), PCM + /// (`2·t_out`), an attribute tensor (`attr·s·s`). + pub output_bytes: u64, + /// The symbols the payload's streams code: one entropy-decode step each. + pub symbols: u64, + /// The most bytes the reference decode (`decode_int` on the reference kernels, + /// `int_kernels::Policy::reference()`) holds at once, beyond the payload and the + /// parameter set as parsed: the payload's tables as parsed, the output, the + /// latents and network activations (8 bytes per element), each network layer's + /// input, output and, for a transposed convolution, its uncropped plane, and a + /// clip's frame list. The fast kernels + /// trade the uncropped plane for a padded copy of the layer's input, at most 16 + /// times the elements the layer reads and writes, and per-worker rows. + pub working_bytes: u64, + /// The multiply-accumulates the decode's networks comprise, layer by layer: a + /// convolution's every output times `c·k·k`, a transposed convolution's every + /// input times `out·k·k` (`out·k` in the audio synthesis), an inverse GDN's every + /// element times `c` (§7). Exact, and 0 for a `WIV1` with keyframe 0, which runs + /// no network. A kernel may skip a zero term, so it performs at most this many. + pub macs: u128, +} + +impl DeclaredCost { + /// A cost from its four figures: what a sink gives [`check_cost`] (and the neural + /// registry's `with_cost_limit`) as its limit. A figure the sink does not bound is + /// its type's maximum. + pub const fn new(output_bytes: u64, symbols: u64, working_bytes: u64, macs: u128) -> Self { + Self { output_bytes, symbols, working_bytes, macs } + } + + /// The first figure of this cost over `limit`'s, by name, or `None` where every + /// figure is within it. + pub fn exceeds(&self, limit: &DeclaredCost) -> Option<&'static str> { + if self.output_bytes > limit.output_bytes { + Some("output bytes") + } else if self.symbols > limit.symbols { + Some("symbols") + } else if self.working_bytes > limit.working_bytes { + Some("working bytes") + } else if self.macs > limit.macs { + Some("multiply-accumulates") + } else { + None + } + } +} + +/// A sink's cost limit (§7, *Memory and work*), applied before any symbol is decoded: +/// [`declared_cost`], with its refusal where the payload or the set fails a check it +/// reads, and `cost_over_limit` where the cost exceeds `limit` in any figure. A +/// capability with no declared cost (`wai.neural.int_synth`, `wai.neural.int_mlicpp`) +/// has nothing to check, `Ok(None)`; it is decoded as before. +pub fn check_cost(cap: &str, payload: &[u8], params: &IntParams<'_>, limit: &DeclaredCost) -> Result, IntPayloadError> { + if crate::codecs::pinned_int_payload(cap).is_none() { + return Ok(None); + } + let cost = declared_cost(cap, payload, params)?; + match cost.exceeds(limit) { + None => Ok(Some(cost)), + Some(what) => Err(IntPayloadError::new(Refusal::CostOverLimit, format!("{cap}: the declared {what} exceed the sink's limit"))), + } +} + +/// [`DeclaredCost`] for `payload` under `cap` against `params`: the parse and every +/// check a decode makes before its first symbol (the set's members and networks +/// planned), with the refusal a decode would give, then the sizes those checks fix. +/// A `WIV1` with keyframe 1 is also refused here when its keyframe's size is not the +/// clip's (`geometry_mismatch`), as a decode refuses it before it decodes the keyframe. +/// So a payload the decode would refuse before its first symbol is refused here with +/// the same code. Only the five +/// capabilities SPEC §5 pins a format to have a declared cost +/// (`unsupported_capability` otherwise). +pub fn declared_cost(cap: &str, payload: &[u8], params: &IntParams<'_>) -> Result { + if !crate::codecs::is_dispatchable(cap) { + return Err(IntPayloadError::new(Refusal::NotDispatchable, format!("{cap} is not dispatchable (SPEC \u{a7}4)"))); + } + match crate::codecs::pinned_int_payload(cap) { + Some(wire::MAGIC_WIV1) => cost_wiv1(cap, payload, params), + Some(wire::MAGIC_WIH1) => { + let k = wire::parse_wih1(payload)?; + let p = cond_cost(cap, &k, params)?; + Ok(DeclaredCost { output_bytes: p.output_bytes(), symbols: p.z + p.y, working_bytes: p.working_bytes(), macs: p.macs() }) + } + Some(wire::MAGIC_WIA1) => cost_wia1(cap, payload, params), + Some(wire::MAGIC_WIS1) => cost_wis1(cap, payload, params), + _ => Err(unsupported(cap, "no declared cost: its payload format is not one SPEC \u{a7}5 pins here")), + } +} + +/// The bytes of one `Vec`'s header on this target, for a table of tables. +#[cfg(any(feature = "audio_int", feature = "splat_int"))] +const VEC: u64 = std::mem::size_of::>() as u64; + +/// The conditional codec's plan for a parsed `WIH1` against the capability's set. +fn cond_cost(cap: &str, k: &wire::Wih1<'_>, params: &IntParams<'_>) -> Result { + let (g_s, h_s, ent) = int_hyper_models(cap, params)?; + let s = crate::int_hyper_synth::CondStream { + m: k.m as usize, sh: k.sh as usize, sw: k.sw as usize, + n_z: k.n_z as usize, zh: k.zh as usize, zw: k.zw as usize, + z_rans: k.z_rans, rans: k.rans, + }; + Ok(crate::int_hyper_synth::cond_plan(&g_s, &h_s, &ent, &s)?) +} + +#[cfg(feature = "video_int")] +fn cost_wiv1(cap: &str, payload: &[u8], params: &IntParams<'_>) -> Result { + use crate::int_video::{check, working_bytes, VideoStream}; + let keyframe = if cap == crate::codecs::CAP_VIDEO_INT_MOTION { 0 } else { 1 }; + let v = wire::parse_wiv1(payload, Some(keyframe))?; + let s = VideoStream { + n_frames: v.n_frames as usize, h: v.h as usize, w: v.w as usize, block: v.block as usize, + q: v.q as i64, offset: v.offset, mv_precision: v.mv_precision, cdf: &v.cdf, mvs: &v.mvs, rans: v.rans, + }; + check(&s).map_err(|e| IntPayloadError::new(Refusal::FieldRange, e))?; + // The parsed table and motion vectors, held for the whole decode. + let tables = 4 * v.cdf.len() as u64 + 4 * v.mvs.len() as u64; + let frame = v.h as u64 * v.w as u64 * 3; + let output_bytes = frame * v.n_frames as u64; + match v.wih1 { + None => Ok(DeclaredCost { output_bytes, symbols: output_bytes, working_bytes: v.keyframe0_working_bytes(), macs: 0 }), + Some(nested) => { + let p = cond_cost(cap, &wire::parse_wih1(nested)?, params)?; + if p.gs.out != (3, v.h as usize, v.w as usize) { + return Err(IntPayloadError::new(Refusal::GeometryMismatch, "the keyframe's size is not the clip's h × w")); + } + // The keyframe's decode, then the clip with the keyframe's image beside it. + let working = p.working_bytes().max(working_bytes(&s) + frame); + Ok(DeclaredCost { output_bytes, symbols: frame * (v.n_frames as u64 - 1) + p.z + p.y, working_bytes: tables + working, macs: p.macs() }) + } + } +} + +#[cfg(not(feature = "video_int"))] +fn cost_wiv1(cap: &str, _: &[u8], _: &IntParams<'_>) -> Result { + Err(unsupported(cap, "this build has no integer video decoder (feature video_int)")) +} + +#[cfg(feature = "audio_int")] +fn cost_wia1(cap: &str, payload: &[u8], params: &IntParams<'_>) -> Result { + use crate::int_audio::{audio_plan, AudioModel, AudioStream}; + let a = wire::parse_wia1(payload)?; + let m = AudioModel::from_bytes(params.member(cap, "model.wam")?).map_err(|e| model(format!("model.wam: {e}")))?; + let s = AudioStream { + afrac: a.afrac as u32, n_ch: a.n_ch as usize, lat_len: a.lat_len as usize, t_out: a.t_out as usize, + offset: a.offset, median_q: &a.median_q, cdfs: &a.cdfs, rans: a.rans, + }; + let (_, working, macs) = audio_plan(&m, &s)?; + // The parsed medians and tables, held for the whole decode. + let tables = a.n_ch as u64 * (8 + VEC + 4 * a.cdfs[0].len() as u64); + Ok(DeclaredCost { output_bytes: 2 * a.t_out as u64, symbols: a.n_ch as u64 * a.lat_len as u64, working_bytes: tables + working, macs }) +} + +#[cfg(not(feature = "audio_int"))] +fn cost_wia1(cap: &str, _: &[u8], _: &IntParams<'_>) -> Result { + Err(unsupported(cap, "this build has no integer audio decoder (feature audio_int)")) +} + +#[cfg(feature = "splat_int")] +fn cost_wis1(cap: &str, payload: &[u8], params: &IntParams<'_>) -> Result { + use crate::int_hyper_synth::IntModel; + use crate::int_splat::{splat_plan, SplatStream}; + let v = wire::parse_wis1(payload)?; + let gs = IntModel::from_bytes(params.member(cap, "g_s.whm")?).map_err(|e| model(format!("g_s.whm: {e}")))?; + let s = SplatStream { + afrac: v.afrac as u32, c: v.c as usize, sh: v.sh as usize, sw: v.sw as usize, s: v.s as usize, + n_attr: v.attr as usize, cdfs: &v.cdfs, offsets: &v.offsets, median_q: &v.median_q, rans: v.rans, + }; + let (working, macs) = splat_plan(&gs, &s)?; + // The parsed ranges, offsets, medians and tables, held for the whole decode. + let cdfs: u64 = v.cdfs.iter().map(|t| 4 * t.len() as u64).sum(); + let tables = 8 * v.attr as u64 + v.c as u64 * (4 + 8 + VEC) + cdfs; + Ok(DeclaredCost { output_bytes: v.attr as u64 * v.s as u64 * v.s as u64, symbols: v.c as u64 * v.sh as u64 * v.sw as u64, working_bytes: tables + working, macs }) +} + +#[cfg(not(feature = "splat_int"))] +fn cost_wis1(cap: &str, _: &[u8], _: &IntParams<'_>) -> Result { + Err(unsupported(cap, "this build has no integer splat decoder (feature splat_int)")) +} + +fn unsupported(cap: &str, why: &str) -> IntPayloadError { + IntPayloadError::new(Refusal::UnsupportedCapability, format!("{cap}: {why}")) +} + +/// The conditional codec's three set members, parsed and checked. +fn int_hyper_models( + cap: &str, + params: &IntParams<'_>, +) -> Result<(crate::int_hyper_synth::IntModel, crate::int_hyper_synth::IntModel, crate::int_hyper_synth::CondEntropy), IntPayloadError> { + use crate::int_hyper_synth::{CondEntropy, IntModel}; + let net = |name: &str| -> Result { + IntModel::from_bytes(params.member(cap, name)?).map_err(|e| model(format!("{name}: {e}"))) + }; + let g_s = net("g_s.whm")?; + let h_s = net("h_s.whm")?; + let ent: serde_json::Value = serde_json::from_slice(params.member(cap, "entropy.json")?) + .map_err(|e| model(format!("entropy.json: {e}")))?; + let ent = CondEntropy::try_from_value(&ent).map_err(|e| model(format!("entropy.json: {e}")))?; + Ok((g_s, h_s, ent)) +} + +/// A parsed `WIH1` decoded to RGB with the capability's set. A `WIV1` keyframe passes +/// the clip's `h × w` as `size`: the keyframe's planned image is checked against it +/// (`geometry_mismatch`) before any of its symbols is decoded (§7). +fn wih1_image(cap: &str, k: &wire::Wih1<'_>, params: &IntParams<'_>, size: Option<(u32, u32)>) -> Result<(Vec, u32, u32), IntPayloadError> { + let (g_s, h_s, ent) = int_hyper_models(cap, params)?; + let s = crate::int_hyper_synth::CondStream { + m: k.m as usize, sh: k.sh as usize, sw: k.sw as usize, + n_z: k.n_z as usize, zh: k.zh as usize, zw: k.zw as usize, + z_rans: k.z_rans, rans: k.rans, + }; + if let Some((h, w)) = size { + let p = crate::int_hyper_synth::cond_plan(&g_s, &h_s, &ent, &s)?; + if p.gs.out != (3, h as usize, w as usize) { + return Err(IntPayloadError::new(Refusal::GeometryMismatch, "the keyframe's size is not the clip's h × w")); + } + } + let (rgb, h, w) = crate::int_hyper_synth::decode_conditional_checked(&g_s, &h_s, &ent, &s)?; + Ok((rgb, h as u32, w as u32)) +} + +fn decode_wih1(cap: &str, payload: &[u8], params: &IntParams<'_>) -> Result { + let k = wire::parse_wih1(payload)?; + let (rgb, height, width) = wih1_image(cap, &k, params, None)?; + Ok(IntDecoded::Image { rgb, width, height }) +} + +#[cfg(feature = "video_int")] +fn decode_wiv1(cap: &str, payload: &[u8], params: &IntParams<'_>) -> Result { + use crate::int_video::{decode_pframes_typed, decode_typed, VideoStream, STREAM_EXHAUSTED, STREAM_NONCANONICAL}; + let keyframe = if cap == crate::codecs::CAP_VIDEO_INT_MOTION { 0 } else { 1 }; + let v = wire::parse_wiv1(payload, Some(keyframe))?; + let s = VideoStream { + n_frames: v.n_frames as usize, h: v.h as usize, w: v.w as usize, block: v.block as usize, + q: v.q as i64, offset: v.offset, mv_precision: v.mv_precision, cdf: &v.cdf, mvs: &v.mvs, rans: v.rans, + }; + let video = |e: &'static str| { + let refusal = match e { + STREAM_EXHAUSTED => Refusal::StreamExhausted, + STREAM_NONCANONICAL => Refusal::StreamNoncanonical, + _ => Refusal::FieldRange, + }; + IntPayloadError::new(refusal, e) + }; + let clip = match v.wih1 { + None => decode_typed(&s).map_err(video)?, + Some(nested) => { + let k = wire::parse_wih1(nested)?; + let (frame0, ..) = wih1_image(cap, &k, params, Some((v.h, v.w)))?; + decode_pframes_typed(&frame0, &s).map_err(video)? + } + }; + Ok(IntDecoded::Video { frames: clip.frames, width: v.w, height: v.h }) +} + +#[cfg(not(feature = "video_int"))] +fn decode_wiv1(cap: &str, _: &[u8], _: &IntParams<'_>) -> Result { + Err(unsupported(cap, "this build has no integer video decoder (feature video_int)")) +} + +#[cfg(feature = "audio_int")] +fn decode_wia1(cap: &str, payload: &[u8], params: &IntParams<'_>) -> Result { + use crate::int_audio::{decode_typed, AudioModel, AudioStream}; + let a = wire::parse_wia1(payload)?; + let m = AudioModel::from_bytes(params.member(cap, "model.wam")?).map_err(|e| model(format!("model.wam: {e}")))?; + let s = AudioStream { + afrac: a.afrac as u32, n_ch: a.n_ch as usize, lat_len: a.lat_len as usize, t_out: a.t_out as usize, + offset: a.offset, median_q: &a.median_q, cdfs: &a.cdfs, rans: a.rans, + }; + let samples = decode_typed(&m, &s)?; + Ok(IntDecoded::Pcm { samples, sample_rate: a.sr, channels: 1 }) +} + +#[cfg(not(feature = "audio_int"))] +fn decode_wia1(cap: &str, _: &[u8], _: &IntParams<'_>) -> Result { + Err(unsupported(cap, "this build has no integer audio decoder (feature audio_int)")) +} + +#[cfg(feature = "splat_int")] +fn decode_wis1(cap: &str, payload: &[u8], params: &IntParams<'_>) -> Result { + use crate::int_hyper_synth::IntModel; + use crate::int_splat::{decode_typed, SplatStream}; + let v = wire::parse_wis1(payload)?; + let gs = IntModel::from_bytes(params.member(cap, "g_s.whm")?).map_err(|e| model(format!("g_s.whm: {e}")))?; + let s = SplatStream { + afrac: v.afrac as u32, c: v.c as usize, sh: v.sh as usize, sw: v.sw as usize, s: v.s as usize, + n_attr: v.attr as usize, cdfs: &v.cdfs, offsets: &v.offsets, median_q: &v.median_q, rans: v.rans, + }; + let d = decode_typed(&gs, &s)?; + Ok(IntDecoded::Attr { bytes: d.attr, n_attr: v.attr, s: v.s, n_gaussians: v.n_gaussians, ranges: v.ranges }) +} + +#[cfg(not(feature = "splat_int"))] +fn decode_wis1(cap: &str, _: &[u8], _: &IntParams<'_>) -> Result { + Err(unsupported(cap, "this build has no integer splat decoder (feature splat_int)")) +} + +// ───────────────────────── converters (JSON tool output → payload) ───────────────────────── + +/// A converter's refusal of its JSON input. +fn json(why: &str) -> IntPayloadError { + IntPayloadError::new(Refusal::FieldRange, format!("the JSON form: {why}")) +} + +#[cfg(any(feature = "video_int", feature = "audio_int", feature = "splat_int"))] +fn to_u32(v: usize, what: &str) -> Result { + u32::try_from(v).map_err(|_| json(&format!("{what} exceeds u32"))) +} + +#[cfg(any(feature = "video_int", feature = "audio_int", feature = "splat_int"))] +fn table_len(len: usize) -> Result<(), IntPayloadError> { + if !(wire::CDF_LEN_MIN..=wire::CDF_LEN_MAX).contains(&len) { + return Err(json("a CDF table's length is outside 3..=4097")); + } + Ok(()) +} + +/// Convert the video encoders' JSON bitstream to `WIV1`: keyframe 0, or keyframe 1 +/// with `keyframe_wih1` (a `WIH1` payload) as its nested I-frame. The JSON's implicit +/// rules are made explicit as [`crate::int_video::VideoJson`] states; `q` must be in +/// `1..=65535`. The result is parsed before it is returned, so a converter never +/// emits a payload a sink would refuse at parse. +#[cfg(feature = "video_int")] +pub fn wiv1_from_json(bs: &serde_json::Value, keyframe_wih1: Option<&[u8]>) -> Result, IntPayloadError> { + let j = crate::int_video::VideoJson::from_value(bs).map_err(json)?; + let q = u32::try_from(j.q).ok().filter(|q| (1..=wire::MAX_Q).contains(q)).ok_or_else(|| json("q is outside 1..=65535"))?; + table_len(j.cdf.len())?; + // A one-frame clip carries no motion vector: its header is written with + // mv_precision 1 and block 1, the values the format fixes for it. + let one = j.n_frames == 1; + let v = wire::Wiv1 { + mv_precision: if one { 1 } else { j.mv_precision }, + keyframe: keyframe_wih1.is_some() as u8, + n_frames: to_u32(j.n_frames, "n_frames")?, + h: to_u32(j.h, "h")?, + w: to_u32(j.w, "w")?, + block: if one { 1 } else { to_u32(j.block, "block")? }, + q, + offset: j.offset, + cdf: j.cdf.clone(), + mvs: j.mvs.clone(), + rans: &j.rans, + wih1: keyframe_wih1, + }; + let out = wire::write_wiv1(&v); + wire::parse_wiv1(&out, None)?; + Ok(out) +} + +/// Convert the conditional codec's JSON bitstream (`cond_bitstream.json`) to `WIH1`. +pub fn wih1_from_json(bs: &serde_json::Value) -> Result, IntPayloadError> { + let dim = |k: &str| bs[k].as_u64().and_then(|v| u32::try_from(v).ok()).ok_or_else(|| json(&format!("`{k}` is missing or out of range"))); + let bytes = |k: &str| -> Result, IntPayloadError> { + bs[k].as_array().and_then(|a| a.iter().map(|b| b.as_u64().and_then(|b| u8::try_from(b).ok())).collect::>>()) + .ok_or_else(|| json(&format!("`{k}` is not an array of bytes"))) + }; + let (z_rans, rans) = (bytes("z_rans")?, bytes("rans")?); + let v = wire::Wih1 { + m: dim("m")?, sh: dim("sh")?, sw: dim("sw")?, n_z: dim("n_z")?, zh: dim("zh")?, zw: dim("zw")?, + z_rans: &z_rans, rans: &rans, + }; + let out = wire::write_wih1(&v); + wire::parse_wih1(&out)?; + Ok(out) +} + +/// Convert the audio export's JSON bitstream (`audio_bitstream.json`) to `WIA1` at +/// the sample rate `sr` (the export's `audio.json` carries it as `sample_rate`; the +/// bitstream does not). The JSON's implicit rules are made explicit as +/// [`crate::int_audio::AudioJson`] states. +#[cfg(feature = "audio_int")] +pub fn wia1_from_json(bs: &serde_json::Value, sr: u32) -> Result, IntPayloadError> { + let j = crate::int_audio::AudioJson::from_value(bs).map_err(json)?; + if j.median_q.len() != j.n_ch || j.cdfs.len() != j.n_ch { + return Err(json("the medians and tables are not one per channel (n_ch)")); + } + table_len(j.cdfs[0].len())?; + let v = wire::Wia1 { + afrac: u8::try_from(j.afrac).map_err(|_| json("afrac exceeds a byte"))?, + sr, + n_ch: to_u32(j.n_ch, "n_ch")?, + lat_len: to_u32(j.lat_len, "lat_len")?, + t_out: to_u32(j.t_out, "t_out")?, + offset: j.offset, + median_q: j.median_q.clone(), + cdfs: j.cdfs.clone(), + rans: &j.rans, + }; + let out = wire::write_wia1(&v); + wire::parse_wia1(&out)?; + Ok(out) +} + +/// Convert the splat export's JSON bitstream (`splat.json`) to `WIS1`, with its +/// per-attribute presentation ranges. The export writes each range as a binary32 +/// value; a range that is not exactly one is refused, so the payload carries what the +/// JSON states. `splat_int` reads JSON numbers correctly rounded +/// (`serde_json/float_roundtrip`), so a decimal that denotes a binary32 value reads as +/// exactly that value, and no tolerance is needed. +#[cfg(feature = "splat_int")] +pub fn wis1_from_json(bs: &serde_json::Value) -> Result, IntPayloadError> { + let j = crate::int_splat::SplatJson::from_value(bs).map_err(json)?; + if j.cdfs.len() != j.c || j.offsets.len() != j.c || j.median_q.len() != j.c { + return Err(json("the tables, offsets and medians are not one per channel (c)")); + } + for t in &j.cdfs { + table_len(t.len())?; + } + let exact = |x: f64| -> Result { + let f = x as f32; + if f as f64 == x { Ok(f) } else { Err(json("a range is not exactly a binary32 value")) } + }; + let ranges = j.ranges.iter().map(|&(lo, hi)| Ok((exact(lo)?, exact(hi)?))).collect::, IntPayloadError>>()?; + let v = wire::Wis1 { + afrac: u8::try_from(j.afrac).map_err(|_| json("afrac exceeds a byte"))?, + c: to_u32(j.c, "c")?, + sh: to_u32(j.sh, "sh")?, + sw: to_u32(j.sw, "sw")?, + s: to_u32(j.s, "s")?, + attr: to_u32(j.n_attr, "attr")?, + n_gaussians: to_u32(j.n_gaussians, "n_gaussians")?, + ranges, + cdfs: j.cdfs.clone(), + offsets: j.offsets.clone(), + median_q: j.median_q.clone(), + rans: &j.rans, + }; + let out = wire::write_wis1(&v); + wire::parse_wis1(&out)?; + Ok(out) +} diff --git a/wai/wai-rs/src/int_payload/tests.rs b/wai/wai-rs/src/int_payload/tests.rs new file mode 100644 index 000000000..445557452 --- /dev/null +++ b/wai/wai-rs/src/int_payload/tests.rs @@ -0,0 +1,777 @@ +//! `int_payload`: the binary forms against the JSON they are converted from, the +//! integer decode path without the neural sink, the closed refusal list, and SPEC +//! §3.1's check 5 through [`super::resolve`]. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use sha2::{Digest, Sha256}; + +use super::*; +use crate::int_kernels::{with_policy, Policy}; + +const POLICIES: [Policy; 5] = [Policy::reference(), Policy::fast(1), Policy::fast(2), Policy::fast(5), Policy::fast(0)]; + +fn vroot() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/vectors") +} +fn read(rel: &str) -> Vec { + let p = vroot().join(rel); + std::fs::read(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())) +} +fn jload(rel: &str) -> serde_json::Value { + serde_json::from_slice(&read(rel)).unwrap_or_else(|e| panic!("parse {rel}: {e}")) +} +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} +fn sha(b: &[u8]) -> String { + hex(&Sha256::digest(b)) +} + +/// Decode under every kernel policy; all outputs must be identical. +fn same_under_every_policy(what: &str, f: impl Fn() -> T) -> T { + let want = with_policy(POLICIES[0], &f); + for p in &POLICIES[1..] { + assert!(with_policy(*p, &f) == want, "{what}: output under {p:?} differs from the reference kernels"); + } + want +} + +/// An unpinned decode's parameters over `held`, at a sink with no key log. +fn unpinned<'a>(cap: &str, held: HeldBytes<'a>) -> IntParams<'a> { + resolve(cap, None, &[held], None).expect("an unpinned set resolves") +} + +/// The tiny conditional codec's set, by member name. +fn int_hyper_set() -> BTreeMap> { + [("entropy.json", "int_hyper/tiny_entropy.json"), ("g_s.whm", "int_hyper/tiny_g_s.bin"), ("h_s.whm", "int_hyper/tiny_h_s.bin")] + .into_iter().map(|(k, v)| (k.to_string(), read(v))).collect() +} +fn view(m: &BTreeMap>) -> BTreeMap { + m.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect() +} + +/// Every frame's SHA-256 and the clip's, against the sidecar entry `key`. +#[cfg(feature = "video_int")] +fn assert_frames_sidecar(frames: &[Vec], key: &str) { + let side = jload("int_video/sha256.json"); + let want = side[key]["frames_sha256"].as_array().unwrap(); + assert_eq!(want.len(), frames.len(), "{key}: frame count"); + let per: Vec<[u8; 32]> = frames.iter().map(|f| Sha256::digest(f).into()).collect(); + for (i, d) in per.iter().enumerate() { + assert_eq!(hex(d), want[i].as_str().unwrap(), "{key}: frame {i}"); + } + assert_eq!(sha(&per.concat()), side[key]["clip_sha256"].as_str().unwrap(), "{key}: clip"); +} + +/// §6.1 test 17, `WIV1`: the converter writes the committed payload; the payload's +/// fields are the JSON's; the payload decodes, under every kernel policy, to the +/// frames the JSON form decodes to through the same typed decoder, and those frames +/// are anchored to output recorded before this revision: the sidecar's SHA-256 and the +/// FNV-1a hashes the encode tools recorded (`int_video.json`, `video_real.json`, +/// `video_nk.json`). The keyframe-1 clip is also checked against the unchecked +/// conditional decode of its keyframe. +#[cfg(feature = "video_int")] +#[test] +fn wiv1_json_equivalence() { + use crate::int_video::{decode, decode_pframes, fnv1a, VideoJson}; + for (name, json, bin, conf) in [ + ("int_video", "int_video/int_video_bitstream.json", "int_video/int_video_bitstream.wiv1", "int_video/int_video.json"), + ("video_real", "int_video/video_real_bitstream.json", "int_video/video_real_bitstream.wiv1", "int_video/video_real.json"), + ] { + let js = jload(json); + let payload = read(bin); + assert_eq!(wiv1_from_json(&js, None).unwrap(), payload, "{name}: the converter writes the committed payload"); + let (v, j) = (wire::parse_wiv1(&payload, Some(0)).unwrap(), VideoJson::from_value(&js).unwrap()); + assert_eq!((v.n_frames as usize, v.h as usize, v.w as usize, v.block as usize, v.q as i64, v.offset, v.mv_precision), + (j.n_frames, j.h, j.w, j.block, j.q, j.offset, j.mv_precision), "{name}: header"); + assert_eq!((&v.cdf, &v.mvs, v.rans), (&j.cdf, &j.mvs, j.rans.as_slice()), "{name}: tables and stream"); + let frames = same_under_every_policy(name, || match decode_int(crate::codecs::CAP_VIDEO_INT_MOTION, &payload, &IntParams::none()) { + Ok(IntDecoded::Video { frames, .. }) => frames, + other => panic!("{name}: {other:?}"), + }); + assert_eq!(frames, decode(&js).unwrap().frames, "{name}: the binary and JSON forms decode alike"); + assert_frames_sidecar(&frames, name); + let want = jload(conf); + for (f, w) in frames.iter().zip(want["frame_hashes"].as_array().unwrap()) { + assert_eq!(format!("{:016x}", fnv1a(f)), w.as_str().unwrap(), "{name}: FNV-1a"); + } + } + // Keyframe 1: the nested WIH1 is the tiny conditional codec's payload. + let (set, wih1) = (int_hyper_set(), read("int_hyper/tiny_bitstream.wih1")); + let js = jload("int_video/video_nk_bitstream.json"); + let payload = read("int_video/video_nk_bitstream.wiv1"); + assert_eq!(wiv1_from_json(&js, Some(&wih1)).unwrap(), payload, "video_nk: the converter writes the committed payload"); + assert_eq!(wire::parse_wiv1(&payload, Some(1)).unwrap().wih1, Some(wih1.as_slice())); + let files = view(&set); + let params = unpinned(crate::codecs::CAP_VIDEO_INT_HYPER, HeldBytes::Set { listing: b"", files: &files }); + let frames = same_under_every_policy("video_nk", || match decode_int(crate::codecs::CAP_VIDEO_INT_HYPER, &payload, ¶ms) { + Ok(IntDecoded::Video { frames, .. }) => frames, + other => panic!("video_nk: {other:?}"), + }); + let legacy = { + use crate::int_hyper_synth::{decode_conditional, CondBitstream, CondEntropy, IntModel}; + let g_s = IntModel::from_json(&jload("int_hyper/tiny_g_s.json")); + let h_s = IntModel::from_json(&jload("int_hyper/tiny_h_s.json")); + let ent = CondEntropy::from_value(&jload("int_hyper/tiny_entropy.json")); + let (f0, h, w) = decode_conditional(&g_s, &h_s, &ent, &CondBitstream::from_value(&jload("int_hyper/tiny_bitstream.json"))).unwrap(); + decode_pframes(&f0, h, w, &js).unwrap().frames + }; + assert_eq!(frames, legacy, "video_nk: the binary and JSON forms decode alike"); + assert_frames_sidecar(&frames, "video_nk"); + let want = jload("int_video/video_nk.json"); + let want = want["frame_hashes"].as_array().unwrap(); + assert_eq!(frames.len(), want.len(), "video_nk: frame count"); + for (f, w) in frames.iter().zip(want) { + assert_eq!(format!("{:016x}", crate::int_video::fnv1a(f)), w.as_str().unwrap(), "video_nk: the recorded FNV-1a"); + } +} + +/// §6.1 test 17, `WIH1`. +#[test] +fn wih1_json_equivalence() { + use crate::int_hyper_synth::{decode_conditional, fnv1a, CondBitstream, CondEntropy, IntModel}; + let js = jload("int_hyper/tiny_bitstream.json"); + let payload = read("int_hyper/tiny_bitstream.wih1"); + assert_eq!(wih1_from_json(&js).unwrap(), payload, "the converter writes the committed payload"); + let (k, b) = (wire::parse_wih1(&payload).unwrap(), CondBitstream::from_value(&js)); + assert_eq!((k.m as usize, k.sh as usize, k.sw as usize, k.n_z as usize, k.zh as usize, k.zw as usize), + (b.m, b.sh, b.sw, b.n_z, b.zh, b.zw)); + assert_eq!((k.z_rans, k.rans), (b.z_rans.as_slice(), b.rans.as_slice())); + let set = int_hyper_set(); + let files = view(&set); + let params = unpinned(crate::codecs::CAP_NEURAL_INT_HYPER, HeldBytes::Set { listing: b"", files: &files }); + let rgb = same_under_every_policy("tiny", || match decode_int(crate::codecs::CAP_NEURAL_INT_HYPER, &payload, ¶ms) { + Ok(IntDecoded::Image { rgb, width: 16, height: 16 }) => rgb, + other => panic!("tiny: {other:?}"), + }); + let g_s = IntModel::from_json(&jload("int_hyper/tiny_g_s.json")); + let h_s = IntModel::from_json(&jload("int_hyper/tiny_h_s.json")); + let ent = CondEntropy::from_value(&jload("int_hyper/tiny_entropy.json")); + assert_eq!(rgb, decode_conditional(&g_s, &h_s, &ent, &b).unwrap().0, "the payload and the unchecked JSON decode agree"); + assert_eq!(sha(&rgb), jload("int_hyper/sha256.json")["tiny"]["sha256"].as_str().unwrap()); + assert_eq!(format!("{:016x}", fnv1a(&rgb)), jload("int_hyper/tiny.json")["rgb_hash"].as_str().unwrap()); + // The checked entropy reader accepts the committed tables and gives what the + // unchecked one gives. + let checked = CondEntropy::try_from_value(&jload("int_hyper/tiny_entropy.json")).unwrap(); + assert_eq!((checked.edges, checked.cdfs, checked.z_median_q), (ent.edges, ent.cdfs, ent.z_median_q)); +} + +/// §6.1 test 17, `WIA1`: the payload and its JSON form decode alike through the typed +/// decoder, to the PCM recorded before this revision. +#[cfg(feature = "audio_int")] +#[test] +fn wia1_json_equivalence() { + use crate::int_audio::{decode, pcm_fnv1a, AudioJson, AudioModel}; + let (js, conf) = (jload("int_audio/audio_bitstream.json"), jload("int_audio/audio.json")); + let sr = conf["sample_rate"].as_u64().unwrap() as u32; + let payload = read("int_audio/audio_bitstream.wia1"); + assert_eq!(wia1_from_json(&js, sr).unwrap(), payload, "the converter writes the committed payload"); + let (a, j) = (wire::parse_wia1(&payload).unwrap(), AudioJson::from_value(&js).unwrap()); + assert_eq!((a.afrac as u32, a.sr, a.n_ch as usize, a.lat_len as usize, a.t_out as usize, a.offset), + (j.afrac, sr, j.n_ch, j.lat_len, j.t_out, j.offset)); + assert_eq!((&a.median_q, &a.cdfs, a.rans), (&j.median_q, &j.cdfs, j.rans.as_slice())); + let model = read("int_audio/audio_model.bin"); + let params = unpinned(crate::codecs::CAP_AUDIO_INT_CODEC, HeldBytes::File(&model)); + let pcm = same_under_every_policy("audio", || match decode_int(crate::codecs::CAP_AUDIO_INT_CODEC, &payload, ¶ms) { + Ok(IntDecoded::Pcm { samples, sample_rate: 16_000, channels: 1 }) => samples, + other => panic!("audio: {other:?}"), + }); + assert_eq!(pcm, decode(&AudioModel::from_bytes(&model).unwrap(), &js).unwrap(), "the binary and JSON forms decode alike"); + // Anchored below to output recorded before this revision: the sidecar and the + // export's FNV-1a (`audio.json`). + let s16le: Vec = pcm.iter().flat_map(|s| s.to_le_bytes()).collect(); + assert_eq!(sha(&s16le), jload("int_audio/sha256.json")["audio"]["sha256"].as_str().unwrap()); + assert_eq!(format!("{:016x}", pcm_fnv1a(&pcm)), conf["pcm_fnv1a"].as_str().unwrap()); +} + +/// §6.1 test 17, `WIS1`, with the presentation ranges the JSON states: the payload and +/// its JSON form decode alike through the typed decoder, to the attributes recorded +/// before this revision. +#[cfg(feature = "splat_int")] +#[test] +fn wis1_json_equivalence() { + use crate::int_hyper_synth::{fnv1a, IntModel}; + use crate::int_splat::{decode, SplatJson}; + let js = jload("int_splat/splat.json"); + let payload = read("int_splat/splat.wis1"); + assert_eq!(wis1_from_json(&js).unwrap(), payload, "the converter writes the committed payload"); + let (v, j) = (wire::parse_wis1(&payload).unwrap(), SplatJson::from_value(&js).unwrap()); + assert_eq!((v.afrac as u32, v.c as usize, v.sh as usize, v.sw as usize, v.s as usize, v.attr as usize, v.n_gaussians as usize), + (j.afrac, j.c, j.sh, j.sw, j.s, j.n_attr, j.n_gaussians)); + assert_eq!((&v.cdfs, &v.offsets, &v.median_q, v.rans), (&j.cdfs, &j.offsets, &j.median_q, j.rans.as_slice())); + for (&(lo, hi), &(jl, jh)) in v.ranges.iter().zip(&j.ranges) { + assert_eq!((lo, hi), (jl as f32, jh as f32), "the ranges are the JSON's binary32 values"); + } + let gs = read("int_splat/splat_gs.bin"); + let params = unpinned(crate::codecs::CAP_SPLAT_INT_CODEC, HeldBytes::File(&gs)); + let (bytes, ranges) = same_under_every_policy("splat", || match decode_int(crate::codecs::CAP_SPLAT_INT_CODEC, &payload, ¶ms) { + Ok(IntDecoded::Attr { bytes, n_attr: 14, s: 32, n_gaussians: 1024, ranges }) => (bytes, ranges), + other => panic!("splat: {other:?}"), + }); + assert_eq!(ranges, v.ranges); + assert_eq!(bytes, decode(&IntModel::from_bytes(&gs).unwrap(), &js).unwrap().attr, "the binary and JSON forms decode alike"); + // Anchored below to output recorded before this revision: the sidecar and the + // export's FNV-1a (`attr_hash`). + assert_eq!(sha(&bytes), jload("int_splat/sha256.json")["splat"]["sha256"].as_str().unwrap()); + assert_eq!(format!("{:016x}", fnv1a(&bytes)), js["attr_hash"].as_str().unwrap()); +} + +/// §6.1 test 18: every integer capability decodes through `decode_int` in a build +/// without the neural sink (`neural`, which pulls the ONNX runtime), to its pinned +/// hashes: the five pinned formats, `wai.neural.int_mlicpp`'s committed envelopes (the +/// form-less pin and the listing pin, verified in memory) and the `wai.neural.int_synth` +/// corpus. +#[cfg(all(feature = "video_int", feature = "audio_int", feature = "splat_int", not(feature = "neural")))] +#[test] +fn decode_int_without_neural() { + use crate::container::Wai; + use crate::codecs as c; + let set = int_hyper_set(); + let files = view(&set); + let hyp = unpinned(c::CAP_NEURAL_INT_HYPER, HeldBytes::Set { listing: b"", files: &files }); + let (model, gs) = (read("int_audio/audio_model.bin"), read("int_splat/splat_gs.bin")); + let cases: [(&str, &str, IntParams<'_>); 6] = [ + (c::CAP_VIDEO_INT_MOTION, "int_video/int_video_bitstream.wiv1", IntParams::none()), + (c::CAP_VIDEO_INT_MOTION, "int_video/video_real_bitstream.wiv1", IntParams::none()), + (c::CAP_VIDEO_INT_HYPER, "int_video/video_nk_bitstream.wiv1", hyp.clone()), + (c::CAP_NEURAL_INT_HYPER, "int_hyper/tiny_bitstream.wih1", hyp.clone()), + (c::CAP_AUDIO_INT_CODEC, "int_audio/audio_bitstream.wia1", unpinned(c::CAP_AUDIO_INT_CODEC, HeldBytes::File(&model))), + (c::CAP_SPLAT_INT_CODEC, "int_splat/splat.wis1", unpinned(c::CAP_SPLAT_INT_CODEC, HeldBytes::File(&gs))), + ]; + for (cap, rel, params) in cases { + assert!(decode_int(cap, &read(rel), ¶ms).is_ok(), "{cap} {rel}"); + } + + // wai.neural.int_mlicpp: the form-less pin over the bundle, and the listing pin, + // each verified in memory by resolve, decode to the committed image. + let d = vroot().join("int_mlicpp"); + let rd = |n: &str| std::fs::read(d.join(n)).unwrap(); + let want: serde_json::Value = serde_json::from_slice(&rd("expected.json")).unwrap(); + let (bundle, listing) = (rd("model.wmb"), rd("model.sha256")); + let set_files: BTreeMap = [("model.wmb".to_string(), bundle.as_slice())].into_iter().collect(); + for (env, held) in [("stream.wai", HeldBytes::File(&bundle)), ("stream_set.wai", HeldBytes::Set { listing: &listing, files: &set_files })] { + let env = Wai::from_bytes(&rd(env)).unwrap(); + let held = [held]; + let params = resolve(c::CAP_NEURAL_INT_MLICPP, env.prior(), &held, None).expect("the pin verifies"); + assert!(params.is_pinned()); + match decode_int(c::CAP_NEURAL_INT_MLICPP, &env.payload, ¶ms) { + Ok(IntDecoded::Image { rgb, .. }) => assert_eq!(sha(&rgb), want["rgb_sha256"].as_str().unwrap()), + other => panic!("int_mlicpp: {other:?}"), + } + } + + // wai.neural.int_synth: the neural-conformance corpus, by its blake3 RGB hash. + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../neural-conformance"); + let manifest: serde_json::Value = serde_json::from_slice(&std::fs::read(dir.join("expected_hashes.json")).unwrap()).unwrap(); + let mut n = 0; + for (name, spec) in manifest.as_object().unwrap() { + let vd = dir.join("vectors").join(name); + let (payload, model) = (std::fs::read(vd.join("latent.bin")).unwrap(), std::fs::read(vd.join("int_synth.json")).unwrap()); + let rgb = match decode_int(c::CAP_NEURAL_INT_SYNTH, &payload, &unpinned(c::CAP_NEURAL_INT_SYNTH, HeldBytes::File(&model))) { + Ok(IntDecoded::Image { rgb, .. }) => rgb, + other => panic!("{name}: {other:?}"), + }; + let mut h = blake3::Hasher::new(); + h.update(b"wai:int-synth-rgb\x01"); + h.update(&rgb); + assert_eq!(h.finalize().to_hex().as_str(), spec["rgb_hash"].as_str().unwrap(), "{name}"); + n += 1; + } + assert!(n >= 4, "the four committed int_synth vectors, found {n}"); +} + +/// The refusal list is closed: every code distinct, snake_case, and read back. +#[test] +fn refusal_codes_are_closed_and_distinct() { + let mut codes: Vec<&str> = Refusal::ALL.iter().map(|r| r.code()).collect(); + for c in &codes { + assert!(c.bytes().all(|b| b.is_ascii_lowercase() || b == b'_'), "{c} is snake_case"); + assert_eq!(Refusal::from_code(c).map(Refusal::code), Some(*c)); + } + codes.sort_unstable(); + codes.dedup(); + assert_eq!(codes.len(), Refusal::ALL.len(), "codes are distinct"); + assert_eq!(Refusal::from_code("prior_not_held"), None); +} + +/// A JSON form under each pinned string is refused as `bad_magic`, before any +/// parameter is read (the parameters here are absent, and would be `set_incomplete`). +#[test] +fn json_form_is_refused_before_any_parameter_is_read() { + for (cap, json) in [ + (crate::codecs::CAP_VIDEO_INT_MOTION, "int_video/int_video_bitstream.json"), + (crate::codecs::CAP_VIDEO_INT_HYPER, "int_video/video_nk_bitstream.json"), + (crate::codecs::CAP_NEURAL_INT_HYPER, "int_hyper/tiny_bitstream.json"), + (crate::codecs::CAP_AUDIO_INT_CODEC, "int_audio/audio_bitstream.json"), + (crate::codecs::CAP_SPLAT_INT_CODEC, "int_splat/splat.json"), + ] { + let built = match cap { + crate::codecs::CAP_NEURAL_INT_HYPER => true, + crate::codecs::CAP_AUDIO_INT_CODEC => cfg!(feature = "audio_int"), + crate::codecs::CAP_SPLAT_INT_CODEC => cfg!(feature = "splat_int"), + _ => cfg!(feature = "video_int"), + }; + let want = if built { "bad_magic" } else { "unsupported_capability" }; + assert_eq!(decode_int(cap, &read(json), &IntParams::none()).map(|_| ()).unwrap_err().code(), want, "{cap}"); + } +} + +/// The refusals that concern the capability or the set rather than the payload's bytes. +#[test] +fn capability_and_set_refusals() { + let code = |r: Result| r.map(|_| "decoded").unwrap_or_else(|e| e.code()); + // A candidate is not dispatchable (SPEC §4); a dispatchable non-integer capability + // has no integer decoder. + assert_eq!(code(decode_int(crate::codecs::CAP_SPLAT_SOG, b"WIS1", &IntParams::none())), "not_dispatchable"); + assert_eq!(code(decode_int(crate::codecs::CAP_IMAGE_PNG, b"\x89PNG", &IntParams::none())), "unsupported_capability"); + // The conditional codec reads three files: a set without h_s.whm is incomplete, + // and the refusal names the file. + let mut set = int_hyper_set(); + set.remove("h_s.whm"); + let files = view(&set); + let r = decode_int(crate::codecs::CAP_NEURAL_INT_HYPER, &read("int_hyper/tiny_bitstream.wih1"), + &unpinned(crate::codecs::CAP_NEURAL_INT_HYPER, HeldBytes::Set { listing: b"", files: &files })); + assert_eq!(r.as_ref().map_err(|e| (e.code(), e.detail())).err(), Some(("set_incomplete", "h_s.whm"))); + // One file cannot hold a several-file set. + let one = read("int_hyper/tiny_g_s.bin"); + assert_eq!(code(decode_int(crate::codecs::CAP_NEURAL_INT_HYPER, &read("int_hyper/tiny_bitstream.wih1"), + &unpinned(crate::codecs::CAP_NEURAL_INT_HYPER, HeldBytes::File(&one)))), "set_incomplete"); + // Entropy tables that break the table rules are the model's fault. + let mut bad = int_hyper_set(); + bad.insert("entropy.json".into(), br#"{"scale_edges":[],"cdfs":[],"cdf_lengths":[],"offsets":[],"z_cdfs":[],"z_cdf_lengths":[],"z_offsets":[],"z_median_q":[]}"#.to_vec()); + let files = view(&bad); + assert_eq!(code(decode_int(crate::codecs::CAP_NEURAL_INT_HYPER, &read("int_hyper/tiny_bitstream.wih1"), + &unpinned(crate::codecs::CAP_NEURAL_INT_HYPER, HeldBytes::Set { listing: b"", files: &files }))), "model_mismatch"); + // int_mlicpp: a payload its decoder refuses, and a bundle that is not one. + let bundle = std::fs::read(vroot().join("int_mlicpp/model.wmb")).unwrap(); + assert_eq!(code(decode_int(crate::codecs::CAP_NEURAL_INT_MLICPP, b"WIM1", &unpinned(crate::codecs::CAP_NEURAL_INT_MLICPP, HeldBytes::File(&bundle)))), "payload_malformed"); + assert_eq!(code(decode_int(crate::codecs::CAP_NEURAL_INT_MLICPP, b"WIM1", &unpinned(crate::codecs::CAP_NEURAL_INT_MLICPP, HeldBytes::File(b"WHM1")))), "model_mismatch"); + // int_synth: a side of 2^64 is refused, not shifted. + let mut p = vec![0u8; 12]; + p[8] = 1; + p[10] = 64; + let spec = std::fs::read(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../neural-conformance/vectors/gradient/int_synth.json")).unwrap(); + assert_eq!(code(decode_int(crate::codecs::CAP_NEURAL_INT_SYNTH, &p, &unpinned(crate::codecs::CAP_NEURAL_INT_SYNTH, HeldBytes::File(&spec)))), "payload_malformed"); +} + +/// `container::check_int_payload_framing` (what every emit path, `wai wrap` and `wai +/// inspect` run, with no `neural_int`) is the wire parser with the capability's +/// keyframe kind: it refuses every payload with the parser's code, a keyframe-0 `WIV1` +/// under `wai.video.int_hyper` and a `WIV1` cut inside its header among them, and +/// accepts what the parser accepts. +#[test] +fn framing_check_is_the_parser() { + let k0 = read("int_video/int_video_bitstream.wiv1"); + let payloads: Vec> = vec![ + k0.clone(), k0[..40].to_vec(), read("int_video/video_nk_bitstream.wiv1"), read("int_hyper/tiny_bitstream.wih1"), + read("int_audio/audio_bitstream.wia1"), read("int_splat/splat.wis1"), + read("int_splat/splat.json"), b"WIV1".to_vec(), b"WIH1\x02".to_vec(), b"WI".to_vec(), Vec::new(), + ]; + let mut refused = std::collections::BTreeSet::new(); + for &(cap, magic) in crate::codecs::PINNED_INT_PAYLOADS { + for p in &payloads { + let parsed = match &magic { + b"WIV1" => wire::parse_wiv1(p, Some((cap == crate::codecs::CAP_VIDEO_INT_HYPER) as u8)).err(), + b"WIH1" => wire::parse_wih1(p).err(), + b"WIA1" => wire::parse_wia1(p).err(), + _ => wire::parse_wis1(p).err(), + }; + let framed = crate::container::check_int_payload_framing(cap, p).err(); + assert_eq!(framed.as_ref().map(|e| (e.code, e.detail)), parsed.map(|e| (e.code(), e.detail)), "{cap}, {} bytes", p.len()); + refused.extend(framed.map(|e| e.code)); + } + } + for code in ["bad_magic", "format_capability_mismatch", "keyframe_capability_mismatch", "version", "stream_length"] { + assert!(refused.contains(code), "no case gave {code}"); + } + let hyp = crate::codecs::CAP_VIDEO_INT_HYPER; + assert_eq!(crate::container::check_int_payload_framing(hyp, &k0).unwrap_err().code, "keyframe_capability_mismatch"); + assert_eq!(crate::container::check_int_payload_framing(crate::codecs::CAP_VIDEO_INT_MOTION, &k0[..40]).unwrap_err().code, "stream_length"); + assert_eq!(crate::container::check_int_payload_framing(crate::codecs::CAP_IMAGE_PNG, b"{}"), Ok(())); +} + +/// A key log stand-in: revokes the digests in `revoked`; where `held_to`, a set it +/// does not grant (`granted`) is not authorised. +#[derive(Debug)] +struct Authority { + revoked: Vec, + granted: Vec, + held_to: bool, +} + +impl crate::pin::PinAuthority for Authority { + fn check(&self, capability: &str, prior: &Prior) -> Result<(), PinRefusal> { + self.check_unpinned(capability, std::slice::from_ref(&prior.sha256)) + } + fn check_unpinned(&self, capability: &str, digests: &[String]) -> Result<(), PinRefusal> { + if digests.iter().any(|d| self.revoked.contains(d)) { + return Err(PinRefusal::Revoked { capability: capability.into(), revoked_by: None }); + } + if self.held_to && !digests.iter().any(|d| self.granted.contains(d)) { + return Err(PinRefusal::NotAuthorised { capability: capability.into(), check: None, reason: None }); + } + Ok(()) + } +} + +/// SPEC §3.1 check 5 through the ungated path: a revoked set and an unauthorised set +/// are refused, pinned and unpinned; with no key log, or with one that grants the set, +/// the same set resolves. +#[test] +fn check_5_through_the_ungated_path() { + use crate::codecs::{CAP_AUDIO_INT_CODEC as AUD, CAP_NEURAL_INT_HYPER as HYP, CAP_PRIOR_SET}; + let set = int_hyper_set(); + let files = view(&set); + let listing = crate::pin::listing_from(&files).unwrap(); + let digest = crate::pin::sha256_hex(&listing); + let held = [HeldBytes::Set { listing: &listing, files: &files }]; + let pin = Prior::new("tiny", digest.clone(), "decode-equivalence").with_form(CAP_PRIOR_SET); + let code = |r: Result, PinRefusal>| r.map(|p| if p.is_pinned() { "pinned" } else { "unpinned" }).unwrap_or_else(|e| e.code()); + let revoking = Authority { revoked: vec![digest.clone()], granted: vec![], held_to: false }; + let ungranting = Authority { revoked: vec![], granted: vec![], held_to: true }; + let granting = Authority { revoked: vec![], granted: vec![digest.clone()], held_to: true }; + for (prior, ok) in [(Some(&pin), "pinned"), (None, "unpinned")] { + assert_eq!(code(resolve(HYP, prior, &held, None)), ok); + assert_eq!(code(resolve(HYP, prior, &held, Some(&revoking))), "prior_revoked"); + assert_eq!(code(resolve(HYP, prior, &held, Some(&ungranting))), "prior_not_authorised"); + assert_eq!(code(resolve(HYP, prior, &held, Some(&granting))), ok); + } + // The pinned decode reads the files its listing names, and decodes. + let p = resolve(HYP, Some(&pin), &held, Some(&granting)).unwrap(); + assert!(decode_int(HYP, &read("int_hyper/tiny_bitstream.wih1"), &p).is_ok()); + // A one-file set, unpinned: checked under its file's digest. + let model = read("int_audio/audio_model.bin"); + let one = [HeldBytes::File(&model)]; + let revoke_model = Authority { revoked: vec![crate::pin::sha256_hex(&model)], granted: vec![], held_to: false }; + assert_eq!(code(resolve(AUD, None, &one, Some(&revoke_model))), "prior_revoked"); + assert_eq!(code(resolve(AUD, None, &one, None)), "unpinned"); + // A capability that reads no set needs none, and no key log is consulted. + assert!(resolve(crate::codecs::CAP_VIDEO_INT_MOTION, None, &[], Some(&revoking)).is_ok_and(|p| p.is_none())); + // An unpinned decode with no set held. + assert_eq!(code(resolve(HYP, None, &[], None)), "prior_not_held"); +} + +/// An unpinned decode is checked under the bytes it reads (SPEC §4, *Module +/// authorisation*): the tiny set is refused under its own listing, and the same files +/// held with an empty listing, or beside a file the decode does not read, are refused +/// too, under the listing of the members read. A listing that does not match its files, +/// or is malformed, is refused before check 5; the decode reads only the members a +/// listing names. +#[test] +fn unpinned_check_5_covers_the_files_the_decode_reads() { + use crate::pin::{listing_from, sha256_hex}; + let hyp = crate::codecs::CAP_NEURAL_INT_HYPER; + let set = int_hyper_set(); + let files = view(&set); + let listing = listing_from(&files).unwrap(); + let revoking = Authority { revoked: vec![sha256_hex(&listing)], granted: vec![], held_to: false }; + let code = |r: Result, PinRefusal>| r.map(|_| "ok").unwrap_or_else(|e| e.code()); + fn held<'a>(listing: &'a [u8], files: &'a BTreeMap) -> HeldBytes<'a> { + HeldBytes::Set { listing, files } + } + let tiny = read("int_hyper/tiny_bitstream.wih1"); + let own = listing.as_slice(); + assert_eq!(code(resolve(hyp, None, &[held(own, &files)], Some(&revoking))), "prior_revoked", "its own listing"); + assert_eq!(code(resolve(hyp, None, &[held(b"", &files)], Some(&revoking))), "prior_revoked", "no listing"); + let mut extra = set.clone(); + extra.insert("notes.txt".into(), b"not a member".to_vec()); + let extra = view(&extra); + assert_eq!(code(resolve(hyp, None, &[held(b"", &extra)], Some(&revoking))), "prior_revoked", "an unread file beside it"); + // With no key log, the same set decodes. + let p = resolve(hyp, None, &[held(b"", &files)], None).unwrap(); + assert!(!p.is_pinned()); + assert!(decode_int(hyp, &tiny, &p).is_ok()); + // A listing whose file is not the one held, and a malformed listing. + let mut tampered = set.clone(); + tampered.get_mut("g_s.whm").unwrap().push(0); + let tampered = view(&tampered); + assert_eq!(code(resolve(hyp, None, &[held(own, &tampered)], None)), "part_mismatch"); + assert_eq!(code(resolve(hyp, None, &[held(b"not a listing\n", &files)], None)), "set_malformed"); + // A listing of two of the three members: the decode reads those two only. + let two: BTreeMap = files.iter().filter(|(k, _)| k.as_str() != "h_s.whm").map(|(k, v)| (k.clone(), *v)).collect(); + let two = listing_from(&two).unwrap(); + let p = resolve(hyp, None, &[held(&two, &files)], None).unwrap(); + let r = decode_int(hyp, &tiny, &p); + assert_eq!(r.as_ref().map_err(|e| (e.code(), e.detail())).err(), Some(("set_incomplete", "h_s.whm"))); +} + +/// The same refusals from the key log a sink holds (`key_log::SinkKeyLog`), as the +/// neural sink's `int_mlicpp_revoked_set_is_refused_through_the_key_log` checks them. +#[cfg(feature = "key_discovery")] +#[test] +fn check_5_with_a_sink_key_log() { + use crate::key_log::{commitment, Anchor, KeyEntry, KeyStatus, ModuleForm, ModuleGrant, Revision, SinkKeyLog}; + use ed25519_dalek::SigningKey; + const P: &str = "https://publisher.example"; + let hyp = crate::codecs::CAP_NEURAL_INT_HYPER; + let key = |s: u8| SigningKey::from_bytes(&[s; 32]); + let set = int_hyper_set(); + let files = view(&set); + let listing = crate::pin::listing_from(&files).unwrap(); + let digest = crate::pin::sha256(&listing); + let held = [HeldBytes::Set { listing: &listing, files: &files }]; + let pin = Prior::new("tiny", hex(&digest), "decode-equivalence").with_form(crate::codecs::CAP_PRIOR_SET); + let sink = |revoked: Vec<[u8; 32]>, statuses: bool| { + let unsigned = Revision { + origin: P.into(), seq: 0, prev: None, issued_at: "2026-09-01T00:00:00Z".into(), root: [0; 32], + next_root: commitment(&key(0x62).verifying_key().to_bytes()), + keys: vec![KeyEntry::active("modd", key(0x6b).verifying_key().to_bytes(), &["module/decoder"])], + revoked_modules: vec![], grant_max_s: 3_600, status_max_age_s: 600, sig: [0; 64], + }; + let r0 = Revision::seal(&key(0x61), unsigned.clone()).unwrap(); + let r1 = Revision::seal(&key(0x61), Revision { seq: 1, prev: Some(r0.revision_hash()), + issued_at: "2026-09-10T00:00:00Z".into(), revoked_modules: revoked, ..unsigned }).unwrap(); + let grant = ModuleGrant::seal(&key(0x6b), "did:web:publisher.example#modd", P, digest, ModuleForm::Decoder, &[hyp], + "2026-09-20T05:30:00Z", "2026-09-20T06:30:00Z").unwrap().to_json(); + let st = if statuses { vec![KeyStatus::seal(&key(0x61), &r1, "2026-09-20T05:55:00Z").to_json()] } else { vec![] }; + SinkKeyLog::new(vec![r0.clone(), r1], vec![Anchor::pinned(P, r0.revision_hash())]) + .with_statuses(st).with_grants(vec![grant]).at("2026-09-20T06:00:00Z").hold_pinned_sets(hyp, &[P]) + }; + let code = |r: Result, PinRefusal>| r.map(|_| "ok").unwrap_or_else(|e| e.code()); + for prior in [Some(&pin), None] { + assert_eq!(code(resolve(hyp, prior, &held, Some(&sink(vec![digest], true)))), "prior_revoked"); + assert_eq!(code(resolve(hyp, prior, &held, Some(&sink(vec![], false)))), "prior_not_authorised"); + assert_eq!(code(resolve(hyp, prior, &held, Some(&sink(vec![], true)))), "ok"); + } +} + +/// A pinned set answers only for the paths its listing names, and a form-less pin's +/// one file only for the capability's one member. +#[test] +fn a_verified_set_answers_only_for_what_it_verified() { + use crate::codecs::CAP_PRIOR_SET; + let mut set = int_hyper_set(); + let listing = crate::pin::listing_from(&view(&set)).unwrap(); + set.insert("extra.bin".into(), b"unverified".to_vec()); + let files = view(&set); + let hyp = crate::codecs::CAP_NEURAL_INT_HYPER; + let pin = Prior::new("tiny", crate::pin::sha256_hex(&listing), "decode-equivalence").with_form(CAP_PRIOR_SET); + let held = [HeldBytes::Set { listing: &listing, files: &files }]; + let p = resolve(hyp, Some(&pin), &held, None).unwrap(); + assert!(p.is_pinned()); + assert!(p.member(hyp, "g_s.whm").is_ok()); + assert_eq!(p.member(hyp, "extra.bin").unwrap_err().code(), "set_incomplete"); + let aud = crate::codecs::CAP_AUDIO_INT_CODEC; + let one = [HeldBytes::File(b"WAM1")]; + let p = resolve(aud, Some(&Prior::new("m", crate::pin::sha256_hex(b"WAM1"), "decode-equivalence")), &one, None).unwrap(); + assert!(p.member(aud, "model.wam").is_ok()); + assert!(p.member(hyp, "g_s.whm").is_err(), "one file is not a several-file set"); +} + +/// The entropy tables of a set obey the table rules of the extension's §2 (§7): a +/// length of 2 or 4098 is refused, as a payload's own table is, and 3 and 4097 are read. +#[test] +fn entropy_tables_obey_the_table_length_rule() { + use crate::int_hyper_synth::CondEntropy; + let table = |n: usize| -> Vec { + let mut t: Vec = (0..n as i64).collect(); + t[n - 1] = 65_536; + t + }; + let ent = |n: usize| serde_json::json!({ + "scale_edges": [1], "cdfs": [table(n)], "cdf_lengths": [n], "offsets": [0], + "z_cdfs": [table(n)], "z_cdf_lengths": [n], "z_offsets": [0], "z_median_q": [0] + }); + for n in [2, 4098] { + assert!(CondEntropy::try_from_value(&ent(n)).is_err(), "a table of {n} entries is refused"); + } + for n in [3, 4097] { + assert!(CondEntropy::try_from_value(&ent(n)).is_ok(), "a table of {n} entries is read"); + } +} + +/// A network is planned in 64-bit arithmetic, layer by layer, so a size past the cap is +/// `too_large` however far past it is, on every target. Planning in `usize` refused a +/// transposed convolution of stride 2^31 on a side of 3 as malformed on a 32-bit sink +/// and as too large on a 64-bit one; the stride `usize::MAX / 2 + 1` (2^63 here) is the +/// same split on this target. A convolution padded past the cap is too large though +/// its output would be one element. Parameters that do not fit, or a padding that +/// leaves no output, are the model's. On the committed networks the plan is the shape +/// they run at. +#[test] +fn network_plans_refuse_alike_on_every_target() { + use crate::int_hyper_synth::{net_shape, plan_net, IntLayer, IntModel, RefusalKind as K}; + let plan = |m: &IntModel, c, h, w| plan_net(m, c, h, w).map(|p| p.out); + let net = |layer: IntLayer| IntModel { afrac: 8, gf: 8, shift: 8, layers: vec![layer] }; + let convt = |stride: usize, pad: usize| net(IntLayer::ConvT { out_ch: 1, k: 1, stride, pad, opad: 0, weight: vec![1], mult: vec![1], bias: vec![0] }); + let kind = |r: Result<(usize, usize, usize), crate::int_hyper_synth::Refused>| r.map_err(|e| e.kind); + for stride in [1usize << 31, usize::MAX / 2 + 1, usize::MAX] { + assert_eq!(kind(plan(&convt(stride, 0), 1, 3, 3)), Err(K::TooLarge), "stride {stride}"); + } + let conv = net(IntLayer::Conv { out_ch: 1, k: 1, stride: usize::MAX, pad: usize::MAX / 4, weight: vec![1], mult: vec![1], bias: vec![0] }); + assert_eq!(kind(plan(&conv, 1, 3, 3)), Err(K::TooLarge)); + assert_eq!(kind(plan(&convt(1, 2), 1, 3, 3)), Err(K::ModelMismatch), "padding that leaves no output"); + let short = net(IntLayer::ConvT { out_ch: 2, k: 1, stride: 1, pad: 0, opad: 0, weight: vec![1], mult: vec![1, 1], bias: vec![0, 0] }); + assert_eq!(kind(plan(&short, 1, 3, 3)), Err(K::ModelMismatch), "weights that do not fit"); + assert_eq!(kind(plan(&convt(2, 0), 1, 3, 3)), Ok((1, 5, 5))); + for (file, c, h, w) in [("int_hyper/tiny_g_s.bin", 4, 4, 4), ("int_hyper/tiny_h_s.bin", 2, 2, 2), ("int_splat/splat_gs.bin", 32, 8, 8)] { + let m = IntModel::from_bytes(&read(file)).unwrap(); + assert_eq!(plan(&m, c, h, w).unwrap(), net_shape(&m, c, h, w).unwrap().out, "{file}"); + } +} + +/// The splat converter carries each presentation range exactly: a JSON decimal that +/// denotes a binary32 value converts to it, and one a single f64 unit away from it, +/// written as its own shortest decimal, is refused. `splat_int` reads numbers +/// correctly rounded, so no tolerance stands in for the reader. +#[cfg(feature = "splat_int")] +#[test] +fn splat_ranges_are_carried_exactly() { + let text = String::from_utf8(read("int_splat/splat.json")).unwrap(); + let js: serde_json::Value = serde_json::from_str(&text).unwrap(); + let lo0 = js["lo"][0].as_f64().unwrap(); + assert_eq!((lo0 as f32) as f64, lo0, "the committed range is a binary32 value, read exactly"); + assert!(wis1_from_json(&js).is_ok()); + for step in [1i64, -1] { + let off = f64::from_bits(((lo0 as f32) as f64).to_bits().wrapping_add_signed(step)); + let mut bad: serde_json::Value = serde_json::from_str(&text).unwrap(); + bad["lo"][0] = serde_json::from_str(&format!("{off:?}")).unwrap(); + assert_eq!(bad["lo"][0].as_f64(), Some(off), "the reader is correctly rounded"); + assert_eq!(wis1_from_json(&bad).unwrap_err().code(), "field_range", "{off:?}"); + } +} + +/// The converters check the JSON's channel count against its tables: a medians list, +/// a table list or an offsets list one short is refused (`field_range`), never written +/// out of step with the header. +#[cfg(all(feature = "audio_int", feature = "splat_int"))] +#[test] +fn converters_check_the_channel_count() { + let audio = jload("int_audio/audio_bitstream.json"); + for key in ["median_q", "cdfs"] { + let mut j = audio.clone(); + j[key].as_array_mut().unwrap().pop(); + let e = wia1_from_json(&j, 16_000).unwrap_err(); + assert_eq!((e.code(), e.detail().contains("one per channel")), ("field_range", true), "audio {key}: {e}"); + } + let splat = jload("int_splat/splat.json"); + for key in ["median_q", "cdfs", "offsets"] { + let mut j = splat.clone(); + j[key].as_array_mut().unwrap().pop(); + let e = wis1_from_json(&j).unwrap_err(); + assert_eq!((e.code(), e.detail().contains("one per channel")), ("field_range", true), "splat {key}: {e}"); + } +} + +/// `WHM1` and `WAM1` hold nothing after their last layer (§9): a committed network with +/// one byte appended is refused, and a decode against it is `model_mismatch`. +#[test] +fn a_model_file_with_bytes_after_its_last_layer_is_refused() { + use crate::int_hyper_synth::IntModel; + let mut g_s = read("int_hyper/tiny_g_s.bin"); + assert!(IntModel::from_bytes(&g_s).is_ok()); + g_s.push(0); + assert_eq!(IntModel::from_bytes(&g_s).err(), Some("bytes follow the last layer")); + let mut set = int_hyper_set(); + set.insert("g_s.whm".into(), g_s); + let files = view(&set); + let hyp = crate::codecs::CAP_NEURAL_INT_HYPER; + let r = decode_int(hyp, &read("int_hyper/tiny_bitstream.wih1"), &unpinned(hyp, HeldBytes::Set { listing: b"", files: &files })); + assert_eq!(r.map(|_| ()).unwrap_err().code(), "model_mismatch"); + #[cfg(feature = "audio_int")] + { + let mut wam = read("int_audio/audio_model.bin"); + assert!(crate::int_audio::AudioModel::from_bytes(&wam).is_ok()); + wam.push(0); + assert_eq!(crate::int_audio::AudioModel::from_bytes(&wam).err(), Some("bytes follow the last layer")); + } +} + +/// Check 5 covers every identity of the bytes a pinned decode reads, not only the pin's +/// digest: a one-file set revoked under its file's digest is refused through a set-form +/// pin over a listing of that file, and a set revoked under the listing of its members +/// is refused through a pin over a listing that adds a file the decode does not read. +#[test] +fn a_pin_by_another_identity_does_not_reach_a_revoked_set() { + use crate::codecs::{CAP_AUDIO_INT_CODEC as AUD, CAP_NEURAL_INT_HYPER as HYP, CAP_PRIOR_SET}; + use crate::pin::{listing_from, sha256_hex}; + let code = |r: Result, PinRefusal>| r.map(|p| if p.is_pinned() { "pinned" } else { "unpinned" }).unwrap_or_else(|e| e.code()); + let model = read("int_audio/audio_model.bin"); + let one: BTreeMap = [("model.wam".to_string(), model.as_slice())].into_iter().collect(); + let listing = listing_from(&one).unwrap(); + let revoke_file = Authority { revoked: vec![sha256_hex(&model)], granted: vec![], held_to: false }; + let set_pin = Prior::new("m", sha256_hex(&listing), "decode-equivalence").with_form(CAP_PRIOR_SET); + let held = [HeldBytes::Set { listing: &listing, files: &one }]; + assert_eq!(code(resolve(AUD, Some(&set_pin), &held, None)), "pinned"); + assert_eq!(code(resolve(AUD, Some(&set_pin), &held, Some(&revoke_file))), "prior_revoked", "a set-form pin over the revoked file"); + + let set = int_hyper_set(); + let three = listing_from(&view(&set)).unwrap(); + let mut extra = set.clone(); + extra.insert("README".into(), b"not a member".to_vec()); + let extra_v = view(&extra); + let four = listing_from(&extra_v).unwrap(); + let revoke_three = Authority { revoked: vec![sha256_hex(&three)], granted: vec![], held_to: false }; + let pin4 = Prior::new("t", sha256_hex(&four), "decode-equivalence").with_form(CAP_PRIOR_SET); + let held4 = [HeldBytes::Set { listing: &four, files: &extra_v }]; + assert_eq!(code(resolve(HYP, Some(&pin4), &held4, None)), "pinned"); + assert_eq!(code(resolve(HYP, Some(&pin4), &held4, Some(&revoke_three))), "prior_revoked", "a pin over the members and a README"); + // A key log that grants the pin's digest still authorises it. + let granting = Authority { revoked: vec![], granted: vec![sha256_hex(&four)], held_to: true }; + assert_eq!(code(resolve(HYP, Some(&pin4), &held4, Some(&granting))), "pinned"); +} + +/// A one-frame clip has one header (§3): of the same one-pixel clip under six +/// `(mv_precision, block)` pairs, only `(1, 1)` parses; the others are `field_range`. +/// The converter writes a one-frame JSON clip with that header whatever the JSON +/// states, and the result decodes as the JSON does. +#[cfg(feature = "video_int")] +#[test] +fn a_one_frame_clip_has_one_header() { + let rans: Vec = [0u32, 4].iter().flat_map(|w| w.to_le_bytes()).collect(); + let clip = |mv_precision: u8, block: u32| wire::write_wiv1(&wire::Wiv1 { + mv_precision, keyframe: 0, n_frames: 1, h: 1, w: 1, block, q: 1, offset: 0, + cdf: vec![0, 32_768, 49_152, 65_536], mvs: vec![], rans: &rans, wih1: None, + }); + for (mvp, block) in [(1u8, 1u32), (1, 4), (1, 7), (1, u32::MAX), (2, 1), (2, 1 << 31)] { + let got = wire::parse_wiv1(&clip(mvp, block), Some(0)).map(|_| ()).map_err(|e| e.code()); + assert_eq!(got, if (mvp, block) == (1, 1) { Ok(()) } else { Err("field_range") }, "mv_precision {mvp}, block {block}"); + } + let js = serde_json::json!({ + "n_frames": 1, "h": 1, "w": 1, "block": 8, "mv_precision": 2, "q": 1, "offset": 0, + "cdf": [0, 32_768, 49_152, 65_536], "mvs": [[]], "rans_b64": "AAAAAAQAAAA=" + }); + let p = wiv1_from_json(&js, None).unwrap(); + let v = wire::parse_wiv1(&p, Some(0)).unwrap(); + assert_eq!((v.mv_precision, v.block), (1, 1)); + match decode_int(crate::codecs::CAP_VIDEO_INT_MOTION, &p, &IntParams::none()) { + Ok(IntDecoded::Video { frames, .. }) => assert_eq!(frames, crate::int_video::decode(&js).unwrap().frames), + other => panic!("{other:?}"), + } +} + +/// An integer range is carried only where it is exactly a binary32 value: `2^60` is, +/// and converts to exactly that; `2^60 + 1` is not (an f64 would round it to `2^60`), +/// and is refused, as the Python verifier refuses it. +#[cfg(feature = "splat_int")] +#[test] +fn splat_integer_ranges_are_carried_exactly() { + let text = String::from_utf8(read("int_splat/splat.json")).unwrap(); + let with = |n: &str| { + let mut j: serde_json::Value = serde_json::from_str(&text).unwrap(); + let big: serde_json::Value = serde_json::from_str(n).unwrap(); + j["lo"][0] = big.clone(); + j["hi"][0] = big; + wis1_from_json(&j) + }; + let p = with("1152921504606846976").unwrap(); + assert_eq!(wire::parse_wis1(&p).unwrap().ranges[0], ((1u64 << 60) as f32, (1u64 << 60) as f32)); + assert_eq!(with("1152921504606846977").unwrap_err().code(), "field_range"); + assert_eq!(with("-1152921504606846977").unwrap_err().code(), "field_range"); +} + +/// The JSON reader refuses a frame whose motion vectors are not a list (a number, an +/// object), as the Python converter does, instead of reading it as an empty list. +#[cfg(feature = "video_int")] +#[test] +fn a_frame_entry_that_is_not_a_list_is_refused() { + let mut js = jload("int_video/int_video_bitstream.json"); + assert!(crate::int_video::VideoJson::from_value(&js).is_ok()); + for bad in [serde_json::json!(5), serde_json::json!({"dy": 0}), serde_json::json!(null)] { + js["mvs"][1] = bad.clone(); + assert!(crate::int_video::VideoJson::from_value(&js).is_err(), "{bad}"); + assert_eq!(wiv1_from_json(&js, None).unwrap_err().code(), "field_range", "{bad}"); + } +} diff --git a/wai/wai-rs/src/int_payload/wire.rs b/wai/wai-rs/src/int_payload/wire.rs new file mode 100644 index 000000000..099af499a --- /dev/null +++ b/wai/wai-rs/src/int_payload/wire.rs @@ -0,0 +1,785 @@ +//! The payload formats of the WAI-native integer codecs ([integer-payloads] §3–§6): +//! `WIV1` (`wai.video.int_motion`, `wai.video.int_hyper`), `WIH1` +//! (`wai.neural.int_hyper`), `WIA1` (`wai.audio.int_codec`) and `WIS1` +//! (`wai.splat.int_codec`). A parser and a writer for each, and the closed list of +//! refusal codes ([`Refusal`]). +//! +//! Every multi-byte field is little-endian. A parser checks the fields in the order +//! the extension lists them and reports the first rule that fails, so two +//! conforming parsers refuse the same bytes with the same code. Every size is +//! formed in `u64` with checked arithmetic and capped by the same constants on +//! every target, `wasm32` included, so a payload is accepted or refused alike +//! everywhere and no parser allocates more than the bytes it holds justify. +//! +//! This file is self-contained: it uses `std`'s `Vec` and nothing from the crate, so +//! the cross-architecture byte-equality crate (`wai/byte-exact-conformance`) +//! `#[path]`-includes it and parses the committed binary vectors with the same code +//! a sink runs. +//! +//! [integer-payloads]: ../../../extensions/integer-payloads.md + +/// `WIV1`: integer inter-frame video. +/// +/// cbindgen:ignore +pub const MAGIC_WIV1: [u8; 4] = *b"WIV1"; +/// `WIH1`: the integer conditional image codec. +/// +/// cbindgen:ignore +pub const MAGIC_WIH1: [u8; 4] = *b"WIH1"; +/// `WIA1`: the integer learned audio codec. +/// +/// cbindgen:ignore +pub const MAGIC_WIA1: [u8; 4] = *b"WIA1"; +/// `WIS1`: the integer learned Gaussian-splat attribute codec. +/// +/// cbindgen:ignore +pub const MAGIC_WIS1: [u8; 4] = *b"WIS1"; +/// The four magics, for telling a payload of another of these formats from one of +/// none ([`Refusal::FormatCapabilityMismatch`] against [`Refusal::BadMagic`]). +/// +/// cbindgen:ignore +pub const MAGICS: [[u8; 4]; 4] = [MAGIC_WIV1, MAGIC_WIH1, MAGIC_WIA1, MAGIC_WIS1]; +/// The one version this revision defines, for every format. +/// +/// cbindgen:ignore +pub const VERSION: u8 = 1; + +/// The most elements any one tensor a payload sizes may hold: a frame (`h·w·3`), a +/// `WIV1` motion-vector field (`2·nb·(n_frames − 1)`), a latent, an attribute tensor, a +/// decoded clip's samples. Also the cap on every activation a decode forms on the way +/// ([integer-payloads] §7). +/// +/// cbindgen:ignore +pub const MAX_TENSOR: u64 = 1 << 26; +/// The most decoded bytes a whole `WIV1` clip may hold (`n_frames·h·w·3`). +/// +/// cbindgen:ignore +pub const MAX_CLIP: u64 = 1 << 30; +/// The most frames a `WIV1` clip may hold. +/// +/// cbindgen:ignore +pub const MAX_FRAMES: u64 = 1 << 20; +/// The largest fixed-point scale a payload may declare (`afrac` in `1..=30`). +/// +/// cbindgen:ignore +pub const MAX_AFRAC: u8 = 30; +/// The largest magnitude of a per-channel median (`|median_q| ≤ 2^47`). +/// +/// cbindgen:ignore +pub const MAX_MEDIAN_Q: i64 = 1 << 47; +/// The bounds on a CDF table's length. +/// +/// cbindgen:ignore +pub const CDF_LEN_MIN: usize = 3; +/// cbindgen:ignore +pub const CDF_LEN_MAX: usize = 4097; +/// The total every CDF table ends at (rANS precision 16). +/// +/// cbindgen:ignore +pub const CDF_TOTAL: u32 = 1 << 16; +/// The largest residual quantisation step. +/// +/// cbindgen:ignore +pub const MAX_Q: u32 = 65_535; + +/// Why a payload was refused: the closed list of [integer-payloads] §8. The first +/// rule that fails, in the order the extension gives, is the reason. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +/// cbindgen:ignore +pub enum Refusal { + /// The payload does not start with any of the four magics (a JSON form does not). + BadMagic, + /// The payload is one of the four formats, but not the one the capability pins. + FormatCapabilityMismatch, + /// A `WIV1` keyframe kind that does not belong to the capability (0 is + /// `wai.video.int_motion`'s, 1 is `wai.video.int_hyper`'s). + KeyframeCapabilityMismatch, + /// A version this revision does not define. + Version, + /// A reserved field that is not zero. + ReservedNonzero, + /// A field outside its range. + FieldRange, + /// A tensor, a clip or an activation larger than the caps allow. + TooLarge, + /// A CDF table that breaks the table rules. + CdfMalformed, + /// The payload ends before a field or section it declares, or a rANS stream's + /// length is not a multiple of 4 or is under 8. + StreamLength, + /// A nested `WIH1` whose declared length is not the rest of the payload. + NestedLength, + /// Bytes after the last section. + Trailing, + /// A rANS stream that ran out before its last symbol. + StreamExhausted, + /// A rANS stream that does not end canonically after its last symbol: a word is + /// left unread, or the state is not `2^31` ([integer-payloads] §2). + StreamNoncanonical, + /// A shape the payload declares disagrees with the shape the decode produces. + GeometryMismatch, + /// The parameter set does not fit the payload. + ModelMismatch, + /// The parameter set does not contain a file the decode reads. + SetIncomplete, + /// The capability is not dispatchable (SPEC §4). + NotDispatchable, + /// The capability has no integer decode in this build. + UnsupportedCapability, + /// A payload of `wai.neural.int_synth` or `wai.neural.int_mlicpp`, whose formats + /// are defined elsewhere, refused by its decoder. + PayloadMalformed, + /// The payload's declared cost exceeds the limit the sink applies before decoding + /// ([integer-payloads] §7, *Memory and work*): the sink's own policy, so dispatch + /// moves on. + CostOverLimit, +} + +impl Refusal { + /// Every refusal, in the order of [integer-payloads] §8. + /// + /// cbindgen:ignore + pub const ALL: [Refusal; 20] = [ + Refusal::BadMagic, + Refusal::FormatCapabilityMismatch, + Refusal::KeyframeCapabilityMismatch, + Refusal::Version, + Refusal::ReservedNonzero, + Refusal::FieldRange, + Refusal::TooLarge, + Refusal::CdfMalformed, + Refusal::StreamLength, + Refusal::NestedLength, + Refusal::Trailing, + Refusal::StreamExhausted, + Refusal::StreamNoncanonical, + Refusal::GeometryMismatch, + Refusal::ModelMismatch, + Refusal::SetIncomplete, + Refusal::NotDispatchable, + Refusal::UnsupportedCapability, + Refusal::PayloadMalformed, + Refusal::CostOverLimit, + ]; + + /// The code [integer-payloads] §8 names. + pub fn code(self) -> &'static str { + match self { + Refusal::BadMagic => "bad_magic", + Refusal::FormatCapabilityMismatch => "format_capability_mismatch", + Refusal::KeyframeCapabilityMismatch => "keyframe_capability_mismatch", + Refusal::Version => "version", + Refusal::ReservedNonzero => "reserved_nonzero", + Refusal::FieldRange => "field_range", + Refusal::TooLarge => "too_large", + Refusal::CdfMalformed => "cdf_malformed", + Refusal::StreamLength => "stream_length", + Refusal::NestedLength => "nested_length", + Refusal::Trailing => "trailing", + Refusal::StreamExhausted => "stream_exhausted", + Refusal::StreamNoncanonical => "stream_noncanonical", + Refusal::GeometryMismatch => "geometry_mismatch", + Refusal::ModelMismatch => "model_mismatch", + Refusal::SetIncomplete => "set_incomplete", + Refusal::NotDispatchable => "not_dispatchable", + Refusal::UnsupportedCapability => "unsupported_capability", + Refusal::PayloadMalformed => "payload_malformed", + Refusal::CostOverLimit => "cost_over_limit", + } + } + + /// The refusal whose code is `code`. + pub fn from_code(code: &str) -> Option { + Refusal::ALL.into_iter().find(|r| r.code() == code) + } +} + +/// A parser's refusal: the code and what failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WireError { + pub refusal: Refusal, + pub detail: &'static str, +} + +impl WireError { + /// cbindgen:ignore + pub const fn new(refusal: Refusal, detail: &'static str) -> Self { + WireError { refusal, detail } + } + pub fn code(&self) -> &'static str { + self.refusal.code() + } +} + +impl std::fmt::Display for WireError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}: {}", self.refusal.code(), self.detail) + } +} + +impl std::error::Error for WireError {} + +const fn err(refusal: Refusal, detail: &'static str) -> WireError { + WireError { refusal, detail } +} + +/// The magic check every parser starts with: `want` passes; another of the four +/// magics is the wrong format for the capability; anything else, a JSON form +/// included, is not one of these formats. +pub fn check_magic(p: &[u8], want: [u8; 4]) -> Result<(), WireError> { + match p.get(0..4) { + Some(m) if m == want => Ok(()), + Some(m) if MAGICS.iter().any(|k| k[..] == *m) => { + Err(err(Refusal::FormatCapabilityMismatch, "the payload is another integer format than the capability's")) + } + _ => Err(err(Refusal::BadMagic, "the payload does not start with the capability's magic")), + } +} + +/// The table rules every CDF obeys ([integer-payloads] §7): `3 ≤ len ≤ 4097`, +/// `cdf[0] = 0`, `cdf[len−1] = 2^16`, strictly increasing. They are the +/// precondition under which the rANS decoder's symbol search and state update stay +/// in range. +pub fn check_cdf(cdf: &[i32]) -> Result<(), WireError> { + if cdf.len() < CDF_LEN_MIN || cdf.len() > CDF_LEN_MAX { + return Err(err(Refusal::CdfMalformed, "a CDF table's length is outside 3..=4097")); + } + if cdf[0] != 0 || cdf[cdf.len() - 1] != CDF_TOTAL as i32 { + return Err(err(Refusal::CdfMalformed, "a CDF table does not run from 0 to 65536")); + } + if cdf.windows(2).any(|w| w[0] >= w[1]) { + return Err(err(Refusal::CdfMalformed, "a CDF table is not strictly increasing")); + } + Ok(()) +} + +/// The product of `dims` in `u64`, or `None` past `u64`. +fn product(dims: &[u64]) -> Option { + dims.iter().try_fold(1u64, |a, &d| a.checked_mul(d)) +} + +/// A product checked against `cap`: [`Refusal::TooLarge`] past it (or past `u64`). +fn capped(dims: &[u64], cap: u64, detail: &'static str) -> Result { + product(dims).filter(|&n| n <= cap).ok_or(err(Refusal::TooLarge, detail)) +} + +/// A bounds-checked little-endian reader. Every read past the end is +/// [`Refusal::StreamLength`]. +struct Cur<'a> { + b: &'a [u8], + p: usize, +} + +impl<'a> Cur<'a> { + fn new(b: &'a [u8], p: usize) -> Self { + Cur { b, p } + } + fn left(&self) -> usize { + self.b.len() - self.p + } + /// The next `n` bytes; `n` is a `u64` the caller formed, so no allocation or + /// slice is ever sized from more than the payload holds. + fn take(&mut self, n: u64, what: &'static str) -> Result<&'a [u8], WireError> { + let n = usize::try_from(n).ok().filter(|&n| n <= self.left()).ok_or(err(Refusal::StreamLength, what))?; + let s = &self.b[self.p..self.p + n]; + self.p += n; + Ok(s) + } + fn u8(&mut self, what: &'static str) -> Result { + Ok(self.take(1, what)?[0]) + } + fn u16(&mut self, what: &'static str) -> Result { + let s = self.take(2, what)?; + Ok(u16::from_le_bytes([s[0], s[1]])) + } + fn u32(&mut self, what: &'static str) -> Result { + let s = self.take(4, what)?; + Ok(u32::from_le_bytes([s[0], s[1], s[2], s[3]])) + } + fn i32(&mut self, what: &'static str) -> Result { + Ok(self.u32(what)? as i32) + } + /// A CDF table of `len` `u32` entries, read and checked. + fn cdf(&mut self, len: u64, what: &'static str) -> Result, WireError> { + let s = self.take(len.checked_mul(4).unwrap_or(u64::MAX), what)?; + let cdf: Vec = s + .chunks_exact(4) + .map(|c| u32::from_le_bytes([c[0], c[1], c[2], c[3]])) + .map(|v| i32::try_from(v).unwrap_or(i32::MAX)) + .collect(); + check_cdf(&cdf)?; + Ok(cdf) + } + /// A length-prefixed rANS stream: `u32 L` then `L` bytes, `L` a multiple of 4 + /// and at least 8. + fn stream(&mut self, what: &'static str) -> Result<&'a [u8], WireError> { + let len = self.u32(what)?; + if len % 4 != 0 || len < 8 { + return Err(err(Refusal::StreamLength, "a rANS stream's length is not a multiple of 4, or is under 8")); + } + self.take(len as u64, what) + } + /// Nothing may follow the last section. + fn end(&self) -> Result<(), WireError> { + if self.left() != 0 { + return Err(err(Refusal::Trailing, "bytes follow the last section")); + } + Ok(()) + } +} + +/// The checks every format opens with: the magic, then the version byte at offset +/// 4, then a fixed header of `header` bytes. +fn open<'a>(p: &'a [u8], magic: [u8; 4], header: usize) -> Result, WireError> { + check_magic(p, magic)?; + let version = *p.get(4).ok_or(err(Refusal::StreamLength, "the payload ends before its version"))?; + if version != VERSION { + return Err(err(Refusal::Version, "a version this revision does not define")); + } + if p.len() < header { + return Err(err(Refusal::StreamLength, "the payload ends inside its fixed header")); + } + Ok(Cur::new(p, 5)) +} + +fn put_u16(o: &mut Vec, v: u16) { + o.extend_from_slice(&v.to_le_bytes()); +} +fn put_u32(o: &mut Vec, v: u32) { + o.extend_from_slice(&v.to_le_bytes()); +} +fn put_i32(o: &mut Vec, v: i32) { + o.extend_from_slice(&v.to_le_bytes()); +} +fn put_i64(o: &mut Vec, v: i64) { + o.extend_from_slice(&v.to_le_bytes()); +} +fn put_cdf(o: &mut Vec, cdf: &[i32]) { + for &v in cdf { + put_u32(o, v as u32); + } +} +fn put_stream(o: &mut Vec, s: &[u8]) { + put_u32(o, s.len() as u32); + o.extend_from_slice(s); +} + +// ───────────────────────────── WIV1 ───────────────────────────── + +/// A parsed `WIV1` payload ([integer-payloads] §3). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Wiv1<'a> { + /// `1` whole-pel, `2` half-pel: a motion vector is in units of `1/mv_precision` pel. + pub mv_precision: u8, + /// `0`: the first frame is predicted from grey and coded in the residual stream + /// (`wai.video.int_motion`). `1`: the first frame is the nested `WIH1`'s decode + /// (`wai.video.int_hyper`). + pub keyframe: u8, + pub n_frames: u32, + pub h: u32, + pub w: u32, + pub block: u32, + pub q: u32, + pub offset: i32, + pub cdf: Vec, + /// Frames `1..n_frames`, each its block grid row-major: `(dy, dx)` per block, + /// `nb = ⌈h/block⌉·⌈w/block⌉` per frame. + pub mvs: Vec<(i16, i16)>, + /// The residual rANS stream. + pub rans: &'a [u8], + /// The nested `WIH1` payload, when `keyframe` is 1. + pub wih1: Option<&'a [u8]>, +} + +/// `WIV1`'s fixed header, in bytes. +/// +/// cbindgen:ignore +pub const WIV1_HEADER: usize = 36; + +impl Wiv1<'_> { + /// Blocks per frame: `⌈h/block⌉·⌈w/block⌉`. + pub fn blocks_per_frame(&self) -> u64 { + blocks(self.h, self.w, self.block) + } + + /// The most bytes the reference decode of a keyframe-0 clip holds at once beyond + /// the payload ([integer-payloads] §7, *Memory and work*): its table and motion + /// vectors as parsed (4 bytes an entry), the clip (one byte per sample) and its + /// frame list (one `Vec` header per frame). The header alone fixes it; a keyframe-1 + /// clip also runs its keyframe's networks, which the parameter set sizes. + pub fn keyframe0_working_bytes(&self) -> u64 { + let clip = self.h as u64 * self.w as u64 * 3 * self.n_frames as u64; + 4 * self.cdf.len() as u64 + 4 * self.mvs.len() as u64 + clip + + std::mem::size_of::>() as u64 * self.n_frames as u64 + } +} + +fn blocks(h: u32, w: u32, block: u32) -> u64 { + (h as u64).div_ceil(block.max(1) as u64) * (w as u64).div_ceil(block.max(1) as u64) +} + +/// Parse a `WIV1` payload. `want_keyframe` is the keyframe kind the capability +/// pins (`Some(0)` for `wai.video.int_motion`, `Some(1)` for `wai.video.int_hyper`); +/// `None` accepts either. +pub fn parse_wiv1(p: &[u8], want_keyframe: Option) -> Result, WireError> { + let mut c = open(p, MAGIC_WIV1, WIV1_HEADER)?; + let mv_precision = c.u8("mv_precision")?; + let keyframe = c.u8("keyframe")?; + let reserved0 = c.u8("reserved")?; + let n_frames = c.u32("n_frames")?; + let h = c.u32("h")?; + let w = c.u32("w")?; + let block = c.u32("block")?; + let q = c.u32("q")?; + let offset = c.i32("offset")?; + let cdf_len = c.u16("cdf_len")?; + let reserved1 = c.u16("reserved")?; + if reserved0 != 0 || reserved1 != 0 { + return Err(err(Refusal::ReservedNonzero, "a reserved field is not zero")); + } + if !matches!(mv_precision, 1 | 2) { + return Err(err(Refusal::FieldRange, "mv_precision is neither 1 nor 2")); + } + if keyframe > 1 { + return Err(err(Refusal::FieldRange, "keyframe is neither 0 nor 1")); + } + if n_frames == 0 || h == 0 || w == 0 { + return Err(err(Refusal::FieldRange, "n_frames, h or w is zero")); + } + if block == 0 { + return Err(err(Refusal::FieldRange, "block is zero")); + } + if q == 0 || q > MAX_Q { + return Err(err(Refusal::FieldRange, "q is outside 1..=65535")); + } + if (cdf_len as usize) < CDF_LEN_MIN || cdf_len as usize > CDF_LEN_MAX { + return Err(err(Refusal::FieldRange, "cdf_len is outside 3..=4097")); + } + // A one-frame clip codes no motion vector, so neither field would change its + // decode; they are fixed so that such a clip has one header. + if n_frames == 1 && (mv_precision != 1 || block != 1) { + return Err(err(Refusal::FieldRange, "a one-frame clip has mv_precision 1 and block 1")); + } + if let Some(k) = want_keyframe { + if k != keyframe { + return Err(err(Refusal::KeyframeCapabilityMismatch, "the keyframe kind is not the capability's")); + } + } + capped(&[h as u64, w as u64, 3], MAX_TENSOR, "a frame exceeds 2^26 elements")?; + capped(&[n_frames as u64, h as u64, w as u64, 3], MAX_CLIP, "the clip exceeds 2^30 bytes")?; + capped(&[n_frames as u64], MAX_FRAMES, "the clip exceeds 2^20 frames")?; + let n_mv = capped(&[2, n_frames as u64 - 1, blocks(h, w, block)], MAX_TENSOR, "the motion vectors exceed 2^26 elements")? / 2; + let cdf = c.cdf(cdf_len as u64, "the CDF table")?; + let raw = c.take(n_mv.checked_mul(4).unwrap_or(u64::MAX), "the motion vectors")?; + let mvs = raw + .chunks_exact(4) + .map(|m| (i16::from_le_bytes([m[0], m[1]]), i16::from_le_bytes([m[2], m[3]]))) + .collect(); + let rans = c.stream("the residual stream")?; + let wih1 = if keyframe == 1 { + let k = c.u32("the keyframe length")?; + if k as usize != c.left() { + return Err(err(Refusal::NestedLength, "the keyframe length is not the rest of the payload")); + } + let nested = c.take(k as u64, "the keyframe")?; + parse_wih1(nested)?; + Some(nested) + } else { + c.end()?; + None + }; + Ok(Wiv1 { mv_precision, keyframe, n_frames, h, w, block, q, offset, cdf, mvs, rans, wih1 }) +} + +/// Write a `WIV1` payload. The caller supplies conforming fields; the converters +/// parse what they write before returning it. +pub fn write_wiv1(v: &Wiv1<'_>) -> Vec { + let mut o = Vec::with_capacity(WIV1_HEADER + v.cdf.len() * 4 + v.mvs.len() * 4 + v.rans.len() + 8); + o.extend_from_slice(&MAGIC_WIV1); + o.extend_from_slice(&[VERSION, v.mv_precision, v.keyframe, 0]); + put_u32(&mut o, v.n_frames); + put_u32(&mut o, v.h); + put_u32(&mut o, v.w); + put_u32(&mut o, v.block); + put_u32(&mut o, v.q); + put_i32(&mut o, v.offset); + put_u16(&mut o, v.cdf.len() as u16); + put_u16(&mut o, 0); + put_cdf(&mut o, &v.cdf); + for &(dy, dx) in &v.mvs { + o.extend_from_slice(&dy.to_le_bytes()); + o.extend_from_slice(&dx.to_le_bytes()); + } + put_stream(&mut o, v.rans); + if let Some(k) = v.wih1 { + put_stream(&mut o, k); + } + o +} + +// ───────────────────────────── WIH1 ───────────────────────────── + +/// A parsed `WIH1` payload ([integer-payloads] §4): the conditional codec's two +/// streams and the shapes of its two latents. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Wih1<'a> { + /// Latent `y`: `m` channels of `sh × sw`. + pub m: u32, + pub sh: u32, + pub sw: u32, + /// Hyper-latent `z`: `n_z` channels of `zh × zw`. + pub n_z: u32, + pub zh: u32, + pub zw: u32, + /// The factorized-prior stream of `z`. + pub z_rans: &'a [u8], + /// The Gaussian-conditional stream of `y`. + pub rans: &'a [u8], +} + +/// `WIH1`'s fixed header, in bytes. +/// +/// cbindgen:ignore +pub const WIH1_HEADER: usize = 32; + +/// Parse a `WIH1` payload. +pub fn parse_wih1(p: &[u8]) -> Result, WireError> { + let mut c = open(p, MAGIC_WIH1, WIH1_HEADER)?; + let reserved = c.take(3, "reserved")?; + let m = c.u32("m")?; + let sh = c.u32("sh")?; + let sw = c.u32("sw")?; + let n_z = c.u32("n_z")?; + let zh = c.u32("zh")?; + let zw = c.u32("zw")?; + if reserved.iter().any(|&b| b != 0) { + return Err(err(Refusal::ReservedNonzero, "a reserved field is not zero")); + } + if [m, sh, sw, n_z, zh, zw].contains(&0) { + return Err(err(Refusal::FieldRange, "a latent dimension is zero")); + } + capped(&[n_z as u64, zh as u64, zw as u64], MAX_TENSOR, "the hyper-latent exceeds 2^26 elements")?; + capped(&[2, m as u64, sh as u64, sw as u64], MAX_TENSOR, "the latent's mean and scale exceed 2^26 elements")?; + let z_rans = c.stream("the z stream")?; + let rans = c.stream("the y stream")?; + c.end()?; + Ok(Wih1 { m, sh, sw, n_z, zh, zw, z_rans, rans }) +} + +/// Write a `WIH1` payload. +pub fn write_wih1(v: &Wih1<'_>) -> Vec { + let mut o = Vec::with_capacity(WIH1_HEADER + v.z_rans.len() + v.rans.len() + 8); + o.extend_from_slice(&MAGIC_WIH1); + o.extend_from_slice(&[VERSION, 0, 0, 0]); + for d in [v.m, v.sh, v.sw, v.n_z, v.zh, v.zw] { + put_u32(&mut o, d); + } + put_stream(&mut o, v.z_rans); + put_stream(&mut o, v.rans); + o +} + +// ───────────────────────────── WIA1 ───────────────────────────── + +/// A parsed `WIA1` payload ([integer-payloads] §5). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Wia1<'a> { + /// The fixed-point scale; it must equal the model's. + pub afrac: u8, + /// The sample rate the PCM is presented at, in hertz. + pub sr: u32, + /// Latent channels and length. + pub n_ch: u32, + pub lat_len: u32, + /// Samples presented: the first `t_out` of the synthesis. + pub t_out: u32, + /// The symbol offset every channel's table is coded at. + pub offset: i32, + pub median_q: Vec, + /// One table per channel, all of one length. + pub cdfs: Vec>, + pub rans: &'a [u8], +} + +/// `WIA1`'s fixed header, in bytes. +/// +/// cbindgen:ignore +pub const WIA1_HEADER: usize = 32; + +/// Parse a `WIA1` payload. +pub fn parse_wia1(p: &[u8]) -> Result, WireError> { + let mut c = open(p, MAGIC_WIA1, WIA1_HEADER)?; + let afrac = c.u8("afrac")?; + let reserved0 = c.u16("reserved")?; + let sr = c.u32("sr")?; + let n_ch = c.u32("n_ch")?; + let lat_len = c.u32("lat_len")?; + let t_out = c.u32("t_out")?; + let offset = c.i32("offset")?; + let cdf_len = c.u16("cdf_len")?; + let reserved1 = c.u16("reserved")?; + if reserved0 != 0 || reserved1 != 0 { + return Err(err(Refusal::ReservedNonzero, "a reserved field is not zero")); + } + if afrac == 0 || afrac > MAX_AFRAC { + return Err(err(Refusal::FieldRange, "afrac is outside 1..=30")); + } + if sr == 0 || n_ch == 0 || lat_len == 0 || t_out == 0 { + return Err(err(Refusal::FieldRange, "sr, n_ch, lat_len or t_out is zero")); + } + if (cdf_len as usize) < CDF_LEN_MIN || cdf_len as usize > CDF_LEN_MAX { + return Err(err(Refusal::FieldRange, "cdf_len is outside 3..=4097")); + } + capped(&[n_ch as u64, lat_len as u64], MAX_TENSOR, "the latent exceeds 2^26 elements")?; + capped(&[t_out as u64], MAX_TENSOR, "t_out exceeds 2^26 samples")?; + let raw = c.take(n_ch as u64 * 8, "the medians")?; + let median_q: Vec = raw.chunks_exact(8).map(|b| i64::from_le_bytes(b.try_into().unwrap())).collect(); + if median_q.iter().any(|m| m.unsigned_abs() > MAX_MEDIAN_Q as u64) { + return Err(err(Refusal::FieldRange, "a median's magnitude exceeds 2^47")); + } + let mut cdfs = Vec::with_capacity(n_ch as usize); + for _ in 0..n_ch { + cdfs.push(c.cdf(cdf_len as u64, "the CDF tables")?); + } + let rans = c.stream("the latent stream")?; + c.end()?; + Ok(Wia1 { afrac, sr, n_ch, lat_len, t_out, offset, median_q, cdfs, rans }) +} + +/// Write a `WIA1` payload. +pub fn write_wia1(v: &Wia1<'_>) -> Vec { + let cdf_len = v.cdfs.first().map_or(0, Vec::len); + let mut o = Vec::with_capacity(WIA1_HEADER + v.median_q.len() * 8 + v.cdfs.len() * cdf_len * 4 + v.rans.len() + 4); + o.extend_from_slice(&MAGIC_WIA1); + o.extend_from_slice(&[VERSION, v.afrac, 0, 0]); + put_u32(&mut o, v.sr); + put_u32(&mut o, v.n_ch); + put_u32(&mut o, v.lat_len); + put_u32(&mut o, v.t_out); + put_i32(&mut o, v.offset); + put_u16(&mut o, cdf_len as u16); + put_u16(&mut o, 0); + for &m in &v.median_q { + put_i64(&mut o, m); + } + for cdf in &v.cdfs { + put_cdf(&mut o, cdf); + } + put_stream(&mut o, v.rans); + o +} + +// ───────────────────────────── WIS1 ───────────────────────────── + +/// A parsed `WIS1` payload ([integer-payloads] §6). +#[derive(Debug, Clone, PartialEq)] +pub struct Wis1<'a> { + /// The fixed-point scale; it must equal the model's. + pub afrac: u8, + /// Latent: `c` channels of `sh × sw`. + pub c: u32, + pub sh: u32, + pub sw: u32, + /// The attribute tensor: `attr` planes of `s × s`. + pub s: u32, + pub attr: u32, + /// Gaussians in the cloud: the first `n_gaussians` cells of each plane. + pub n_gaussians: u32, + /// Per attribute, the range `(lo, hi)` its `u8` plane is presented over. + /// Presentation only: outside the splat-equivalence digest. + pub ranges: Vec<(f32, f32)>, + /// Per channel, its table, its symbol offset and its median. + pub cdfs: Vec>, + pub offsets: Vec, + pub median_q: Vec, + pub rans: &'a [u8], +} + +/// `WIS1`'s fixed header, in bytes. +/// +/// cbindgen:ignore +pub const WIS1_HEADER: usize = 32; + +/// Parse a `WIS1` payload. +pub fn parse_wis1(p: &[u8]) -> Result, WireError> { + let mut cur = open(p, MAGIC_WIS1, WIS1_HEADER)?; + let afrac = cur.u8("afrac")?; + let reserved = cur.u16("reserved")?; + let c = cur.u32("c")?; + let sh = cur.u32("sh")?; + let sw = cur.u32("sw")?; + let s = cur.u32("s")?; + let attr = cur.u32("attr")?; + let n_gaussians = cur.u32("n_gaussians")?; + if reserved != 0 { + return Err(err(Refusal::ReservedNonzero, "a reserved field is not zero")); + } + if afrac == 0 || afrac > MAX_AFRAC { + return Err(err(Refusal::FieldRange, "afrac is outside 1..=30")); + } + if [c, sh, sw, s, attr, n_gaussians].contains(&0) { + return Err(err(Refusal::FieldRange, "a dimension is zero")); + } + if n_gaussians as u64 > s as u64 * s as u64 { + return Err(err(Refusal::FieldRange, "n_gaussians exceeds s·s")); + } + capped(&[c as u64, sh as u64, sw as u64], MAX_TENSOR, "the latent exceeds 2^26 elements")?; + capped(&[attr as u64, s as u64, s as u64], MAX_TENSOR, "the attribute tensor exceeds 2^26 elements")?; + let raw = cur.take(attr as u64 * 8, "the attribute ranges")?; + let mut ranges = Vec::with_capacity(attr as usize); + for r in raw.chunks_exact(8) { + let lo = f32::from_bits(u32::from_le_bytes([r[0], r[1], r[2], r[3]])); + let hi = f32::from_bits(u32::from_le_bytes([r[4], r[5], r[6], r[7]])); + if !lo.is_finite() || !hi.is_finite() || lo > hi { + return Err(err(Refusal::FieldRange, "an attribute range is not finite, or its lo exceeds its hi")); + } + ranges.push((lo, hi)); + } + let raw = cur.take(c as u64 * 2, "the CDF lengths")?; + let lens: Vec = raw.chunks_exact(2).map(|b| u16::from_le_bytes([b[0], b[1]])).collect(); + if lens.iter().any(|&l| (l as usize) < CDF_LEN_MIN || l as usize > CDF_LEN_MAX) { + return Err(err(Refusal::FieldRange, "a CDF length is outside 3..=4097")); + } + let raw = cur.take(c as u64 * 4, "the symbol offsets")?; + let offsets = raw.chunks_exact(4).map(|b| i32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect(); + let raw = cur.take(c as u64 * 8, "the medians")?; + let median_q: Vec = raw.chunks_exact(8).map(|b| i64::from_le_bytes(b.try_into().unwrap())).collect(); + if median_q.iter().any(|m| m.unsigned_abs() > MAX_MEDIAN_Q as u64) { + return Err(err(Refusal::FieldRange, "a median's magnitude exceeds 2^47")); + } + let mut cdfs = Vec::with_capacity(c as usize); + for &l in &lens { + cdfs.push(cur.cdf(l as u64, "the CDF tables")?); + } + let rans = cur.stream("the latent stream")?; + cur.end()?; + Ok(Wis1 { afrac, c, sh, sw, s, attr, n_gaussians, ranges, cdfs, offsets, median_q, rans }) +} + +/// Write a `WIS1` payload. +pub fn write_wis1(v: &Wis1<'_>) -> Vec { + let mut o = Vec::with_capacity(WIS1_HEADER + v.ranges.len() * 8 + v.rans.len() + 64); + o.extend_from_slice(&MAGIC_WIS1); + o.extend_from_slice(&[VERSION, v.afrac, 0, 0]); + for d in [v.c, v.sh, v.sw, v.s, v.attr, v.n_gaussians] { + put_u32(&mut o, d); + } + for &(lo, hi) in &v.ranges { + put_u32(&mut o, lo.to_bits()); + put_u32(&mut o, hi.to_bits()); + } + for cdf in &v.cdfs { + put_u16(&mut o, cdf.len() as u16); + } + for &off in &v.offsets { + put_i32(&mut o, off); + } + for &m in &v.median_q { + put_i64(&mut o, m); + } + for cdf in &v.cdfs { + put_cdf(&mut o, cdf); + } + put_stream(&mut o, v.rans); + o +} diff --git a/wai/wai-rs/src/int_splat.rs b/wai/wai-rs/src/int_splat.rs index a6cbbe99c..9eb63a0cb 100644 --- a/wai/wai-rs/src/int_splat.rs +++ b/wai/wai-rs/src/int_splat.rs @@ -11,13 +11,9 @@ use serde_json::Value; -use crate::int_hyper_synth::{fnv1a, run_net, IntModel}; +use crate::int_hyper_synth::{fnv1a, refused, run_net_checked, IntModel, Refused, RefusalKind, MAX_AFRAC, MAX_TENSOR}; use crate::rans::RansDecoder; -fn i64s(v: &Value) -> Vec { - v.as_array().map(|a| a.iter().filter_map(|n| n.as_i64()).collect()).unwrap_or_default() -} - /// A decoded splat: the `[attr · s · s]` `u8` attribute image (channel-major), to be /// un-normalized + un-sorted into Gaussians by the presentation layer. pub struct DecodedSplat { @@ -33,58 +29,184 @@ impl DecodedSplat { } } -/// Decode a learned-splat **bitstream → integer attribute tensor**, fully integer: -/// per channel, factorized rANS-decode the symbol `s` and rebuild the fixed-point -/// latent `y_q = s·2^afrac + median_q`; integer synthesis `g_s` (transposed convs + -/// IGDN) → `[attr, S, S]` at `2^afrac`; round-half-up → `u8`. `gs` is the WHM1 model -/// (`splat_gs.bin`); `bs` the export's `splat.json`. Byte-identical on every machine. -pub fn decode(gs: &IntModel, bs: &Value) -> Result { - let c = bs["c"].as_u64().ok_or("c")? as usize; - let sh = bs["sh"].as_u64().ok_or("sh")? as usize; - let sw = bs["sw"].as_u64().ok_or("sw")? as usize; - let s = bs["s"].as_u64().ok_or("s")? as usize; - let n_attr = bs["attr"].as_u64().ok_or("attr")? as usize; - let afrac = gs.afrac; - let one = 1i64 << afrac; - let median_q = i64s(&bs["median_q"]); - let offsets: Vec = bs["offsets"].as_array().ok_or("offsets")?.iter().map(|n| n.as_i64().unwrap() as i32).collect(); - if median_q.len() != c || offsets.len() != c { - return Err("median_q / offsets length disagrees with channel count"); - } - let cdfs: Vec> = bs["cdfs"] - .as_array() - .ok_or("cdfs")? - .iter() - .map(|row| row.as_array().unwrap().iter().map(|n| n.as_i64().unwrap() as i32).collect()) - .collect(); - let rans = b64_decode(bs["rans_b64"].as_str().ok_or("rans_b64")?)?; +/// A cloud's coded form, typed: what a `WIS1` payload carries +/// (`extensions/integer-payloads.md` §6), and what the tool's JSON form carries, read +/// by [`SplatJson`]. Per latent channel: a table, a symbol offset and a median. +#[derive(Debug, Clone, Copy)] +pub struct SplatStream<'a> { + /// The fixed-point scale the stream was coded at; it must equal the model's. + pub afrac: u32, + pub c: usize, + pub sh: usize, + pub sw: usize, + pub s: usize, + pub n_attr: usize, + pub cdfs: &'a [Vec], + pub offsets: &'a [i32], + pub median_q: &'a [i64], + pub rans: &'a [u8], +} - // 1. per-channel factorized rANS-decode the latent, rebuild y_q. - let plane = sh * sw; - let mut y = vec![0i64; c * plane]; - let mut dec = RansDecoder::new(&rans)?; - for ch in 0..c { - let cdf_len = cdfs[ch].len(); - for p in 0..plane { - let sym = dec.decode(&cdfs[ch], cdf_len, offsets[ch]); - y[ch * plane + p] = sym as i64 * one + median_q[ch]; +/// The tool's JSON bitstream (`tools/wai_int_splat_export.py`'s `splat.json`), read +/// into the typed form, with the members only presentation reads: `n_gaussians` and +/// the per-attribute ranges `lo`, `hi`. A converter input and a debugging aid; the +/// payload is `WIS1`. Its `afrac` is read (the old JSON decoder ignored it) and must +/// be the model's. +#[derive(Debug, Clone)] +pub struct SplatJson { + pub afrac: u32, + pub c: usize, + pub sh: usize, + pub sw: usize, + pub s: usize, + pub n_attr: usize, + pub n_gaussians: usize, + pub cdfs: Vec>, + pub offsets: Vec, + pub median_q: Vec, + /// Per attribute, `(lo, hi)` as the JSON carries them; an integer an `f64` does not + /// hold exactly is refused. + pub ranges: Vec<(f64, f64)>, + pub rans: Vec, +} + +impl SplatJson { + /// Read the tool's JSON bitstream. + pub fn from_value(bs: &Value) -> Result { + let uz = |k: &'static str| bs[k].as_u64().and_then(|v| usize::try_from(v).ok()).ok_or(k); + let (c, sh, sw, s, n_attr) = (uz("c")?, uz("sh")?, uz("sw")?, uz("s")?, uz("attr")?); + let n_gaussians = uz("n_gaussians")?; + let afrac = bs["afrac"].as_u64().and_then(|v| u32::try_from(v).ok()).ok_or("afrac")?; + let offsets = bs["offsets"].as_array().ok_or("offsets")?.iter() + .map(|n| n.as_i64().and_then(|v| i32::try_from(v).ok())).collect::>>().ok_or("offsets")?; + let median_q = bs["median_q"].as_array().ok_or("median_q")?.iter() + .map(Value::as_i64).collect::>>().ok_or("median_q")?; + let cdfs = bs["cdfs"].as_array().ok_or("cdfs")?.iter() + .map(|row| row.as_array().and_then(|r| r.iter().map(|n| n.as_i64().and_then(|v| i32::try_from(v).ok())).collect::>>())) + .collect::>>().ok_or("cdfs holds a value outside i32")?; + // A range as the JSON states it: an integer only where an f64 holds it exactly + // (so 2^60 + 1 is refused rather than rounded to 2^60), a decimal as read. + let exact = |v: &Value| -> Option { + let int = v.as_i64().map(i128::from).or(v.as_u64().map(i128::from)); + match int { + Some(i) => Some(i as f64).filter(|&f| f as i128 == i), + None => v.as_f64(), + } + }; + let f = |k: &'static str| bs[k].as_array().ok_or(k)?.iter().map(exact).collect::>>().ok_or(k); + let (lo, hi) = (f("lo")?, f("hi")?); + if lo.len() != n_attr || hi.len() != n_attr { + return Err("lo and hi are not one per attribute"); + } + let rans = b64_decode(bs["rans_b64"].as_str().ok_or("rans_b64")?)?; + Ok(SplatJson { afrac, c, sh, sw, s, n_attr, n_gaussians, cdfs, offsets, median_q, ranges: lo.into_iter().zip(hi).collect(), rans }) + } + + /// The typed form, borrowing this one's tables. + pub fn stream(&self) -> SplatStream<'_> { + SplatStream { + afrac: self.afrac, c: self.c, sh: self.sh, sw: self.sw, s: self.s, n_attr: self.n_attr, + cdfs: &self.cdfs, offsets: &self.offsets, median_q: &self.median_q, rans: &self.rans, } } +} + +/// Decode a typed cloud → the integer attribute tensor, byte-exact on every machine, +/// refusing rather than panicking or overflowing on any input: +/// +/// 1. the stream's `afrac` is the model's, in `1..=30`; the medians, offsets and +/// tables are one per channel and every table is well formed; `g_s` maps the +/// latent's shape to `[n_attr, s, s]` with no buffer over +/// [`crate::int_hyper_synth::MAX_TENSOR`] elements — all before anything is +/// allocated; +/// 2. per channel, rANS-decode the symbol and rebuild `s·2^afrac + median_q` +/// ([`crate::int_entropy::dequantize`]); a stream that runs out before its last +/// symbol is refused; +/// 3. integer synthesis `g_s` through [`crate::int_hyper_synth::run_net_checked`]; +/// 4. round-half-up → `u8`, channel-major `[attr, s, s]`. +pub fn decode_typed(gs: &IntModel, st: &SplatStream<'_>) -> Result { + use crate::int_entropy::dequantize; + use RefusalKind::*; + splat_plan(gs, st)?; + let (afrac, c, sh, sw, s, n_attr) = (gs.afrac, st.c, st.sh, st.sw, st.s, st.n_attr); + + // 1. per-channel factorized rANS-decode the latent, rebuild y_q. The latent grows + // a symbol at a time, and the first symbol that runs the stream out refuses the + // payload before its value is used. + let plane = sh * sw; + let mut y = Vec::new(); + let mut dec = RansDecoder::new(st.rans).map_err(|_| refused(FieldRange, "the latent stream is not a rANS stream"))?; + for ch in 0..c { + for _ in 0..plane { + let sym = dec.decode(&st.cdfs[ch], st.cdfs[ch].len(), st.offsets[ch]); + if dec.is_exhausted() { + return Err(refused(StreamExhausted, "the latent stream ended before its last symbol")); + } + crate::int_hyper_synth::push_capped(&mut y, dequantize(sym, afrac, st.median_q[ch]).ok_or(refused(FieldRange, "a latent value leaves i64"))?, c * plane); + } + } + if !dec.is_canonical_end() { + return Err(refused(StreamNoncanonical, "the latent stream does not end canonically after its last symbol")); + } // 2. integer synthesis → attribute tensor at 2^afrac. - let (x, oc, oh, ow) = run_net(gs, &y, c, sh, sw); - if oc != n_attr || oh != s || ow != s { - return Err("g_s output shape disagrees with the splat header"); - } + let (x, ..) = run_net_checked(gs, &y, c, sh, sw).map_err(|r| { + if r.out_of_range { + refused(FieldRange, "a decoded latent could take a layer outside the exact integer range") + } else { + refused(ModelMismatch, "g_s's parameters do not fit the running shapes") + } + })?; + drop(y); // 3. round-half-up → u8 (channel-major [attr, s, s]). - let half = 1i64 << (afrac - 1); - let mut attr = vec![0u8; n_attr * s * s]; - for (i, v) in x.iter().enumerate() { - let q = (*v).clamp(0, one); - attr[i] = ((q * 255 + half) >> afrac).clamp(0, 255) as u8; - } + let (one, half) = (1i64 << afrac, 1i64 << (afrac - 1)); + let attr = x.iter().map(|&v| ((v.clamp(0, one) * 255 + half) >> afrac).clamp(0, 255) as u8).collect(); Ok(DecodedSplat { attr, n_attr, s }) } +/// What [`decode_typed`] checks before it decodes a symbol, and the most bytes the +/// decode then holds at once on the reference kernels beyond the payload and the +/// parsed model: the latent (`i64`) with `g_s`'s buffers while it runs, then `g_s`'s +/// output (`i64`) with the attribute tensor (`u8`); and `g_s`'s multiply-accumulates +/// ([`crate::int_hyper_synth::NetPlan`]). +pub(crate) fn splat_plan(gs: &IntModel, st: &SplatStream<'_>) -> Result<(u64, u128), Refused> { + use RefusalKind::*; + let afrac = gs.afrac; + if !(1..=MAX_AFRAC).contains(&afrac) { + return Err(refused(ModelMismatch, "the model's fixed-point scale is outside 1..=30")); + } + if st.afrac != afrac { + return Err(refused(ModelMismatch, "the stream's afrac is not the model's")); + } + let (c, sh, sw, s, n_attr) = (st.c, st.sh, st.sw, st.s, st.n_attr); + if [c, sh, sw, s, n_attr].contains(&0) || st.median_q.len() != c || st.offsets.len() != c || st.cdfs.len() != c { + return Err(refused(FieldRange, "a dimension is zero, or the medians, offsets or tables are not one per channel")); + } + if st.median_q.iter().any(|m| m.unsigned_abs() > 1 << 47) { + return Err(refused(FieldRange, "a median's magnitude exceeds 2^47")); + } + let ok = |t: &Vec| (3..=4097).contains(&t.len()) && t[0] == 0 && t[t.len() - 1] == 1 << 16 && t.windows(2).all(|p| p[0] < p[1]); + if !st.cdfs.iter().all(ok) { + return Err(refused(CdfMalformed, "a table is not a CDF from 0 to 65536, strictly increasing")); + } + let size = |d: &[usize]| d.iter().try_fold(1usize, |a, &x| a.checked_mul(x)).filter(|&n| n <= MAX_TENSOR); + let (Some(latent), Some(out)) = (size(&[c, sh, sw]), size(&[n_attr, s, s])) else { + return Err(refused(TooLarge, "the latent or the attribute tensor exceeds 2^26 elements")); + }; + let p = crate::int_hyper_synth::plan_net(gs, c, sh, sw)?; + if p.out != (n_attr, s, s) { + return Err(refused(GeometryMismatch, "g_s does not give the attribute shape the payload declares")); + } + let (latent, out) = (latent as u64, out as u64); + Ok(((8 * (latent + p.live)).max(8 * out + out), p.macs)) +} + +/// Decode the tool's JSON bitstream (`splat.json`) → the integer attribute tensor: +/// [`SplatJson`] then [`decode_typed`]. A debugging and converter path; the payload is +/// `WIS1`. `gs` is the WHM1 model (`splat_gs.bin`). Byte-identical on every machine. +pub fn decode(gs: &IntModel, bs: &Value) -> Result { + decode_typed(gs, &SplatJson::from_value(bs)?.stream()).map_err(|r| r.detail) +} + /// Dependency-free base64 decode (standard alphabet) — the rANS bytes ride in JSON as /// base64, same as `int_audio` / `int_video`. fn b64_decode(s: &str) -> Result, &'static str> { diff --git a/wai/wai-rs/src/int_synth.rs b/wai/wai-rs/src/int_synth.rs index 0c5dcd329..916f2fb3b 100644 --- a/wai/wai-rs/src/int_synth.rs +++ b/wai/wai-rs/src/int_synth.rs @@ -244,6 +244,14 @@ pub fn decode(payload: &[u8], model_path: &Path) -> Result Result { + decode_from_spec(payload, spec_json) +} + /// Decode an int_synth payload against the model spec supplied as bytes (the /// `int_synth.json` contents). No filesystem, no float, no ONNX — the /// integer-exact decode a browser / WASM sink runs directly, host-supplied @@ -256,12 +264,16 @@ pub fn decode_from_spec(payload: &[u8], spec_json: &[u8]) -> Result usize { - v[k].as_u64().unwrap_or(0) as usize +/// A JSON member as a `usize`, refused when it is absent, not an integer, or does not +/// fit the target's `usize` (so a 32-bit sink refuses what it cannot index rather than +/// truncating it). +fn u(v: &Value, k: &'static str) -> Result { + v[k].as_u64().and_then(|n| usize::try_from(n).ok()).ok_or(k) +} + +/// The most elements one decoded frame may hold (`h·w·3`), the same on every target. +/// +/// cbindgen:ignore +pub const MAX_FRAME: usize = 1 << 26; +/// The most bytes a whole decoded clip may hold (`n_frames·h·w·3`). +/// +/// cbindgen:ignore +pub const MAX_CLIP: usize = 1 << 30; +/// The most frames a clip may hold. +/// +/// cbindgen:ignore +pub const MAX_FRAMES: usize = 1 << 20; +/// The most elements the motion-vector field may hold (`2·nb·(n_frames − 1)`). +/// +/// cbindgen:ignore +pub const MAX_MV: usize = 1 << 26; + +/// The sizes a clip's header declares, checked in the order the extension's §3 caps +/// them, with checked arithmetic: the frame `h·w·3`, the clip `n_frames·h·w·3`, the +/// frame count, and the motion-vector field `2·nb·(n_frames − 1)`. Returns the frame's +/// element count and `nb`. +fn caps(n_frames: usize, h: usize, w: usize, block: usize) -> Result<(usize, usize), &'static str> { + let frame = h.checked_mul(w).and_then(|v| v.checked_mul(3)).filter(|&n| n <= MAX_FRAME) + .ok_or("a frame exceeds 2^26 elements")?; + frame.checked_mul(n_frames).filter(|&n| n <= MAX_CLIP).ok_or("the clip exceeds 2^30 bytes")?; + if n_frames > MAX_FRAMES { + return Err("the clip exceeds 2^20 frames"); + } + // h·w ≤ 2^26, so nb fits. + let nb = h.div_ceil(block) * w.div_ceil(block); + nb.checked_mul(n_frames.saturating_sub(1)).and_then(|v| v.checked_mul(2)).filter(|&n| n <= MAX_MV) + .ok_or("the motion vectors exceed 2^26 elements")?; + Ok((frame, nb)) +} +/// The refusal [`decode_typed`] and [`decode_pframes_typed`] give when the residual +/// stream runs out before the clip's last symbol. +/// +/// cbindgen:ignore +pub const STREAM_EXHAUSTED: &str = "the residual stream ended before its last symbol"; +/// The refusal [`decode_typed`] and [`decode_pframes_typed`] give when the residual +/// stream does not end canonically after the clip's last symbol: a word left unread, +/// or a state other than `2^31` (`extensions/integer-payloads.md` §2). +/// +/// cbindgen:ignore +pub const STREAM_NONCANONICAL: &str = "the residual stream does not end canonically after its last symbol"; + +/// A clip's coded form, typed: what the `WIV1` payload carries (and what the tool's +/// JSON form carries, read by [`VideoJson`]). Both decode entry points take it. +/// +/// `mvs` holds frames `1..n_frames` in order, each its block grid row-major +/// (`⌈h/block⌉·⌈w/block⌉` entries), as `(dy, dx)` in units of `1/mv_precision` pel. +#[derive(Debug, Clone, Copy)] +pub struct VideoStream<'a> { + pub n_frames: usize, + pub h: usize, + pub w: usize, + pub block: usize, + pub q: i64, + pub offset: i32, + /// `1` whole-pel, `2` half-pel. + pub mv_precision: u8, + pub cdf: &'a [i32], + pub mvs: &'a [(i16, i16)], + pub rans: &'a [u8], +} + +/// The tool's JSON bitstream (`tools/wai_int_video_*encode.py`), read into the typed +/// form. The JSON form is a converter input and a debugging aid; the payload a +/// capability carries is `WIV1` (`extensions/integer-payloads.md`). What the old JSON +/// decoder left implicit is made explicit here, so a JSON bitstream and the `WIV1` +/// converted from it decode alike: an absent `mv_precision` is 1 (only 1 and 2 are +/// read); `mvs` has one list per frame and the first is empty; a frame's list shorter +/// than its block grid is padded with `(0, 0)`, the vector the old decoder read for a +/// missing block; a vector outside `i16`, or a list longer than the grid, is refused. +/// `rmax` is not read. +#[derive(Debug, Clone)] +pub struct VideoJson { + pub n_frames: usize, + pub h: usize, + pub w: usize, + pub block: usize, + pub q: i64, + pub offset: i32, + pub mv_precision: u8, + pub cdf: Vec, + pub mvs: Vec<(i16, i16)>, + pub rans: Vec, +} + +impl VideoJson { + /// Read the tool's JSON bitstream. + pub fn from_value(bs: &Value) -> Result { + Self::read(bs, None) + } + + /// [`Self::from_value`] at the geometry `(h, w)` instead of the bitstream's own + /// (the P-frames of a clip whose I-frame came from elsewhere). + fn read(bs: &Value, geometry: Option<(usize, usize)>) -> Result { + let (block, nf) = (u(bs, "block")?, u(bs, "n_frames")?); + let (h, w) = match geometry { + Some(g) => g, + None => (u(bs, "h")?, u(bs, "w")?), + }; + if nf == 0 || h == 0 || w == 0 || block == 0 { + return Err("n_frames, h, w or block is zero"); + } + // Every size is capped before anything is sized from it: a short list of + // motion vectors is padded below, so the field's size is the header's, not the + // JSON's. + let (_, nb) = caps(nf, h, w, block)?; + let q = bs["q"].as_i64().ok_or("q")?; + let offset = bs["offset"].as_i64().and_then(|o| i32::try_from(o).ok()).ok_or("offset")?; + let cdf = bs["cdf"].as_array().ok_or("cdf")?.iter() + .map(|n| n.as_i64().and_then(|v| i32::try_from(v).ok())) + .collect::>>().ok_or("cdf holds a value outside i32")?; + let mv_precision = match bs.get("mv_precision") { + None => 1, + Some(p) => match p.as_u64() { + Some(1) => 1, + Some(2) => 2, + _ => return Err("mv_precision is neither 1 nor 2"), + }, + }; + let lists = bs["mvs"].as_array().ok_or("mvs")?; + if lists.len() != nf { + return Err("mvs has not one list per frame"); + } + if lists.first().and_then(Value::as_array).is_some_and(|l| !l.is_empty()) { + return Err("the first frame carries motion vectors"); + } + let mut mvs = Vec::with_capacity(nb * (nf - 1)); + for fr in lists.iter().skip(1) { + let list = fr.as_array().ok_or("a frame's motion vectors are not a list")?; + if list.len() > nb { + return Err("a frame carries more motion vectors than blocks"); + } + for m in list { + let comp = |i: usize| m.get(i).and_then(Value::as_i64).and_then(|v| i16::try_from(v).ok()); + mvs.push((comp(0).ok_or("a motion vector is outside i16")?, comp(1).ok_or("a motion vector is outside i16")?)); + } + mvs.resize(mvs.len() + nb - list.len(), (0, 0)); + } + let rans = b64_decode(bs["rans_b64"].as_str().ok_or("rans_b64")?)?; + Ok(VideoJson { n_frames: nf, h, w, block, q, offset, mv_precision, cdf, mvs, rans }) + } + + /// The typed form, borrowing this one's tables. + pub fn stream(&self) -> VideoStream<'_> { + VideoStream { + n_frames: self.n_frames, h: self.h, w: self.w, block: self.block, q: self.q, offset: self.offset, + mv_precision: self.mv_precision, cdf: &self.cdf, mvs: &self.mvs, rans: &self.rans, + } + } } /// Predict one channel of one pixel from the reference with a motion vector @@ -90,11 +251,11 @@ fn u(v: &Value, k: &str) -> usize { /// sub-pixel motion (real footage) predicts well while the decode stays exact. #[inline] #[allow(clippy::too_many_arguments)] -fn predict(rf: &[i64], y: usize, x: usize, dy: i64, dx: i64, c: usize, prec: i64, h: usize, w: usize) -> i64 { +fn predict(rf: &[u8], y: usize, x: usize, dy: i64, dx: i64, c: usize, prec: i64, h: usize, w: usize) -> i64 { let p = |yy: i64, xx: i64| -> i64 { let cy = yy.clamp(0, h as i64 - 1) as usize; let cx = xx.clamp(0, w as i64 - 1) as usize; - rf[(cy * w + cx) * 3 + c] + rf[(cy * w + cx) * 3 + c] as i64 }; if prec == 1 { return p(y as i64 + dy, x as i64 + dx); @@ -110,100 +271,133 @@ fn predict(rf: &[i64], y: usize, x: usize, dy: i64, dx: i64, c: usize, prec: i64 } } -/// Decode an integer inter-frame bitstream (`int_video_bitstream.json`) → frames, -/// byte-exact on every machine. Pure i64. -pub fn decode(bs: &Value) -> Result { - let (block, q, h, w, nf) = (u(bs, "block"), bs["q"].as_i64().ok_or("q")?, u(bs, "h"), u(bs, "w"), u(bs, "n_frames")); - let offset = bs["offset"].as_i64().ok_or("offset")? as i32; - let cdf: Vec = bs["cdf"].as_array().ok_or("cdf")?.iter().map(|n| n.as_i64().unwrap() as i32).collect(); - let cdf_len = cdf.len(); - let rans = b64_decode(bs["rans_b64"].as_str().ok_or("rans_b64")?)?; - // mvs[f][blk] = [dy, dx] - let mvs: Vec> = bs["mvs"].as_array().ok_or("mvs")?.iter().map(|fr| { - fr.as_array().map(|blks| { - blks.iter().map(|m| { - let a = m.as_array().unwrap(); - (a[0].as_i64().unwrap(), a[1].as_i64().unwrap()) - }).collect() - }).unwrap_or_default() - }).collect(); - - let nbx = w.div_ceil(block); - let prec = bs["mv_precision"].as_i64().unwrap_or(1); // 1 whole-pel, 2 half-pel - let mut dec = RansDecoder::new(&rans)?; - let mut frames: Vec> = Vec::with_capacity(nf); - let mut prev = vec![0i64; h * w * 3]; // previous reconstructed frame (HWC, 0..255) - for f in 0..nf { - let mut frame = vec![0i64; h * w * 3]; - let mvs_f = mvs.get(f); - for y in 0..h { - for x in 0..w { - // motion vector for this block (P-frames only) - let (dy, dx) = if f == 0 { - (0, 0) - } else { - mvs_f.and_then(|m| m.get((y / block) * nbx + (x / block)).copied()).unwrap_or((0, 0)) - }; - let base = (y * w + x) * 3; - for c in 0..3 { - let pred = if f == 0 { 128 } else { predict(&prev, y, x, dy, dx, c, prec, h, w) }; - let r = dec.decode(&cdf, cdf_len, offset) as i64; // residual symbol - frame[base + c] = (pred + r * q).clamp(0, 255); - } - } - } - frames.push(frame.iter().map(|&v| v as u8).collect()); - prev = frame; +/// The checks both decode entry points make before allocating a frame: every field +/// in range, the CDF a table the rANS decoder is total over, the frame, clip, frame +/// count and motion-vector field under their caps ([`caps`]), and the motion-vector +/// count the block grid's, all formed with checked arithmetic. Returns the frame's +/// element count and `nb`. +pub(crate) fn check(s: &VideoStream<'_>) -> Result<(usize, usize), &'static str> { + if s.n_frames == 0 || s.h == 0 || s.w == 0 || s.block == 0 { + return Err("n_frames, h, w or block is zero"); } - Ok(DecodedClip { frames, h, w }) + if !(1..=65_535).contains(&s.q) { + return Err("q is outside 1..=65535"); + } + if !matches!(s.mv_precision, 1 | 2) { + return Err("mv_precision is neither 1 nor 2"); + } + let c = s.cdf; + if c.len() < 3 || c.len() > 4097 || c[0] != 0 || c[c.len() - 1] != 1 << 16 || c.windows(2).any(|p| p[0] >= p[1]) { + return Err("the CDF is not a table from 0 to 65536, strictly increasing, of 3..=4097 entries"); + } + let (frame, nb) = caps(s.n_frames, s.h, s.w, s.block)?; + if (s.n_frames - 1).checked_mul(nb) != Some(s.mvs.len()) { + return Err("the motion vectors are not one per block for frames 1..n_frames"); + } + Ok((frame, nb)) } -/// Decode the **P-frames** of a clip on top of an already-decoded I-frame -/// (`i_frame`, RGB HWC `u8`) — the integer block-MC + quantized residual loop, -/// seeded with frame 0 from any source. The NEURAL-KEYFRAME video composes this +/// One frame (from a grey `prev` of `None`, the first frame of keyframe 0), appended +/// to `out` sample by sample as its residual symbols are read (raster `y, x, c`) and +/// added to the prediction. The frame is never allocated ahead of the symbols that +/// fill it, and the first symbol during whose decode the stream runs out refuses the +/// clip ([`STREAM_EXHAUSTED`]); its value is not used. `r·q` is under `2^47` in +/// magnitude (the decoder saturates a symbol to `i32`, `q ≤ 65535`), so the sum fits +/// `i64`. +fn frame(s: &VideoStream<'_>, dec: &mut RansDecoder<'_>, prev: Option<&[u8]>, mvs: &[(i16, i16)], out: &mut Vec) -> Result<(), &'static str> { + let (h, w, block) = (s.h, s.w, s.block); + let n = h * w * 3; + let nbx = w.div_ceil(block); + let prec = s.mv_precision as i64; + for y in 0..h { + for x in 0..w { + let (dy, dx) = mvs.get((y / block) * nbx + (x / block)).map_or((0, 0), |&(a, b)| (a as i64, b as i64)); + for c in 0..3 { + let pred = match prev { + None => 128, + Some(rf) => predict(rf, y, x, dy, dx, c, prec, h, w), + }; + let r = dec.decode(s.cdf, s.cdf.len(), s.offset) as i64; // residual symbol + if dec.is_exhausted() { + return Err(STREAM_EXHAUSTED); + } + crate::int_hyper_synth::push_capped(out, (pred + r * s.q).clamp(0, 255) as u8, n); + } + } + } + Ok(()) +} + +/// Frames `first..n_frames` of a clip, each predicted from the last frame of `frames` +/// (which holds the frames before `first`), appended to `frames`. Each frame grows a +/// sample at a time to its `h·w·3` bytes and no further, so the clip holds its +/// declared bytes and the frame list ([`working_bytes`]). +fn decode_frames(s: &VideoStream<'_>, nb: usize, first: usize, frames: &mut Vec>) -> Result<(), &'static str> { + let mut dec = RansDecoder::new(s.rans)?; + for f in first..s.n_frames { + let mut cur = Vec::new(); + match f { + 0 => frame(s, &mut dec, None, &[], &mut cur)?, + _ => frame(s, &mut dec, frames.last().map(Vec::as_slice), &s.mvs[(f - 1) * nb..f * nb], &mut cur)?, + } + frames.push(cur); + } + if !dec.is_canonical_end() { + return Err(STREAM_NONCANONICAL); + } + Ok(()) +} + +/// The most bytes [`decode_typed`] holds at once, beyond the payload and its parsed +/// tables, on a stream [`check`] admits: the clip (`n_frames·h·w·3`) and the frame list +/// (one `Vec` header of 24 bytes per frame on a 64-bit target). +pub fn working_bytes(s: &VideoStream<'_>) -> u64 { + let frame = (s.h * s.w * 3) as u64; + frame * s.n_frames as u64 + std::mem::size_of::>() as u64 * s.n_frames as u64 +} + +/// Decode a typed clip whose I-frame is predicted from grey (`WIV1` keyframe 0, +/// `wai.video.int_motion`) → frames, byte-exact on every machine. Refuses, rather +/// than panicking on or misreading, a stream that breaks [`check`]'s rules or runs +/// out before its last symbol ([`STREAM_EXHAUSTED`], at the first symbol that runs it +/// out). Pure i64. +pub fn decode_typed(s: &VideoStream<'_>) -> Result { + let (_, nb) = check(s)?; + let mut frames = Vec::with_capacity(s.n_frames); + decode_frames(s, nb, 0, &mut frames)?; + Ok(DecodedClip { frames, h: s.h, w: s.w }) +} + +/// Decode the **P-frames** of a typed clip on top of an already-decoded I-frame +/// (`i_frame`, RGB HWC `u8`, `h·w·3` bytes at the stream's `h × w`) — `WIV1` +/// keyframe 1 (`wai.video.int_hyper`), whose I-frame is the nested `WIH1`'s decode. +/// The residual stream holds frames `1..n_frames` only. Pure i64. +pub fn decode_pframes_typed(i_frame: &[u8], s: &VideoStream<'_>) -> Result { + let (n, nb) = check(s)?; + if i_frame.len() != n { + return Err("i_frame size disagrees with h·w·3"); + } + let mut frames = Vec::with_capacity(s.n_frames); + frames.push(i_frame.to_vec()); + decode_frames(s, nb, 1, &mut frames)?; + Ok(DecodedClip { frames, h: s.h, w: s.w }) +} + +/// Decode the tool's JSON bitstream (`int_video_bitstream.json`) → frames, byte-exact +/// on every machine: [`VideoJson`] then [`decode_typed`]. A debugging and converter +/// path; the payload is `WIV1`. Pure i64. +pub fn decode(bs: &Value) -> Result { + decode_typed(&VideoJson::from_value(bs)?.stream()) +} + +/// Decode the **P-frames** of a JSON bitstream on top of an already-decoded I-frame +/// (`i_frame`, RGB HWC `u8`) at `h × w` — the integer block-MC + quantized residual +/// loop, seeded with frame 0 from any source. The NEURAL-KEYFRAME video composes this /// with the learned `int_hyper` I-frame decode (`int_hyper_synth::decode_conditional` /// → frame 0): a learned, byte-exact keyframe + integer inter-frames, every frame /// reproducible by hash. Pure i64. `bs`'s `n_frames` counts the I-frame; `mvs[0]` -/// is empty; the rANS stream holds only frames `1..n_frames`'s residuals. +/// is empty; the rANS stream holds only frames `1..n_frames`'s residuals. A debugging +/// and converter path; the payload is `WIV1` with keyframe 1. pub fn decode_pframes(i_frame: &[u8], h: usize, w: usize, bs: &Value) -> Result { - let (block, q, nf) = (u(bs, "block"), bs["q"].as_i64().ok_or("q")?, u(bs, "n_frames")); - if i_frame.len() != h * w * 3 { - return Err("i_frame size disagrees with h·w·3"); - } - let offset = bs["offset"].as_i64().ok_or("offset")? as i32; - let cdf: Vec = bs["cdf"].as_array().ok_or("cdf")?.iter().map(|n| n.as_i64().unwrap() as i32).collect(); - let cdf_len = cdf.len(); - let rans = b64_decode(bs["rans_b64"].as_str().ok_or("rans_b64")?)?; - let mvs: Vec> = bs["mvs"].as_array().ok_or("mvs")?.iter().map(|fr| { - fr.as_array().map(|blks| { - blks.iter().map(|m| { - let a = m.as_array().unwrap(); - (a[0].as_i64().unwrap(), a[1].as_i64().unwrap()) - }).collect() - }).unwrap_or_default() - }).collect(); - - let nbx = w.div_ceil(block); - let prec = bs["mv_precision"].as_i64().unwrap_or(1); - let mut dec = RansDecoder::new(&rans)?; - let mut frames: Vec> = Vec::with_capacity(nf); - frames.push(i_frame.to_vec()); - let mut prev: Vec = i_frame.iter().map(|&b| b as i64).collect(); - for mvs_f in mvs.iter().take(nf).skip(1) { - let mut frame = vec![0i64; h * w * 3]; - for y in 0..h { - for x in 0..w { - let (dy, dx) = mvs_f.get((y / block) * nbx + (x / block)).copied().unwrap_or((0, 0)); - let base = (y * w + x) * 3; - for c in 0..3 { - let pred = predict(&prev, y, x, dy, dx, c, prec, h, w); - let r = dec.decode(&cdf, cdf_len, offset) as i64; - frame[base + c] = (pred + r * q).clamp(0, 255); - } - } - } - frames.push(frame.iter().map(|&v| v as u8).collect()); - prev = frame; - } - Ok(DecodedClip { frames, h, w }) + decode_pframes_typed(i_frame, &VideoJson::read(bs, Some((h, w)))?.stream()) } diff --git a/wai/wai-rs/src/jwp_delivery.rs b/wai/wai-rs/src/jwp_delivery.rs index 51ecd8064..f71858b7e 100644 --- a/wai/wai-rs/src/jwp_delivery.rs +++ b/wai/wai-rs/src/jwp_delivery.rs @@ -43,7 +43,7 @@ //! This crate seals group receipts only to generate its conformance corpus; //! [`GroupBuilder`] is not a publishing path. -use crate::claim_receipt::{object_content_hash, verify_ed25519_strict, ClaimBody, ClaimReceipt, Reason as ClaimReason}; +use crate::claim_receipt::{verify_ed25519_strict, ClaimBody, ClaimReceipt, Reason as ClaimReason}; use crate::energy_binding::Coverage; use crate::energy_class::{ check_label, json_fields, label_from_json, AcquisitionError, AcquisitionKind, CounterFiltering, EnergyProvenance, @@ -663,7 +663,7 @@ impl ObjectReceipt { /// the class's evidence and the origin. pub fn accept(&self, group: &GroupReceipt, envelope: &[u8]) -> Result<(), DeliveryError> { group.verify()?; - if object_content_hash(envelope) != self.content_hash { + if crate::jwp_hash::content_hash(envelope) != self.content_hash { return Err(DeliveryError::at(4, Reason::ContentHashMismatch)); } self.verify_inclusion(group) diff --git a/wai/wai-rs/src/jwp_delivery/corpus.rs b/wai/wai-rs/src/jwp_delivery/corpus.rs index 915f401aa..3513f264b 100644 --- a/wai/wai-rs/src/jwp_delivery/corpus.rs +++ b/wai/wai-rs/src/jwp_delivery/corpus.rs @@ -63,7 +63,7 @@ fn build( let mut b = GroupBuilder::default(); for (i, (j, l)) in parts.iter().enumerate() { let n = first + i as u8; - b.add(object_content_hash(&envelope(n)), *j, &origin(n), l.clone()); + b.add(crate::jwp_hash::content_hash(&envelope(n)), *j, &origin(n), l.clone()); } b.finish(&key(0x11), PUBLISHER_ID, 7, group_id, T0, T1, parent, labelled).expect("the corpus's groups compose") } diff --git a/wai/wai-rs/src/jwp_delivery/tests.rs b/wai/wai-rs/src/jwp_delivery/tests.rs index 158b6bc6a..cf9874fb5 100644 --- a/wai/wai-rs/src/jwp_delivery/tests.rs +++ b/wai/wai-rs/src/jwp_delivery/tests.rs @@ -28,7 +28,7 @@ const T1: &str = "2026-09-30T12:00:01Z"; fn group(parts: &[(u64, Option)], labelled: bool) -> (GroupReceipt, Vec) { let mut b = GroupBuilder::default(); for (i, (j, l)) in parts.iter().enumerate() { - b.add(object_content_hash(&env(i as u8)), *j, "origin.example", l.clone()); + b.add(crate::jwp_hash::content_hash(&env(i as u8)), *j, "origin.example", l.clone()); } b.finish(&key(0x11), "did:example:publisher", 7, 3, T0, T1, None, labelled).expect("composes") } @@ -48,9 +48,9 @@ fn lab(p: EnergyProvenance) -> Option { #[test] fn the_legacy_leaf_and_tree_are_the_ones_every_profile_already_builds() { - let h = object_content_hash(b"one"); + let h = crate::jwp_hash::content_hash(b"one"); assert_eq!(root(&[leaf_legacy(&h)]), crate::world::receipt::merkle_root(&[h])); - let hs: Vec<[u8; 32]> = (0..5u8).map(|i| object_content_hash(&[i])).collect(); + let hs: Vec<[u8; 32]> = (0..5u8).map(|i| crate::jwp_hash::content_hash(&[i])).collect(); let leaves: Vec<[u8; 32]> = hs.iter().map(leaf_legacy).collect(); assert_eq!(root(&leaves), crate::world::receipt::merkle_root(&hs)); assert_eq!(root(&[]), [0u8; 32]); @@ -58,7 +58,7 @@ fn the_legacy_leaf_and_tree_are_the_ones_every_profile_already_builds() { #[test] fn the_labelled_leaf_binds_the_figure_the_class_and_the_origin() { - let h = object_content_hash(b"x"); + let h = crate::jwp_hash::content_hash(b"x"); let base = leaf_labelled(&h, 1_000, &ObjectLabel::Class(chip(20_000, 1_000_000)), "o"); assert_ne!(base, leaf_legacy(&h)); for other in [ @@ -429,7 +429,7 @@ fn a_non_canonical_key_encoding_is_refused() { fn the_parent_link() { let (g0, _) = group(&[(1, lab(EnergyProvenance::ModelBased))], true); let mut b = GroupBuilder::default(); - b.add(object_content_hash(&env(0)), 1, "o", lab(EnergyProvenance::ModelBased)); + b.add(crate::jwp_hash::content_hash(&env(0)), 1, "o", lab(EnergyProvenance::ModelBased)); let (g1, _) = b.finish(&key(0x11), "did:example:publisher", 7, 4, T0, T1, Some(g0.link_hash()), true).unwrap(); assert_eq!(g1.check_parent(Some(&g0)), Ok(())); assert_eq!(g1.check_parent(None), Err(DeliveryError::at(3, Reason::ParentNotHeld))); diff --git a/wai/wai-rs/src/jwp_hash.rs b/wai/wai-rs/src/jwp_hash.rs new file mode 100644 index 000000000..d20bc358c --- /dev/null +++ b/wai/wai-rs/src/jwp_hash.rs @@ -0,0 +1,34 @@ +//! jwp-receipts §2.1: the content hash of a delivered object. +//! +//! `content_hash = BLAKE3("moq-jwp:object\x01" ‖ envelope)`, over the envelope bytes +//! exactly as they travel. It is an object's identity in a JWP object receipt, the +//! subject of a claim about that object, and the leaf a labelled delivery receipt +//! commits to. This module holds it alone, gated on nothing but BLAKE3, so every +//! receipt family that names an object computes it in one place. + +/// jwp-receipts §2.1: the per-object content hash domain. +pub const DOMAIN_OBJECT: &[u8] = b"moq-jwp:object\x01"; + +/// jwp-receipts §2.1 content hash of a WAI envelope: BLAKE3 over [`DOMAIN_OBJECT`] +/// followed by the envelope bytes. +pub fn content_hash(envelope_bytes: &[u8]) -> [u8; 32] { + let mut h = blake3::Hasher::new(); + h.update(DOMAIN_OBJECT); + h.update(envelope_bytes); + *h.finalize().as_bytes() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The known answer claim-conformance's independent verifier computes for its + /// canonical fixture object (`claim_verify.py`, BLAKE3 over the domain and the + /// object), so the hoisted function is the one the corpus froze. + #[test] + fn content_hash_known_answer() { + let h = content_hash(b"claim-conformance: canonical fixture object"); + let hex: String = h.iter().map(|b| format!("{b:02x}")).collect(); + assert_eq!(hex, "af904358ba08fb542a845ce37edb3e5ee7f56eb362dfb1ee10b209e167d420de"); + } +} diff --git a/wai/wai-rs/src/key_log.rs b/wai/wai-rs/src/key_log.rs index 6e470c643..ab6768e97 100644 --- a/wai/wai-rs/src/key_log.rs +++ b/wai/wai-rs/src/key_log.rs @@ -1888,6 +1888,10 @@ impl crate::pin::PinAuthority for SinkKeyLog { let digests: Vec<[u8; 32]> = digests.iter().filter_map(|d| hash32(d)).collect(); self.check_unpinned_set(capability, &digests) } + + fn requires_grant(&self, capability: &str) -> bool { + self.holds_to_section_7_9(capability) + } } // ───────────────────────────── published keys ───────────────────────────── diff --git a/wai/wai-rs/src/key_log/corpus.rs b/wai/wai-rs/src/key_log/corpus.rs index e6991a33d..29d2affe3 100644 --- a/wai/wai-rs/src/key_log/corpus.rs +++ b/wai/wai-rs/src/key_log/corpus.rs @@ -6,7 +6,7 @@ use super::*; use crate::claim_receipt::{ - object_content_hash, verify_claim_json, BindingType, ClaimBody, ClaimReceipt, DeclaredUncertainty, EnergyClaim, + verify_claim_json, BindingType, ClaimBody, ClaimReceipt, DeclaredUncertainty, EnergyClaim, FigureHeader, LawfulBasisClaim, MeasurementClaim, OperatingPointClaim, Purpose, SoftBindingClaim, SubjectType, VectorResult, }; @@ -196,7 +196,7 @@ fn logs() -> Logs { // ───────────────────────────── items and claims ───────────────────────────── fn subject() -> [u8; 32] { - object_content_hash(b"signer-conformance: the object a claim is about") + crate::jwp_hash::content_hash(b"signer-conformance: the object a claim is about") } fn energy_body(class: &str) -> ClaimBody { @@ -221,7 +221,7 @@ fn energy_body(class: &str) -> ClaimBody { fn figure_header(class: &str) -> FigureHeader { let ClaimBody::Energy(e) = energy_body(class) else { unreachable!("energy_body is an energy claim") }; let provenance = e.acquisition_class().expect("energy_body's class is well formed"); - let group = object_content_hash(b"signer-conformance: the group a figure was delivered in"); + let group = crate::jwp_hash::content_hash(b"signer-conformance: the group a figure was delivered in"); FigureHeader::new("moq-jwp:object", Some(0), Some(group), e.joules_micro, &provenance) } diff --git a/wai/wai-rs/src/key_log/tests.rs b/wai/wai-rs/src/key_log/tests.rs index c91a2132c..c714519d9 100644 --- a/wai/wai-rs/src/key_log/tests.rs +++ b/wai/wai-rs/src/key_log/tests.rs @@ -310,7 +310,7 @@ fn from_group_reads_the_classes_a_verified_group_carries() { let build = |labels: &[Option], labelled: bool| { let mut b = GroupBuilder::default(); for (i, l) in labels.iter().enumerate() { - b.add(crate::claim_receipt::object_content_hash(&[i as u8]), 12_000, "origin.example", l.clone()); + b.add(crate::jwp_hash::content_hash(&[i as u8]), 12_000, "origin.example", l.clone()); } b.finish(&key(0x66), sid, 7, 3, "2026-09-18T11:00:00Z", "2026-09-18T12:00:00Z", None, labelled).expect("composes") }; @@ -383,7 +383,7 @@ fn from_family_reads_a_verified_receipt() { &key(0x64), "did:web:publisher.example#m1", crate::claim_receipt::SubjectType::Object, - crate::claim_receipt::object_content_hash(b"subject"), + crate::jwp_hash::content_hash(b"subject"), 0, "2026-09-18T12:00:00Z", crate::claim_receipt::ClaimBody::SoftBinding(crate::claim_receipt::SoftBindingClaim { diff --git a/wai/wai-rs/src/lib.rs b/wai/wai-rs/src/lib.rs index 3922b7d44..450522584 100644 --- a/wai/wai-rs/src/lib.rs +++ b/wai/wai-rs/src/lib.rs @@ -164,6 +164,23 @@ pub mod int_world; #[cfg(feature = "splat_int")] pub mod int_splat; +// The parsers and writers of the integer payload formats (`WIV1`, `WIH1`, `WIA1`, +// `WIS1`), with no dependency on the rest of the crate and no feature: the container's +// emit paths check a payload with them (`container::check_int_payload_framing`), and +// `int_payload` re-exports them as `int_payload::wire`. +#[doc(hidden)] +#[path = "int_payload/wire.rs"] +pub mod int_wire; + +// The payloads of the WAI-native integer codecs (`extensions/integer-payloads.md`): +// the WIV1, WIH1, WIA1 and WIS1 parsers and writers, the JSON converters, and +// `decode_int`, the one integer decode path, with the pin and key-log checks over +// parameter sets held in memory. No ONNX runtime and no filesystem: the neural sink +// delegates the integer capabilities to it, and it builds for wasm32. Its video, audio +// and splat arms sit behind their own features. +#[cfg(feature = "neural_int")] +pub mod int_payload; + #[cfg(feature = "neural")] pub mod neural; // The neural C FFI uses the `ffi` feature's shared buffer and error types, so it @@ -277,6 +294,11 @@ pub mod energy_binding; #[cfg(feature = "c2pa_emit")] pub mod c2pa_emit; +// jwp-receipts §2.1: an object's content hash, BLAKE3 over a domain and the envelope. +// One module, gated only on BLAKE3, so every receipt family computes it in one place. +#[cfg(feature = "jwp_hash")] +pub mod jwp_hash; + // Typed receipt classes (extensions/jwp-receipts.md §2.4.3–§2.4.7, §2.6, §3.1–§3.2): // energy, lawful-basis, log-inclusion, soft-binding and binding-recovery statements // about delivered objects, on one `claim` envelope and one signature rule beside the diff --git a/wai/wai-rs/src/neural/int_mlicpp.rs b/wai/wai-rs/src/neural/int_mlicpp.rs deleted file mode 100644 index 3ead0610a..000000000 --- a/wai/wai-rs/src/neural/int_mlicpp.rs +++ /dev/null @@ -1,21 +0,0 @@ -//! Neural-dispatch shim for `wai.neural.int_mlicpp`. -//! -//! The decoder lives at the crate root in [`crate::int_mlicpp`] behind the light -//! `neural_int` feature (no ONNX runtime). This shim reads the model bundle and maps -//! the result onto the neural module's types. The parameter set is one file, the -//! `WMB1` bundle (`model.wmb` within a set): a pinned decode parses the very bytes -//! its pin verified, so nothing is hashed or read twice. - -use super::{image::DecodedImage, params::ParamSource, DecodeError}; - -pub(crate) fn decode(payload: &[u8], params: &ParamSource) -> Result { - let bytes = params.model("model.wmb").map_err(|e| match e { - // An unreadable unpinned model stays an invalid payload, as before. - DecodeError::InvalidPayload(why) => DecodeError::InvalidPayload(format!("int_mlicpp model {why}")), - other => other, - })?; - let model = crate::int_mlicpp::IntMlicModel::from_bundle(&bytes) - .map_err(|e| DecodeError::InvalidPayload(e.to_string()))?; - let img = crate::int_mlicpp::decode(&model, payload).map_err(|e| DecodeError::InvalidPayload(e.to_string()))?; - Ok(DecodedImage { rgb: img.rgb, width: img.width as u32, height: img.height as u32 }) -} diff --git a/wai/wai-rs/src/neural/int_synth.rs b/wai/wai-rs/src/neural/int_synth.rs deleted file mode 100644 index 01dfaa623..000000000 --- a/wai/wai-rs/src/neural/int_synth.rs +++ /dev/null @@ -1,23 +0,0 @@ -//! Neural-dispatch shim for the integer synthesis decoder. -//! -//! The actual integer-exact decode lives at the crate root in -//! [`crate::int_synth`] behind the light `neural_int` feature — it needs no -//! ONNX runtime, so it stays buildable (and cross-arch-checkable) without -//! `ort`. This shim adapts its image and error types onto the neural module's. -//! -//! The parameter set is one file, the model specification `int_synth.json` -//! (SPEC §5). A pinned decode reads it from the bytes the pin verified; an unpinned -//! one reads `int_synth.json` beside the registered path, as before. - -use super::{image::DecodedImage, params::ParamSource, DecodeError}; - -pub(crate) fn decode(payload: &[u8], params: &ParamSource) -> Result { - let r = match params { - ParamSource::Unpinned(path) => crate::int_synth::decode(payload, path), - ParamSource::Pinned(_) => crate::int_synth::decode_from_spec(payload, ¶ms.model("int_synth.json")?), - }; - match r { - Ok(img) => Ok(DecodedImage { rgb: img.rgb, width: img.width, height: img.height }), - Err(crate::int_synth::IntSynthError::InvalidPayload(e)) => Err(DecodeError::InvalidPayload(e)), - } -} diff --git a/wai/wai-rs/src/neural/mod.rs b/wai/wai-rs/src/neural/mod.rs index c797c21a3..254548871 100644 --- a/wai/wai-rs/src/neural/mod.rs +++ b/wai/wai-rs/src/neural/mod.rs @@ -35,8 +35,6 @@ pub mod jpegai_header; pub mod jpegai_hyper; pub mod jpegai_mans; pub mod jpegai_bitstream; -mod int_synth; -mod int_mlicpp; mod video; mod video_dcvc; pub(crate) use crate::rans; @@ -54,23 +52,37 @@ pub use video::DecodedVideo; /// capability (a set and its successor, SPEC §3.1). For a pin the registry offers, /// for each registered and held path: /// -/// - form-less: the file itself — for `wai.neural.int_synth`, the `int_synth.json` -/// beside it, the one file its decode reads; +/// - form-less: the file itself; for an integer capability whose set is one file, +/// the one file its decode reads (inside the path when the path is a directory; for +/// `wai.neural.int_synth`, the `int_synth.json` beside a registered file); /// - `wai.prior.set`: every `*.sha256` file directly inside the path's directory /// (the path itself if it is a directory), each a listing rooted there. /// /// A key log, where set ([`ModelRegistry::with_key_log`]), runs SPEC §3.1's check 5: /// a set whose digest it revokes is refused, and the sets of the capabilities the /// sink's relying party holds to jwp-receipts §7.9 must be held under §7.9. It runs -/// over the set an unpinned decode reads too (SPEC §4, *Module authorisation*). A -/// registry without one does not run check 5. +/// over the set an unpinned decode reads too (SPEC §4, *Module authorisation*), and the +/// decode then reads only bytes check 5 ran over, read once: for an integer capability, +/// the members its decode reads ([`ParamSource::UnpinnedRead`]); for another, the +/// registered file of a one-file set or every file below the directory its decoder +/// reads from, and, for a capability held to §7.9, the part of that §7.9 authorised +/// ([`ParamSource::Checked`]). A registry without one does not run check 5, and an +/// unpinned decode reads the registered path as it always has. +/// +/// A capability whose decode reads no parameter set (`wai.video.int_motion`) is +/// advertised with [`ModelRegistry::builtin`] instead, and holds no path. +/// +/// A cost limit, where set ([`ModelRegistry::with_cost_limit`]), refuses an integer +/// payload whose declared cost exceeds it before any of its symbols is decoded. /// /// The browser equivalent is the `NEURAL_MODELS` map in `wai-web/demo/index.html`. #[derive(Default, Debug, Clone)] pub struct ModelRegistry { pub paths: HashMap, pub held: HashMap>, + builtin: Vec, key_log: Option>, + cost_limit: Option, } impl ModelRegistry { @@ -93,6 +105,21 @@ impl ModelRegistry { self } + /// Refuse, before any of its symbols is decoded, an integer payload whose declared + /// cost (`int_payload::declared_cost`, `extensions/integer-payloads.md` §7, *Memory + /// and work*) exceeds `limit` in any figure: output bytes, symbols, working bytes or + /// multiply-accumulates (`DeclaredCost::new`, a figure left unbounded at its type's + /// maximum). The refusal is `cost_over_limit` ([`DecodeError::OverCostLimit`]), the + /// sink's own policy, so dispatch moves on as for a capability the sink cannot + /// decode (§8): a `WAI1` to its fallback, a `WAI2` to its next entry. A capability + /// with no declared cost — `wai.neural.int_synth`, `wai.neural.int_mlicpp`, and + /// every capability outside the integer path — is not checked. A registry without + /// a limit refuses nothing for its cost. + pub fn with_cost_limit(mut self, limit: crate::int_payload::DeclaredCost) -> Self { + self.cost_limit = Some(limit); + self + } + /// Hold a further artifact a pin over `capability` may resolve against. pub fn hold(mut self, capability: impl Into, path: impl Into) -> Self { self.held.entry(capability.into()).or_default().push(path.into()); @@ -103,9 +130,21 @@ impl ModelRegistry { self.paths.get(capability) } - /// Whether this sink advertises `capability` (it is registered). + /// Advertise `capability`, whose decode reads no parameter set and so has no path + /// to register: `wai.video.int_motion`, decoded by `int_payload::decode_int`. A + /// capability that decodes against a parameter set is advertised only through + /// [`Self::register`]; given here, it is ignored. + pub fn builtin(mut self, capability: impl Into) -> Self { + let c = capability.into(); + if crate::codecs::is_known_capability(&c) && !crate::codecs::decodes_against_parameter_set(&c) { + self.builtin.push(c); + } + self + } + + /// Whether this sink advertises `capability` (it is registered, or built in). pub fn advertises(&self, capability: &str) -> bool { - self.paths.contains_key(capability) + self.paths.contains_key(capability) || self.builtin.iter().any(|c| c == capability) } /// The artifacts a pin over `capability` is checked against (see the type's doc). @@ -117,28 +156,275 @@ impl ModelRegistry { /// The parameters an unpinned decode through `capability` reads: its registered /// artifact. At a registry with a key log, for a capability a pin applies to, check /// 5 runs first over that artifact (SPEC §4, *Module authorisation*), identified by - /// each digest a pin over it could name: its file's SHA-256 where the capability's - /// set is one file, and the SHA-256 of each listing the registry finds for it in the - /// form `wai.prior.set`. The digests are read when the decode starts; an unpinned - /// decode then reads the path, as it always does. `capability` is registered. + /// each digest a pin over it could name. + /// + /// For an integer capability (one `int_payload::decode_int` decodes), the members + /// its decode reads are read here, once, from the paths [`int_member_path`] gives, + /// and returned as [`ParamSource::UnpinnedRead`]: check 5 runs over exactly those + /// bytes (`int_payload::unpinned_read`: a one-file set's file, and the listing of + /// the members read), and the decode reads nothing else. A listing beside the path + /// (the form `wai.prior.set`) adds its digest only when it names, with their + /// digests, every member the decode reads ([`listing_digests`]). + /// + /// For any other capability what the decode can read is read here, once + /// ([`read_once`]): the registered file of a one-file set; otherwise every file + /// below the directory the decoder reads its files from (the registered directory, + /// or the registered file's), as `pin::listing_of_dir` lists it. Check 5 runs over + /// every digest a pin over it could name: the file's, or the directory's derived + /// listing and each member's; and each listing beside it that matches what was read. + /// The decode then reads only those bytes ([`ParamSource::Checked`]), never the path + /// again, so a file changed or added after the check is not read, and a file + /// outside the set is `set_incomplete`. Where the key log holds the capability's sets + /// to §7.9, the decode reads only the part §7.9 authorised: the whole, the files a + /// granted listing names, or one granted file. `capability` is registered. fn unpinned(&self, capability: &str) -> Result { - let path = &self.paths[capability]; - if let Some(k) = self.key_log.as_deref().filter(|_| crate::container::pin_applies_to(capability)) { - let one_file = crate::container::pin_shape(capability) == crate::codecs::ParameterSetShape::OneFile; - let mut digests = Vec::new(); - let files = if one_file { artifacts_at(capability, [path], false) } else { Vec::new() }; - for h in files.iter().chain(artifacts_at(capability, [path], true).iter()) { - let (crate::pin::Held::File(f) | crate::pin::Held::Listing(f)) = h; - if let Ok(bytes) = std::fs::read(f) { - digests.push(crate::pin::sha256_hex(&bytes)); - } + let Some(path) = self.paths.get(capability) else { + // A built-in capability reads no parameter set: there is nothing to check. + return Ok(ParamSource::Unpinned(PathBuf::new())); + }; + let key_log = self.key_log.as_deref().filter(|_| crate::container::pin_applies_to(capability)); + if !crate::int_payload::set_members(capability).is_empty() { + let files = read_int_members(capability, path); + if let Some(k) = key_log { + let view: std::collections::BTreeMap = files.iter().map(|(n, b)| (n.clone(), b.as_slice())).collect(); + let (mut digests, _) = crate::int_payload::unpinned_read(capability, &crate::pin::HeldBytes::Set { listing: b"", files: &view })?; + digests.extend(listing_digests(capability, path, &view)); + k.check_unpinned(capability, &digests)?; } - k.check_unpinned(capability, &digests)?; + return Ok(ParamSource::UnpinnedRead(files)); } - Ok(ParamSource::Unpinned(path.clone())) + let Some(k) = key_log else { return Ok(ParamSource::Unpinned(path.clone())) }; + let read = read_once(capability, path)?; + k.check_unpinned(capability, &read.identities)?; + if !k.requires_grant(capability) { + return Ok(read.source(Part::Whole)); + } + // Check 5 held the set under one of these digests: the decode reads that part. + let i = read.parts.iter().position(|(d, _)| k.check_unpinned(capability, std::slice::from_ref(d)).is_ok()) + .ok_or_else(|| PinRefusal::NotAuthorised { capability: capability.to_string(), check: None, reason: None })?; + let mut read = read; + let (_, part) = read.parts.swap_remove(i); + Ok(read.source(part)) } } +/// What an unpinned decode through a capability outside the integer path can read, at +/// a registry with a key log, read once ([`read_once`]). +struct ReadOnce { + /// The bytes read: the one file, or every file below the decoder's directory. + set: crate::pin::Verified, + /// The registered file's name within `set`, where a file was registered. + model: Option, + /// Every digest a pin over what was read could name, for check 5. + identities: Vec, + /// The parts §7.9 can hold the decode under, each with its digest, in order: the + /// whole; each matching listing (naming the registered file, where a file was + /// registered); each member file (the registered one, where a file was registered). + parts: Vec<(String, Part)>, +} + +/// A part of what [`read_once`] read. +enum Part { + Whole, + /// These files of the set read. + Names(Vec), + /// A set read apart: a listing beside a one-file set's file, with its files. + Read(crate::pin::Verified), +} + +impl ReadOnce { + /// The parameters a decode of `part` reads. + fn source(self, part: Part) -> ParamSource { + use crate::pin::Verified; + let set = match (part, self.set) { + (Part::Whole, set) => set, + (Part::Names(names), Verified::Set { root, mut files }) => Verified::Set { + root, + files: names.into_iter().filter_map(|n| files.remove(&n).map(|b| (n, b))).collect(), + }, + (Part::Names(_), file) => file, + (Part::Read(v), _) => v, + }; + let model = match set { + Verified::File(_) => None, + Verified::Set { .. } => self.model, + }; + ParamSource::Checked { set, model } + } +} + +/// Read, once, what an unpinned decode through `capability` (outside the integer path) +/// at `registered` can read, with every digest a pin over it could name (SPEC §4, +/// *Module authorisation*): +/// +/// - a one-file capability registered as its file: that file, identified by its +/// SHA-256 and by each listing beside it that names it and matches the files it +/// names (each such listing's files read with it, [`listed_sets`]); +/// - otherwise every file below the directory the decoder reads its files from — the +/// registered directory, or the registered file's — as `pin::listing_of_dir` lists +/// it (`pin::read_dir_set`: a symbolic link is `set_malformed`), identified by that +/// derived listing, by each member's SHA-256, and by each listing beside the files +/// that matches them. The decoders outside the integer path read files by name from +/// that directory, some chosen by the payload (`wai.neural.bmshj2018` reads the +/// tables of the payload's quality), so the set is the directory, not a fixed list. +/// +/// A registered path that cannot be read is `set_incomplete`. +fn read_once(capability: &str, registered: &Path) -> Result { + use crate::pin::{listing_from, parse_listing, read_dir_set, sha256, sha256_hex, Held, Verified}; + let missing = |p: &Path| PinRefusal::SetIncomplete { path: p.display().to_string() }; + let file_name = registered.is_file().then(|| registered.file_name().and_then(|n| n.to_str()).map(str::to_string)).flatten(); + if crate::container::pin_shape(capability) == crate::codecs::ParameterSetShape::OneFile && registered.is_file() { + let bytes = std::fs::read(registered).map_err(|_| missing(registered))?; + let d = sha256_hex(&bytes); + let mut out = ReadOnce { set: Verified::File(bytes), model: file_name, identities: vec![d.clone()], parts: vec![(d, Part::Whole)] }; + for (d, v) in listed_sets(capability, registered) { + out.identities.push(d.clone()); + out.parts.push((d, Part::Read(v))); + } + return Ok(out); + } + if !registered.is_dir() && file_name.is_none() { + return Err(missing(registered)); + } + let root = if registered.is_dir() { registered.to_path_buf() } else { ParamSource::dir(registered) }; + let files = read_dir_set(&root)?; + if let Some(m) = &file_name { + if !files.contains_key(m) { + return Err(missing(registered)); + } + } + let view: std::collections::BTreeMap = files.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect(); + let derived = sha256_hex(&listing_from(&view)?); + let mut identities = vec![derived.clone()]; + let mut parts = vec![(derived, Part::Whole)]; + for h in artifacts_at(capability, [®istered.to_path_buf()], true) { + let Held::Listing(f) = h else { continue }; + let Ok(listing) = std::fs::read(&f) else { continue }; + let Ok(lines) = parse_listing(&listing) else { continue }; + if lines.iter().all(|(p, d)| files.get(p).is_some_and(|b| sha256(b) == *d)) { + let d = sha256_hex(&listing); + identities.push(d.clone()); + if file_name.as_ref().is_none_or(|m| lines.iter().any(|(p, _)| p == m)) { + parts.push((d, Part::Names(lines.into_iter().map(|(p, _)| p).collect()))); + } + } + } + for (name, b) in &files { + let d = sha256_hex(b); + identities.push(d.clone()); + if file_name.as_ref().is_none_or(|m| m == name) { + parts.push((d, Part::Names(vec![name.clone()]))); + } + } + Ok(ReadOnce { set: Verified::Set { root, files }, model: file_name, identities, parts }) +} + +/// The digests of the listings beside `registered` (each `*.sha256` in its directory, +/// the form `wai.prior.set`) that identify the members an integer capability's +/// unpinned decode reads, `read`, for check 5. A listing is read as a pinned set's +/// would be (SPEC §3.1 checks 7 and 8): malformed (`set_malformed`), or naming a file +/// whose bytes are not its digest (`part_mismatch`), it is refused and contributes no +/// digest. A listed file that is not a member is read from beside the listing; the +/// listing goes through `int_payload::unpinned_read` and counts only when it names, +/// with their digests, every member the decode reads. +fn listing_digests(capability: &str, registered: &Path, read: &std::collections::BTreeMap) -> Vec { + use crate::pin::{parse_listing, Held, HeldBytes}; + let mut out = Vec::new(); + for h in artifacts_at(capability, [®istered.to_path_buf()], true) { + let Held::Listing(f) = h else { continue }; + let Ok(listing) = std::fs::read(&f) else { continue }; + let Ok(lines) = parse_listing(&listing) else { continue }; + let root = f.parent().map(Path::to_path_buf).unwrap_or_default(); + let others: std::collections::BTreeMap> = lines.iter() + .filter(|(p, _)| !read.contains_key(p)) + .filter_map(|(p, _)| std::fs::read(root.join(p)).ok().map(|b| (p.clone(), b))) + .collect(); + let mut view: std::collections::BTreeMap = others.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect(); + view.extend(read.iter().map(|(k, v)| (k.clone(), *v))); + if let Ok((digests, Some(paths))) = crate::int_payload::unpinned_read(capability, &HeldBytes::Set { listing: &listing, files: &view }) { + if paths.len() == read.len() { + out.extend(digests); + } + } + } + out +} + +/// The sets the listings beside the registered file of a one-file capability outside +/// the integer path identify (each `*.sha256` in its directory, the form +/// `wai.prior.set`): each listing's digest with the files it names, read once from +/// beside it. A listing is read as a pinned set's would be (SPEC §3.1 checks 7 and 8): +/// malformed, or naming a file that is absent or whose bytes are not its digest, it is +/// refused and identifies nothing; a listing that does not name the registered file +/// identifies nothing either, since the decode reads that file. Only a listing check 5 +/// then authorises is a set the decode reads, and it reads nothing outside it. +fn listed_sets(capability: &str, registered: &Path) -> Vec<(String, crate::pin::Verified)> { + use crate::pin::{parse_listing, sha256, sha256_hex, Held, Verified}; + let mut out = Vec::new(); + let file_name = registered.is_file().then(|| registered.file_name().and_then(|n| n.to_str())).flatten(); + for h in artifacts_at(capability, [®istered.to_path_buf()], true) { + let Held::Listing(f) = h else { continue }; + let Ok(listing) = std::fs::read(&f) else { continue }; + let Ok(lines) = parse_listing(&listing) else { continue }; + if registered.is_file() && file_name.is_none_or(|n| !lines.iter().any(|(p, _)| p == n)) { + continue; + } + let root = f.parent().map(Path::to_path_buf).unwrap_or_default(); + let files: Option>> = lines.iter() + .map(|(p, d)| std::fs::read(root.join(p)).ok().filter(|b| sha256(b) == *d).map(|b| (p.clone(), b))) + .collect(); + if let Some(files) = files { + out.push((sha256_hex(&listing), Verified::Set { root, files })); + } + } + out +} + +/// Every identity of the bytes a pin verified, besides the pin's own digest: for an +/// integer capability, those of the members its decode reads +/// (`int_payload::read_identities`); for another, the one file's digest, or the +/// listing of every file the verified set holds. +fn verified_identities(capability: &str, v: &crate::pin::Verified) -> Vec { + use crate::pin::{listing_from, sha256_hex, Verified}; + if !crate::int_payload::set_members(capability).is_empty() { + let read = crate::int_payload::IntParams::verified(v).read(capability); + return crate::int_payload::read_identities(capability, &read); + } + match v { + Verified::File(b) => vec![sha256_hex(b)], + Verified::Set { files, .. } => { + let view: std::collections::BTreeMap = files.iter().map(|(k, b)| (k.clone(), b.as_slice())).collect(); + listing_from(&view).map(|l| vec![sha256_hex(&l)]).unwrap_or_default() + } + } +} + +/// The file an integer capability's decode reads for its set member `member`, at the +/// registered path `registered`: inside it when it is a directory; the registered file +/// itself for a one-file set (except `wai.neural.int_synth`, whose registered path +/// names the model and whose set is the `int_synth.json` beside it); otherwise the +/// file named `member` beside it. A pin over the set, check 5 and the decode all read +/// this one path, so they cover the same bytes. +fn int_member_path(capability: &str, registered: &Path, member: &str) -> PathBuf { + let one_file = crate::container::pin_shape(capability) == crate::codecs::ParameterSetShape::OneFile; + if registered.is_dir() { + registered.join(member) + } else if one_file && capability != crate::codecs::CAP_NEURAL_INT_SYNTH { + registered.to_path_buf() + } else { + registered.parent().map(|d| d.join(member)).unwrap_or_else(|| PathBuf::from(member)) + } +} + +/// The members of `capability`'s set read from beside `registered`, once +/// ([`int_member_path`]); a member that cannot be read is absent, and the decode then +/// refuses with `set_incomplete` after it has checked the payload's framing. +fn read_int_members(capability: &str, registered: &Path) -> std::collections::BTreeMap> { + crate::int_payload::set_members(capability).iter() + .filter_map(|m| std::fs::read(int_member_path(capability, registered, m)).ok().map(|b| (m.to_string(), b))) + .collect() +} + /// The artifacts at `paths` a pin over `capability` is checked against: see /// [`ModelRegistry`]. fn artifacts_at<'a>(capability: &str, paths: impl IntoIterator, set_form: bool) -> Vec { @@ -153,10 +439,9 @@ fn artifacts_at<'a>(capability: &str, paths: impl IntoIterator Result { let set_form = prior.form() == Some(crate::codecs::CAP_PRIOR_SET); let key_log = self.key_log.as_deref().map(|k| k as &dyn crate::pin::PinAuthority); - crate::pin::verify_with(capability, prior, &self.held_artifacts(capability, set_form), READABLE_PRIOR_FORMS, key_log) + let v = crate::pin::verify_with(capability, prior, &self.held_artifacts(capability, set_form), READABLE_PRIOR_FORMS, key_log)?; + // Check 5 ran over the pin's digest; it runs again over every identity of the + // verified bytes (SPEC §3.1 check 5), so a set revoked under another identity + // is not reached through this pin. + if let Some(k) = key_log { + let mut ids = verified_identities(capability, &v); + ids.push(prior.sha256.clone()); + k.check_unpinned(capability, &ids)?; + } + Ok(v) } /// The parameters a decode through `capability` reads: verified against `prior` /// when a pin applies to the capability ([`crate::container::pin_applies_to`]), /// otherwise the registered artifact, unverified — the honest meaning of an - /// unpinned decode — once a key log the registry holds has not refused it. + /// unpinned decode — once a key log the registry holds has not refused it; with a + /// key log, the bytes check 5 ran over, read once, and where that key log holds + /// the capability's sets to jwp-receipts §7.9 only the part §7.9 authorised + /// ([`ParamSource::Checked`]). pub fn resolve_verified(&self, capability: &str, prior: Option<&Prior>) -> Result { - if !self.paths.contains_key(capability) { + if !self.advertises(capability) { return Err(DecodeError::ModelNotRegistered(capability.to_string())); } match prior { @@ -215,12 +512,17 @@ pub fn model_sha256(path: &Path) -> Result { /// Output of a neural-capability decode. The variant is determined by /// the capability's media class — audio caps produce `Audio`, image caps -/// produce `Image`, video caps produce `Video`. +/// produce `Image`, video caps produce `Video`. The integer capabilities decode +/// through `int_payload::decode_int`: an image gives `Image`, and integer video, +/// PCM and splat attributes give `Int`, in the integer form their digests are +/// taken over (RGB8 frames, `i16` PCM, the attribute tensor), never converted. #[derive(Debug, Clone)] +#[non_exhaustive] pub enum Decoded { Audio(DecodedAudio), Image(DecodedImage), Video(DecodedVideo), + Int(crate::int_payload::IntDecoded), } #[derive(Debug)] @@ -243,6 +545,15 @@ pub enum DecodeError { Ort(String), InvalidPayload(String), Zstd(String), + /// An integer payload whose declared cost exceeds the registry's limit + /// ([`ModelRegistry::with_cost_limit`]), refused before any symbol was decoded + /// (`cost_over_limit`). The sink's own policy: dispatch moves on. + OverCostLimit(String), + /// The manifest declares an encryption this sink cannot open (SPEC §6.4 step 2, + /// `unsupported_encryption`): the envelope is inert here, and its ciphertext reaches + /// no codec. The reference sink opens no scheme; a sink that does decrypts first + /// and decodes the plaintext envelope. + Encrypted(String), } impl std::fmt::Display for DecodeError { @@ -262,6 +573,8 @@ impl std::fmt::Display for DecodeError { Self::Ort(e) => write!(f, "onnxruntime error: {e}"), Self::InvalidPayload(e) => write!(f, "invalid payload: {e}"), Self::Zstd(e) => write!(f, "zstd error: {e}"), + Self::OverCostLimit(e) => write!(f, "cost_over_limit: {e}"), + Self::Encrypted(e) => f.write_str(e), } } } @@ -271,9 +584,52 @@ fn is_set_incomplete(e: &DecodeError) -> bool { matches!(e, DecodeError::PriorMismatch(PinRefusal::SetIncomplete { .. })) } +/// Whether a decode was refused for its declared cost, before it decoded anything +/// (`cost_over_limit`): the sink's policy, so dispatch moves on. +fn is_over_cost(e: &DecodeError) -> bool { + matches!(e, DecodeError::OverCostLimit(_)) +} + +/// Whether `capability` decodes to an integer output other than an image: integer +/// video (`WIV1`), PCM (`WIA1`) or splat attributes (`WIS1`), which +/// [`decode_payload`] returns as [`Decoded::Int`]. +pub fn decodes_to_int(capability: &str) -> bool { + use crate::int_wire::{MAGIC_WIA1, MAGIC_WIS1, MAGIC_WIV1}; + matches!(crate::codecs::pinned_int_payload(capability), Some(m) if [MAGIC_WIV1, MAGIC_WIA1, MAGIC_WIS1].contains(&m)) +} + +/// Whether a decode through `cap` failed because this build has no decoder for it +/// (`unsupported_capability`): the sink cannot decode `cap` at all, so SPEC §4 does not +/// take step 3 (or 5) with it, and dispatch moves on as for a capability the sink does +/// not advertise. +fn is_unsupported(e: &DecodeError, cap: &str) -> bool { + matches!(e, DecodeError::UnknownCapability(c) if c == cap) +} + /// Decode `payload` through `capability` against `params` — the per-capability /// dispatch, shared by the `WAI1` and `WAI2` paths. pub fn decode_payload(capability: &str, payload: &[u8], params: &ParamSource) -> Result { + decode_payload_within(capability, payload, params, None) +} + +/// What decoding `payload` through the integer capability `capability` against +/// `params` will cost, read before any symbol is decoded +/// (`int_payload::declared_cost`, `extensions/integer-payloads.md` §7): over the +/// bytes [`ModelRegistry::resolve_verified`] returns, so a pinned decode's cost is +/// read from the set the pin verified, as its decode would read it. A capability +/// with no declared cost is [`DecodeError::UnknownCapability`]; a payload or set the +/// cost's checks refuse gives the decode's refusal. +pub fn declared_cost(capability: &str, payload: &[u8], params: &ParamSource) -> Result { + with_int_params(capability, params, |p| { + crate::int_payload::declared_cost(capability, payload, p).map_err(|e| int_error(capability, e)) + }) +} + +/// [`decode_payload`] with a cost limit applied to the integer capabilities first +/// ([`ModelRegistry::with_cost_limit`]). +fn decode_payload_within( + capability: &str, payload: &[u8], params: &ParamSource, limit: Option<&crate::int_payload::DeclaredCost>, +) -> Result { match capability { "wai.neural.encodec32" => audio::decode(payload, params, 32_000).map(Decoded::Audio), "wai.neural.dac" => audio::decode(payload, params, 44_100).map(Decoded::Audio), @@ -283,10 +639,13 @@ pub fn decode_payload(capability: &str, payload: &[u8], params: &ParamSource) -> "wai.neural.snac" => snac::decode(payload, params).map(Decoded::Audio), "wai.neural.bmshj2018" => image::decode(payload, params).map(Decoded::Image), "wai.neural.mbt2018_mean" => hyperprior::decode(payload, params).map(Decoded::Image), - "wai.neural.int_synth" => int_synth::decode(payload, params).map(Decoded::Image), - // The MLIC++ entropy topology, integer end to end (crate::int_mlicpp), over - // its one-file WMB1 bundle. - "wai.neural.int_mlicpp" => int_mlicpp::decode(payload, params).map(Decoded::Image), + // The integer capabilities: the one integer decode path (crate::int_payload), + // which needs no ONNX runtime. int_synth reads its one-file spec, int_mlicpp its + // WMB1 bundle, int_hyper (and the int_hyper keyframe) three files, the audio and + // splat codecs their one model file; int_motion reads none. + "wai.neural.int_synth" | "wai.neural.int_mlicpp" | "wai.neural.int_hyper" + | "wai.video.int_motion" | "wai.video.int_hyper" | "wai.audio.int_codec" + | "wai.splat.int_codec" => int_decode(capability, payload, params, limit), "wai.neural.elic" => context::decode(payload, params).map(Decoded::Image), "wai.neural.mlicpp" => mlicpp::decode(payload, params).map(Decoded::Image), // MLICv2 is the SAME multi-reference channel-autoregressive decode; its gains @@ -338,13 +697,76 @@ pub fn decode_payload(capability: &str, payload: &[u8], params: &ParamSource) -> } } +/// An integer capability through `int_payload::decode_int`, against the parameters +/// `params` names ([`with_int_params`]). The payload's framing is checked before any +/// member is read, and a member the set lacks is `set_incomplete`, so dispatch moves +/// on as it does for any decoder. With a cost limit, the payload's declared cost is +/// checked against it first (`int_payload::check_cost`): one over it is refused +/// before any symbol is decoded (`cost_over_limit`). +fn int_decode(cap: &str, payload: &[u8], params: &ParamSource, limit: Option<&crate::int_payload::DeclaredCost>) -> Result { + use crate::int_payload::{check_cost, decode_int, IntDecoded}; + with_int_params(cap, params, |p| { + if let Some(limit) = limit { + check_cost(cap, payload, p, limit).map_err(|e| int_error(cap, e))?; + } + match decode_int(cap, payload, p) { + Ok(IntDecoded::Image { rgb, width, height }) => Ok(Decoded::Image(DecodedImage { rgb, width, height })), + Ok(other) => Ok(Decoded::Int(other)), + Err(e) => Err(int_error(cap, e)), + } + }) +} + +/// An integer decode's refusal as a [`DecodeError`]: `set_incomplete` as SPEC §3.1's, +/// `unsupported_capability` as a capability with no decoder, `cost_over_limit` as the +/// sink's policy, and every other code as the payload's. +fn int_error(cap: &str, e: crate::int_payload::IntPayloadError) -> DecodeError { + use crate::int_payload::Refusal; + match e.refusal() { + Refusal::SetIncomplete => DecodeError::PriorMismatch(PinRefusal::SetIncomplete { path: e.detail().to_string() }), + Refusal::UnsupportedCapability => DecodeError::UnknownCapability(cap.to_string()), + Refusal::CostOverLimit => DecodeError::OverCostLimit(e.detail().to_string()), + _ => DecodeError::InvalidPayload(format!("{cap}: {e}")), + } +} + +/// `f` with the integer parameters `params` names: for a pin, or a set §7.9 +/// authorised, the verified bytes; for an unpinned decode, the set the registry read +/// when the decode started ([`ParamSource::UnpinnedRead`]), or, given a path directly, +/// the set's members read from beside it now ([`int_member_path`]). +fn with_int_params( + cap: &str, params: &ParamSource, f: impl FnOnce(&crate::int_payload::IntParams<'_>) -> Result, +) -> Result { + use crate::int_payload::{resolve, IntParams}; + let read_now; + let unpinned = match params { + ParamSource::UnpinnedRead(files) => Some(files), + ParamSource::Unpinned(path) => { + read_now = read_int_members(cap, path); + Some(&read_now) + } + _ => None, + }; + let view: std::collections::BTreeMap = + unpinned.into_iter().flatten().map(|(k, v)| (k.clone(), v.as_slice())).collect(); + let p = match params { + ParamSource::Pinned(v) | ParamSource::Checked { set: v, .. } => IntParams::verified(v), + _ => resolve(cap, None, &[crate::pin::HeldBytes::Set { listing: b"", files: &view }], None) + .map_err(DecodeError::PriorMismatch)?, + }; + f(&p) +} + /// SPEC §4 for a parsed `WAI1` envelope at a sink whose parameter sets `registry` /// holds, through the capabilities `gate` admits. A refused pin makes its capability /// unsupported for this envelope and dispatch moves on to the fallback; a pinned -/// decode refused with `set_incomplete` does too, and so does an unpinned decode whose -/// set the registry's key log refuses. When nothing decodes, the error is step 6's: -/// the reason kept for the capability (step 3) before the one kept for the fallback -/// (step 5), whether that reason is a refused pin, `set_incomplete` or the key log's. +/// decode refused with `set_incomplete` does too, and so do an unpinned decode whose +/// set the registry's key log refuses, a capability this build has no decoder for +/// (`DecodeError::UnknownCapability`, an integer decode's `unsupported_capability`) and +/// an integer payload over the registry's cost limit (`cost_over_limit`). +/// When nothing decodes, the error is step 6's: the reason kept for the capability +/// (step 3) before the one kept for the fallback (step 5), whether that reason is a +/// refused pin, `set_incomplete`, the key log's, a missing decoder or the cost limit. fn decode_envelope_with( env: &Wai, registry: &ModelRegistry, @@ -370,6 +792,7 @@ fn decode_envelope_with( _ => DecodeError::PriorMismatch(refusal), }); } + Err(e @ Inert::Encrypted { .. }) => return Err(DecodeError::Encrypted(e.to_string())), Err(_) => { return Err(incomplete.map(|(_, e)| e) .unwrap_or_else(|| DecodeError::ModelNotRegistered(env.capability().to_string()))); @@ -389,8 +812,8 @@ fn decode_envelope_with( } }, }; - match decode_payload(d.capability, &env.payload, ¶ms) { - Err(e) if is_set_incomplete(&e) => { + match decode_payload_within(d.capability, &env.payload, ¶ms, registry.cost_limit.as_ref()) { + Err(e) if is_set_incomplete(&e) || is_unsupported(&e, d.capability) || is_over_cost(&e) => { excluded.push(d.capability.to_string()); if incomplete.is_none() { incomplete = Some((d.capability.to_string(), e)); @@ -404,7 +827,8 @@ fn decode_envelope_with( /// Decode a parsed `WAI1` envelope (SPEC §4): the capability if the registry holds /// it and, where a pin applies, a parameter set that satisfies the pin (SPEC §3.1); /// else the fallback on the same terms. A pinned decode reads only the bytes it -/// verified. Returns [`DecodeError::PriorMismatch`] with the first refusal when no +/// verified. An envelope whose manifest declares an encryption is inert here +/// ([`DecodeError::Encrypted`], SPEC §6.4 step 2): this sink opens no scheme. Returns [`DecodeError::PriorMismatch`] with the first refusal when no /// capability's pin resolves, and [`DecodeError::ModelNotRegistered`] when the /// registry holds neither capability. pub fn decode_envelope(env: &Wai, registry: &ModelRegistry) -> Result { @@ -474,10 +898,12 @@ pub fn decode_envelope_strict(env: &Wai, registry: &ModelRegistry) -> Result Result<(usize, Decoded), DecodeError> { decode_multi_where(m, registry, |_| Ok(())) } @@ -514,11 +940,15 @@ fn decode_multi_where( } }, }; - match decode_payload(&p.meta.capability, p.payload, ¶ms) { + match decode_payload_within(&p.meta.capability, p.payload, ¶ms, registry.cost_limit.as_ref()) { Err(e) if is_set_incomplete(&e) => { let DecodeError::PriorMismatch(r) = &e else { unreachable!() }; incomplete.push((p.index, format!("{}: {r}", r.code()))); } + Err(e) if is_over_cost(&e) => incomplete.push((p.index, e.to_string())), + Err(e) if is_unsupported(&e, &p.meta.capability) => { + incomplete.push((p.index, format!("unsupported_capability: {e}"))); + } Err(e) => return Err(e), Ok(d) => return Ok((p.index, d)), } diff --git a/wai/wai-rs/src/neural/params.rs b/wai/wai-rs/src/neural/params.rs index 504d58888..7e6c78695 100644 --- a/wai/wai-rs/src/neural/params.rs +++ b/wai/wai-rs/src/neural/params.rs @@ -5,7 +5,9 @@ //! sink always has. A pinned decode reads only the bytes the pin verified //! ([`crate::pin::Verified`]) — the very bytes that were hashed, never a second read //! of the same path — and a file the verified set does not contain is refused with -//! `set_incomplete` instead of being read from disk unverified. +//! `set_incomplete` instead of being read from disk unverified. So does an unpinned +//! decode at a sink that holds a key log: it reads only the bytes check 5 ran over, +//! read once ([`ParamSource::Checked`], [`ParamSource::UnpinnedRead`]). //! //! Sessions are built for determinism (`crate::ort_session`): one intra-op and one //! inter-op thread and a fixed graph-optimization level, because multi-threaded @@ -14,6 +16,7 @@ //! hardware, which only an integer decode is (SPEC §7). use std::borrow::Cow; +use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use ort::session::Session; @@ -25,12 +28,28 @@ use super::DecodeError; /// The parameters one decode reads. #[derive(Debug, Clone)] +#[non_exhaustive] pub enum ParamSource { /// No pin applied: the registered path, and the files beside it (the path /// itself when it is a directory). Unpinned(PathBuf), /// A pin verified this set; the decode reads nothing else. Pinned(Verified), + /// No pin applied, and the sink holds a key log (SPEC §4, *Module authorisation*): + /// the set the decode reads, read once when the decode started, over whose every + /// identity check 5 ran ([`super::ModelRegistry`]) — the registered file of a + /// one-file set, or every file below the directory the decoder reads from; where + /// the key log holds the capability's sets to jwp-receipts §7.9, the set §7.9 + /// authorised. `model` is the registered file's name within a set, where a file + /// was registered. The decode reads nothing else, as a pinned decode does: a file + /// the set does not contain is `set_incomplete`. + Checked { set: Verified, model: Option }, + /// No pin applied, and the set an integer capability's decode reads was read into + /// memory once, when the decode started: its members (`int_payload::set_members`) + /// by the names a set lists them under, a member the sink could not read being + /// absent. A key log the registry holds ran check 5 over exactly these bytes, and + /// the decode reads nothing else ([`super::ModelRegistry`]). + UnpinnedRead(BTreeMap>), } fn incomplete(name: &str) -> DecodeError { @@ -39,7 +58,7 @@ fn incomplete(name: &str) -> DecodeError { impl ParamSource { /// The directory an unpinned decode reads siblings from. - fn dir(path: &Path) -> PathBuf { + pub(crate) fn dir(path: &Path) -> PathBuf { if path.is_dir() { path.to_path_buf() } else { @@ -53,19 +72,28 @@ impl ParamSource { } /// The artifact the capability registers, named `set_name` within a set: for an - /// unpinned decode the registered file itself; for a form-less pin the one - /// verified file; for a set, the file it lists as `set_name`. + /// unpinned decode the registered file itself; for one verified file, that file; + /// for a set read where a file was registered, that file; for another set, the + /// file it lists as `set_name`. pub(crate) fn model(&self, set_name: &str) -> Result, DecodeError> { + fn from<'a>(v: &'a Verified, name: &str) -> Result, DecodeError> { + match v { + Verified::File(b) => Ok(Cow::Borrowed(b.as_slice())), + v => v.file(name).map(Cow::Borrowed).map_err(DecodeError::PriorMismatch), + } + } match self { Self::Unpinned(p) => std::fs::read(p).map(Cow::Owned) .map_err(|e| DecodeError::InvalidPayload(format!("{}: {e}", p.display()))), - Self::Pinned(Verified::File(b)) => Ok(Cow::Borrowed(b)), - Self::Pinned(v) => v.file(set_name).map(Cow::Borrowed).map_err(DecodeError::PriorMismatch), + Self::UnpinnedRead(files) => files.get(set_name).map(|b| Cow::Borrowed(b.as_slice())).ok_or_else(|| incomplete(set_name)), + Self::Pinned(v) => from(v, set_name), + Self::Checked { set, model } => from(set, model.as_deref().unwrap_or(set_name)), } } /// The file `name` beside the registered artifact (unpinned), or in the verified - /// set. A form-less pin verified one file, so it holds no sibling. + /// set. One verified file (a form-less pin, or a one-file set §7.9 authorised) + /// holds no sibling. pub(crate) fn sibling(&self, name: &str) -> Result, DecodeError> { match self { Self::Unpinned(p) => { @@ -73,8 +101,8 @@ impl ParamSource { std::fs::read(&f).map(Cow::Owned) .map_err(|e| DecodeError::InvalidPayload(format!("{}: {e}", f.display()))) } - Self::Pinned(Verified::File(_)) => Err(incomplete(name)), - Self::Pinned(v) => v.file(name).map(Cow::Borrowed).map_err(DecodeError::PriorMismatch), + Self::UnpinnedRead(files) => files.get(name).map(|b| Cow::Borrowed(b.as_slice())).ok_or_else(|| incomplete(name)), + Self::Pinned(v) | Self::Checked { set: v, .. } => v.file(name).map(Cow::Borrowed).map_err(DecodeError::PriorMismatch), } } @@ -83,8 +111,8 @@ impl ParamSource { pub(crate) fn try_sibling(&self, name: &str) -> Option> { match self { Self::Unpinned(p) => std::fs::read(Self::dir(p).join(name)).ok().map(Cow::Owned), - Self::Pinned(Verified::File(_)) => None, - Self::Pinned(v) => v.file(name).ok().map(Cow::Borrowed), + Self::UnpinnedRead(files) => files.get(name).map(|b| Cow::Borrowed(b.as_slice())), + Self::Pinned(v) | Self::Checked { set: v, .. } => v.file(name).ok().map(Cow::Borrowed), } } @@ -102,7 +130,7 @@ impl ParamSource { pub(crate) fn session_model(&self, set_name: &str) -> Result { match self { Self::Unpinned(p) => crate::ort_session::load_deterministic(p).map_err(DecodeError::Ort), - Self::Pinned(_) => session_from_verified(&self.model(set_name)?), + Self::Pinned(_) | Self::Checked { .. } | Self::UnpinnedRead(_) => session_from_verified(&self.model(set_name)?), } } @@ -110,7 +138,7 @@ impl ParamSource { pub(crate) fn session_sibling(&self, name: &str) -> Result { match self { Self::Unpinned(p) => crate::ort_session::load_deterministic(&Self::dir(p).join(name)).map_err(DecodeError::Ort), - Self::Pinned(_) => session_from_verified(&self.sibling(name)?), + Self::Pinned(_) | Self::Checked { .. } | Self::UnpinnedRead(_) => session_from_verified(&self.sibling(name)?), } } } diff --git a/wai/wai-rs/src/pin.rs b/wai/wai-rs/src/pin.rs index ad41b48a3..dfbba4eb7 100644 --- a/wai/wai-rs/src/pin.rs +++ b/wai/wai-rs/src/pin.rs @@ -188,7 +188,17 @@ pub fn write_listing(root: &Path, paths: &[String]) -> Result, PinRefusa /// `.` and the `*.sha256` files directly in `root` (the listings themselves). A /// symbolic link, or a name a listing path cannot carry, is refused. pub fn listing_of_dir(root: &Path) -> Result, PinRefusal> { - fn walk(root: &Path, rel: &str, out: &mut Vec) -> Result<(), PinRefusal> { + let files = read_dir_set(root)?; + let view: BTreeMap = files.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect(); + listing_from(&view) +} + +/// Every file [`listing_of_dir`] lists below `root`, read once, by its listing path: +/// the set a sink holds as that directory, with the bytes its listing is derived +/// from. A symbolic link, or a name a listing path cannot carry, is refused +/// (`set_malformed`). +pub fn read_dir_set(root: &Path) -> Result>, PinRefusal> { + fn walk(root: &Path, rel: &str, out: &mut BTreeMap>) -> Result<(), PinRefusal> { let dir = if rel.is_empty() { root.to_path_buf() } else { root.join(rel) }; let rd = std::fs::read_dir(&dir).map_err(|e| malformed(format!("{}: {e}", dir.display())))?; for entry in rd { @@ -207,14 +217,18 @@ pub fn listing_of_dir(root: &Path) -> Result, PinRefusal> { } else if rel.is_empty() && name.ends_with(".sha256") { continue; } else { - out.push(path); + if !is_listing_path(&path) { + return Err(malformed(format!("'{path}' is not a listing path"))); + } + let b = std::fs::read(root.join(&path)).map_err(|e| malformed(format!("{path}: {e}")))?; + out.insert(path, b); } } Ok(()) } - let mut paths = Vec::new(); - walk(root, "", &mut paths)?; - write_listing(root, &paths) + let mut files = BTreeMap::new(); + walk(root, "", &mut files)?; + Ok(files) } /// A parameter set a sink holds for a capability, in memory: one file, or a @@ -246,6 +260,14 @@ pub trait PinAuthority: std::fmt::Debug { /// `prior_revoked` when any of them is revoked, and, for a capability held to /// §7.9, with `prior_not_authorised` unless §7.9 holds the set under one of them. fn check_unpinned(&self, capability: &str, digests: &[String]) -> Result<(), PinRefusal>; + + /// Whether the sink holds `capability`'s sets to §7.9, so that an unpinned decode + /// through it reads a set only under a digest §7.9 holds. The default asks + /// [`Self::check_unpinned`] over no digest, which such a sink refuses and any other + /// accepts (no digest is revoked). + fn requires_grant(&self, capability: &str) -> bool { + self.check_unpinned(capability, &[]).is_err() + } } /// Which held artifact satisfied a pin: its index in the `held` slice given to diff --git a/wai/wai-rs/src/rans.rs b/wai/wai-rs/src/rans.rs index 06ace9bce..5303e1f16 100644 --- a/wai/wai-rs/src/rans.rs +++ b/wai/wai-rs/src/rans.rs @@ -39,6 +39,16 @@ impl<'a> RansDecoder<'a> { self.exhausted } + /// Whether the stream has ended canonically: it never ran out, every word has been + /// read, and the state is `2^31`, the state an encoder starts from. Called after a + /// decode's last symbol. A stream that ends otherwise (an unread word after the + /// last symbol, or a state the encoder did not start from) codes the same symbols + /// as the canonical stream, so a decoder that requires this accepts one stream per + /// sequence of symbols. + pub fn is_canonical_end(&self) -> bool { + !self.exhausted && self.state == RANS64_L && self.idx * 4 == self.words.len() + } + #[inline] fn renorm(&mut self) { if self.state < RANS64_L { let off = self.idx * 4; diff --git a/wai/wai-rs/src/video_receipt.rs b/wai/wai-rs/src/video_receipt.rs index 0560e2d33..a35364324 100644 --- a/wai/wai-rs/src/video_receipt.rs +++ b/wai/wai-rs/src/video_receipt.rs @@ -1,18 +1,25 @@ //! Video receipt — a JWP profile for `wai.video.int_motion`, the integer -//! inter-frame codec. The **codestream hash** is the identity; the group's -//! objects are the per-frame decoded-pixel hashes; one Ed25519 signature covers -//! codestream hash + frame hashes + exact reconstruction work + measured joules. -//! Chains via `parent_receipt_hash` exactly as the other classes — reuse, not -//! reinvention. +//! inter-frame codec. The **codestream hash** is the identity: BLAKE3 over the +//! `WIV1` payload an envelope carries (`extensions/integer-payloads.md` §3); the +//! group's objects are the per-frame decoded-pixel hashes; one Ed25519 signature +//! covers codestream hash + frame hashes + exact reconstruction work + measured +//! joules. Chains via `parent_receipt_hash` exactly as the other classes — reuse, +//! not reinvention. [`VideoReceipt::seal_decode_wiv1`] seals one and +//! [`VideoReceipt::verify_reconstruction_wiv1`] re-derives it. A receipt sealed +//! before the payload was pinned identifies the tool's JSON form instead; the +//! deprecated [`VideoReceipt::seal_decode`] and +//! [`VideoReceipt::verify_reconstruction`] still read it, so such a receipt keeps +//! verifying byte for byte. //! //! **What makes this profile different from its siblings.** In every other //! cargo class the reconstruction is deterministic *by construction* and taken -//! on trust. Here it is deterministic *and proven on real silicon*: the -//! cross-architecture byte-equality matrix decodes these same vectors on -//! linux-x86_64, linux-aarch64, macos-arm64 and windows-x86_64 and asserts -//! identical bytes. So a verifier holding only the codestream can independently -//! recompute both `clip_hash` and `work_pixel_ops` — see -//! [`VideoReceipt::verify_reconstruction`] — and every field except the +//! on trust. Here it is deterministic *and checked on real silicon*: the +//! cross-architecture byte-equality matrix (`wai-byte-equality.yml`) parses and +//! decodes the committed `WIV1` payloads, and the JSON forms they were converted +//! from, on every platform in its matrix and asserts identical bytes. So a +//! verifier holding only the codestream can independently recompute both +//! `clip_hash` and `work_pixel_ops` — see +//! [`VideoReceipt::verify_reconstruction_wiv1`] — and every field except the //! measurement stops being a claim and becomes a checkable fact. //! //! That leaves the measurement attested-not-verifiable — `joules_micro` and the @@ -99,9 +106,29 @@ pub fn clip_hash(h: u32, w: u32, frames: &[FrameHash]) -> [u8; 32] { *hs.finalize().as_bytes() } +/// Decode a `WIV1` payload of `wai.video.int_motion` (keyframe 0): parsed with +/// `int_payload::wire` and decoded through [`crate::int_video::decode_typed`], each +/// refusal reported by its code (`extensions/integer-payloads.md` §8): the parse's, +/// or the decode's (`stream_exhausted`, `stream_noncanonical`, and `field_range` for +/// a rule of the typed stream). +pub fn decode_wiv1(payload: &[u8]) -> Result { + use crate::int_video::{STREAM_EXHAUSTED, STREAM_NONCANONICAL}; + let v = crate::int_payload::wire::parse_wiv1(payload, Some(0)).map_err(|e| e.refusal.code())?; + let s = crate::int_video::VideoStream { + n_frames: v.n_frames as usize, h: v.h as usize, w: v.w as usize, block: v.block as usize, + q: v.q as i64, offset: v.offset, mv_precision: v.mv_precision, cdf: &v.cdf, mvs: &v.mvs, rans: v.rans, + }; + crate::int_video::decode_typed(&s).map_err(|e| match e { + STREAM_EXHAUSTED => crate::int_payload::Refusal::StreamExhausted.code(), + STREAM_NONCANONICAL => crate::int_payload::Refusal::StreamNoncanonical.code(), + _ => crate::int_payload::Refusal::FieldRange.code(), + }) +} + #[derive(Clone, Debug, PartialEq)] pub struct VideoReceipt { - /// Identity: BLAKE3 over the canonical codestream bytes as they were read. + /// Identity: BLAKE3 over the codestream bytes exactly as they were read — the + /// `WIV1` payload, or, in a receipt sealed before it was pinned, the JSON form. pub codestream_hash: [u8; 32], /// The reconstruction result: the portable whole-clip commitment. pub clip_hash: [u8; 32], @@ -200,12 +227,31 @@ impl VideoReceipt { r } - /// Decode `codestream` and seal a receipt over what it actually produces. + /// Decode the `WIV1` payload `payload` (keyframe 0) and seal a receipt over what + /// it actually produces. The identity is BLAKE3 over `payload` exactly as an + /// envelope carries it, so it identifies the artifact a verifier will be handed. + pub fn seal_decode_wiv1( + signer: &SigningKey, + signer_id: impl Into, + payload: &[u8], + joules_micro: u64, + energy_provenance: Option, + parent_receipt_hash: Option<[u8; 32]>, + ) -> Result { + let clip = decode_wiv1(payload)?; + let codestream_hash = *blake3::hash(payload).as_bytes(); + Ok(Self::build(signer, signer_id, codestream_hash, &clip, joules_micro, energy_provenance, parent_receipt_hash)) + } + + /// Decode `codestream`, the tool's JSON form, and seal a receipt over what it + /// actually produces, identified by BLAKE3 over `raw`, the JSON bytes as read. /// - /// The identity is hashed from `raw` — the codestream bytes exactly as they - /// arrived — rather than from a re-serialisation of the parsed value, so the - /// hash identifies the artifact a verifier will be handed rather than this - /// process's idea of it. + /// The JSON form is no longer a payload (`extensions/integer-payloads.md` §10): + /// a new receipt is sealed over the `WIV1` payload with + /// [`Self::seal_decode_wiv1`]. This stays so that a receipt sealed over the JSON + /// form can still be re-derived and its pinned bytes reproduced. + #[deprecated(note = "the payload of wai.video.int_motion is WIV1 (extensions/integer-payloads.md); \ + seal with `seal_decode_wiv1`. Kept so receipts sealed over the JSON form still verify")] pub fn seal_decode( signer: &SigningKey, signer_id: impl Into, @@ -264,20 +310,38 @@ impl VideoReceipt { k.verify(&self.signing_payload(), &Signature::from_bytes(&self.sig)).is_ok() } - /// The independent check: re-decode the codestream and confirm this receipt - /// describes it — identity, geometry, every frame, the clip commitment and - /// the work figure. + /// The independent check: re-decode the `WIV1` payload and confirm this receipt + /// describes it — identity, geometry, every frame, the clip commitment and the + /// work figure. /// /// Byte-equality across architectures is what makes this meaningful rather /// than circular: the caller need not trust the machine that sealed the - /// receipt, because the same bitstream reconstructs the same bytes on any of - /// the proven architectures. `joules_micro` is untouched here — it is not + /// receipt, because the same payload reconstructs the same bytes on every + /// platform the matrix checks. `joules_micro` is untouched here — it is not /// reproducible and is not meant to be. + pub fn verify_reconstruction_wiv1(&self, payload: &[u8]) -> bool { + if *blake3::hash(payload).as_bytes() != self.codestream_hash { + return false; + } + let Ok(clip) = decode_wiv1(payload) else { return false }; + self.describes(&clip) + } + + /// [`Self::verify_reconstruction_wiv1`] for a receipt sealed over the tool's + /// JSON form: `raw` is the JSON bytes its identity was hashed from. + #[deprecated(note = "the payload of wai.video.int_motion is WIV1 (extensions/integer-payloads.md); \ + verify with `verify_reconstruction_wiv1`. Kept so receipts sealed over the JSON form still verify")] pub fn verify_reconstruction(&self, raw: &[u8], codestream: &Value) -> bool { if *blake3::hash(raw).as_bytes() != self.codestream_hash { return false; } let Ok(clip) = decode(codestream) else { return false }; + self.describes(&clip) + } + + /// Geometry, every frame and then [`Self::verify`]: what a re-decoded clip must + /// match. + fn describes(&self, clip: &DecodedClip) -> bool { if clip.h as u32 != self.h || clip.w as u32 != self.w { return false; } diff --git a/wai/wai-rs/tests/cli_wrap_intent.rs b/wai/wai-rs/tests/cli_wrap_intent.rs index 908b16551..bb8834d9c 100644 --- a/wai/wai-rs/tests/cli_wrap_intent.rs +++ b/wai/wai-rs/tests/cli_wrap_intent.rs @@ -156,3 +156,81 @@ fn wrap_holds_a_companion_envelope_to_the_same_intent_rule() { assert!(stderr.contains("--fallback names a WAI1 alternative"), "{stderr}"); assert!(!out.exists(), "a refused wrap must not write an envelope"); } + +/// SPEC §5 pins the payload of each WAI-native integer codec to a binary format +/// (`extensions/integer-payloads.md`). `wai wrap` parses the payload as a sink does +/// and writes nothing when it is not its capability's format: a tool's JSON output +/// (`bad_magic`, converted first, never wrapped), another format, a `WIV1` cut inside +/// its header (`stream_length`), and a keyframe-0 `WIV1` under `wai.video.int_hyper` +/// (`keyframe_capability_mismatch`), whose magic and version are right. The binary +/// payloads wrap; `wai inspect` reports what a parse finds in an envelope another +/// writer made. +#[test] +fn wrap_refuses_a_payload_that_is_not_the_pinned_integer_format() { + let v = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors"); + let wrap_file = |cap: &str, input: &Path, out: &Path, extra: &[&str]| -> Output { + Command::new(env!("CARGO_BIN_EXE_wai")) + .arg("wrap").arg(cap).arg(input).arg("-o").arg(out).args(extra) + .output() + .expect("run wai") + }; + for (cap, json, bin, code) in [ + ("wai.video.int_motion", "int_video/int_video_bitstream.json", "int_video/int_video_bitstream.wiv1", "bad_magic"), + ("wai.neural.int_hyper", "int_hyper/tiny_bitstream.json", "int_hyper/tiny_bitstream.wih1", "bad_magic"), + ("wai.audio.int_codec", "int_video/video_nk_bitstream.wiv1", "int_audio/audio_bitstream.wia1", "format_capability_mismatch"), + ("wai.splat.int_codec", "int_splat/splat.json", "int_splat/splat.wis1", "bad_magic"), + ("wai.video.int_hyper", "int_video/int_video_bitstream.wiv1", "int_video/video_nk_bitstream.wiv1", "keyframe_capability_mismatch"), + ] { + let out = scratch("int-payload.wai"); + let run = wrap_file(cap, &v.join(json), &out, &["--intent", "improve"]); + let stderr = String::from_utf8_lossy(&run.stderr); + assert!(!run.status.success(), "{cap} {json}: wrap must refuse; stderr: {stderr}"); + assert!(stderr.contains(code) && stderr.contains("integer-payloads"), "{cap}: {stderr}"); + assert!(!out.exists(), "{cap}: a refused wrap must not write an envelope"); + let run = wrap_file(cap, &v.join(bin), &out, &["--intent", "improve"]); + assert!(run.status.success(), "{cap} {bin}: {}", String::from_utf8_lossy(&run.stderr)); + assert_eq!(Wai::read(&out).unwrap().payload, std::fs::read(v.join(bin)).unwrap()); + } + // A WIV1 cut inside its fixed header. + let cut = scratch("int-payload-cut.wiv1"); + std::fs::write(&cut, &std::fs::read(v.join("int_video/int_video_bitstream.wiv1")).unwrap()[..30]).unwrap(); + let out = scratch("int-payload-cut.wai"); + let run = wrap_file("wai.video.int_motion", &cut, &out, &["--intent", "improve"]); + assert!(!run.status.success() && String::from_utf8_lossy(&run.stderr).contains("stream_length"), "{}", String::from_utf8_lossy(&run.stderr)); + assert!(!out.exists()); + // The same rule for a WAI2 base. + let out = scratch("int-payload-hdr.wai"); + let run = wrap_file("wai.video.int_motion", &v.join("int_video/int_video_bitstream.json"), &out, + &["--companion", "wai.meta.hdr10plus", payload().to_str().unwrap()]); + assert!(!run.status.success() && String::from_utf8_lossy(&run.stderr).contains("bad_magic")); + assert!(!out.exists()); + // inspect reports an envelope that carries the JSON form. + let out = scratch("int-payload-json.wai"); + wai::container::Wai::new( + wai::container::Manifest::new("video", "improve", wai::container::ModelRequirement::new("wai.video.int_motion"), + wai::container::Conditioning::new("bytes")), + std::fs::read(v.join("int_video/int_video_bitstream.json")).unwrap(), + ).write(&out).unwrap(); + let run = Command::new(env!("CARGO_BIN_EXE_wai")).arg("inspect").arg(&out).output().expect("run wai"); + let stdout = String::from_utf8_lossy(&run.stdout); + assert!(stdout.contains("integer payload:") && stdout.contains("bad_magic"), "{stdout}"); + // ...and one whose keyframe-0 WIV1 sits under wai.video.int_hyper. + let out = scratch("int-payload-k0.wai"); + wai::container::Wai::new( + wai::container::Manifest::new("video", "improve", wai::container::ModelRequirement::new("wai.video.int_hyper"), + wai::container::Conditioning::new("bytes")), + std::fs::read(v.join("int_video/int_video_bitstream.wiv1")).unwrap(), + ).write(&out).unwrap(); + let run = Command::new(env!("CARGO_BIN_EXE_wai")).arg("inspect").arg(&out).output().expect("run wai"); + let stdout = String::from_utf8_lossy(&run.stdout); + assert!(stdout.contains("integer payload:") && stdout.contains("keyframe_capability_mismatch"), "{stdout}"); + // ...and an encrypted one, whose ciphertext is checked only after decryption. + let out = scratch("int-payload-encrypted.wai"); + let mut m = wai::container::Manifest::new("video", "improve", wai::container::ModelRequirement::new("wai.video.int_motion"), + wai::container::Conditioning::new("bytes")); + m.security = Some(wai::container::Security::new("aes-256-gcm", 7)); + wai::container::Wai::new(m, vec![0x5a; 64]).write(&out).unwrap(); + let run = Command::new(env!("CARGO_BIN_EXE_wai")).arg("inspect").arg(&out).output().expect("run wai"); + let stdout = String::from_utf8_lossy(&run.stdout); + assert!(stdout.contains("integer payload: encrypted"), "{stdout}"); +} diff --git a/wai/wai-rs/tests/ffi_envelope.rs b/wai/wai-rs/tests/ffi_envelope.rs index ad4c1642c..6696b4e09 100644 --- a/wai/wai-rs/tests/ffi_envelope.rs +++ b/wai/wai-rs/tests/ffi_envelope.rs @@ -180,3 +180,81 @@ fn envelope_pack_refuses_a_candidate() { assert!(out.data.is_null()); } } + +/// SPEC §5, *Payloads of the WAI-native integer codecs*: the C packers refuse a payload +/// that is not the format SPEC §5 pins its integer capability (or fallback, or `WAI2` +/// component) to, parsed as a sink parses it: a tool's JSON form, a `WIV1` cut inside +/// its header, and a keyframe-0 `WIV1` under `wai.video.int_hyper`, whose magic and +/// version are right. The committed payloads are written. +#[test] +fn envelope_pack_refuses_an_integer_payload_that_is_not_its_format() { + let vec = |rel: &str| std::fs::read(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors").join(rel)).unwrap(); + let (k0, k1, json) = (vec("int_video/int_video_bitstream.wiv1"), vec("int_video/video_nk_bitstream.wiv1"), vec("int_video/int_video_bitstream.json")); + let man = |cap: &str, fb: &str| CString::new(format!( + r#"{{"wai":"1.0","media":"video","intent":"improve","model_requirement":{{"capability":"{cap}","fallback":{fb}}},"conditioning":{{"kind":"bytes"}},"target":null}}"#)).unwrap(); + let pack = |m: &CString, p: &[u8]| { + let mut out = empty(); + let rc = unsafe { wai_envelope_pack(m.as_ptr(), p.as_ptr(), p.len(), &mut out) }; + if rc == WAI_OK { take(out); } else { assert!(out.data.is_null()); } + (rc, if rc == WAI_OK { String::new() } else { last_error() }) + }; + let (mot, hyp) = ("wai.video.int_motion", "wai.video.int_hyper"); + for (m, p, code) in [ + (man(mot, "null"), &json[..], "bad_magic"), + (man(mot, "null"), &k0[..40], "stream_length"), + (man(hyp, "null"), &k0[..], "keyframe_capability_mismatch"), + (man("wai.video.av1", &format!("\"{mot}\"")), &k1[..], "keyframe_capability_mismatch"), + ] { + let (rc, why) = pack(&m, p); + assert_eq!(rc, WAI_ERR_INVALID_ARG, "{code}"); + assert!(why.contains(&format!("({code}:")), "{why}"); + } + assert_eq!(pack(&man(mot, "null"), &k0).0, WAI_OK); + assert_eq!(pack(&man(hyp, "null"), &k1).0, WAI_OK); + + let multi = CString::new(format!( + r#"{{"wai":"1.1","media":"video","intent":"improve","renditions":[{{"capability":"{mot}","kind":"bytes"}},{{"capability":"{hyp}","kind":"bytes"}}],"target":null}}"#)).unwrap(); + let pack2 = |a: &[u8], b: &[u8]| { + let (ptrs, lens) = ([a.as_ptr(), b.as_ptr()], [a.len(), b.len()]); + let mut out = empty(); + let rc = unsafe { wai_envelope_pack_multi(multi.as_ptr(), ptrs.as_ptr(), lens.as_ptr(), 2, &mut out) }; + if rc == WAI_OK { take(out); } else { assert!(out.data.is_null()); } + (rc, if rc == WAI_OK { String::new() } else { last_error() }) + }; + assert_eq!(pack2(&k0, &k1).0, WAI_OK); + let (rc, why) = pack2(&k0, &k0); + assert_eq!(rc, WAI_ERR_INVALID_ARG); + assert!(why.contains("component 1:") && why.contains("(keyframe_capability_mismatch:"), "{why}"); +} + +/// `wai_int_payload_declared` reads what an integer payload's header declares, before +/// any decode: a `WIV1`'s clip bytes and symbols, and, for keyframe 0, which runs no +/// network, its working bytes and no multiply-accumulates; a `WIH1`'s symbols, its +/// image size and its cost being its networks'; and refuses a payload that is not the +/// capability's format, or a capability with none. +#[test] +fn int_payload_declared_reads_the_header() { + let vec = |rel: &str| std::fs::read(std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors").join(rel)).unwrap(); + let declared = |cap: &str, p: &[u8]| { + let c = CString::new(cap).unwrap(); + let mut out = WaiIntDeclared::default(); + let rc = unsafe { wai_int_payload_declared(c.as_ptr(), p.as_ptr(), p.len(), std::ptr::null(), &mut out) }; + (rc, out, if rc == WAI_OK { String::new() } else { last_error() }) + }; + let k0 = vec("int_video/int_video_bitstream.wiv1"); + let v = wai::int_wire::parse_wiv1(&k0, Some(0)).unwrap(); + let clip = v.n_frames as u64 * v.h as u64 * v.w as u64 * 3; + let (rc, out, _) = declared("wai.video.int_motion", &k0); + assert_eq!((rc, out.output_bytes, out.output_known, out.symbols), (WAI_OK, clip, 1, clip)); + assert_eq!((out.cost_known, out.working_bytes, out.macs_lo, out.macs_hi), (1, v.keyframe0_working_bytes(), 0, 0)); + assert!(out.working_bytes > clip, "the clip, its table, its motion vectors and its frame list"); + let wih1 = vec("int_hyper/tiny_bitstream.wih1"); + let k = wai::int_wire::parse_wih1(&wih1).unwrap(); + let (rc, out, _) = declared("wai.neural.int_hyper", &wih1); + assert_eq!((rc, out.output_known, out.symbols), (WAI_OK, 0, (k.n_z * k.zh * k.zw + k.m * k.sh * k.sw) as u64)); + assert_eq!(out.cost_known, 0, "a WIH1's cost is its networks'"); + let (rc, _, why) = declared("wai.video.int_motion", &vec("int_video/int_video_bitstream.json")); + assert!(rc == WAI_ERR_INVALID_ARG && why.contains("(bad_magic:"), "{why}"); + let (rc, _, _) = declared("wai.image.png", &k0); + assert_eq!(rc, WAI_ERR_INVALID_ARG); +} diff --git a/wai/wai-rs/tests/int_kernels_vectors.rs b/wai/wai-rs/tests/int_kernels_vectors.rs index 797852c1d..586c4ba4a 100644 --- a/wai/wai-rs/tests/int_kernels_vectors.rs +++ b/wai/wai-rs/tests/int_kernels_vectors.rs @@ -222,6 +222,7 @@ fn every_decoded_output_states_a_sha256() { ("pin-conformance", "parameter-set pins: every digest there is a SHA-256 of a parameter set"), ("signer-conformance", "key logs, grants and statuses: signed documents, no decoded output"), ("byte-exact-conformance", "goldens over tests/vectors, whose sidecars state the SHA-256"), + ("int-payload-conformance", "refusal fixtures and the index of the payloads under tests/vectors, whose sidecars state the SHA-256"), ("vectors/int_mlicpp", "expected.json states rgb_sha256 (RGB8)"), ("vectors/jpegai", "inputs only; the decoded pixels' SHA-256 is pinned by the JPEG AI e2e job"), ]; diff --git a/wai/wai-rs/tests/int_payload.rs b/wai/wai-rs/tests/int_payload.rs new file mode 100644 index 000000000..beda78f74 --- /dev/null +++ b/wai/wai-rs/tests/int_payload.rs @@ -0,0 +1,139 @@ +//! The payloads of the WAI-native integer codecs (`extensions/integer-payloads.md`) +//! against the committed corpus `wai/int-payload-conformance/`: +//! +//! - every refusal fixture is refused with the code its index states, at the layer it +//! states (parsing alone, or the decode); +//! - every strict prefix of every committed payload is refused, and every changed byte +//! in a sweep over each payload is refused or decoded — never a panic, in either +//! profile (CI runs this file with overflow checks on and off); +//! - a tool's JSON output carried as an envelope's payload under each pinned string is +//! refused (`bad_magic`) by the framing check `wai wrap` runs and by `decode_int`. +#![cfg(all(feature = "video_int", feature = "audio_int", feature = "splat_int"))] + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use serde_json::Value; + +use wai::container::{Conditioning, Manifest, ModelRequirement, Rendition, Wai, WaiMulti}; +use wai::int_payload::{decode_int, wire, IntParams, Refusal}; +use wai::pin::HeldBytes; + +fn wai_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..") +} +fn read(rel: &str) -> Vec { + let p = wai_dir().join(rel); + std::fs::read(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())) +} +fn index() -> Value { + serde_json::from_slice(&read("int-payload-conformance/vectors.json")).expect("the index parses") +} + +/// The set an index entry names, read: member name → bytes. +fn set_of(entry: &Value) -> BTreeMap> { + entry["set"].as_object().into_iter().flatten().map(|(k, v)| (k.clone(), read(v.as_str().unwrap()))).collect() +} + +fn decode(cap: &str, payload: &[u8], set: &BTreeMap>) -> Result { + let view: BTreeMap = set.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect(); + let params = wai::int_payload::resolve(cap, None, &[HeldBytes::Set { listing: b"", files: &view }], None) + .unwrap_or_else(|_| IntParams::none()); + decode_int(cap, payload, ¶ms) +} + +fn parse_err(cap: &str, p: &[u8]) -> Option { + match wai::codecs::pinned_int_payload(cap)? { + wire::MAGIC_WIV1 => wire::parse_wiv1(p, Some((cap == wai::codecs::CAP_VIDEO_INT_HYPER) as u8)).err(), + wire::MAGIC_WIH1 => wire::parse_wih1(p).err(), + wire::MAGIC_WIA1 => wire::parse_wia1(p).err(), + _ => wire::parse_wis1(p).err(), + } +} + +/// Every fixture is refused with its code, at its layer; and every code a payload's +/// bytes can cause has a fixture. +#[test] +fn every_refusal_fixture_is_refused_with_its_code() { + let idx = index(); + let mut seen = std::collections::BTreeSet::new(); + for f in idx["refusals"].as_array().unwrap() { + let (name, cap, code) = (f["name"].as_str().unwrap(), f["capability"].as_str().unwrap(), f["code"].as_str().unwrap()); + let payload = read(f["payload"].as_str().unwrap()); + let parsed = parse_err(cap, &payload); + match f["layer"].as_str().unwrap() { + "parse" => assert_eq!(parsed.map(|e| e.code()), Some(code), "{name}: refused by parsing alone"), + _ => assert_eq!(parsed, None, "{name}: parses, and is refused by the decode"), + } + let r = decode(cap, &payload, &set_of(f)); + assert_eq!(r.as_ref().map(|_| ()).map_err(|e| e.code()), Err(code), "{name}"); + seen.insert(code); + } + for code in ["bad_magic", "format_capability_mismatch", "keyframe_capability_mismatch", "version", "reserved_nonzero", + "field_range", "too_large", "cdf_malformed", "stream_length", "nested_length", "trailing", + "stream_exhausted", "stream_noncanonical", "geometry_mismatch", "model_mismatch"] { + assert!(Refusal::from_code(code).is_some(), "{code} is a code"); + assert!(seen.contains(code), "no fixture is refused with {code}"); + } +} + +/// Every strict prefix of every committed payload is refused; a sweep of single-byte +/// changes over each is refused or decoded, and nothing panics. The sweep changes +/// every byte of the fixed header and 96 bytes spread evenly over the rest (low bit +/// and high bit by turns), so each payload's tables, length fields and stream are +/// all reached while the decodes stay few enough for a build with overflow checks. +#[test] +fn every_prefix_and_byte_change_is_refused_or_decoded() { + let idx = index(); + for v in idx["vectors"].as_array().unwrap() { + let (name, cap) = (v["name"].as_str().unwrap(), v["capability"].as_str().unwrap()); + let payload = read(v["payload"].as_str().unwrap()); + let set = set_of(v); + assert!(decode(cap, &payload, &set).is_ok(), "{name}: the committed payload decodes"); + for n in 0..payload.len() { + assert!(parse_err(cap, &payload[..n]).is_some(), "{name}: the {n}-byte prefix parses"); + } + let (mut refused, mut decoded) = (0, 0); + const HEADER: usize = 48; + let stride = (payload.len() - HEADER).div_ceil(96).max(1); + for (n, at) in (0..payload.len()).filter(|&i| i < HEADER || (i - HEADER) % stride == 0).enumerate() { + let mut p = payload.clone(); + p[at] ^= if n % 2 == 0 { 0x01 } else { 0x80 }; + match decode(cap, &p, &set) { + Ok(_) => decoded += 1, + Err(_) => refused += 1, + } + } + assert!(refused > 0, "{name}: no change was refused"); + println!("{name}: {refused} changes refused, {decoded} decoded, no panic"); + } +} + +/// A tool's JSON output wrapped as the payload under each pinned string: the framing +/// check (`Wai::check_int_payload_framing`, `WaiMulti::check_int_payload_framing`, +/// which `wai wrap` and `wai inspect` run) and `decode_int` both refuse it as +/// `bad_magic`; its binary conversion passes both. +#[test] +fn a_json_form_in_an_envelope_is_refused() { + use wai::codecs as c; + for (cap, json, bin) in [ + (c::CAP_VIDEO_INT_MOTION, "wai-rs/tests/vectors/int_video/int_video_bitstream.json", "wai-rs/tests/vectors/int_video/int_video_bitstream.wiv1"), + (c::CAP_VIDEO_INT_HYPER, "wai-rs/tests/vectors/int_video/video_nk_bitstream.json", "wai-rs/tests/vectors/int_video/video_nk_bitstream.wiv1"), + (c::CAP_NEURAL_INT_HYPER, "wai-rs/tests/vectors/int_hyper/tiny_bitstream.json", "wai-rs/tests/vectors/int_hyper/tiny_bitstream.wih1"), + (c::CAP_AUDIO_INT_CODEC, "wai-rs/tests/vectors/int_audio/audio_bitstream.json", "wai-rs/tests/vectors/int_audio/audio_bitstream.wia1"), + (c::CAP_SPLAT_INT_CODEC, "wai-rs/tests/vectors/int_splat/splat.json", "wai-rs/tests/vectors/int_splat/splat.wis1"), + ] { + let wrap = |payload: Vec| Wai::new(Manifest::new("binary", "improve", ModelRequirement::new(cap), Conditioning::new("bytes")), payload); + let env = Wai::from_bytes(&wrap(read(json)).to_bytes().unwrap()).unwrap(); + assert_eq!(env.check_int_payload_framing().map_err(|e| e.code), Err("bad_magic"), "{cap}"); + assert_eq!(decode_int(cap, &env.payload, &IntParams::none()).map(|_| ()).map_err(|e| e.code()), Err("bad_magic"), "{cap}"); + assert_eq!(wrap(read(bin)).check_int_payload_framing(), Ok(()), "{cap}: the binary form"); + let multi = WaiMulti::new("binary", "improve", Value::Null, vec![Rendition::media(cap, "bytes", read(json))]); + assert_eq!(multi.check_int_payload_framing().map_err(|(i, e)| (i, e.code)), Err((0, "bad_magic")), "{cap}"); + } + // The fallback decodes the same payload (SPEC §4 step 5), so its framing is checked too. + let env = Wai::new( + Manifest::new("video", "improve", ModelRequirement::new(c::CAP_VIDEO_AV1).with_fallback(c::CAP_VIDEO_INT_MOTION), Conditioning::new("bytes")), + read("wai-rs/tests/vectors/int_video/int_video_bitstream.json")); + assert_eq!(env.check_int_payload_framing().map_err(|e| e.code), Err("bad_magic")); +} diff --git a/wai/wai-rs/tests/int_payload_cost.rs b/wai/wai-rs/tests/int_payload_cost.rs new file mode 100644 index 000000000..27494e371 --- /dev/null +++ b/wai/wai-rs/tests/int_payload_cost.rs @@ -0,0 +1,177 @@ +//! The declared cost of an integer decode (`int_payload::declared_cost`, +//! `extensions/integer-payloads.md` §7, *Memory and work*) against what the decode +//! holds, measured with a global allocator that tracks the bytes live and their peak. +//! +//! A sink reads the declared cost before it decodes and applies its own size policy, +//! so the cost must be one it can rely on: the decode, on the reference kernels, holds +//! at least the output and at most the declared working bytes beside the payload and +//! the parameter set as parsed. For a `WIV1` with keyframe 0, which runs no network, +//! the declared working bytes are what the decode holds, to within a few bytes. +//! +//! The clip a few kilobytes can code: sixteen frames of 4729 × 4729 whose stream is +//! one symbol of frequency 65535/65536 over and over, about 3 KB, decoding to a clip at +//! the 2^30-byte cap. It is decoded in release builds only (CI's release step); a debug +//! build decodes a smaller clip of the same kind. +//! +//! Its own test binary with a single test, so that no other test's allocations land in +//! the window it measures. +#![cfg(all(feature = "video_int", feature = "audio_int", feature = "splat_int"))] + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; + +use wai::int_kernels::{with_policy, Policy}; +use wai::int_payload::{declared_cost, decode_int, resolve, wire, IntParams}; +use wai::pin::HeldBytes; + +/// The system allocator, tracking the bytes live and the most live at once. +struct Live; +static CUR: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +unsafe impl GlobalAlloc for Live { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + PEAK.fetch_max(CUR.fetch_add(l.size(), Relaxed) + l.size(), Relaxed); + unsafe { System.alloc(l) } + } + unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 { + PEAK.fetch_max(CUR.fetch_add(l.size(), Relaxed) + l.size(), Relaxed); + unsafe { System.alloc_zeroed(l) } + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { + if new >= l.size() { + PEAK.fetch_max(CUR.fetch_add(new - l.size(), Relaxed) + (new - l.size()), Relaxed); + } else { + CUR.fetch_sub(l.size() - new, Relaxed); + } + unsafe { System.realloc(p, l, new) } + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + CUR.fetch_sub(l.size(), Relaxed); + unsafe { System.dealloc(p, l) } + } +} + +#[global_allocator] +static ALLOC: Live = Live; + +/// The most bytes `f` holds at once beyond what was live when it started, and its result +/// (still held when the measure is taken). +fn peak_during(f: impl FnOnce() -> R) -> (R, u64) { + let base = CUR.load(Relaxed); + PEAK.store(base, Relaxed); + let r = f(); + (r, (PEAK.load(Relaxed) - base) as u64) +} + +fn read(rel: &str) -> Vec { + let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/vectors").join(rel); + std::fs::read(&p).unwrap_or_else(|e| panic!("{}: {e}", p.display())) +} + +/// A rANS stream of `n` symbols, symbol `i` coded with `(start, freq)` from `sym(i)`, +/// from the state 2^31, so it ends canonically: what an encoder writes. +fn encode(n: u64, sym: impl Fn(u64) -> (u64, u64)) -> Vec { + const L: u64 = 1 << 31; + let mut state = L; + let mut renorm: Vec = Vec::new(); + let mut i = n; + while i > 0 { + i -= 1; + let (start, freq) = sym(i); + let x_max = ((L >> 16) << 32) * freq; + while state >= x_max { + renorm.push(state as u32); + state >>= 32; + } + state = ((state / freq) << 16) + (state % freq) + start; + } + renorm.reverse(); + let mut words = vec![state as u32, (state >> 32) as u32]; + words.extend(renorm); + words.iter().flat_map(|w| w.to_le_bytes()).collect() +} + +/// A keyframe-0 clip of `n_frames` frames of `h × w`, one block per frame, whose every +/// residual is 0 at frequency 65535/65536: a valid stream of a few bytes per 2^20 +/// symbols. +fn constant_clip(n_frames: u32, h: u32, w: u32) -> Vec { + let rans = encode(n_frames as u64 * h as u64 * w as u64 * 3, |_| (0, 65_535)); + wire::write_wiv1(&wire::Wiv1 { + mv_precision: 1, keyframe: 0, n_frames, h, w, block: h.max(w), q: 1, offset: 0, cdf: vec![0, 65_535, 65_536], + mvs: vec![(0, 0); n_frames as usize - 1], rans: &rans, wih1: None, + }) +} + +#[test] +fn the_declared_cost_bounds_what_the_decode_holds() { + use wai::codecs as c; + let hyp: BTreeMap> = [("entropy.json", "int_hyper/tiny_entropy.json"), ("g_s.whm", "int_hyper/tiny_g_s.bin"), ("h_s.whm", "int_hyper/tiny_h_s.bin")] + .into_iter().map(|(k, v)| (k.to_string(), read(v))).collect(); + let hyp_view: BTreeMap = hyp.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect(); + let (model, gs) = (read("int_audio/audio_model.bin"), read("int_splat/splat_gs.bin")); + let unpinned = |cap, held| resolve(cap, None, &[held], None).expect("an unpinned set resolves"); + let none = IntParams::none(); + let hyp_p = unpinned(c::CAP_NEURAL_INT_HYPER, HeldBytes::Set { listing: b"", files: &hyp_view }); + let (aud, spl) = (unpinned(c::CAP_AUDIO_INT_CODEC, HeldBytes::File(&model)), unpinned(c::CAP_SPLAT_INT_CODEC, HeldBytes::File(&gs))); + // The parameter set as a decode parses it, which the declared cost leaves out. + let set_bytes = |cap: &str| -> u64 { + let (r, peak) = peak_during(|| match cap { + c::CAP_NEURAL_INT_HYPER | c::CAP_VIDEO_INT_HYPER => { + let e: serde_json::Value = serde_json::from_slice(&hyp["entropy.json"]).unwrap(); + let t = wai::int_hyper_synth::CondEntropy::try_from_value(&e).unwrap(); + drop(e); + Box::new((wai::int_hyper_synth::IntModel::from_bytes(&hyp["g_s.whm"]).unwrap(), + wai::int_hyper_synth::IntModel::from_bytes(&hyp["h_s.whm"]).unwrap(), t)) as Box + } + c::CAP_AUDIO_INT_CODEC => Box::new(wai::int_audio::AudioModel::from_bytes(&model).unwrap()) as Box, + c::CAP_SPLAT_INT_CODEC => Box::new(wai::int_hyper_synth::IntModel::from_bytes(&gs).unwrap()) as Box, + _ => Box::new(()) as Box, + }); + drop(r); + peak + }; + let small_clip = constant_clip(4, 300, 200); + let mut cases: Vec<(&str, &str, Vec, &IntParams<'_>)> = vec![ + ("int_video", c::CAP_VIDEO_INT_MOTION, read("int_video/int_video_bitstream.wiv1"), &none), + ("video_real", c::CAP_VIDEO_INT_MOTION, read("int_video/video_real_bitstream.wiv1"), &none), + ("4 frames of 300 × 200, a constant stream", c::CAP_VIDEO_INT_MOTION, small_clip, &none), + ("video_nk", c::CAP_VIDEO_INT_HYPER, read("int_video/video_nk_bitstream.wiv1"), &hyp_p), + ("tiny", c::CAP_NEURAL_INT_HYPER, read("int_hyper/tiny_bitstream.wih1"), &hyp_p), + ("audio", c::CAP_AUDIO_INT_CODEC, read("int_audio/audio_bitstream.wia1"), &aud), + ("splat", c::CAP_SPLAT_INT_CODEC, read("int_splat/splat.wis1"), &spl), + ]; + if !cfg!(debug_assertions) { + let big = constant_clip(16, 4729, 4729); + assert!(big.len() < 4096, "the clip at the cap is coded in {} bytes", big.len()); + cases.push(("16 frames of 4729 × 4729 in about 3 KB", c::CAP_VIDEO_INT_MOTION, big, &none)); + } + for (what, cap, payload, params) in cases { + let cost = declared_cost(cap, &payload, params).unwrap_or_else(|e| panic!("{what}: {e}")); + // The header-only reading agrees where the header fixes the output. + let size = wai::container::declared_size(cap, &payload).unwrap().unwrap(); + assert_eq!(size.symbols, cost.symbols, "{what}: symbols"); + if let Some(o) = size.output_bytes { + assert_eq!(o, cost.output_bytes, "{what}: output bytes"); + } + if let (Some(w), Some(m)) = (size.working_bytes, size.macs) { + assert_eq!((w, m), (cost.working_bytes, cost.macs), "{what}: working bytes and multiply-accumulates"); + } + let set = set_bytes(cap); + let (r, peak) = with_policy(Policy::reference(), || peak_during(|| decode_int(cap, &payload, params))); + r.unwrap_or_else(|e| panic!("{what}: {e}")); + assert!(peak >= cost.output_bytes, "{what}: the decode held {peak} bytes, under its {} output bytes", cost.output_bytes); + assert!(peak <= cost.working_bytes + set, "{what}: the decode held {peak} bytes, over the declared {} and the set's {set}", + cost.working_bytes); + if cap == c::CAP_VIDEO_INT_MOTION { + assert!(cost.working_bytes - peak <= 4096, "{what}: declared {}, held {peak}", cost.working_bytes); + assert_eq!(cost.macs, 0, "{what}: a keyframe-0 clip runs no network"); + } else { + assert!(cost.macs > 0, "{what}: its networks run"); + } + println!("{what}: declared {} working bytes ({} output, {} symbols, {} multiply-accumulates); held {peak}", + cost.working_bytes, cost.output_bytes, cost.symbols, cost.macs); + } +} diff --git a/wai/wai-rs/tests/int_payload_preflight.rs b/wai/wai-rs/tests/int_payload_preflight.rs new file mode 100644 index 000000000..68c0d4951 --- /dev/null +++ b/wai/wai-rs/tests/int_payload_preflight.rs @@ -0,0 +1,243 @@ +//! An integer payload whose header asks for more than the caps allow, or more than its +//! parameter set can fit, is refused before the decode allocates anything sized from +//! it (`extensions/integer-payloads.md` §7). +//! +//! Each case below is a committed payload with header fields changed. The parsers cap +//! every tensor a header sizes (`too_large`) before reading a section, and the decode +//! derives each network's shapes from its layer geometry before it allocates a latent; +//! this file checks both directly with a global allocator that records the largest +//! single allocation, and asserts that refusing each payload allocates at once no more +//! than parsing its parameter set needs (each parsed weight is held as an `i64`, so at +//! most four times the set's bytes) or a few kilobytes, whichever is more: nothing +//! sized from the header. +//! +//! Its own test binary with a single test, so that no other test's allocations land in +//! the window it measures. +#![cfg(all(feature = "video_int", feature = "audio_int", feature = "splat_int"))] + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::collections::BTreeMap; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; + +use wai::int_payload::{decode_int, resolve, IntParams}; +use wai::pin::HeldBytes; + +/// The system allocator, recording the largest single request. +struct Largest; +static LARGEST: AtomicUsize = AtomicUsize::new(0); + +unsafe impl GlobalAlloc for Largest { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + LARGEST.fetch_max(l.size(), Relaxed); + unsafe { System.alloc(l) } + } + unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 { + LARGEST.fetch_max(l.size(), Relaxed); + unsafe { System.alloc_zeroed(l) } + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { + LARGEST.fetch_max(new, Relaxed); + unsafe { System.realloc(p, l, new) } + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + unsafe { System.dealloc(p, l) } + } +} + +#[global_allocator] +static ALLOC: Largest = Largest; + +/// The largest single allocation `f` makes, and its result. +fn largest_in(f: impl FnOnce() -> R) -> (R, usize) { + LARGEST.store(0, Relaxed); + let r = f(); + (r, LARGEST.load(Relaxed)) +} + +fn read(rel: &str) -> Vec { + let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/vectors").join(rel); + std::fs::read(&p).unwrap_or_else(|e| panic!("{}: {e}", p.display())) +} + +/// `p` with the u32 fields at the given offsets replaced. +fn with(p: &[u8], fields: &[(usize, u32)]) -> Vec { + let mut p = p.to_vec(); + for &(off, v) in fields { + p[off..off + 4].copy_from_slice(&v.to_le_bytes()); + } + p +} + +const FEW_KB: usize = 16 << 10; + +/// A rANS stream of `n` symbols, symbol `i` coded with `(start, freq)` from `sym(i)`, +/// from the state 2^31, so it ends canonically: what an encoder writes. +fn encode(n: u64, sym: impl Fn(u64) -> (u64, u64)) -> Vec { + const L: u64 = 1 << 31; + let mut state = L; + let mut renorm: Vec = Vec::new(); + let mut i = n; + while i > 0 { + i -= 1; + let (start, freq) = sym(i); + let x_max = ((L >> 16) << 32) * freq; + while state >= x_max { + renorm.push(state as u32); + state >>= 32; + } + state = ((state / freq) << 16) + (state % freq) + start; + } + renorm.reverse(); + let mut words = vec![state as u32, (state >> 32) as u32]; + words.extend(renorm); + words.iter().flat_map(|w| w.to_le_bytes()).collect() +} + +#[test] +fn a_header_past_the_caps_or_the_model_is_refused_before_anything_is_allocated() { + use wai::codecs as c; + let hyp: BTreeMap> = [("entropy.json", "int_hyper/tiny_entropy.json"), ("g_s.whm", "int_hyper/tiny_g_s.bin"), ("h_s.whm", "int_hyper/tiny_h_s.bin")] + .into_iter().map(|(k, v)| (k.to_string(), read(v))).collect(); + let hyp_view: BTreeMap = hyp.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect(); + let unpinned = |cap, held| resolve(cap, None, &[held], None).expect("an unpinned set resolves"); + let hyp = unpinned(c::CAP_NEURAL_INT_HYPER, HeldBytes::Set { listing: b"", files: &hyp_view }); + let (model, gs) = (read("int_audio/audio_model.bin"), read("int_splat/splat_gs.bin")); + let none = IntParams::none(); + let (aud, spl) = (unpinned(c::CAP_AUDIO_INT_CODEC, HeldBytes::File(&model)), unpinned(c::CAP_SPLAT_INT_CODEC, HeldBytes::File(&gs))); + let (iv, nk, ih) = (read("int_video/int_video_bitstream.wiv1"), read("int_video/video_nk_bitstream.wiv1"), read("int_hyper/tiny_bitstream.wih1")); + let (au, sp) = (read("int_audio/audio_bitstream.wia1"), read("int_splat/splat.wis1")); + + // The counter sees a decode's allocations: the committed clip's largest is at least + // one 48 × 48 frame of RGB8. + let (r, largest) = largest_in(|| decode_int(c::CAP_VIDEO_INT_MOTION, &iv, &IntParams::none())); + r.expect("the committed clip decodes"); + assert!(largest >= 48 * 48 * 3, "the counter saw a {largest}-byte allocation at most"); + + let hyp_bytes: usize = hyp_view.values().map(|v| v.len()).sum(); + let cases: Vec<(&str, &str, Vec, &IntParams<'_>, &str, usize)> = vec![ + // A 2^20-frame clip of 48 × 48: each frame is within the cap, the clip is not. + ("a 2^20-frame clip", c::CAP_VIDEO_INT_MOTION, with(&iv, &[(8, 1 << 20)]), &none, "too_large", 0), + // An 8192 × 8192 frame. + ("an 8192 × 8192 frame", c::CAP_VIDEO_INT_HYPER, with(&nk, &[(12, 8192), (16, 8192)]), &hyp, "too_large", hyp_bytes), + // A hyper-latent of 2 × 6000 × 6000 = 72 million elements, past the parser's + // 2^26 cap. + ("a 72-million-element hyper-latent", c::CAP_NEURAL_INT_HYPER, with(&ih, &[(24, 6000), (28, 6000)]), &hyp, "too_large", hyp_bytes), + // A hyper-latent of 2 × 2000 × 2000, within the cap, that h_s would expand to + // 8 × 4000 × 4000, past it. + ("a hyper-latent h_s expands past the cap", c::CAP_NEURAL_INT_HYPER, with(&ih, &[(24, 2000), (28, 2000)]), &hyp, "too_large", hyp_bytes), + // An audio latent of 16 × 2^20, within the cap, that the synthesis would + // upsample past it. + ("an audio latent the synthesis expands past the cap", c::CAP_AUDIO_INT_CODEC, with(&au, &[(16, 1 << 20)]), &aud, "too_large", model.len()), + // A splat latent of 32 × 1024 × 1024, within the cap, that g_s would expand to + // 14 × 4096 × 4096, past it. + ("a splat latent g_s expands past the cap", c::CAP_SPLAT_INT_CODEC, with(&sp, &[(12, 1024), (16, 1024)]), &spl, "too_large", gs.len()), + ]; + let parsed = wai::int_payload::wire::parse_wih1(&with(&ih, &[(24, 6000), (28, 6000)])).map(|_| ()).map_err(|e| e.code()); + assert_eq!(parsed, Err("too_large"), "the parser's cap refuses the 72-million-element hyper-latent"); + for (what, cap, bad, params, code, set_bytes) in cases { + let (r, largest) = largest_in(|| decode_int(cap, &bad, params)); + let bound = FEW_KB.max(4 * set_bytes); + assert!(largest <= bound, "{what}: refusing it allocated {largest} bytes at once, over {bound} ({r:?})"); + assert_eq!(r.map(|_| ()).map_err(|e| e.code()), Err(code), "{what}"); + } + + // Headers every cap admits, whose streams are 8 bytes and cannot carry what the + // header declares (§2, §7): each is refused at the first symbol that runs its + // stream out, having allocated only what it decoded. Before this, the 16-frame + // clip of 4729 × 4729 ran for about a minute and reached gigabytes before it was + // refused at its last symbol. + use wai::int_payload::wire; + let (zeros, ones) = ([0u8; 8], [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f]); + let clip = |n_frames: u32, h: u32, w: u32, block: u32, cdf: Vec, rans: &[u8]| wire::write_wiv1(&wire::Wiv1 { + mv_precision: 1, keyframe: 0, n_frames, h, w, block, q: 1, offset: 0, cdf, + mvs: vec![(0, 0); n_frames as usize - 1], rans, wih1: None, + }); + let wih1 = { + // The tiny h_s doubles the hyper-latent's sides. + let mut k = wire::parse_wih1(&ih).unwrap(); + (k.zh, k.zw, k.sh, k.sw) = (64, 64, 128, 128); + k.z_rans = &ones[..]; + wire::write_wih1(&k) + }; + let wia1 = { + // The committed synthesis gives 16 samples per latent sample (512 → 8192), so + // t_out is what a latent of 4096 presents. + let mut a = wire::parse_wia1(&au).unwrap(); + a.lat_len = 1 << 12; + a.t_out = 16 << 12; + a.rans = &ones[..]; + wire::write_wia1(&a) + }; + let wis1 = { + // The splat g_s quadruples the latent's sides. + let mut v = wire::parse_wis1(&sp).unwrap(); + (v.sh, v.sw, v.s) = (64, 64, 256); + v.rans = &ones[..]; + wire::write_wis1(&v) + }; + let starved: Vec<(&str, &str, Vec, &IntParams<'_>, usize)> = vec![ + // Small enough that a decode which ran on past the first exhausted symbol would + // finish here quickly, and allocate a 256 × 256 frame. + ("16 frames of 256 × 256", c::CAP_VIDEO_INT_MOTION, clip(16, 256, 256, 1 << 31, vec![0, 1, 65_536], &zeros), &none, 0), + ("16 frames of 4729 × 4729", c::CAP_VIDEO_INT_MOTION, clip(16, 4729, 4729, 1 << 31, vec![0, 1, 65_536], &zeros), &none, 0), + ("one frame of 4096 × 5461", c::CAP_VIDEO_INT_MOTION, clip(1, 4096, 5461, 1, vec![0, 65_534, 65_535, 65_536], &ones), &none, 0), + ("a hyper-latent of 2 × 64 × 64", c::CAP_NEURAL_INT_HYPER, wih1, &hyp, hyp_bytes), + ("an audio latent of 16 × 4096", c::CAP_AUDIO_INT_CODEC, wia1, &aud, model.len()), + ("a splat latent of 32 × 64 × 64", c::CAP_SPLAT_INT_CODEC, wis1, &spl, gs.len()), + ]; + for (what, cap, bad, params, set_bytes) in starved { + assert!(wai::container::check_int_payload_framing(cap, &bad).is_ok(), "{what}: the payload parses"); + let (r, largest) = largest_in(|| decode_int(cap, &bad, params)); + let bound = FEW_KB.max(4 * set_bytes); + assert!(largest <= bound, "{what}: refusing it allocated {largest} bytes at once, over {bound} ({r:?})"); + assert_eq!(r.map(|_| ()).map_err(|e| e.code()), Err("stream_exhausted"), "{what}"); + } + + // A WIA1 of 384 bytes whose stream is fully valid (8.4 million symbols of frequency + // 65535/65536) for a latent of 16 × 524287 that presents one sample: its synthesis + // would hold gigabytes for that one sample. The latent is longer than t_out needs, + // so the plan refuses it (geometry_mismatch) before a symbol is decoded, and the + // declared cost refuses it alike. + let wia1 = { + let mut a = wire::parse_wia1(&au).unwrap(); + let n = a.n_ch as usize; + a.lat_len = 524_287; + a.t_out = 1; + a.offset = 0; + a.median_q = vec![0; n]; + a.cdfs = vec![vec![0, 65_535, 65_536]; n]; + let rans = encode(n as u64 * 524_287, |_| (0, 65_535)); + let mut b = a.clone(); + b.rans = &rans; + wire::write_wia1(&b) + }; + assert!(wia1.len() <= 400, "the payload is {} bytes", wia1.len()); + assert!(wai::container::check_int_payload_framing(c::CAP_AUDIO_INT_CODEC, &wia1).is_ok(), "it parses"); + let (r, largest) = largest_in(|| decode_int(c::CAP_AUDIO_INT_CODEC, &wia1, &aud)); + let bound = FEW_KB.max(4 * model.len()); + assert!(largest <= bound, "the 384-byte WIA1: refusing it allocated {largest} bytes at once, over {bound}"); + assert_eq!(r.map(|_| ()).map_err(|e| e.code()), Err("geometry_mismatch"), "the 384-byte WIA1"); + assert_eq!(wai::int_payload::declared_cost(c::CAP_AUDIO_INT_CODEC, &wia1, &aud).map(|_| ()).map_err(|e| e.code()), + Err("geometry_mismatch"), "the declared cost refuses it too"); + + // The tools' JSON form, read by int_video::VideoJson (the converter's and the JSON + // decode's reader): a header past the caps is refused before anything is sized + // from it, on every target. A side of 2^31 made the frame size overflow and the + // reader panic; sixteen frames of 4096 × 4096 at block 1, each with an empty list, + // were padded to 2^28 motion vectors before any cap was checked. + use serde_json::json; + let js = |side: u64, nf: usize| json!({ + "block": 1, "q": 1, "offset": 0, "cdf": [0, 1, 65_536], "n_frames": nf, "h": side, "w": side, + "mvs": vec![json!([]); nf], "rans_b64": "AAAAAAAAAAA=" + }); + for (what, j) in [("h = w = 2^31", js(1 << 31, 2)), ("2^28 padded motion vectors", js(4096, 16))] { + let (r, largest) = largest_in(|| std::panic::catch_unwind(|| ( + wai::int_video::VideoJson::from_value(&j).is_err(), + wai::int_payload::wiv1_from_json(&j, None).map(|_| ()).map_err(|e| e.code()), + wai::int_video::decode(&j).is_err(), + ))); + assert_eq!(r.expect("the JSON reader must not panic"), (true, Err("field_range"), true), "{what}"); + assert!(largest <= FEW_KB, "{what}: refusing it allocated {largest} bytes at once"); + } +} diff --git a/wai/wai-rs/tests/neural_cost_limit.rs b/wai/wai-rs/tests/neural_cost_limit.rs new file mode 100644 index 000000000..4450594bd --- /dev/null +++ b/wai/wai-rs/tests/neural_cost_limit.rs @@ -0,0 +1,230 @@ +//! The reference neural sink applies the declared cost of an integer payload before it +//! decodes a symbol of it (`extensions/integer-payloads.md` §7, *Memory and work*; +//! `cost_over_limit`, §8), and its C ABI does not decode an integer output it has no +//! kind for. Measured with a global allocator that tracks the bytes live and their +//! peak, so "before decoding" is what the allocator sees: nothing sized from the +//! header is allocated. +//! +//! The payload is a clip a few kilobytes can code: sixteen frames whose stream is one +//! symbol of frequency 65535/65536 over and over. At 4729 × 4729 it decodes to a clip at +//! the 2^30-byte cap (release builds); a debug build uses 1024 × 1024 frames, 48 MiB. +//! +//! The tests share the allocator's counters, so they run one at a time. +#![cfg(all(feature = "neural", feature = "ffi", feature = "video_int"))] + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::collections::BTreeMap; +use std::ffi::{CStr, CString}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; +use std::sync::Mutex; + +use wai::codecs::{CAP_NEURAL_INT_HYPER, CAP_VIDEO_INT_MOTION}; +use wai::container::{Conditioning, Manifest, ModelRequirement, Rendition, Wai, WaiMulti}; +use wai::ffi::{wai_int_payload_declared, wai_last_error, WaiBuffer, WaiIntDeclared, WAI_OK}; +use wai::ffi_neural::{wai_neural_decode_ex, wai_neural_decode_video, WaiDecoded}; +use wai::int_payload::{wire, DeclaredCost, IntDecoded}; +use wai::neural::{decode_envelope, decode_multi, DecodeError, Decoded, ModelRegistry}; + +/// The system allocator, tracking the bytes live and the most live at once. +struct Live; +static CUR: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +unsafe impl GlobalAlloc for Live { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + PEAK.fetch_max(CUR.fetch_add(l.size(), Relaxed) + l.size(), Relaxed); + unsafe { System.alloc(l) } + } + unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 { + PEAK.fetch_max(CUR.fetch_add(l.size(), Relaxed) + l.size(), Relaxed); + unsafe { System.alloc_zeroed(l) } + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { + if new >= l.size() { + PEAK.fetch_max(CUR.fetch_add(new - l.size(), Relaxed) + (new - l.size()), Relaxed); + } else { + CUR.fetch_sub(l.size() - new, Relaxed); + } + unsafe { System.realloc(p, l, new) } + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + CUR.fetch_sub(l.size(), Relaxed); + unsafe { System.dealloc(p, l) } + } +} + +#[global_allocator] +static ALLOC: Live = Live; + +static SERIAL: Mutex<()> = Mutex::new(()); + +/// The most bytes `f` holds at once beyond what was live when it started. +fn peak_during(f: impl FnOnce() -> R) -> (R, usize) { + let base = CUR.load(Relaxed); + PEAK.store(base, Relaxed); + let r = f(); + (r, PEAK.load(Relaxed) - base) +} + +/// Refusing before decoding allocates no more than this: the parsed table, a few +/// motion vectors, the refusal's message. +const SMALL: usize = 64 << 10; + +fn vectors() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/vectors") +} + +fn read(rel: &str) -> Vec { + std::fs::read(vectors().join(rel)).unwrap() +} + +fn last_error() -> String { + let p = wai_last_error(); + if p.is_null() { String::new() } else { unsafe { CStr::from_ptr(p) }.to_string_lossy().into_owned() } +} + +/// A rANS stream of `n` symbols, each coded with `(start, freq)`, from the state 2^31, +/// so it ends canonically: what an encoder writes. +fn encode(n: u64, (start, freq): (u64, u64)) -> Vec { + const L: u64 = 1 << 31; + let mut state = L; + let mut renorm: Vec = Vec::new(); + for _ in 0..n { + let x_max = ((L >> 16) << 32) * freq; + while state >= x_max { + renorm.push(state as u32); + state >>= 32; + } + state = ((state / freq) << 16) + (state % freq) + start; + } + renorm.reverse(); + let mut words = vec![state as u32, (state >> 32) as u32]; + words.extend(renorm); + words.iter().flat_map(|w| w.to_le_bytes()).collect() +} + +/// A keyframe-0 clip of sixteen frames whose every residual is 0 at frequency +/// 65535/65536: a valid stream of a few bytes per 2^20 symbols. +fn bomb() -> Vec { + let side = if cfg!(debug_assertions) { 1024 } else { 4729 }; + let n = 16u32; + let rans = encode(n as u64 * side as u64 * side as u64 * 3, (0, 65_535)); + wire::write_wiv1(&wire::Wiv1 { + mv_precision: 1, keyframe: 0, n_frames: n, h: side, w: side, block: side, q: 1, offset: 0, cdf: vec![0, 65_535, 65_536], + mvs: vec![(0, 0); n as usize - 1], rans: &rans, wih1: None, + }) +} + +fn video(cap: &str, payload: Vec) -> Wai { + Wai::new(Manifest::new("video", "improve", ModelRequirement::new(cap), Conditioning::new("bytes")), payload) +} + +/// A registry with a cost limit refuses a payload over it before decoding anything +/// (`cost_over_limit`), and dispatch moves on: a `WAI2` decodes its next entry. The +/// cost it applied is `neural::declared_cost` over the parameters it resolves. +#[test] +fn a_payload_over_the_cost_limit_is_refused_before_any_symbol_is_decoded() { + let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let bomb = bomb(); + assert!(bomb.len() < 4096, "the clip is coded in {} bytes", bomb.len()); + let limit = DeclaredCost::new(16 << 20, u64::MAX, u64::MAX, u128::MAX); + let reg = ModelRegistry::new().builtin(CAP_VIDEO_INT_MOTION).with_cost_limit(limit); + + let env = video(CAP_VIDEO_INT_MOTION, bomb.clone()); + let (r, peak) = peak_during(|| decode_envelope(&env, ®)); + match r { + Err(e @ DecodeError::OverCostLimit(_)) => assert!(e.to_string().starts_with("cost_over_limit: ") && e.to_string().contains("output bytes"), "{e}"), + Err(e) => panic!("expected cost_over_limit, got {e}"), + Ok(_) => panic!("expected cost_over_limit, the clip decoded"), + } + assert!(peak < SMALL, "refusing it held {peak} bytes"); + + let small = read("int_video/int_video_bitstream.wiv1"); + let n_frames = wire::parse_wiv1(&small, Some(0)).unwrap().n_frames as usize; + let m = WaiMulti::new("video", "improve", serde_json::Value::Null, + vec![Rendition::media(CAP_VIDEO_INT_MOTION, "bytes", bomb.clone()), Rendition::media(CAP_VIDEO_INT_MOTION, "bytes", small)]); + let (r, peak) = peak_during(|| decode_multi(&m, ®)); + match r { + Ok((1, Decoded::Int(IntDecoded::Video { frames, .. }))) => assert_eq!(frames.len(), n_frames), + Ok((i, _)) => panic!("decoded entry {i}"), + Err(e) => panic!("the entry within the limit must decode: {e}"), + } + assert!(peak < 1 << 20, "skipping the first entry and decoding the second held {peak} bytes"); + + let params = reg.resolve_verified(CAP_VIDEO_INT_MOTION, None).unwrap(); + let cost = wai::neural::declared_cost(CAP_VIDEO_INT_MOTION, &bomb, ¶ms).unwrap(); + assert_eq!(cost.exceeds(&limit), Some("output bytes")); + assert_eq!(cost.output_bytes, 16 * 3 * if cfg!(debug_assertions) { 1024 * 1024 } else { 4729 * 4729 }); +} + +/// The C ABI has no output kind for integer video, so it does not decode it: the +/// clip is refused before anything is allocated for it, by the registry entry point +/// and by the one-model video entry point. +#[test] +fn the_c_abi_refuses_an_integer_video_before_decoding_it() { + let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let env = video(CAP_VIDEO_INT_MOTION, bomb()).to_emit_bytes().unwrap(); + let reg = CString::new(r#"{"wai.video.int_motion": ["/nonexistent"]}"#).unwrap(); + let mut out = std::mem::MaybeUninit::::uninit(); + let (rc, peak) = peak_during(|| unsafe { wai_neural_decode_ex(env.as_ptr(), env.len(), reg.as_ptr(), 0, out.as_mut_ptr()) }); + let why = last_error(); + assert_ne!(rc, WAI_OK); + assert!(why.contains("no wai_decoded_t kind"), "{why}"); + assert!(peak < SMALL, "decode_ex held {peak} bytes refusing it"); + unsafe { wai::ffi::wai_buffer_free(out.assume_init().data) }; + + let model = CString::new("/nonexistent").unwrap(); + let mut frames = WaiBuffer { data: std::ptr::null_mut(), len: 0, cap: 0 }; + let null = std::ptr::null_mut(); + let (rc, peak) = peak_during(|| unsafe { wai_neural_decode_video(env.as_ptr(), env.len(), model.as_ptr(), &mut frames, null, null, null, null) }); + let why = last_error(); + assert_ne!(rc, WAI_OK); + assert!(why.contains("no wai_decoded_t kind"), "{why}"); + assert!(peak < SMALL, "decode_video held {peak} bytes refusing it"); +} + +/// `neural::declared_cost` over the set a registry resolves is the library's over the +/// same bytes, and so is the C ABI's given the set's directory: every figure, the +/// multiply-accumulates in two halves. A set without a member is `set_incomplete`. +#[test] +fn the_declared_cost_over_a_registrys_set_and_through_the_c_abi() { + let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner()); + let dir = std::env::temp_dir().join(format!("wai-cost-set-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let set: BTreeMap> = [("entropy.json", "int_hyper/tiny_entropy.json"), ("g_s.whm", "int_hyper/tiny_g_s.bin"), ("h_s.whm", "int_hyper/tiny_h_s.bin")] + .into_iter().map(|(k, f)| (k.to_string(), read(f))).collect(); + for (k, b) in &set { + std::fs::write(dir.join(k), b).unwrap(); + } + let payload = read("int_hyper/tiny_bitstream.wih1"); + let view: BTreeMap = set.iter().map(|(k, b)| (k.clone(), b.as_slice())).collect(); + let held = wai::int_payload::resolve(CAP_NEURAL_INT_HYPER, None, &[wai::pin::HeldBytes::Set { listing: b"", files: &view }], None).unwrap(); + let want = wai::int_payload::declared_cost(CAP_NEURAL_INT_HYPER, &payload, &held).unwrap(); + assert!(want.macs > 0); + + let reg = ModelRegistry::new().register(CAP_NEURAL_INT_HYPER, dir.clone()); + let params = reg.resolve_verified(CAP_NEURAL_INT_HYPER, None).unwrap(); + assert_eq!(wai::neural::declared_cost(CAP_NEURAL_INT_HYPER, &payload, ¶ms).unwrap(), want); + + let cap = CString::new(CAP_NEURAL_INT_HYPER).unwrap(); + let path = CString::new(dir.to_str().unwrap()).unwrap(); + let declared = |set: *const std::ffi::c_char| { + let mut out = WaiIntDeclared::default(); + let rc = unsafe { wai_int_payload_declared(cap.as_ptr(), payload.as_ptr(), payload.len(), set, &mut out) }; + (rc, out) + }; + let (rc, out) = declared(path.as_ptr()); + assert_eq!(rc, WAI_OK, "{}", last_error()); + assert_eq!((out.output_known, out.output_bytes, out.symbols, out.cost_known, out.working_bytes), + (1, want.output_bytes, want.symbols, 1, want.working_bytes)); + assert_eq!(((out.macs_hi as u128) << 64) | out.macs_lo as u128, want.macs); + let (rc, out) = declared(std::ptr::null()); + assert_eq!((rc, out.output_known, out.cost_known), (WAI_OK, 0, 0), "the header alone"); + + std::fs::remove_file(dir.join("h_s.whm")).unwrap(); + let (rc, _) = declared(path.as_ptr()); + assert_ne!(rc, WAI_OK); + assert!(last_error().contains("set_incomplete"), "{}", last_error()); + std::fs::remove_dir_all(&dir).ok(); +} diff --git a/wai/wai-rs/tests/neural_native_decode.rs b/wai/wai-rs/tests/neural_native_decode.rs index 425c33d8e..5ec8433c7 100644 --- a/wai/wai-rs/tests/neural_native_decode.rs +++ b/wai/wai-rs/tests/neural_native_decode.rs @@ -550,6 +550,70 @@ fn int_synth_pin_covers_the_spec_it_reads() { assert_eq!(refusal(decode_envelope(&env(&"ab".repeat(32)), ®)), "prior_not_held"); } +/// A key log stand-in that revokes the digests it holds. +#[derive(Debug)] +struct Revokes(Vec); + +impl wai::pin::PinAuthority for Revokes { + fn check(&self, cap: &str, prior: &Prior) -> Result<(), wai::container::PinRefusal> { + self.check_unpinned(cap, std::slice::from_ref(&prior.sha256)) + } + fn check_unpinned(&self, cap: &str, digests: &[String]) -> Result<(), wai::container::PinRefusal> { + if digests.iter().any(|d| self.0.contains(d)) { + return Err(wai::container::PinRefusal::Revoked { capability: cap.into(), revoked_by: None }); + } + Ok(()) + } +} + +/// `wai.neural.int_synth` registered as a model directory, or as the `int_synth.json` +/// inside it: the decode reads that one file, a form-less pin covers it, and check 5 +/// runs over it, so a key log that revokes the spec refuses the unpinned decode either +/// way (`prior_revoked`). Without the key log both registrations decode to the corpus +/// hash. The unpinned decode reads the bytes check 5 ran over: the spec replaced on +/// disk after the registry read it does not change the image. +#[test] +fn int_synth_check_5_covers_the_spec_a_model_directory_holds() { + let cap = "wai.neural.int_synth"; + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../neural-conformance/vectors/gradient"); + let spec = std::fs::read(src.join("int_synth.json")).unwrap(); + let latent = std::fs::read(src.join("latent.bin")).unwrap(); + let digest = wai::pin::sha256_hex(&spec); + let dir = std::env::temp_dir().join(format!("wai-int-synth-dir-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("int_synth.json"), &spec).unwrap(); + let env = |prior: Option| { + let mut cond = Conditioning::new("int_synth"); + if let Some(p) = prior { + cond = cond.with_prior(p); + } + Wai::new(Manifest::new("image", "improve", ModelRequirement::new(cap), cond), latent.clone()) + }; + let rgb_hash = |r: Result| match r { + Ok(Decoded::Image(img)) => { + let mut h = blake3::Hasher::new(); + h.update(b"wai:int-synth-rgb\x01"); + h.update(&img.rgb); + h.finalize().to_hex().to_string() + } + other => refusal(other), + }; + let want = "1282839b699ad3e79fc810a92ef4220fec7a57a573bb6c844709af4c1fb3f638"; + for path in [dir.clone(), dir.join("int_synth.json")] { + let revoking = ModelRegistry::new().register(cap, path.clone()).with_key_log(Revokes(vec![digest.clone()])); + assert_eq!(refusal(decode_envelope(&env(None), &revoking)), "prior_revoked", "{}", path.display()); + let plain = ModelRegistry::new().register(cap, path.clone()); + assert_eq!(rgb_hash(decode_envelope(&env(None), &plain)), want, "{}", path.display()); + let pin = Prior::new("gradient", digest.clone(), "int-synth/i64"); + assert_eq!(rgb_hash(decode_envelope(&env(Some(pin)), &plain)), want, "{}: a form-less pin", path.display()); + } + let reg = ModelRegistry::new().register(cap, dir.clone()).with_key_log(Revokes(vec![])); + let params = reg.resolve_verified(cap, None).expect("the unrevoked spec resolves"); + std::fs::copy(src.join("../bloom/int_synth.json"), dir.join("int_synth.json")).unwrap(); + assert_eq!(rgb_hash(wai::neural::decode_payload(cap, &latent, ¶ms)), want, "the decode read what check 5 ran over"); + std::fs::remove_dir_all(&dir).ok(); +} + /// For every several-file capability the reference sink dispatches, a form-less pin /// is refused with `form_required` before any file is read — even when its digest is /// the stand-in's own, and even for a capability registered as a directory. @@ -1082,3 +1146,485 @@ fn int_synth_trained_model_matches_integer_forward() { println!(" trained int_synth: Rust vs numpy integer-forward max|Δ|={maxd} (must be 0 — integer-exact)"); assert_eq!(maxd, 0, "Rust integer decode must be bit-identical to the reference"); } + +/// A temporary directory holding the tiny conditional codec's set under its member +/// names (`g_s.whm`, `h_s.whm`, `entropy.json`) and a listing, `set.sha256`, of the +/// members `listed`. Returns the directory and the listing's SHA-256. +fn int_hyper_set_dir(tag: &str, listed: &[&str]) -> (PathBuf, String) { + let v = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors/int_hyper"); + let dir = std::env::temp_dir().join(format!("wai-int-hyper-{tag}-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let mut files = std::collections::BTreeMap::new(); + for (member, src) in [("entropy.json", "tiny_entropy.json"), ("g_s.whm", "tiny_g_s.bin"), ("h_s.whm", "tiny_h_s.bin")] { + let b = std::fs::read(v.join(src)).unwrap(); + std::fs::write(dir.join(member), &b).unwrap(); + if listed.contains(&member) { + files.insert(member.to_string(), b); + } + } + let view: std::collections::BTreeMap = files.iter().map(|(k, v)| (k.clone(), v.as_slice())).collect(); + let listing = wai::pin::listing_from(&view).unwrap(); + std::fs::write(dir.join("set.sha256"), &listing).unwrap(); + (dir, wai::pin::sha256_hex(&listing)) +} + +/// `wai.neural.int_hyper` through the neural dispatch, which delegates it to +/// `int_payload::decode_int`: its `WIH1` payload decodes from the three-file set, pinned +/// through the set's listing (under the strict `replicate` gate) and unpinned, to the +/// committed image. A pinned set whose listing leaves out `h_s.whm` is refused with +/// `set_incomplete` naming it. The tool's JSON output carried as the payload is refused +/// (`bad_magic`), and the refusal is final: the fallback the envelope names is not tried. +/// An unpinned set without `h_s.whm` refuses a JSON form as `bad_magic` too, the payload +/// being checked before any member is read, and a `WIH1` as `set_incomplete`. +#[test] +fn int_hyper_decodes_through_the_neural_dispatch() { + use wai::codecs::{DeclaredTier, CAP_NEURAL_INT_HYPER as HYP, CAP_NEURAL_INT_SYNTH, CAP_PRIOR_SET}; + let v = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors/int_hyper"); + let payload = std::fs::read(v.join("tiny_bitstream.wih1")).unwrap(); + let rgb_sha = "dce7282d46f88f03d5dc0fd42f325756d310fe1fc982b6e1792a1fbfbeb482cf"; + let env = |intent: &str, prior: Option, fallback: Option<&str>, payload: Vec| { + let mut req = ModelRequirement::new(HYP); + if let Some(f) = fallback { + req = req.with_fallback(f); + } + let mut cond = Conditioning::new("int_hyper"); + if let Some(p) = prior { + cond = cond.with_prior(p); + } + Wai::new(Manifest::new("image", intent, req, cond), payload) + }; + let sha = |r: Result| match r { + Ok(Decoded::Image(img)) => wai::pin::sha256_hex(&img.rgb), + other => refusal(other), + }; + let (dir, digest) = int_hyper_set_dir("all", &["entropy.json", "g_s.whm", "h_s.whm"]); + let reg = ModelRegistry::new().register(HYP, dir.join("g_s.whm")); + let pin = Prior::new("tiny", digest, "decode-equivalence").with_form(CAP_PRIOR_SET).with_tier(DeclaredTier::DecodeEquivalence); + assert_eq!(sha(decode_envelope_strict(&env("replicate", Some(pin), None, payload.clone()), ®)), rgb_sha); + assert_eq!(sha(decode_envelope(&env("improve", None, None, payload.clone()), ®)), rgb_sha); + + let (part, digest) = int_hyper_set_dir("part", &["entropy.json", "g_s.whm"]); + let pin = Prior::new("tiny", digest, "decode-equivalence").with_form(CAP_PRIOR_SET); + let r = decode_envelope(&env("improve", Some(pin), None, payload.clone()), &ModelRegistry::new().register(HYP, part.join("g_s.whm"))); + assert!(matches!(r, Err(DecodeError::PriorMismatch(wai::container::PinRefusal::SetIncomplete { ref path })) if path == "h_s.whm"), "{}", refusal(r)); + + let json = std::fs::read(v.join("tiny_bitstream.json")).unwrap(); + let spec = Path::new(env!("CARGO_MANIFEST_DIR")).join("../neural-conformance/vectors/gradient/model"); + let both = ModelRegistry::new().register(HYP, dir.join("g_s.whm")).register(CAP_NEURAL_INT_SYNTH, spec); + for fallback in [None, Some(CAP_NEURAL_INT_SYNTH)] { + match decode_envelope(&env("improve", None, fallback, json.clone()), &both) { + Err(DecodeError::InvalidPayload(why)) => assert!(why.contains("bad_magic") && why.contains(HYP), "{why}"), + other => panic!("a JSON form must be refused as bad_magic, got {}", refusal(other)), + } + } + // An unpinned set that lacks a member: the payload is checked first, so a JSON form + // is still bad_magic, and a WIH1 is set_incomplete, naming the member, so dispatch + // moves on as it does for any decoder. + let (lack, _) = int_hyper_set_dir("lack", &["entropy.json", "g_s.whm", "h_s.whm"]); + std::fs::remove_file(lack.join("h_s.whm")).unwrap(); + let reg = ModelRegistry::new().register(HYP, lack.join("g_s.whm")); + match decode_envelope(&env("improve", None, None, json.clone()), ®) { + Err(DecodeError::InvalidPayload(why)) => assert!(why.contains("bad_magic"), "{why}"), + other => panic!("a JSON form must be refused as bad_magic, got {}", refusal(other)), + } + let r = decode_envelope(&env("improve", None, None, payload.clone()), ®); + assert!(matches!(r, Err(DecodeError::PriorMismatch(wai::container::PinRefusal::SetIncomplete { ref path })) if path == "h_s.whm"), "{}", refusal(r)); + std::fs::remove_dir_all(&dir).ok(); + std::fs::remove_dir_all(&part).ok(); + std::fs::remove_dir_all(&lack).ok(); +} + +/// The integer video, audio and splat capabilities through the neural dispatch: +/// `wai.video.int_motion`, which reads no parameter set, advertised with +/// `ModelRegistry::builtin`; `wai.audio.int_codec` and `wai.splat.int_codec`, each +/// registered as its one file, unpinned and under a form-less pin over that file. Each +/// decodes to `Decoded::Int` in the integer form its digest is taken over, with the +/// committed SHA-256. `builtin` does not advertise a capability that reads a set. +#[cfg(all(feature = "video_int", feature = "audio_int", feature = "splat_int"))] +#[test] +fn integer_media_decode_through_the_neural_dispatch() { + use wai::codecs::{CAP_AUDIO_INT_CODEC as AUD, CAP_NEURAL_INT_HYPER, CAP_SPLAT_INT_CODEC as SPL, CAP_VIDEO_INT_MOTION as MOT}; + use wai::int_payload::IntDecoded; + let v = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors"); + let rd = |rel: &str| std::fs::read(v.join(rel)).unwrap(); + let side = |rel: &str, key: &str, field: &str| -> String { + let j: serde_json::Value = serde_json::from_slice(&rd(rel)).unwrap(); + j[key][field].as_str().unwrap().to_string() + }; + let env = |cap: &str, prior: Option, payload: Vec| { + let mut cond = Conditioning::new("bytes"); + if let Some(p) = prior { + cond = cond.with_prior(p); + } + Wai::new(Manifest::new("binary", "improve", ModelRequirement::new(cap), cond), payload) + }; + let reg = ModelRegistry::new() + .builtin(MOT) + .register(AUD, v.join("int_audio/audio_model.bin")) + .register(SPL, v.join("int_splat/splat_gs.bin")); + assert!(reg.advertises(MOT)); + assert!(!ModelRegistry::new().builtin(CAP_NEURAL_INT_HYPER).advertises(CAP_NEURAL_INT_HYPER)); + + match decode_envelope(&env(MOT, None, rd("int_video/int_video_bitstream.wiv1")), ®) { + Ok(Decoded::Int(IntDecoded::Video { frames, width: 48, height: 48 })) => { + let per: Vec = frames.iter().flat_map(|f| wai::pin::sha256(f)).collect(); + assert_eq!(wai::pin::sha256_hex(&per), side("int_video/sha256.json", "int_video", "clip_sha256")); + } + other => panic!("int_motion: {}", refusal(other)), + } + let model = rd("int_audio/audio_model.bin"); + for prior in [None, Some(Prior::new("audio", wai::pin::sha256_hex(&model), "decode-equivalence"))] { + match decode_envelope(&env(AUD, prior, rd("int_audio/audio_bitstream.wia1")), ®) { + Ok(Decoded::Int(IntDecoded::Pcm { samples, sample_rate: 16_000, channels: 1 })) => { + let le: Vec = samples.iter().flat_map(|s| s.to_le_bytes()).collect(); + assert_eq!(wai::pin::sha256_hex(&le), side("int_audio/sha256.json", "audio", "sha256")); + } + other => panic!("int_codec: {}", refusal(other)), + } + } + let gs = rd("int_splat/splat_gs.bin"); + for prior in [None, Some(Prior::new("splat", wai::pin::sha256_hex(&gs), "decode-equivalence"))] { + match decode_envelope(&env(SPL, prior, rd("int_splat/splat.wis1")), ®) { + Ok(Decoded::Int(IntDecoded::Attr { bytes, n_attr: 14, s: 32, .. })) => { + assert_eq!(wai::pin::sha256_hex(&bytes), side("int_splat/sha256.json", "splat", "sha256")); + } + other => panic!("splat: {}", refusal(other)), + } + } +} + +/// An integer capability this build has no decoder for (here `wai.video.int_motion`, +/// without `video_int`) is refused with `unsupported_capability`, which is not final: +/// SPEC §4 does not take step 3 with a capability the sink cannot decode, so dispatch +/// moves on to the fallback, which decodes the payload. Without a fallback the +/// capability is reported as unknown. +#[cfg(not(feature = "video_int"))] +#[test] +fn an_integer_capability_without_a_decoder_moves_on_to_the_fallback() { + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("../neural-conformance/vectors/gradient"); + let latent = std::fs::read(src.join("latent.bin")).unwrap(); + let (mot, synth) = ("wai.video.int_motion", "wai.neural.int_synth"); + let reg = ModelRegistry::new().builtin(mot).register(synth, src.join("model")); + let env = |fallback: Option<&str>| { + let mut req = ModelRequirement::new(mot); + if let Some(f) = fallback { + req = req.with_fallback(f); + } + Wai::new(Manifest::new("image", "improve", req, Conditioning::new("int_synth")), latent.clone()) + }; + match decode_envelope(&env(Some(synth)), ®) { + Ok(Decoded::Image(img)) => assert_eq!((img.width, img.height), (38, 38)), + other => panic!("the fallback must decode, got {}", refusal(other)), + } + assert!(matches!(decode_envelope(&env(None), ®), Err(DecodeError::UnknownCapability(ref c)) if c == mot)); +} + +/// A key log stand-in that holds the capability's sets to §7.9 and grants only the +/// digests it holds: an unpinned set is authorised only under one of them. +#[derive(Debug)] +struct Grants(Vec); + +impl wai::pin::PinAuthority for Grants { + fn check(&self, cap: &str, prior: &Prior) -> Result<(), wai::container::PinRefusal> { + self.check_unpinned(cap, std::slice::from_ref(&prior.sha256)) + } + fn check_unpinned(&self, cap: &str, digests: &[String]) -> Result<(), wai::container::PinRefusal> { + if digests.iter().any(|d| self.0.contains(d)) { + return Ok(()); + } + Err(wai::container::PinRefusal::NotAuthorised { capability: cap.into(), check: None, reason: None }) + } +} + +/// A listing beside the registered path identifies a set only when it matches the +/// files the decode reads. A key log that grants only the publisher's listing refuses +/// an unpinned decode of other files (`prior_not_authorised`), with a copy of that +/// listing beside them or not; the publisher's own files beside the same listing +/// decode. The same holds for a one-file capability registered as its file, and for a +/// capability outside the integer path (resolved, not decoded: no model is needed). +#[test] +fn a_listing_that_does_not_match_the_files_read_authorises_nothing() { + use wai::codecs::{CAP_AUDIO_INT_CODEC as AUD, CAP_NEURAL_INT_HYPER as HYP}; + let code = |r: Result| match r { + Ok(_) => "decoded".to_string(), + Err(e) => refusal(Err(e)), + }; + let v = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors"); + let root = std::env::temp_dir().join(format!("wai-stale-listing-{}", std::process::id())); + let set: std::collections::BTreeMap> = [("entropy.json", "int_hyper/tiny_entropy.json"), ("g_s.whm", "int_hyper/tiny_g_s.bin"), ("h_s.whm", "int_hyper/tiny_h_s.bin")] + .into_iter().map(|(k, f)| (k.to_string(), std::fs::read(v.join(f)).unwrap())).collect(); + let view: std::collections::BTreeMap = set.iter().map(|(k, b)| (k.clone(), b.as_slice())).collect(); + let granted = wai::pin::listing_from(&view).unwrap(); + let write = |dir: &Path, files: &std::collections::BTreeMap>, listing: bool| { + std::fs::create_dir_all(dir).unwrap(); + for (k, b) in files { + std::fs::write(dir.join(k), b).unwrap(); + } + if listing { + std::fs::write(dir.join("publisher.sha256"), &granted).unwrap(); + } + }; + let mut other = set.clone(); + other.get_mut("entropy.json").unwrap().push(b' '); + let env = Wai::new(Manifest::new("image", "improve", ModelRequirement::new(HYP), Conditioning::new("bytes")), + std::fs::read(v.join("int_hyper/tiny_bitstream.wih1")).unwrap()); + let reg = |dir: &Path| ModelRegistry::new().register(HYP, dir.to_path_buf()).with_key_log(Grants(vec![wai::pin::sha256_hex(&granted)])); + for (name, files, listing, want) in [ + ("other files", &other, false, "prior_not_authorised"), + ("other files beside the granted listing", &other, true, "prior_not_authorised"), + ("the granted files beside their listing", &set, true, "decoded"), + ] { + let dir = root.join(name.replace(' ', "-")); + write(&dir, files, listing); + assert_eq!(code(decode_envelope(&env, ®(&dir))), want, "{name}"); + } + + // One file: the publisher's model.wam is granted under its listing; the sink's file + // differs in its last byte. + let model = std::fs::read(v.join("int_audio/audio_model.bin")).unwrap(); + let one: std::collections::BTreeMap = [("model.wam".to_string(), model.as_slice())].into_iter().collect(); + let pub_listing = wai::pin::listing_from(&one).unwrap(); + let mut tampered = model.clone(); + *tampered.last_mut().unwrap() ^= 1; + let dir = root.join("audio"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("model.wam"), &tampered).unwrap(); + std::fs::write(dir.join("publisher.sha256"), &pub_listing).unwrap(); + let envw = Wai::new(Manifest::new("audio", "improve", ModelRequirement::new(AUD), Conditioning::new("bytes")), + std::fs::read(v.join("int_audio/audio_bitstream.wia1")).unwrap()); + let reg = ModelRegistry::new().register(AUD, dir.join("model.wam")).with_key_log(Grants(vec![wai::pin::sha256_hex(&pub_listing)])); + assert_eq!(code(decode_envelope(&envw, ®)), "prior_not_authorised", "a one-file set beside a stale listing"); + + // A capability outside the integer path: the listing must match the files beside it + // and name the registered file. + let cap = "wai.neural.bmshj2018"; + let dir = root.join("onnx"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("decoder.onnx"), b"publisher").unwrap(); + let onnx: std::collections::BTreeMap = [("decoder.onnx".to_string(), &b"publisher"[..])].into_iter().collect(); + let l = wai::pin::listing_from(&onnx).unwrap(); + std::fs::write(dir.join("publisher.sha256"), &l).unwrap(); + let reg = || ModelRegistry::new().register(cap, dir.join("decoder.onnx")).with_key_log(Grants(vec![wai::pin::sha256_hex(&l)])); + assert!(reg().resolve_verified(cap, None).is_ok(), "the granted file beside its listing resolves"); + std::fs::write(dir.join("decoder.onnx"), b"other").unwrap(); + assert!(matches!(reg().resolve_verified(cap, None), Err(DecodeError::PriorMismatch(ref r)) if r.code() == "prior_not_authorised"), + "another file beside the granted listing is not authorised"); + std::fs::remove_dir_all(&root).ok(); +} + +/// Outside the integer path, a granted listing authorises the files it names and +/// nothing else: at a sink whose key log holds the capability's sets to §7.9, the +/// unpinned decode reads only the part §7.9 authorised of what it read once +/// (`ParamSource::Checked`), and a file the decode needs that the part does not name +/// is `set_incomplete`, never read from disk. A `wai.neural.bmshj2018` payload reads +/// its CDF tables (`cdfs_q3.json`) before it loads the decoder, so a decode that read +/// the unauthorised tables beside the decoder would fail parsing them instead. +/// +/// - the decoder registered as its file, beside a granted listing that names only +/// `decoder.onnx` and an unauthorised `cdfs_q3.json`: `set_incomplete`; +/// - the directory registered, beside a granted listing that names only `notes.txt`: +/// `set_incomplete`; +/// - a granted listing of both files: the decode reads the tables from the set; +/// - a one-file capability granted under its file's digest: that file alone; +/// - a key log that only revokes: every file of the decoder's directory, read once. +#[test] +fn a_granted_listing_authorises_only_the_files_it_names() { + use wai::neural::ParamSource; + use wai::pin::Verified; + let bm = "wai.neural.bmshj2018"; + let dir = std::env::temp_dir().join(format!("wai-granted-listing-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let files: [(&str, &[u8]); 3] = [("decoder.onnx", b"the publisher's decoder"), ("cdfs_q3.json", b"{\"unauthorised\": true}"), ("notes.txt", b"an unrelated file")]; + for (n, b) in files { + std::fs::write(dir.join(n), b).unwrap(); + } + let listing = |names: &[&str]| { + let set: std::collections::BTreeMap = + files.iter().filter(|(n, _)| names.contains(n)).map(|(n, b)| (n.to_string(), *b)).collect(); + let l = wai::pin::listing_from(&set).unwrap(); + std::fs::write(dir.join("granted.sha256"), &l).unwrap(); + wai::pin::sha256_hex(&l) + }; + // h 1, w 1, an empty stream, quality 3, one channel, a 1 × 1 latent. + let mut payload = Vec::new(); + for v in [1u32, 1, 0, 3] { + payload.extend_from_slice(&v.to_le_bytes()); + } + payload.extend_from_slice(&[1, 0]); + let env = Wai::new(Manifest::new("image", "improve", ModelRequirement::new(bm), Conditioning::new("bytes")), payload); + let names = |p: &ParamSource| match p { + ParamSource::Checked { set: Verified::Set { files, .. }, .. } => files.keys().cloned().collect::>().join(","), + ParamSource::Checked { set: Verified::File(_), .. } => "one file".into(), + ParamSource::Unpinned(_) => "the path".into(), + other => format!("{other:?}"), + }; + let case = |path: PathBuf, granted: &[&str]| { + let reg = ModelRegistry::new().register(bm, path).with_key_log(Grants(vec![listing(granted)])); + (names(®.resolve_verified(bm, None).unwrap()), refusal(decode_envelope(&env, ®))) + }; + let incomplete = |r: &str| r == "set_incomplete"; + + let (set, r) = case(dir.join("decoder.onnx"), &["decoder.onnx"]); + assert_eq!(set, "decoder.onnx"); + assert!(incomplete(&r), "a listing of the decoder alone does not authorise the tables beside it: {r}"); + let (set, r) = case(dir.clone(), &["notes.txt"]); + assert_eq!(set, "notes.txt"); + assert!(incomplete(&r), "a listing of another file does not authorise the directory: {r}"); + let (set, r) = case(dir.join("decoder.onnx"), &["cdfs_q3.json", "decoder.onnx"]); + assert_eq!(set, "cdfs_q3.json,decoder.onnx"); + assert!(r.contains("cdf parse cdfs_q3.json"), "the listed tables are read from the set: {r}"); + + // A one-file capability granted under its file's digest decodes from that file alone. + std::fs::remove_file(dir.join("granted.sha256")).unwrap(); + let enc = "wai.neural.encodec32"; + let digest = wai::pin::sha256_hex(files[0].1); + let reg = ModelRegistry::new().register(enc, dir.join("decoder.onnx")).with_key_log(Grants(vec![digest.clone()])); + match reg.resolve_verified(enc, None).unwrap() { + ParamSource::Checked { set: Verified::File(b), .. } => assert_eq!(b, files[0].1), + other => panic!("expected the authorised file, got {other:?}"), + } + // A key log that only revokes holds nothing to §7.9: the decode reads every file of + // the decoder's directory, read once. + let reg = ModelRegistry::new().register(bm, dir.join("decoder.onnx")).with_key_log(Revokes(vec!["00".repeat(32)])); + assert_eq!(names(®.resolve_verified(bm, None).unwrap()), "cdfs_q3.json,decoder.onnx,notes.txt"); + std::fs::remove_dir_all(&dir).ok(); +} + +/// At a sink whose key log only revokes, an unpinned decode outside the integer path +/// is checked over every digest a pin over what it reads could name, and reads those +/// bytes alone (SPEC §4, *Module authorisation*). With no listing beside the files, a +/// key log that revokes the listing of `{decoder.onnx, cdfs_q3.json}` still refuses +/// them (`prior_revoked`) through the listing derived from the directory, whether the +/// decoder or the directory is registered; one that revokes a member's own digest +/// refuses them too. The bytes were read once: tables swapped on disk after the check +/// are not the tables the decode reads. With no key log, the decode reads the path. +#[test] +fn a_revoke_only_key_log_checks_and_reads_the_set_once() { + use wai::neural::{decode_payload, ParamSource}; + use wai::pin::Verified; + let bm = "wai.neural.bmshj2018"; + let root = std::env::temp_dir().join(format!("wai-revoke-only-{}", std::process::id())); + let (decoder, cdfs_a, cdfs_b): (&[u8], &[u8], &[u8]) = (b"the publisher's decoder bytes", b"cdfs A: not JSON", b"{\"channels\": 1}"); + let mk = |name: &str| { + let d = root.join(name); + std::fs::create_dir_all(&d).unwrap(); + std::fs::write(d.join("decoder.onnx"), decoder).unwrap(); + std::fs::write(d.join("cdfs_q3.json"), cdfs_a).unwrap(); + d + }; + let set: std::collections::BTreeMap = + [("decoder.onnx".to_string(), decoder), ("cdfs_q3.json".to_string(), cdfs_a)].into_iter().collect(); + let listing = wai::pin::listing_from(&set).unwrap(); + let revoked = wai::pin::sha256_hex(&listing); + let mut payload = Vec::new(); + for v in [1u32, 1, 0, 3] { + payload.extend_from_slice(&v.to_le_bytes()); + } + payload.extend_from_slice(&[1, 0]); + let env = Wai::new(Manifest::new("image", "improve", ModelRequirement::new(bm), Conditioning::new("bytes")), payload.clone()); + + let r = mk("revoked"); + std::fs::write(r.join("set.sha256"), &listing).unwrap(); + let file_reg = || ModelRegistry::new().register(bm, r.join("decoder.onnx")); + assert_eq!(refusal(decode_envelope(&env, &file_reg().with_key_log(Revokes(vec![revoked.clone()])))), "prior_revoked", + "the revoked set beside its listing"); + std::fs::remove_file(r.join("set.sha256")).unwrap(); + assert_eq!(refusal(decode_envelope(&env, &file_reg().with_key_log(Revokes(vec![revoked.clone()])))), "prior_revoked", + "the decoder registered, no listing beside it"); + let dir_reg = ModelRegistry::new().register(bm, r.clone()).with_key_log(Revokes(vec![revoked.clone()])); + assert_eq!(refusal(decode_envelope(&env, &dir_reg)), "prior_revoked", "the directory registered, no listing beside it"); + let member = Revokes(vec![wai::pin::sha256_hex(cdfs_a)]); + assert_eq!(refusal(decode_envelope(&env, &file_reg().with_key_log(member))), "prior_revoked", "a member's own digest revoked"); + + // Read once: the tables swapped after the check are not read. The decode reads A + // (not JSON: "expected value"); B would parse and lack a field. + let t = mk("toctou"); + let reg = ModelRegistry::new().register(bm, t.join("decoder.onnx")).with_key_log(Revokes(vec![])); + let params = reg.resolve_verified(bm, None).unwrap(); + std::fs::write(t.join("cdfs_q3.json"), cdfs_b).unwrap(); + std::fs::write(t.join("cdfs_q4.json"), cdfs_b).unwrap(); + let got = refusal(decode_payload(bm, &payload, ¶ms)); + assert!(got.contains("cdf parse cdfs_q3.json: expected value"), "the decode read the tables checked: {got}"); + match ¶ms { + ParamSource::Checked { set: Verified::Set { files, .. }, model } => { + assert_eq!(files.keys().collect::>(), ["cdfs_q3.json", "decoder.onnx"]); + assert_eq!(model.as_deref(), Some("decoder.onnx")); + } + other => panic!("expected the files read once, got {other:?}"), + } + + // No key log: no check 5, and the decode reads the path, as it always has. + assert!(matches!(ModelRegistry::new().register(bm, t.join("decoder.onnx")).resolve_verified(bm, None), Ok(ParamSource::Unpinned(_)))); + std::fs::remove_dir_all(&root).ok(); +} + +/// The reference sink opens no encryption scheme, so an envelope whose manifest +/// declares one is inert at it (SPEC §6.4 step 2, `unsupported_encryption`): a +/// payload sent in the clear under a claim of encryption is not decoded as plaintext, +/// and a tool's JSON form under that claim is not read as a payload either. The same +/// payload without the claim decodes; a `WAI2` gives every entry the reason, also when +/// its `security` member is not a security object. +#[test] +fn an_encrypted_envelope_is_inert_at_the_reference_sink() { + use wai::codecs::CAP_NEURAL_INT_HYPER as HYP; + use wai::container::{Rendition, Security, WaiMulti}; + use wai::neural::decode_multi; + let v = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors"); + let dir = std::env::temp_dir().join(format!("wai-encrypted-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + for (k, f) in [("entropy.json", "int_hyper/tiny_entropy.json"), ("g_s.whm", "int_hyper/tiny_g_s.bin"), ("h_s.whm", "int_hyper/tiny_h_s.bin")] { + std::fs::copy(v.join(f), dir.join(k)).unwrap(); + } + let reg = ModelRegistry::new().register(HYP, dir.clone()); + let (wih1, json) = (std::fs::read(v.join("int_hyper/tiny_bitstream.wih1")).unwrap(), std::fs::read(v.join("int_hyper/tiny_bitstream.json")).unwrap()); + let env = |payload: &[u8], security: Option| { + let mut m = Manifest::new("image", "improve", ModelRequirement::new(HYP), Conditioning::new("bytes")); + m.security = security; + Wai::from_bytes(&Wai::new(m, payload.to_vec()).to_emit_bytes().unwrap()).unwrap() + }; + assert!(matches!(decode_envelope(&env(&wih1, None), ®), Ok(Decoded::Image(_))), "in the clear, it decodes"); + for payload in [&wih1, &json] { + match decode_envelope(&env(payload, Some(Security::new("aes-256-gcm", 7))), ®) { + Err(DecodeError::Encrypted(why)) => assert!(why.starts_with("unsupported_encryption: "), "{why}"), + other => panic!("an encrypted envelope must be inert, got {}", refusal(other)), + } + } + + let multi = |security: Option| { + let mut m = WaiMulti::new("image", "improve", serde_json::Value::Null, vec![Rendition::media(HYP, "bytes", wih1.clone())]); + if let Some(s) = security { + m.manifest.extra.insert("security".into(), s); + } + WaiMulti::from_bytes(&m.to_bytes().unwrap()).unwrap() + }; + assert!(matches!(decode_multi(&multi(None), ®), Ok((0, Decoded::Image(_))))); + for s in [serde_json::json!({"encryption": "mls-sframe", "kid": 0}), serde_json::json!(true)] { + match decode_multi(&multi(Some(s.clone())), ®) { + Err(DecodeError::NoRendition(skips)) => assert_eq!(skips, vec![(0, "unsupported_encryption".to_string())], "{s}"), + other => panic!("{s}: expected every entry inert, got {:?}", other.map(|(i, _)| i)), + } + } + std::fs::remove_dir_all(&dir).ok(); +} + +/// The neural registry's pinned path runs check 5 over every identity of the bytes it +/// verified: a one-file set whose file digest a key log revokes is refused through a +/// set-form pin over a listing of that file. +#[test] +fn a_pinned_set_revoked_under_its_file_digest_is_refused_through_a_listing_pin() { + use wai::codecs::{CAP_AUDIO_INT_CODEC as AUD, CAP_PRIOR_SET}; + let v = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors"); + let model = std::fs::read(v.join("int_audio/audio_model.bin")).unwrap(); + let dir = std::env::temp_dir().join(format!("wai-listing-pin-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("model.wam"), &model).unwrap(); + let one: std::collections::BTreeMap = [("model.wam".to_string(), model.as_slice())].into_iter().collect(); + let listing = wai::pin::listing_from(&one).unwrap(); + std::fs::write(dir.join("model.sha256"), &listing).unwrap(); + let pin = Prior::new("m", wai::pin::sha256_hex(&listing), "decode-equivalence").with_form(CAP_PRIOR_SET); + let plain = ModelRegistry::new().register(AUD, dir.join("model.wam")); + assert!(plain.verify(AUD, &pin).is_ok(), "the pin verifies"); + let revoking = ModelRegistry::new().register(AUD, dir.join("model.wam")).with_key_log(Revokes(vec![wai::pin::sha256_hex(&model)])); + assert_eq!(revoking.verify(AUD, &pin).map(|_| ()).map_err(|r| r.code()), Err("prior_revoked")); + std::fs::remove_dir_all(&dir).ok(); +} diff --git a/wai/wai-rs/tests/vectors/int_audio/audio_bitstream.wia1 b/wai/wai-rs/tests/vectors/int_audio/audio_bitstream.wia1 new file mode 100644 index 000000000..05d5af267 Binary files /dev/null and b/wai/wai-rs/tests/vectors/int_audio/audio_bitstream.wia1 differ diff --git a/wai/wai-rs/tests/vectors/int_hyper/tiny_bitstream.wih1 b/wai/wai-rs/tests/vectors/int_hyper/tiny_bitstream.wih1 new file mode 100644 index 000000000..f9004d337 Binary files /dev/null and b/wai/wai-rs/tests/vectors/int_hyper/tiny_bitstream.wih1 differ diff --git a/wai/wai-rs/tests/vectors/int_splat/splat.wis1 b/wai/wai-rs/tests/vectors/int_splat/splat.wis1 new file mode 100644 index 000000000..72a429a3d Binary files /dev/null and b/wai/wai-rs/tests/vectors/int_splat/splat.wis1 differ diff --git a/wai/wai-rs/tests/vectors/int_video/int_video_bitstream.wiv1 b/wai/wai-rs/tests/vectors/int_video/int_video_bitstream.wiv1 new file mode 100644 index 000000000..76b0ffddd Binary files /dev/null and b/wai/wai-rs/tests/vectors/int_video/int_video_bitstream.wiv1 differ diff --git a/wai/wai-rs/tests/vectors/int_video/video_nk_bitstream.wiv1 b/wai/wai-rs/tests/vectors/int_video/video_nk_bitstream.wiv1 new file mode 100644 index 000000000..f948456f9 Binary files /dev/null and b/wai/wai-rs/tests/vectors/int_video/video_nk_bitstream.wiv1 differ diff --git a/wai/wai-rs/tests/vectors/int_video/video_real_bitstream.wiv1 b/wai/wai-rs/tests/vectors/int_video/video_real_bitstream.wiv1 new file mode 100644 index 000000000..7775f68be Binary files /dev/null and b/wai/wai-rs/tests/vectors/int_video/video_real_bitstream.wiv1 differ diff --git a/wai/wai-rs/tests/video_receipt.rs b/wai/wai-rs/tests/video_receipt.rs index c8ce13d98..b697e6eab 100644 --- a/wai/wai-rs/tests/video_receipt.rs +++ b/wai/wai-rs/tests/video_receipt.rs @@ -5,6 +5,10 @@ //! attested by the signature alone. Each half is tested for what it actually //! promises, and the tamper cases confirm the checkable half cannot be bent. #![cfg(feature = "video_receipt")] +// Most tests here seal over the tool's JSON form through the deprecated entry points: +// they are the legacy receipts, whose pinned bytes must keep verifying. The tests of +// the WIV1 payload, the identity a receipt has now, are at the end. +#![allow(deprecated)] use std::path::PathBuf; @@ -277,3 +281,70 @@ fn refined_labels_keep_their_signed_bytes() { ] ); } + +/// A receipt sealed over the `WIV1` payload (`extensions/integer-payloads.md` §3): +/// its identity is BLAKE3 over the payload, it re-derives from the payload alone, +/// and every reconstruction field — frames, clip commitment, geometry, work — is the +/// one the JSON form's receipt carries, since both decode to the same frames. +#[test] +fn a_wiv1_receipt_identifies_the_payload_and_reconstructs_the_same_clip() { + for (bin, json) in [("int_video_bitstream.wiv1", "int_video_bitstream.json"), ("video_real_bitstream.wiv1", "video_real_bitstream.json")] { + let payload = raw(bin); + let r = VideoReceipt::seal_decode_wiv1(&key(7), "did:example:meter", &payload, 4_200, Some(chip()), None).expect("seal"); + assert!(r.verify(), "{bin}"); + assert!(r.verify_reconstruction_wiv1(&payload), "{bin}: re-decode must reproduce every frame"); + assert_eq!(r.codestream_hash, *blake3::hash(&payload).as_bytes(), "{bin}: the identity is the payload's"); + let (legacy, json_bytes, _) = seal(json, 4_200); + assert_eq!((&r.frames, r.clip_hash, r.h, r.w, r.work_pixel_ops), (&legacy.frames, legacy.clip_hash, legacy.h, legacy.w, legacy.work_pixel_ops), + "{bin}: the same reconstruction as the JSON form's receipt"); + assert_ne!(r.codestream_hash, legacy.codestream_hash); + // Neither receipt vouches for the other's codestream. + assert!(!r.verify_reconstruction_wiv1(&json_bytes), "{bin}: a JSON form is not a WIV1 payload"); + assert!(!legacy.verify_reconstruction_wiv1(&payload), "{bin}: the legacy receipt names the JSON bytes"); + let mut other = payload.clone(); + let at = other.len() - 5; + other[at] ^= 1; + assert!(!r.verify_reconstruction_wiv1(&other), "{bin}: one changed byte"); + } +} + +/// The signed bytes of receipts sealed over the `WIV1` payload, captured from the +/// code as this identity was introduced: a metered one with the on-chip class, an +/// unmetered one, and one chained to the first. Never edited to make a test pass. +#[test] +fn wiv1_receipts_keep_their_signed_bytes() { + let payload = raw("int_video_bitstream.wiv1"); + let metered = VideoReceipt::seal_decode_wiv1(&key(7), "did:example:meter", &payload, 4_200, Some(chip()), None).unwrap(); + let unmetered = VideoReceipt::seal_decode_wiv1(&key(7), "did:example:meter", &payload, 0, None, None).unwrap(); + let chained = VideoReceipt::seal_decode_wiv1(&key(7), "did:example:meter", &payload, 0, None, Some(metered.receipt_hash())).unwrap(); + assert!(metered.verify() && unmetered.verify() && chained.verify()); + let got = [ + hex(&metered.codestream_hash), + hex(&metered.receipt_hash()), + hex(blake3::hash(metered.to_json().as_bytes()).as_bytes()), + hex(&unmetered.receipt_hash()), + hex(&chained.receipt_hash()), + ]; + assert_eq!( + got, + [ + "277865b574029f700f4fac023de3ab06ae598ee6f8ed726444e91442a6bd59e9", + "52c0ca1c9b383b9ad83226d423b36ecafbef92bf431432c7426a484c0406a742", + "97a7cb3b2598f609d205159e2ef1b2d7fb2d35e429ee161a6a8fbbd8ffdcae52", + "f19f86c9106981096bdbb5ea06926e3543112c26a5f737732938feb51c2b6119", + "58ca49ab785116aec8d2f3b990722e962967cbca6b4d6fe41087343872bd1628", + ] + ); +} + +/// `video_receipt::decode_wiv1` reports a refusal by its code +/// (`extensions/integer-payloads.md` §8), the decode's as well as the parse's: the +/// refusal fixtures of a run-out stream and an unread word, and a JSON form. +#[test] +fn decode_wiv1_reports_refusal_codes() { + let fixture = |name: &str| std::fs::read(PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../int-payload-conformance/refusals").join(name)).unwrap(); + let code = |p: &[u8]| wai::video_receipt::decode_wiv1(p).map(|_| ()).unwrap_err(); + assert_eq!(code(&fixture("wiv1_exhausted.bin")), "stream_exhausted"); + assert_eq!(code(&fixture("wiv1_unread_word.bin")), "stream_noncanonical"); + assert_eq!(code(&raw("int_video_bitstream.json")), "bad_magic"); +} diff --git a/wai/wai-web/demo/int_hyper.html b/wai/wai-web/demo/int_hyper.html index 8350ca207..fcbf75333 100644 --- a/wai/wai-web/demo/int_hyper.html +++ b/wai/wai-web/demo/int_hyper.html @@ -24,8 +24,10 @@

WAI integer-exact conditional codec — in-browser, no ML runtime

- A wai.neural.int_hyper bitstream arrived: a factorized side-stream z and the - Gaussian-conditional main stream y, both rANS-coded. This tab decodes it to RGB with the + This tab decodes the export tool's debug form of a wai.neural.int_hyper bitstream + (cond_bitstream.json; the payload an envelope carries is WIH1, which + decodeWih1 reads): a factorized side-stream z and the + Gaussian-conditional main stream y, both rANS-coded. It decodes them to RGB with the same pure-i64 Rust that decodes natively, compiled to wasm32 — no onnxruntime-web, no WebGPU, no float anywhere. The whole path is integer: factorized z → integer hyper-decoder → integer-σ entropy bucketing → integer synthesis. Because of that (the integer-entropy diff --git a/wai/wai-web/demo/verify_int_payload_wasm.mjs b/wai/wai-web/demo/verify_int_payload_wasm.mjs new file mode 100644 index 000000000..8f161f85e --- /dev/null +++ b/wai/wai-web/demo/verify_int_payload_wasm.mjs @@ -0,0 +1,69 @@ +// The wasm32 decode of the integer payloads (extensions/integer-payloads.md) equals +// the native decode: each committed WIV1, WIH1, WIA1 and WIS1 payload, decoded by +// decodeWiv1 / decodeWih1 / decodeWia1 / decodeWis1 in the browser crate built for +// wasm32, must give the SHA-256 its vector's sha256.json sidecar states, and a +// tool's JSON form must be refused with bad_magic. +// +// node verify_int_payload_wasm.mjs [path/to/wai_web.js] [path/to/wai-rs/tests/vectors] +// +// Exit 0 iff every check passes. + +import { readFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const here = dirname(fileURLToPath(import.meta.url)); +const pkg = resolve(here, process.argv[2] ?? './pkg-node/wai_web.js'); +const vectors = resolve(process.argv[3] ?? resolve(here, '../../wai-rs/tests/vectors')); +const wai = await import(pathToFileURL(pkg).href); + +const rd = (rel) => new Uint8Array(readFileSync(`${vectors}/${rel}`)); +const side = (rel, key) => JSON.parse(readFileSync(`${vectors}/${rel}`, 'utf8'))[key]; +const sha = (b) => createHash('sha256').update(b).digest(); +const hex = (b) => Buffer.from(b).toString('hex'); +const clip = (v) => { + const n = v.width * v.height * 3, rgb = v.rgb, digests = []; + for (let f = 0; f < v.nFrames; f++) { + const one = v.frame(f); + if (one.length !== n || !one.every((b, i) => b === rgb[f * n + i])) throw new Error(`frame ${f} differs from rgb`); + digests.push(sha(one)); + } + return hex(sha(Buffer.concat(digests))); +}; +const [g, h, e] = [rd('int_hyper/tiny_g_s.bin'), rd('int_hyper/tiny_h_s.bin'), rd('int_hyper/tiny_entropy.json')]; + +const checks = [ + ['int_video (WIV1, keyframe 0)', () => clip(wai.decodeWiv1('wai.video.int_motion', rd('int_video/int_video_bitstream.wiv1'))), + side('int_video/sha256.json', 'int_video').clip_sha256], + ['video_real (WIV1, half-pel)', () => clip(wai.decodeWiv1('wai.video.int_motion', rd('int_video/video_real_bitstream.wiv1'))), + side('int_video/sha256.json', 'video_real').clip_sha256], + ['video_nk (WIV1, keyframe 1)', () => clip(wai.decodeWiv1('wai.video.int_hyper', rd('int_video/video_nk_bitstream.wiv1'), g, h, e)), + side('int_video/sha256.json', 'video_nk').clip_sha256], + ['tiny (WIH1)', () => hex(sha(wai.decodeWih1(rd('int_hyper/tiny_bitstream.wih1'), g, h, e).rgb)), + side('int_hyper/sha256.json', 'tiny').sha256], + ['audio (WIA1)', () => { + const a = wai.decodeWia1(rd('int_audio/audio_bitstream.wia1'), rd('int_audio/audio_model.bin')); + if (a.sampleRate !== 16000) return `sample rate ${a.sampleRate}`; + const pcm = a.pcm; + return hex(sha(Buffer.from(pcm.buffer, pcm.byteOffset, pcm.byteLength))); + }, side('int_audio/sha256.json', 'audio').sha256], + ['splat (WIS1)', () => { + const s = wai.decodeWis1(rd('int_splat/splat.wis1'), rd('int_splat/splat_gs.bin')); + if (s.ranges.length !== 2 * s.nAttr || s.nGaussians !== 1024) return 'ranges or Gaussian count'; + return hex(sha(s.attr)); + }, side('int_splat/sha256.json', 'splat').sha256], + ['a JSON form is refused', () => { + try { wai.decodeWiv1('wai.video.int_motion', rd('int_video/int_video_bitstream.json')); return 'decoded'; } + catch (err) { return String(err.message ?? err).split(':')[0]; } + }, 'bad_magic'], +]; + +let pass = true; +for (const [name, run, want] of checks) { + const got = run(); + const ok = got === want; + pass = pass && ok; + console.log(`${ok ? 'PASS' : 'FAIL'} ${name.padEnd(30)} ${String(got).slice(0, 16)}${ok ? '' : ` want ${want}`}`); +} +process.exit(pass ? 0 : 1); diff --git a/wai/wai-web/src/audio_int.rs b/wai/wai-web/src/audio_int.rs index c149738f4..fc846daad 100644 --- a/wai/wai-web/src/audio_int.rs +++ b/wai/wai-web/src/audio_int.rs @@ -15,6 +15,7 @@ use wasm_bindgen::prelude::*; #[wasm_bindgen] pub struct DecodedAudio { pcm: Vec, + sample_rate: u32, } #[wasm_bindgen] @@ -28,16 +29,35 @@ impl DecodedAudio { pub fn n_samples(&self) -> u32 { self.pcm.len() as u32 } + /// The sample rate the `WIA1` payload states, in hertz; `0` from the JSON form, + /// which carries none. + #[wasm_bindgen(getter, js_name = sampleRate)] + pub fn sample_rate(&self) -> u32 { + self.sample_rate + } } -/// Decode a learned-audio model (`audio_model.bin`, WAM1) + bitstream -/// (`audio_bitstream.json`) to int16 PCM in the browser, byte-identical to the -/// native reference. No float, no ML runtime. +/// Decode a `WIA1` payload (`wai.audio.int_codec`, `extensions/integer-payloads.md` +/// §5) to int16 PCM in the browser, byte-identical to the native reference, with its +/// one-file parameter set (`model.wam`, the `WAM1` synthesis). A refusal's message +/// starts with its code (`bad_magic` for a JSON form). No float, no ML runtime. +#[wasm_bindgen(js_name = decodeWia1)] +pub fn decode_wia1(payload: &[u8], model_wam: &[u8]) -> Result { + match crate::int_payload::decode("wai.audio.int_codec", payload, &[("model.wam", model_wam)]).map_err(|e| JsError::new(&e))? { + wai::int_payload::IntDecoded::Pcm { samples, sample_rate, .. } => Ok(DecodedAudio { pcm: samples, sample_rate }), + _ => Err(JsError::new("unsupported_capability: not PCM")), + } +} + +/// Decode a learned-audio model (`audio_model.bin`, WAM1) + the tool's JSON +/// bitstream (`audio_bitstream.json`) to int16 PCM in the browser. For debugging and +/// conversion only: the payload `wai.audio.int_codec` carries is `WIA1` +/// ([`decode_wia1`]). Byte-identical to the native reference. #[wasm_bindgen(js_name = decodeIntAudio)] pub fn decode_int_audio(model_bin: &[u8], bitstream_json: &[u8]) -> Result { let model = wai::int_audio::AudioModel::from_bytes(model_bin).map_err(JsError::new)?; let bs: serde_json::Value = serde_json::from_slice(bitstream_json).map_err(|e| JsError::new(&e.to_string()))?; let pcm = wai::int_audio::decode(&model, &bs).map_err(JsError::new)?; - Ok(DecodedAudio { pcm }) + Ok(DecodedAudio { pcm, sample_rate: 0 }) } diff --git a/wai/wai-web/src/int_payload.rs b/wai/wai-web/src/int_payload.rs new file mode 100644 index 000000000..716c95981 --- /dev/null +++ b/wai/wai-web/src/int_payload.rs @@ -0,0 +1,170 @@ +//! The payloads SPEC §5 pins to the WAI-native integer codecs +//! (`extensions/integer-payloads.md`), decoded in the browser: `WIV1` +//! (`wai.video.int_motion`, `wai.video.int_hyper`), `WIH1` (`wai.neural.int_hyper`), +//! `WIA1` (`wai.audio.int_codec`) and `WIS1` (`wai.splat.int_codec`). +//! +//! Every entry point (`decodeWiv1`, `decodeWih1`, `decodeWia1`, `decodeWis1`) runs +//! `wai::int_payload::decode_int`, the same Rust a native sink runs, over the parameter +//! files the host passes in, named as a set lists them. A refusal comes back as an +//! error whose message starts with its code from the extension's closed list +//! (`bad_magic`, `stream_exhausted`, …). The tools' JSON forms are not payloads: the +//! `decodeInt*` entry points that read them are for debugging and conversion only. + +use std::collections::BTreeMap; + +use wai::int_payload::{decode_int, DeclaredCost, IntDecoded}; +use wai::pin::HeldBytes; +use wasm_bindgen::prelude::*; + +/// Decode `payload` through `cap` against the set members `set` (name, bytes): the +/// pure-Rust half of the bindings, which the native tests run. +pub(crate) fn decode(cap: &str, payload: &[u8], set: &[(&str, &[u8])]) -> Result { + let files: BTreeMap = set.iter().map(|(k, v)| (k.to_string(), *v)).collect(); + let params = wai::int_payload::resolve(cap, None, &[HeldBytes::Set { listing: b"", files: &files }], None) + .map_err(|e| format!("{}: {e}", e.code()))?; + decode_int(cap, payload, ¶ms).map_err(|e| e.to_string()) +} + +/// What decoding `payload` through `cap` against the set members `set` will cost, +/// before any symbol is decoded (`int_payload::declared_cost`): the pure-Rust half of +/// `intPayloadCost`. +pub(crate) fn cost(cap: &str, payload: &[u8], set: &[(&str, &[u8])]) -> Result { + let files: BTreeMap = set.iter().map(|(k, v)| (k.to_string(), *v)).collect(); + let params = wai::int_payload::resolve(cap, None, &[HeldBytes::Set { listing: b"", files: &files }], None) + .map_err(|e| format!("{}: {e}", e.code()))?; + wai::int_payload::declared_cost(cap, payload, ¶ms).map_err(|e| e.to_string()) +} + +/// What a decode will cost, read before it starts (`intPayloadCost`): the output's +/// bytes, the symbols the streams code, the most bytes the reference decode holds at +/// once, and the networks' multiply-accumulates (`extensions/integer-payloads.md` §7, +/// *Memory and work*). A page applies its own size policy to these and does not +/// decode a payload past it. The byte and symbol counts are under `2^53` and exact as +/// numbers; `macs` is a `BigInt`. +#[wasm_bindgen] +pub struct IntPayloadCost { + output_bytes: f64, + symbols: f64, + working_bytes: f64, + macs: u128, +} + +#[wasm_bindgen] +impl IntPayloadCost { + #[wasm_bindgen(getter, js_name = outputBytes)] + pub fn output_bytes(&self) -> f64 { + self.output_bytes + } + #[wasm_bindgen(getter)] + pub fn symbols(&self) -> f64 { + self.symbols + } + #[wasm_bindgen(getter, js_name = workingBytes)] + pub fn working_bytes(&self) -> f64 { + self.working_bytes + } + #[wasm_bindgen(getter)] + pub fn macs(&self) -> u128 { + self.macs + } +} + +/// The declared cost of decoding `payload` through `capability`, before decoding it. +/// The set members are given as the decoders take them: `g_s`, `h_s` and `entropy` +/// for `wai.neural.int_hyper` and `wai.video.int_hyper`, `model` for +/// `wai.audio.int_codec`, `g_s` for `wai.splat.int_codec`, none for +/// `wai.video.int_motion`. A refusal's message starts with its code. +#[wasm_bindgen(js_name = intPayloadCost)] +pub fn int_payload_cost( + capability: &str, payload: &[u8], g_s: Option>, h_s: Option>, entropy: Option>, + model: Option>, +) -> Result { + let (g, h, e, m) = (g_s.unwrap_or_default(), h_s.unwrap_or_default(), entropy.unwrap_or_default(), model.unwrap_or_default()); + let hyper = int_hyper_set(&g, &h, &e); + let set: Vec<(&str, &[u8])> = match capability { + "wai.neural.int_hyper" | "wai.video.int_hyper" => hyper.to_vec(), + "wai.audio.int_codec" => vec![("model.wam", &m)], + "wai.splat.int_codec" => vec![("g_s.whm", &g)], + _ => vec![], + }; + let c = cost(capability, payload, &set).map_err(|e| JsError::new(&e))?; + Ok(IntPayloadCost { output_bytes: c.output_bytes as f64, symbols: c.symbols as f64, working_bytes: c.working_bytes as f64, macs: c.macs }) +} + +/// The conditional codec's three set members, from the host's bytes. +pub(crate) fn int_hyper_set<'a>(g_s: &'a [u8], h_s: &'a [u8], entropy: &'a [u8]) -> [(&'static str, &'a [u8]); 3] { + [("entropy.json", entropy), ("g_s.whm", g_s), ("h_s.whm", h_s)] +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(rel: &str) -> Vec { + let p = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../wai-rs/tests/vectors").join(rel); + std::fs::read(&p).unwrap_or_else(|e| panic!("{}: {e}", p.display())) + } + fn side(rel: &str, key: &str, field: &str) -> String { + let j: serde_json::Value = serde_json::from_slice(&read(rel)).unwrap(); + j[key][field].as_str().unwrap().to_string() + } + + /// Each committed payload decodes, through the browser crate's path, to the + /// SHA-256 its sidecar states; a JSON form is refused with `bad_magic`. + #[test] + fn the_committed_payloads_decode_to_their_sidecars() { + let (g, h, e) = (read("int_hyper/tiny_g_s.bin"), read("int_hyper/tiny_h_s.bin"), read("int_hyper/tiny_entropy.json")); + let set = int_hyper_set(&g, &h, &e); + let clip = |d: IntDecoded| match d { + IntDecoded::Video { frames, .. } => wai::pin::sha256_hex(&frames.iter().flat_map(|f| wai::pin::sha256(f)).collect::>()), + other => panic!("{other:?}"), + }; + assert_eq!(clip(decode("wai.video.int_motion", &read("int_video/int_video_bitstream.wiv1"), &[]).unwrap()), + side("int_video/sha256.json", "int_video", "clip_sha256")); + assert_eq!(clip(decode("wai.video.int_hyper", &read("int_video/video_nk_bitstream.wiv1"), &set).unwrap()), + side("int_video/sha256.json", "video_nk", "clip_sha256")); + match decode("wai.neural.int_hyper", &read("int_hyper/tiny_bitstream.wih1"), &set).unwrap() { + IntDecoded::Image { rgb, .. } => assert_eq!(wai::pin::sha256_hex(&rgb), side("int_hyper/sha256.json", "tiny", "sha256")), + other => panic!("{other:?}"), + } + let model = read("int_audio/audio_model.bin"); + match decode("wai.audio.int_codec", &read("int_audio/audio_bitstream.wia1"), &[("model.wam", &model)]).unwrap() { + IntDecoded::Pcm { samples, sample_rate: 16_000, .. } => { + let le: Vec = samples.iter().flat_map(|s| s.to_le_bytes()).collect(); + assert_eq!(wai::pin::sha256_hex(&le), side("int_audio/sha256.json", "audio", "sha256")); + } + other => panic!("{other:?}"), + } + let gs = read("int_splat/splat_gs.bin"); + match decode("wai.splat.int_codec", &read("int_splat/splat.wis1"), &[("g_s.whm", &gs)]).unwrap() { + IntDecoded::Attr { bytes, ranges, .. } => { + assert_eq!(wai::pin::sha256_hex(&bytes), side("int_splat/sha256.json", "splat", "sha256")); + assert_eq!(ranges.len(), 14); + } + other => panic!("{other:?}"), + } + let refused = decode("wai.video.int_motion", &read("int_video/int_video_bitstream.json"), &[]).unwrap_err(); + assert!(refused.starts_with("bad_magic"), "{refused}"); + } + + /// The browser's declared cost is the library's: the committed clip's output is its + /// frames' bytes, its symbols one per byte, and its working set the clip and the + /// frame list beside the parsed table and motion vectors; a JSON form is refused. + #[test] + fn the_declared_cost_is_read_before_decoding() { + let k0 = read("int_video/int_video_bitstream.wiv1"); + let c = cost("wai.video.int_motion", &k0, &[]).unwrap(); + let clip: usize = match decode("wai.video.int_motion", &k0, &[]).unwrap() { + IntDecoded::Video { frames, .. } => frames.iter().map(Vec::len).sum(), + other => panic!("{other:?}"), + }; + assert_eq!((c.output_bytes, c.symbols), (clip as u64, clip as u64)); + assert!(c.working_bytes > c.output_bytes); + assert_eq!(c.macs, 0, "a keyframe-0 clip runs no network"); + let (g, h, e) = (read("int_hyper/tiny_g_s.bin"), read("int_hyper/tiny_h_s.bin"), read("int_hyper/tiny_entropy.json")); + let img = cost("wai.neural.int_hyper", &read("int_hyper/tiny_bitstream.wih1"), &int_hyper_set(&g, &h, &e)).unwrap(); + assert_eq!(img.output_bytes, 16 * 16 * 3); + assert!(img.macs > 0, "the image's networks run"); + assert!(cost("wai.video.int_motion", &read("int_video/int_video_bitstream.json"), &[]).unwrap_err().starts_with("bad_magic")); + } +} diff --git a/wai/wai-web/src/int_splat.rs b/wai/wai-web/src/int_splat.rs index 9e557b7cf..a1e6b462a 100644 --- a/wai/wai-web/src/int_splat.rs +++ b/wai/wai-web/src/int_splat.rs @@ -14,12 +14,14 @@ use wai::int_hyper_synth::IntModel; /// A decoded splat: the `[n_attr · s · s]` `u8` attribute tensor (channel-major) — /// un-normalized to Gaussians (pos/scale/rot/color/opacity) on the JS side using the -/// per-attribute ranges in the bitstream. +/// per-attribute ranges the payload carries (`ranges`). #[wasm_bindgen] pub struct DecodedSplat { attr: Vec, n_attr: u32, s: u32, + n_gaussians: u32, + ranges: Vec, } #[wasm_bindgen] @@ -36,16 +38,47 @@ impl DecodedSplat { pub fn s(&self) -> u32 { self.s } + /// Gaussians in the cloud: the first `nGaussians` cells of each `s × s` plane. + #[wasm_bindgen(getter, js_name = nGaussians)] + pub fn n_gaussians(&self) -> u32 { + self.n_gaussians + } + /// Per attribute, `lo` then `hi`: the range its `u8` plane is presented over. + /// Presentation only: the splat-equivalence digest covers `attr`, not these. + #[wasm_bindgen(getter)] + pub fn ranges(&self) -> Box<[f32]> { + self.ranges.clone().into_boxed_slice() + } } -/// Decode a learned-splat model (`splat_gs.bin`, WHM1) + bitstream (`splat.json`) to -/// the integer attribute tensor in the browser, byte-identical to the native -/// reference. No float, no ML runtime. +/// Decode a `WIS1` payload (`wai.splat.int_codec`, `extensions/integer-payloads.md` +/// §6) to the integer attribute tensor in the browser, byte-identical to the native +/// reference, with its one-file parameter set (`g_s.whm`, the `WHM1` synthesis), and +/// the per-attribute ranges and Gaussian count the payload carries. A refusal's +/// message starts with its code (`bad_magic` for a JSON form). No float, no ML runtime. +#[wasm_bindgen(js_name = decodeWis1)] +pub fn decode_wis1(payload: &[u8], g_s: &[u8]) -> Result { + match crate::int_payload::decode("wai.splat.int_codec", payload, &[("g_s.whm", g_s)]).map_err(|e| JsError::new(&e))? { + wai::int_payload::IntDecoded::Attr { bytes, n_attr, s, n_gaussians, ranges } => Ok(DecodedSplat { + attr: bytes, n_attr, s, n_gaussians, ranges: ranges.iter().flat_map(|&(lo, hi)| [lo, hi]).collect(), + }), + _ => Err(JsError::new("unsupported_capability: not an attribute tensor")), + } +} + +/// Decode a learned-splat model (`splat_gs.bin`, WHM1) + the tool's JSON bitstream +/// (`splat.json`) to the integer attribute tensor in the browser. For debugging and +/// conversion only: the payload `wai.splat.int_codec` carries is `WIS1` +/// ([`decode_wis1`]). Byte-identical to the native reference. #[wasm_bindgen(js_name = decodeIntSplat)] pub fn decode_int_splat(gs_bin: &[u8], bitstream_json: &[u8]) -> Result { let gs = IntModel::from_bytes(gs_bin).map_err(JsError::new)?; let bs: serde_json::Value = serde_json::from_slice(bitstream_json).map_err(|e| JsError::new(&e.to_string()))?; - let d = wai::int_splat::decode(&gs, &bs).map_err(JsError::new)?; - Ok(DecodedSplat { attr: d.attr, n_attr: d.n_attr as u32, s: d.s as u32 }) + let j = wai::int_splat::SplatJson::from_value(&bs).map_err(JsError::new)?; + let d = wai::int_splat::decode_typed(&gs, &j.stream()).map_err(|r| JsError::new(r.detail))?; + Ok(DecodedSplat { + attr: d.attr, n_attr: d.n_attr as u32, s: d.s as u32, n_gaussians: j.n_gaussians as u32, + ranges: j.ranges.iter().flat_map(|&(lo, hi)| [lo as f32, hi as f32]).collect(), + }) } diff --git a/wai/wai-web/src/lib.rs b/wai/wai-web/src/lib.rs index 7d3768260..c7c5f5a5e 100644 --- a/wai/wai-web/src/lib.rs +++ b/wai/wai-web/src/lib.rs @@ -37,6 +37,11 @@ pub mod neural_int; // portable hash. pub mod generate; +// The payloads SPEC §5 pins to the integer codecs (WIV1, WIH1, WIA1, WIS1), decoded +// through wai::int_payload::decode_int: the shared half of decodeWiv1, decodeWih1, +// decodeWia1 and decodeWis1. +mod int_payload; + // Byte-exact integer inter-frame VIDEO (wai.video.int_motion / int_hyper) — pure // i64, no ML runtime; frame-state-equivalence, decoded + per-frame-verified in-page. pub mod video; @@ -430,8 +435,9 @@ impl WaiEnvelope { /// SPEC §4: the capability that decodes this envelope at a page that /// advertises `advertised` and holds the parameter sets `held` (see /// `verifyPin`); `undefined` when it is inert here — including when its - /// capability is companion-only, or when its pin is refused and no fallback - /// remains. + /// capability is companion-only, when its pin is refused and no fallback + /// remains, or when its manifest declares an encryption, which this page does not + /// open (SPEC §6.4 step 2). #[wasm_bindgen(js_name = dispatchCapability)] pub fn dispatch_capability(&self, advertised: Vec, held: JsValue) -> Result, JsError> { let held = held_from_js(&held)?; @@ -455,19 +461,50 @@ pub fn parse(bytes: &[u8]) -> Result { Ok(WaiEnvelope { inner, manifest_json: String::from_utf8_lossy(m).into_owned() }) } +/// What [`pack`] refuses before it writes: a manifest that names a candidate as its +/// capability or fallback (SPEC §5, *Candidate capabilities*), and a payload that is +/// not the format SPEC §5 pins its integer capability or fallback to (*Payloads of the +/// WAI-native integer codecs*), checked by the wire parser a sink runs. The pure-Rust +/// half of the binding, which the native tests run. +fn pack_refusal(m: &wai::container::Manifest, payload: &[u8]) -> Result<(), String> { + let req = &m.model_requirement; + wai::container::check_candidates_v1(&req.capability, req.fallback.as_deref()).map_err(|e| e.to_string())?; + // An encrypted payload is ciphertext; a sink checks it after decrypting (SPEC §6.4). + if !m.security.as_ref().is_none_or(wai::container::Security::is_plaintext) { + return Ok(()); + } + for cap in std::iter::once(req.capability.as_str()).chain(req.fallback.as_deref()) { + wai::container::check_int_payload_framing(cap, payload).map_err(|e| e.to_string())?; + } + Ok(()) +} + +/// What [`pack_multi`] refuses before it writes, once it holds the payloads: a +/// `security` member that is not a security object (SPEC §6.1), and a component whose +/// payload is not the format SPEC §5 pins its integer capability to. +fn pack_multi_refusal(manifest: &wai::container::ManifestV2, payloads: &[Vec]) -> Result<(), String> { + wai::container::manifest_v2_security(manifest)?; + if !wai::container::manifest_v2_plaintext(manifest) { + return Ok(()); + } + for (i, (r, p)) in manifest.renditions.iter().zip(payloads).enumerate() { + wai::container::check_int_payload_framing(&r.capability, p).map_err(|e| format!("component {i}: {e}"))?; + } + Ok(()) +} + /// Build a WAI envelope from a manifest JSON string + a payload byte /// array. Mirrors `wai_envelope_pack` from the native SDK so JS code /// can produce .wai files in the browser too: like it, refuses a manifest /// that names a candidate capability as its capability or fallback, which an -/// encoder MUST NOT emit (SPEC §5, *Candidate capabilities*). The manifest is +/// encoder MUST NOT emit (SPEC §5, *Candidate capabilities*), and a payload that is +/// not the format SPEC §5 pins its integer capability or fallback to. The manifest is /// written as given. #[wasm_bindgen(js_name = pack)] pub fn pack(manifest_json: &str, payload: &[u8]) -> Result, JsError> { let m: wai::container::Manifest = serde_json::from_str(manifest_json) .map_err(|e| JsError::new(&format!("manifest parse: {e}")))?; - wai::container::check_candidates_v1(&m.model_requirement.capability, - m.model_requirement.fallback.as_deref()) - .map_err(|e| JsError::new(&e.to_string()))?; + pack_refusal(&m, payload).map_err(|e| JsError::new(&e))?; let mb = manifest_json.as_bytes(); let mut out = Vec::with_capacity(12 + mb.len() + payload.len()); out.extend_from_slice(wai::container::MAGIC); @@ -612,7 +649,8 @@ pub fn parse_multi(bytes: &[u8]) -> Result { /// The manifest's `renditions` array MUST be the same length and order as the /// payloads. The manifest is written as given. Like `wai_envelope_pack_multi`, /// refuses a table with a component, in either role, whose capability is a -/// candidate (SPEC §5). +/// candidate, or whose payload is not the format SPEC §5 pins its integer capability +/// to (SPEC §5). #[wasm_bindgen(js_name = packMulti)] pub fn pack_multi(manifest_json: &str, payloads: js_sys::Array) -> Result, JsError> { let manifest: wai::container::ManifestV2 = serde_json::from_str(manifest_json) @@ -636,6 +674,7 @@ pub fn pack_multi(manifest_json: &str, payloads: js_sys::Array) -> Result| { + let mut req = ModelRequirement::new(cap); + if let Some(f) = fallback { + req = req.with_fallback(f); + } + Manifest::new("video", "improve", req, Conditioning::new("bytes")) + }; + // The refusal names the code: "... (: ; ...)". + let refused = |r: Result<(), String>| r.err().and_then(|e| e.split_once(" to (").map(|(_, t)| t.split(':').next().unwrap_or_default().to_string())); + let (mot, hyp) = ("wai.video.int_motion", "wai.video.int_hyper"); + assert_eq!(super::pack_refusal(&v1(mot, None), &k0), Ok(())); + assert_eq!(super::pack_refusal(&v1(hyp, None), &k1), Ok(())); + assert_eq!(refused(super::pack_refusal(&v1(mot, None), &read("int_video/int_video_bitstream.json"))).as_deref(), Some("bad_magic")); + assert_eq!(refused(super::pack_refusal(&v1(mot, None), &k0[..40])).as_deref(), Some("stream_length")); + assert_eq!(refused(super::pack_refusal(&v1(hyp, None), &k0)).as_deref(), Some("keyframe_capability_mismatch")); + assert_eq!(refused(super::pack_refusal(&v1("wai.video.av1", Some(mot)), &k1)).as_deref(), Some("keyframe_capability_mismatch")); + let m2: ManifestV2 = serde_json::from_value(serde_json::json!({ + "wai": "1.1", "media": "video", "intent": "improve", + "renditions": [{"capability": mot, "kind": "bytes"}, {"capability": hyp, "kind": "bytes"}] + })).unwrap(); + assert_eq!(super::pack_multi_refusal(&m2, &[k0.clone(), k1.clone()]), Ok(())); + let e = super::pack_multi_refusal(&m2, &[k0.clone(), k0.clone()]).unwrap_err(); + assert!(e.starts_with("component 1:") && e.contains("keyframe_capability_mismatch"), "{e}"); + // An encrypted envelope's payload is ciphertext, checked after decryption + // (SPEC §6.4): it is written. + let cipher: Vec = k0.iter().enumerate().map(|(i, b)| b ^ (i as u8).wrapping_mul(151).wrapping_add(7)).collect(); + let enc = v1(mot, None).with_security(wai::container::Security::new("aes-256-gcm", 7)); + assert_eq!(super::pack_refusal(&enc, &cipher), Ok(())); + let mut m2e = m2.clone(); + m2e.extra.insert("security".into(), serde_json::json!({"encryption": "mls-sframe", "kid": 0})); + assert_eq!(super::pack_multi_refusal(&m2e, &[cipher.clone(), cipher.clone()]), Ok(())); + // A `security` member that is not a security object is not written: it would + // claim an encryption no sink can read. + for bad in [serde_json::json!(true), serde_json::json!("aes"), serde_json::json!([]), serde_json::json!({"encryption": 5})] { + m2e.extra.insert("security".into(), bad.clone()); + let e = super::pack_multi_refusal(&m2e, &[cipher.clone(), cipher.clone()]).unwrap_err(); + assert!(e.contains("is not a security object"), "{bad}: {e}"); + } + } + /// A companion-only component is never presented as media: not picked, not /// returned by `rendition`, and flagged `selectable: false` in the metadata. #[test] diff --git a/wai/wai-web/src/neural_int.rs b/wai/wai-web/src/neural_int.rs index 7194af16c..46524534b 100644 --- a/wai/wai-web/src/neural_int.rs +++ b/wai/wai-web/src/neural_int.rs @@ -54,7 +54,23 @@ pub fn decode_int_synth(payload: &[u8], spec_json: &[u8]) -> Result Result { + let set = crate::int_payload::int_hyper_set(g_s, h_s, entropy); + match crate::int_payload::decode("wai.neural.int_hyper", payload, &set).map_err(|e| JsError::new(&e))? { + wai::int_payload::IntDecoded::Image { rgb, width, height } => Ok(DecodedImage { rgb, width, height }), + _ => Err(JsError::new("unsupported_capability: not an image")), + } +} + +/// Decode the tool's JSON conditional bitstream → RGB in the browser. For debugging +/// and conversion only: the payload `wai.neural.int_hyper` carries is `WIH1` +/// ([`decode_wih1`]). The rest of this note describes the decode both share: /// integer-exact (`wai.neural.int_hyper`). The host installs the model /// (`int_synth_sota_model.json` = g_s, `int_hs_model.json` = h_s, /// `cond_entropy.json` = the entropy tables); WAI carries the bitstream @@ -70,13 +86,14 @@ pub fn decode_int_hyper( let parse = |b: &[u8]| -> Result { serde_json::from_slice(b).map_err(|e| JsError::new(&e.to_string())) }; - cond_decode(IntModel::from_json(&parse(g_s_json)?), IntModel::from_json(&parse(h_s_json)?), entropy_json, bitstream_json) + let net = |b: &[u8]| IntModel::try_from_json(&parse(b)?).map_err(|k| JsError::new(&format!("model_mismatch: a model's `{k}`"))); + cond_decode(net(g_s_json)?, net(h_s_json)?, entropy_json, bitstream_json) } -/// Same decode from the **compact binary** model (`*_model.bin`, ~½ the JSON -/// size — int12 weights as `i16`). This is the path the in-browser demo uses: -/// the model download is the bottleneck, and the binary roughly halves it while -/// the decode stays byte-identical to the JSON model and to the native reference. +/// Same decode from the **compact binary** model (`*_model.bin`, int12 weights as +/// `i16`), of the tool's JSON bitstream. For debugging and conversion only: the +/// payload is `WIH1` ([`decode_wih1`]). Byte-identical to the JSON model and to the +/// native reference. #[wasm_bindgen(js_name = decodeIntHyperBin)] pub fn decode_int_hyper_bin( g_s_bin: &[u8], h_s_bin: &[u8], entropy_json: &[u8], bitstream_json: &[u8], @@ -91,16 +108,61 @@ fn cond_decode( g_s: wai::int_hyper_synth::IntModel, h_s: wai::int_hyper_synth::IntModel, entropy_json: &[u8], bitstream_json: &[u8], ) -> Result { - use wai::int_hyper_synth::{decode_conditional, CondBitstream, CondEntropy}; - let parse = |b: &[u8]| -> Result { - serde_json::from_slice(b).map_err(|e| JsError::new(&e.to_string())) - }; - let ent = CondEntropy::from_value(&parse(entropy_json)?); - let bs = CondBitstream::from_value(&parse(bitstream_json)?); - let (rgb, height, width) = decode_conditional(&g_s, &h_s, &ent, &bs).map_err(JsError::new)?; + let (rgb, height, width) = cond_decode_checked(&g_s, &h_s, entropy_json, bitstream_json).map_err(|e| JsError::new(&e))?; Ok(DecodedImage { rgb, width: width as u32, height: height as u32 }) } +/// The debug JSON decode, through the checked readers (`CondEntropy::try_from_value`, +/// `CondBitstream::try_from_value`) and `decode_conditional_checked`, so a JSON the +/// export tool did not write is refused, never a panic: the pure-Rust half the native +/// tests run. +pub(crate) fn cond_decode_checked( + g_s: &wai::int_hyper_synth::IntModel, h_s: &wai::int_hyper_synth::IntModel, entropy_json: &[u8], bitstream_json: &[u8], +) -> Result<(Vec, usize, usize), String> { + use wai::int_hyper_synth::{decode_conditional_checked, CondBitstream, CondEntropy}; + let parse = |b: &[u8]| serde_json::from_slice::(b).map_err(|e| e.to_string()); + let ent = CondEntropy::try_from_value(&parse(entropy_json)?).map_err(|e| format!("model_mismatch: {e}"))?; + let bs = CondBitstream::try_from_value(&parse(bitstream_json)?).map_err(|k| format!("field_range: `{k}`"))?; + decode_conditional_checked(g_s, h_s, &ent, &bs.stream()).map_err(|r| r.to_string()) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + use wai::int_hyper_synth::IntModel; + + fn read(rel: &str) -> Vec { + let p = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../wai-rs/tests/vectors").join(rel); + std::fs::read(&p).unwrap_or_else(|e| panic!("{}: {e}", p.display())) + } + + /// The debug JSON decode refuses, rather than panics on, a JSON the export tool + /// did not write: an empty model or bitstream, a hyper-latent of 2^61 elements, + /// more hyper-latent channels than tables. The committed tiny codec decodes. + #[test] + fn the_debug_json_decode_refuses_instead_of_panicking() { + assert!(IntModel::try_from_json(&json!({})).is_err()); + let (g, h) = (IntModel::from_bytes(&read("int_hyper/tiny_g_s.bin")).unwrap(), IntModel::from_bytes(&read("int_hyper/tiny_h_s.bin")).unwrap()); + let ent = read("int_hyper/tiny_entropy.json"); + let got = super::cond_decode_checked(&g, &h, &ent, &read("int_hyper/tiny_bitstream.json")).unwrap(); + assert_eq!((got.1, got.2), (16, 16)); + // The JSON models read through the checked reader decode alike. + let net = |rel: &str| IntModel::try_from_json(&serde_json::from_slice(&read(rel)).unwrap()).unwrap(); + let from_json = super::cond_decode_checked(&net("int_hyper/tiny_g_s.json"), &net("int_hyper/tiny_h_s.json"), &ent, &read("int_hyper/tiny_bitstream.json")).unwrap(); + assert_eq!(from_json, got); + let bs = |v: serde_json::Value| serde_json::to_vec(&v).unwrap(); + let zeros = vec![0u8; 8]; + for (what, b) in [ + ("{}", bs(json!({}))), + ("n_z·zh·zw = 2^61", bs(json!({"z_rans": zeros, "rans": zeros, "n_z": 1u64 << 20, "zh": 1u64 << 20, "zw": 1u64 << 21, "m": 1, "sh": 1, "sw": 1}))), + ("more z channels than tables", bs(json!({"z_rans": zeros, "rans": zeros, "n_z": 4096, "zh": 1, "zw": 1, "m": 1, "sh": 1, "sw": 1}))), + ] { + let r = std::panic::catch_unwind(|| super::cond_decode_checked(&g, &h, &ent, &b)); + assert!(matches!(r, Ok(Err(_))), "{what}: {r:?}"); + } + } +} + // ---- int_entropy: the deterministic scale→bucket primitive, in the browser ---- /// Precompute a CompressAI-style ascending `scale_table` as fixed-point integer diff --git a/wai/wai-web/src/video.rs b/wai/wai-web/src/video.rs index 1e6f70d7d..8fe707d52 100644 --- a/wai/wai-web/src/video.rs +++ b/wai/wai-web/src/video.rs @@ -9,26 +9,32 @@ use wasm_bindgen::prelude::*; -/// A decoded clip: all frames concatenated (`n_frames · width · height · 3` -/// bytes), to be sliced per frame on the JS side and painted to a canvas. +/// A decoded clip: its frames, each `width · height · 3` bytes of RGB, as the +/// decoder produced them. `frame(i)` hands JS one frame; `rgb` concatenates them all +/// into one new buffer on each read, so a page that paints frame by frame holds the +/// clip once. #[wasm_bindgen] pub struct DecodedVideo { - rgb: Vec, - n_frames: u32, + frames: Vec>, width: u32, height: u32, } #[wasm_bindgen] impl DecodedVideo { - /// All frames, concatenated: frame `f` is `rgb[f·w·h·3 .. (f+1)·w·h·3]`. + /// All frames, concatenated into a new buffer: frame `f` is + /// `rgb[f·w·h·3 .. (f+1)·w·h·3]`. #[wasm_bindgen(getter)] pub fn rgb(&self) -> Box<[u8]> { - self.rgb.clone().into_boxed_slice() + self.frames.concat().into_boxed_slice() + } + /// Frame `index` (`width · height · 3` bytes), or an empty buffer past the last. + pub fn frame(&self, index: u32) -> Box<[u8]> { + self.frames.get(index as usize).cloned().unwrap_or_default().into_boxed_slice() } #[wasm_bindgen(getter, js_name = nFrames)] pub fn n_frames(&self) -> u32 { - self.n_frames + self.frames.len() as u32 } #[wasm_bindgen(getter)] pub fn width(&self) -> u32 { @@ -40,10 +46,29 @@ impl DecodedVideo { } } -/// Decode a deterministic-integer inter-frame bitstream (`int_video_bitstream.json`) -/// to RGB frames in the browser, byte-identical to the native reference. No float, -/// no ML runtime. `fnv1aHex` (the `generate` binding) hashes each frame for the -/// in-page receipt check. +/// Decode a `WIV1` payload (`extensions/integer-payloads.md` §3) to RGB frames in +/// the browser, byte-identical to the native reference: `capability` is the +/// envelope's, `wai.video.int_motion` (keyframe 0, no parameters) or +/// `wai.video.int_hyper` (keyframe 1, whose nested `WIH1` keyframe decodes with the +/// conditional codec's `g_s.whm`, `h_s.whm` and `entropy.json`). A refusal's message +/// starts with its code (`bad_magic` for a JSON form). No float, no ML runtime. +#[wasm_bindgen(js_name = decodeWiv1)] +pub fn decode_wiv1( + capability: &str, payload: &[u8], g_s: Option>, h_s: Option>, entropy: Option>, +) -> Result { + let (g, h, e) = (g_s.unwrap_or_default(), h_s.unwrap_or_default(), entropy.unwrap_or_default()); + let set = crate::int_payload::int_hyper_set(&g, &h, &e); + let set: &[(&str, &[u8])] = if capability == "wai.video.int_hyper" { &set } else { &[] }; + match crate::int_payload::decode(capability, payload, set).map_err(|e| JsError::new(&e))? { + wai::int_payload::IntDecoded::Video { frames, width, height } => Ok(DecodedVideo { frames, width, height }), + _ => Err(JsError::new("unsupported_capability: not an integer video capability")), + } +} + +/// Decode the tool's JSON bitstream (`int_video_bitstream.json`) to RGB frames. For +/// debugging and conversion only: the payload `wai.video.int_motion` carries is +/// `WIV1` ([`decode_wiv1`]), and a JSON form is not one. Byte-identical to the +/// native reference; `fnv1aHex` (the `generate` binding) hashes each frame. #[wasm_bindgen(js_name = decodeIntVideo)] pub fn decode_int_video(bitstream_json: &[u8]) -> Result { let bs: serde_json::Value = @@ -52,10 +77,12 @@ pub fn decode_int_video(bitstream_json: &[u8]) -> Result Ok(flatten(clip)) } -/// Decode the **P-frames** of a clip on top of an already-decoded I-frame +/// Decode the **P-frames** of a JSON bitstream on top of an already-decoded I-frame /// (`i_frame`, RGB HWC `u8`, e.g. from `decodeIntHyperBin`) — the NEURAL-KEYFRAME /// path (`wai.video.int_hyper`): a learned `int_hyper` keyframe + integer -/// inter-frames, the whole clip byte-exact. No float, no ML runtime. +/// inter-frames, the whole clip byte-exact. For debugging and conversion only: the +/// payload is `WIV1` with its keyframe nested ([`decode_wiv1`]). No float, no ML +/// runtime. #[wasm_bindgen(js_name = decodeIntVideoPframes)] pub fn decode_int_video_pframes(i_frame: &[u8], w: u32, h: u32, bitstream_json: &[u8]) -> Result { let bs: serde_json::Value = @@ -65,9 +92,5 @@ pub fn decode_int_video_pframes(i_frame: &[u8], w: u32, h: u32, bitstream_json: } fn flatten(clip: wai::int_video::DecodedClip) -> DecodedVideo { - let mut rgb = Vec::with_capacity(clip.frames.len() * clip.h * clip.w * 3); - for f in &clip.frames { - rgb.extend_from_slice(f); - } - DecodedVideo { rgb, n_frames: clip.frames.len() as u32, width: clip.w as u32, height: clip.h as u32 } + DecodedVideo { width: clip.w as u32, height: clip.h as u32, frames: clip.frames } }