From e8f8e0033781397147aaf9f5864f5b9edc816ca7 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Wed, 4 Nov 2026 23:09:19 -0500 Subject: [PATCH] wai/det-source-fixes, converted to the layout of the fix: no shared append point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The change of wai/det-source-fixes (152aab5c7..9c4159958, 15 commits), on the fix branch: its changelog entries as files of wai/changes/, its wai-conformance.yml edits as step files, corpus files and test tags, and every other file as the branch has it. - a5bb23634 wai-rs: the inverse GDN's integer roots take no floating point - b9463b4d0 wai-rs: the splat ranges and JSON form are read in integer arithmetic - b20be5196 wai-rs: int_entropy's scale conversions are formed in integers - 437669081 wai-rs: a test that the integer decode paths hold no floating point - 36aade4f1 wai: the integer decode paths without floating point, in the changelog and integer-payloads §10 - f781bf091 wai-rs: the AVX-512 root's comment states the bound s + bit keeps - 799ab8476 wai-rs: float_free reads the other integer entropy decoders too - b79447e64 wai-rs: splat ranges one last digit apart decode to adjacent values - 43f75cc36 wai: the changelog names the decoders float_free reads and the last-digit pairs - 2e308fd28 wai-rs: the parameter sets' JSON is read by the strict reader - 42f848583 wai-rs: int_payload re-exports Binary32 - d1dbe4936 wai-rs: float_free reads the operator set, every integer module, and serde_json's parser - 931699331 wai: the parameter sets' JSON read strictly, in integer-payloads §9 and §14 and the changelog - 97d133380 wai-rs: float_free names what a decode reaches beyond what it reads - 9c4159958 wai: the changelog names the general JSON reader once Co-Authored-By: Claude Opus 5.5 --- wai/byte-exact-conformance/Cargo.toml | 14 +- .../src/int_codec_golden.rs | 3 +- .../src/int_payload_golden.rs | 4 +- wai/byte-exact-conformance/src/lib.rs | 5 + ...4-integer-decode-paths-hold-no-floating.md | 72 ++++ wai/extensions/integer-payloads.md | 28 +- wai/wai-rs/Cargo.toml | 11 +- wai/wai-rs/examples/int_splat_decode.rs | 4 +- wai/wai-rs/src/bin/wai_decode_bench.rs | 6 +- wai/wai-rs/src/bin/wai_int_payload_vectors.rs | 6 +- wai/wai-rs/src/float_free.rs | 366 ++++++++++++++++++ wai/wai-rs/src/int_entropy.rs | 71 +++- wai/wai-rs/src/int_hyper_synth.rs | 24 +- wai/wai-rs/src/int_kernels.rs | 163 ++++++-- wai/wai-rs/src/int_kernels/x86.rs | 28 +- wai/wai-rs/src/int_mlicpp.rs | 63 ++- wai/wai-rs/src/int_ops.rs | 2 +- wai/wai-rs/src/int_payload/mod.rs | 35 +- wai/wai-rs/src/int_payload/tests.rs | 366 ++++++++++++++++-- wai/wai-rs/src/int_payload/wire.rs | 41 +- wai/wai-rs/src/int_splat.rs | 317 +++++++++++++-- wai/wai-rs/src/lib.rs | 12 +- wai/wai-rs/src/splat_exact.rs | 9 +- wai/wai-rs/src/strict_json.rs | 27 +- wai/wai-rs/tests/int_hyper_ops.rs | 8 +- wai/wai-rs/tests/int_kernels_vectors.rs | 3 +- wai/wai-rs/tests/int_payload_cost.rs | 6 +- wai/wai-rs/tests/int_splat.rs | 6 +- wai/wai-rs/tests/int_splat_ops.rs | 4 +- wai/wai-rs/tests/ops_gpu.rs | 4 +- wai/wai-web/src/int_splat.rs | 19 +- wai/wai-web/src/neural_int.rs | 14 +- 32 files changed, 1512 insertions(+), 229 deletions(-) create mode 100644 wai/changes/2026-11-04-integer-decode-paths-hold-no-floating.md create mode 100644 wai/wai-rs/src/float_free.rs diff --git a/wai/byte-exact-conformance/Cargo.toml b/wai/byte-exact-conformance/Cargo.toml index e4fe6d6a8..af483155e 100644 --- a/wai/byte-exact-conformance/Cargo.toml +++ b/wai/byte-exact-conformance/Cargo.toml @@ -40,4 +40,16 @@ audio_int = [] # the goldens below decode `WIV1` and `WIR1` through on every leg of the matrix as well: # the arithmetic a 32-bit fabric runs, held to the same digests. On by default. i32_forms = ["wai-ops/i32_forms"] -default = ["splat_int", "audio_int", "i32_forms"] +# wai-rs's features that the included strict JSON reader (`strict_json.rs`, which the splat +# decoder reads its JSON form with) names for its other users there; declared so the source's +# `cfg` names known features. Never enabled here. +prior_wire = [] +scene_map = [] +conform_result = [] +fleet = [] +video_receipt = [] +channel_lineup = [] +# wai-rs's feature for the integer decoders this crate includes, which the strict JSON reader +# names for them (they read their parameter sets' JSON with it): on by default, as they are. +neural_int = [] +default = ["neural_int", "splat_int", "audio_int", "i32_forms"] diff --git a/wai/byte-exact-conformance/src/int_codec_golden.rs b/wai/byte-exact-conformance/src/int_codec_golden.rs index d385226af..d9195448c 100644 --- a/wai/byte-exact-conformance/src/int_codec_golden.rs +++ b/wai/byte-exact-conformance/src/int_codec_golden.rs @@ -100,7 +100,8 @@ fn splat_wis1_decodes_to_its_attribute_golden() { assert_eq!(jload("int_splat/sha256.json")["splat"]["sha256"].as_str(), Some(SPLAT_ATTR_SHA256), "the sidecar"); let bs = jload("int_splat/splat.json"); - let j = SplatJson::from_value(&bs).expect("the JSON form reads"); + let j = SplatJson::read(&read("int_splat/splat.json")).expect("the JSON form reads"); + assert_eq!(j.ranges, v.ranges, "the JSON form's ranges are the payload's, read in integers"); let from_json = both_kernels("the splat JSON form", || decode_splat(&gs, &j.stream()).expect("the JSON form decodes").attr); assert_eq!(from_json, attr, "the JSON form and the payload decode alike"); let fnv = crate::int_hyper_synth::fnv1a(&attr); diff --git a/wai/byte-exact-conformance/src/int_payload_golden.rs b/wai/byte-exact-conformance/src/int_payload_golden.rs index 213d9fa0d..7a8a813ab 100644 --- a/wai/byte-exact-conformance/src/int_payload_golden.rs +++ b/wai/byte-exact-conformance/src/int_payload_golden.rs @@ -68,7 +68,7 @@ fn assert_clip(frames: &[Vec], key: &str) -> String { fn keyframe(k: &Wih1<'_>) -> (Vec, usize, usize) { let g_s = IntModel::from_bytes(&read("int_hyper/tiny_g_s.bin")).expect("g_s"); let h_s = IntModel::from_bytes(&read("int_hyper/tiny_h_s.bin")).expect("h_s"); - let ent = CondEntropy::try_from_value(&jload("int_hyper/tiny_entropy.json")).expect("entropy tables"); + let ent = CondEntropy::read(&read("int_hyper/tiny_entropy.json")).expect("entropy tables"); let s = CondStream { m: k.m as usize, sh: k.sh as usize, sw: k.sw as usize, n_z: k.n_z as usize, zh: k.zh as usize, zw: k.zw as usize, z_rans: k.z_rans, rans: k.rans, @@ -123,7 +123,7 @@ fn wih1_payload_is_byte_identical() { fn keyframe_through_the_lowering(k: &Wih1<'_>) -> (Vec, usize, usize) { let g_s = IntModel::from_bytes(&read("int_hyper/tiny_g_s.bin")).expect("g_s"); let h_s = IntModel::from_bytes(&read("int_hyper/tiny_h_s.bin")).expect("h_s"); - let ent = CondEntropy::try_from_value(&jload("int_hyper/tiny_entropy.json")).expect("entropy tables"); + let ent = CondEntropy::read(&read("int_hyper/tiny_entropy.json")).expect("entropy tables"); let s = CondStream { m: k.m as usize, sh: k.sh as usize, sw: k.sw as usize, n_z: k.n_z as usize, zh: k.zh as usize, zw: k.zw as usize, z_rans: k.z_rans, rans: k.rans, diff --git a/wai/byte-exact-conformance/src/lib.rs b/wai/byte-exact-conformance/src/lib.rs index 58f9ca321..c9e374b79 100644 --- a/wai/byte-exact-conformance/src/lib.rs +++ b/wai/byte-exact-conformance/src/lib.rs @@ -84,6 +84,11 @@ pub mod int_mlicpp; pub mod int_audio; #[path = "../../wai-rs/src/int_splat.rs"] pub mod int_splat; +// The strict JSON reader the conditional and MLIC++ decoders read their entropy tables with, +// and the splat decoder its JSON form, every number in integer arithmetic (so the matrix reads +// the same values on every platform). Self-contained. +#[path = "../../wai-rs/src/strict_json.rs"] +pub(crate) mod strict_json; // The wire parser of the integer payloads SPEC §5 pins (WIV1, WIH1, WIA1, WIS1). // Self-contained — std's Vec only, nothing from the crate — so the matrix parses the diff --git a/wai/changes/2026-11-04-integer-decode-paths-hold-no-floating.md b/wai/changes/2026-11-04-integer-decode-paths-hold-no-floating.md new file mode 100644 index 000000000..2221bc8c1 --- /dev/null +++ b/wai/changes/2026-11-04-integer-decode-paths-hold-no-floating.md @@ -0,0 +1,72 @@ +### The integer decode paths hold no floating point in their source, and read no JSON with a float parser + +The integer square root of the inverse GDN, the splat codec's ranges and JSON form, and `int_entropy`'s scale +conversions no longer use floating point; the decode reads a parameter set's JSON with the strict reader, which +converts no number to a float; and a test of the sources keeps the integer decode paths, and the operator set they +run their networks through, free of floating point and of a float parser. No payload, corpus or decoded output +changes: the integer corpora regenerate byte for byte, on arm64 macOS and x86_64 Linux. The test reads source, not a +compiled decode. + +- **The inverse GDN's root.** The narrow kernel sets seeded each integer square root from the IEEE-754 square root + (`(n as f64).sqrt()`), and the AVX-512 kernel took eight at a time with a vector square root; the integer + correction after each decided the result. The seed is now a fixed-point reciprocal square root (a table on the + norm's top bits and two Newton steps, formed exactly in `u64` and `u128`), within one of the root, and the + AVX-512 roots are taken digit by digit in integer lanes. The roots are the same integers. The test holds the root + to its definition, an independent digit-by-digit root and `u64::isqrt` on the edges of `u64` (0, 1, every + `k² − 1`, `k²` and `k² + 1` around the powers of two and across the range, `u64::MAX`), and an ignored test + checks every boundary `k² − 1`, `k²` for `k` below `2^32`. +- **`WIS1` ranges** are read, checked and written as binary32 bits (`int_payload::wire::Binary32`): finite, and + `lo ≤ hi` as IEEE 754 compares two finite values (`−0` equal to `+0`), in integer arithmetic. The rule + (integer-payloads §6) is unchanged; a test holds it to the float comparison on every class of bit pattern. +- **The splat JSON form** (`splat.json`) is read from its text with the strict JSON reader: its integers as integer + spellings, and each range by `int_splat::binary32_of` in exact integer arithmetic (integer-payloads §10, which + now says how a number denotes a binary32 value). A number spelled as an integer is that integer, any other + spelling denotes the binary64 value nearest it, and either must be a binary32 value; this is how the independent + converter, `int_payload_verify.py`, reads a number. The reader before went through serde_json, whose float + reading is correctly rounded only with its `float_roundtrip` feature (which `splat_int` turned on for the whole + crate, and the byte-equality crate never had), and it disagreed with the independent converter in two places: + it kept the sign of the integer `-0` (zero, so `+0`), and it rounded an integer past `u64` (refused unless it is + a binary32 value). A range one digit from the binary64 tie between `1` and the next binary64 value now reads as + binary64 rounds it on each side of the tie, with a test, which also holds two spellings one last digit apart + (`4194304.0` and `4194304.5`, `8.388608e6` and `8.388609e6`) to the two adjacent values the payload's decode + presents. The JSON decode refuses a range that is not a binary32 + value, as the converter always did, and the text is read strictly (a member named twice, or bytes after the + value, refuse it). `splat_int` no longer turns on `float_roundtrip`; `splat_exact`, which reads an asset's + `POSITION` bounds as JSON numbers, does. +- **`int_entropy::fixed_scale_edges` and `quantize_scale`** form `⌊x·2^frac⌋` from the binary32 bits in integers, + the values the binary64 product and its floor gave, which a test checks at every exponent and every `frac`. +- **The parameter sets' JSON** (`entropy.json` of `wai.neural.int_hyper` and `wai.video.int_hyper`, integer-payloads + §9, and a `WMB1` bundle's tables, §14) is read by the strict JSON reader, each integer from its spelling. The + general JSON reader the crate depends on (serde_json) read it before: its number parser is binary64 code, which ran + on a fraction, an exponent or an integer past `u64` before the integer reads refused them, so a decoder built into + a module with no floating point could not hold it. §9 now says how the text is read, and two readings change: `-0` + is zero, as §9's integer rule already gave (the general reader read it as a float, and the set was refused), and a + member named twice, or a value more than 64 levels deep, refuses the set (`model_mismatch`), where the general + reader kept the last of two members and read deeper values. The committed sets read as before, with tests of each + reading through `decode_int` and through a `WMB1` bundle. +- **A test of the sources** (`float_free`) fails on a floating-point type, literal or vector intrinsic in any module + on the integer decode paths, outside its tests and model-checker harnesses: the kernels, the operator-set bridge, + the payloads, the integer codecs, the rANS coder and its bucketing, the strict JSON reader, and the other integer + entropy decoders and the decodes built on them (JPEG AI's mANS decoder, `WIR1`, the NNC tensor's arithmetic decoder + and its tensor decode, `dred-int/1` but its floating-point reference and measurement, and `wai.avatar.face`); in + every file of the operator set (`wai-ops`); and on a call of the general reader's parser in those modules, but two + that are not decodes: the direct `int_synth` decode kept for the differential tests, and the face verifier's read + of an envelope's manifest. Every `int_*` file, and every file under `int_kernels/` and `int_payload/`, must be + among those it reads, so a new one fails it until it is listed. The tools' JSON forms (`from_value`, the + `*_from_json` converters) still take a `serde_json::Value`; they are converters, not decodes. Outside these paths, + `exact_tonemap`'s ST 2094-50 reader still forms binary64 slopes for its measurement, beside the Q30 slopes its + integer derivation uses. +- **Breaking:** `api:rust:wai::int_payload::IntDecoded::Attr`: its `ranges` are `(Binary32, Binary32)` pairs, the + bits the payload carries, so the decode forms no float. A sink presenting a range takes `f32::from_bits(r.0)`. + `Binary32` is re-exported as `int_payload::Binary32`. +- **Breaking:** `api:rust:wai::int_payload::wire::Wis1::ranges`, and so `wire::parse_wis1`'s result and + `wire::write_wis1`'s argument: `(Binary32, Binary32)` bits, where they were `(f32, f32)`. Build one with + `Binary32(x.to_bits())` and read it with `f32::from_bits(b.0)`. +- **Breaking:** `api:rust:wai::int_hyper_synth::CondEntropy::try_from_value`: replaced by `CondEntropy::read`, which + takes the file's bytes and reads them as a set's decode does, so no float parser reads the tables first. Call it + with `entropy.json`'s bytes in place of a parsed value. +- **Breaking:** `api:rust:wai::int_splat::SplatJson::ranges`: the binary32 values `lo` and `hi` denote, as + `(Binary32, Binary32)`, where they were `(f64, f64)` as a float parser read them. +- **Breaking:** `api:rust:wai::int_splat::SplatJson::from_value`, `api:rust:wai::int_splat::decode` and + `api:rust:wai::int_payload::wis1_from_json`: the JSON form is read from its text, so no float parser reads it + first. Call `SplatJson::read`, `decode` and `wis1_from_json` with the file's bytes in place of a parsed value. diff --git a/wai/extensions/integer-payloads.md b/wai/extensions/integer-payloads.md index a945c7c1f..dd5ef2cd4 100644 --- a/wai/extensions/integer-payloads.md +++ b/wai/extensions/integer-payloads.md @@ -527,7 +527,9 @@ arithmetic is [integer-operators](integer-operators.md) §13.2 (`WHM1`) and §13 **`entropy.json`**, the tables of `wai.neural.int_hyper` and `wai.video.int_hyper`: a JSON object with these members. Other members are ignored. An integer is a JSON number -with no fraction and no exponent. +with no fraction and no exponent, and `-0` is zero. A file that is not a JSON text +(RFC 8259, UTF-8, no member named twice, at most 64 levels, nothing after the value) is +refused (`model_mismatch`). The reference decode reads the text with no float parser. | member | value | |---|---| @@ -568,9 +570,15 @@ implicit: - `WIA1`: `sr` is the export's `sample_rate` (in `audio.json`, which the bitstream does not carry); `lo` is the symbol offset; every channel's table must have the first table's length; `afrac` is carried and must be the model's. -- `WIS1`: the ranges `lo` and `hi` must each be a binary32 value, carried exactly (a - JSON integer included: `2^60 + 1` is refused, not rounded to `2^60`); - `afrac` is carried and must be the model's. +- `WIS1`: the ranges `lo` and `hi` must each be a binary32 value, carried exactly. A + number spelled as an integer (no fraction, no exponent) is that integer: `2^60 + 1` is + refused, not rounded to `2^60`, and `-0` is zero, so `+0`. Any other spelling denotes + the binary64 value nearest it (ties to even; one that rounds to zero is a zero of its + sign, and one past the binary64 range is no value), and that value must be a binary32 + value: the export writes each range as the shortest decimal of a binary32 value + widened to binary64. The reference converter reads the JSON text in integer + arithmetic, so no float parser decides a range. `afrac` is carried and must be the + model's. A payload and the JSON it is converted from decode to the same bytes; the conformance vectors check that for every committed JSON bitstream. @@ -698,12 +706,12 @@ multiple of 4, at least 8) when the decode reaches it. The set is one file, a `WMB1` bundle: the magic `WMB1`, then four sections, each a `u32` length followed by that many bytes, in the order `h_s`, `entropy`, `g_s` (each a `WHM1` network, §9) and the tables. The tables are a JSON object with the members of -`entropy.json` (§9), and `afrac`. Nothing follows the fourth section. The bundle's -rules: the three networks' `afrac` are equal, lie in `1..=30`, and equal the tables' -`afrac`; `1 ≤ N ≤ 65536`; and every table, over its length, follows -[integer-operators](integer-operators.md) E1's `t2` rule (`2 ≤ n_b`, `n_b` no more than -its array's length, `cdf[0] = 0`, `cdf[n_b − 1] = 65536`, strictly increasing), in -place of §2's rule. +`entropy.json` (§9), and `afrac`, read as §9 reads `entropy.json`. Nothing follows the +fourth section. The bundle's rules: the three networks' `afrac` are equal, lie in +`1..=30`, and equal the tables' `afrac`; `1 ≤ N ≤ 65536`; and every table, over its +length, follows [integer-operators](integer-operators.md) E1's `t2` rule (`2 ≤ n_b`, +`n_b` no more than its array's length, `cdf[0] = 0`, `cdf[n_b − 1] = 65536`, strictly +increasing), in place of §2's rule. The decode: diff --git a/wai/wai-rs/Cargo.toml b/wai/wai-rs/Cargo.toml index 8cc3bcd14..a045b4787 100644 --- a/wai/wai-rs/Cargo.toml +++ b/wai/wai-rs/Cargo.toml @@ -207,8 +207,10 @@ audio_int = ["neural_int"] world_int = ["neural_int"] # Integer-exact LEARNED Gaussian-splat ATTRIBUTE codec ("the int_hyper of 3D"): # learned transform over the [14,S,S] attribute image → byte-exact attribute tensor -# (splat-equivalence, a portable cloud hash). Reuses int_hyper_synth (neural_int). -splat_int = ["neural_int", "serde_json/float_roundtrip"] +# (splat-equivalence, a portable cloud hash). Reuses int_hyper_synth (neural_int). Its +# JSON form is read with the strict reader, every number in integer arithmetic, so it +# needs nothing of serde_json's float parsing. +splat_int = ["neural_int"] neural = ["neural_int", "dep:ort", "dep:ndarray", "dep:zstd-safe"] # The perceptual / generative-semantic path: CLIP-similarity conformance, # regenerate-at-sink. Reuses the ONNX runtime for the reference CLIP embedder; @@ -478,8 +480,9 @@ robustness = ["synthesis_mark"] # the commitment to the frames at canonical poses; a sink's view statement; a view's class under # synthesis's lattice; and a float renderer of the same model, for measurement only. Reuses # splat_int (the decode) and synthesis (origin claims). SHA-256, BLAKE3 and Ed25519 only; no new -# dependency. -splat_exact = ["splat_int", "synthesis"] +# dependency. An asset's POSITION min and max are read as JSON numbers and compared exactly with +# the decode's, so serde_json reads them correctly rounded (`float_roundtrip`). +splat_exact = ["splat_int", "synthesis", "serde_json/float_roundtrip"] # Faces from keypoints (extensions/avatars.md §6 to §13): `wai.avatar.face` — a keypoint track # (WFK1 segments) and an identity prior named by SHA-256 (WFI1) generate a talking head that is # the same bytes on every sink: the driver's integer warp and keypoint maps, then the generator's diff --git a/wai/wai-rs/examples/int_splat_decode.rs b/wai/wai-rs/examples/int_splat_decode.rs index 3023da4da..3d483d863 100644 --- a/wai/wai-rs/examples/int_splat_decode.rs +++ b/wai/wai-rs/examples/int_splat_decode.rs @@ -11,8 +11,6 @@ use wai::int_splat::decode; fn main() { let dir = std::env::args().nth(1).expect("usage: int_splat_decode "); let gs = IntModel::from_bytes(&std::fs::read(format!("{dir}/splat_gs.bin")).unwrap()).unwrap(); - let bs: serde_json::Value = - serde_json::from_slice(&std::fs::read(format!("{dir}/splat.json")).unwrap()).unwrap(); - let clip = decode(&gs, &bs).expect("decode"); + let clip = decode(&gs, &std::fs::read(format!("{dir}/splat.json")).unwrap()).expect("decode"); println!("{} gaussians × {} attrs ({}²) attr_hash {:016x}", clip.s * clip.s, clip.n_attr, clip.s, clip.attr_hash()); } diff --git a/wai/wai-rs/src/bin/wai_decode_bench.rs b/wai/wai-rs/src/bin/wai_decode_bench.rs index 928f04922..f758f702a 100644 --- a/wai/wai-rs/src/bin/wai_decode_bench.rs +++ b/wai/wai-rs/src/bin/wai_decode_bench.rs @@ -387,13 +387,13 @@ fn vector_cases(cases: &mut Vec) { { let d = format!("{VECTORS}/int_splat"); let gs = IntModel::from_bytes(&std::fs::read(format!("{d}/splat_gs.bin")).unwrap()).unwrap(); - let bs = read_json(&format!("{d}/splat.json")); - let n = bs["n_gaussians"].as_u64().unwrap(); + let text = std::fs::read(format!("{d}/splat.json")).unwrap(); + let n = read_json(&format!("{d}/splat.json"))["n_gaussians"].as_u64().unwrap(); cases.push(Case { name: format!("vector · int_splat trained attribute decode ({n} Gaussians, committed)"), per_run: (n as f64 / 1e3, "kGaussians"), macs: None, - run: Box::new(move || Out::B(wai::int_splat::decode(&gs, &bs).unwrap().attr)), + run: Box::new(move || Out::B(wai::int_splat::decode(&gs, &text).unwrap().attr)), }); } // int_world: trained transition + synthesis per generated frame. diff --git a/wai/wai-rs/src/bin/wai_int_payload_vectors.rs b/wai/wai-rs/src/bin/wai_int_payload_vectors.rs index df9d4aa46..c3f9ff222 100644 --- a/wai/wai-rs/src/bin/wai_int_payload_vectors.rs +++ b/wai/wai-rs/src/bin/wai_int_payload_vectors.rs @@ -128,7 +128,7 @@ fn vectors(t: &mut Tree) -> Vec { set: vec![("model.wam", format!("{V}/int_audio/audio_model.bin"))], sidecar: format!("{V}/int_audio/sha256.json"), }); - let wis1 = wis1_from_json(&t.json(&format!("{V}/int_splat/splat.json"))).expect("splat WIS1"); + let wis1 = wis1_from_json(&t.read(&format!("{V}/int_splat/splat.json"))).expect("splat WIS1"); t.put(&format!("{V}/int_splat/splat.wis1"), wis1); out.push(Vector { name: "splat", @@ -316,8 +316,8 @@ fn fixtures(t: &mut Tree) -> Vec { Some(wire::write_wis1(&v)) }; fx(t, "wis1_json_form", SPL, None, Some(format!("{V}/int_splat/splat.json")), "bad_magic", "parse"); - fx(t, "wis1_range_order", SPL, wis(&|v| v.ranges[0] = (1.0, -1.0)), None, "field_range", "parse"); - fx(t, "wis1_range_nan", SPL, wis(&|v| v.ranges[0].0 = f32::NAN), None, "field_range", "parse"); + fx(t, "wis1_range_order", SPL, wis(&|v| v.ranges[0] = (wire::Binary32(1f32.to_bits()), wire::Binary32((-1f32).to_bits()))), None, "field_range", "parse"); + fx(t, "wis1_range_nan", SPL, wis(&|v| v.ranges[0].0 = wire::Binary32(f32::NAN.to_bits())), None, "field_range", "parse"); fx(t, "wis1_n_gaussians", SPL, wis(&|v| v.n_gaussians = v.s * v.s + 1), None, "field_range", "parse"); fx(t, "wis1_cdf_len", SPL, edit(&sp, &|b| b[32 + 14 * 8..32 + 14 * 8 + 2].copy_from_slice(&2u16.to_le_bytes())), None, "field_range", "parse"); fx(t, "wis1_trailing", SPL, edit(&sp, &|b| b.push(0)), None, "trailing", "parse"); diff --git a/wai/wai-rs/src/float_free.rs b/wai/wai-rs/src/float_free.rs new file mode 100644 index 000000000..0eb2dc457 --- /dev/null +++ b/wai/wai-rs/src/float_free.rs @@ -0,0 +1,366 @@ +//! No floating point on the integer decode paths (a test of the sources). +//! +//! The integer capabilities decode to the same bytes on every machine because no step of +//! their decode rounds: every value is an integer, and so is every operation on it. A +//! floating-point step on such a path is exact only by an argument someone must make and +//! keep, and a decoder compiled to a module that admits no floating point cannot hold one +//! at all. So these paths hold none in their source, and this test reads the source of +//! every module on them, and of the operator set they run their networks through +//! (`wai-ops`, every file), outside tests and model-checker harnesses, for a +//! floating-point type, literal or vector intrinsic: `f32` or `f64` (alone or in a name +//! such as `vaddq_f32` or `f32x4`), a numeric literal with a fraction or an exponent, or an +//! x86 intrinsic on packed or scalar floats (`_mm*_pd`, `_ps`, `_sd`, `_ss`, `_ph`). +//! +//! The one place a float may be named is a conversion from a float source by contract, +//! listed in [`BOUNDARIES`] with its reason: it reads its argument's bits, and every step +//! after is integer. +//! +//! It reads source, not a dependency's code, so a dependency could bring floating point +//! back unseen. The one that did is serde_json: its number parser is binary64 code, which +//! runs on a fraction, an exponent or an integer past `u64` before an integer read refuses +//! it. So the modules read here call no serde_json parser either (but on the lines in +//! [`JSON_BOUNDARIES`]): the parameter sets' JSON (`entropy.json`, a `WMB1` bundle's +//! tables) is read by [`crate::strict_json`], which is read here, and `int_synth.json` by +//! the operator set's own reader. The tools' JSON forms (`from_value` and the +//! `*_from_json` converters, over a `serde_json::Value`) are converters and debugging aids, +//! not decodes: their callers parse the text. Beyond these modules and the operator set, a +//! decode reaches the capability table's lookups (`codecs`), the pin checks and their +//! SHA-256 (`pin`, whose listing is text) and `std`, none of which reads JSON; this test +//! does not read them. +//! +//! The test fails when a file of the integer modules (`int_*.rs`, and the files under +//! `int_kernels/` and `int_payload/` but its tests) is not in [`SOURCES`], so a new one is +//! read from the start. +//! +//! Outside these paths, recorded and not read: `exact_tonemap`'s ST 2094-50 reader +//! (`read_metadata_set`) forms `Alternate::m_f64`, the slopes in binary64 (a tangent, and a +//! rational's quotient), for §6's measurement alone, beside the Q30 slopes its integer +//! derivation uses. The derivation's output does not depend on them, but its reader cannot +//! be built into a module with no floating point until they are formed apart from it. + +/// The modules on the integer decode paths: the kernels, the operator-set bridge, the +/// payload formats and their decode, the integer codecs, the entropy coder and its +/// bucketing, and the strict reader the decoders read their parameter sets' JSON with +/// (and the splat codec its JSON form); and the other integer entropy decoders and the +/// integer decodes built on them: JPEG AI's mANS decoder, the residual refinement +/// (`WIR1`) and its step, the NNC tensor's arithmetic decoder and its tensor decode, +/// DRED's range decoder, payload, tables, weights, lowering and stream (all of +/// `dred-int/1` but its floating-point reference and its measurement), and the faces +/// from keypoints. +const SOURCES: &[(&str, &str)] = &[ + ("int_kernels.rs", include_str!("int_kernels.rs")), + ("int_kernels/narrow.rs", include_str!("int_kernels/narrow.rs")), + ("int_kernels/x86.rs", include_str!("int_kernels/x86.rs")), + ("int_ops.rs", include_str!("int_ops.rs")), + ("int_payload/wire.rs", include_str!("int_payload/wire.rs")), + ("int_payload/mod.rs", include_str!("int_payload/mod.rs")), + ("int_synth.rs", include_str!("int_synth.rs")), + ("int_hyper_synth.rs", include_str!("int_hyper_synth.rs")), + ("int_mlicpp.rs", include_str!("int_mlicpp.rs")), + ("int_video.rs", include_str!("int_video.rs")), + ("int_video_enc.rs", include_str!("int_video_enc.rs")), + ("int_audio.rs", include_str!("int_audio.rs")), + ("int_splat.rs", include_str!("int_splat.rs")), + ("int_world.rs", include_str!("int_world.rs")), + ("int_sr.rs", include_str!("int_sr.rs")), + ("int_set_cache.rs", include_str!("int_set_cache.rs")), + ("int_transform.rs", include_str!("int_transform.rs")), + ("int_transformer.rs", include_str!("int_transformer.rs")), + ("int_entropy.rs", include_str!("int_entropy.rs")), + ("rans.rs", include_str!("rans.rs")), + ("strict_json.rs", include_str!("strict_json.rs")), + ("jpegai_mans.rs", include_str!("jpegai_mans.rs")), + ("staged/wir1.rs", include_str!("staged/wir1.rs")), + ("staged/wir1_arith.rs", include_str!("staged/wir1_arith.rs")), + ("prior_wire/cabac.rs", include_str!("prior_wire/cabac.rs")), + ("prior_wire/nnc.rs", include_str!("prior_wire/nnc.rs")), + ("dred_exact/ec.rs", include_str!("dred_exact/ec.rs")), + ("dred_exact/packet.rs", include_str!("dred_exact/packet.rs")), + ("dred_exact/payload.rs", include_str!("dred_exact/payload.rs")), + ("dred_exact/tables.rs", include_str!("dred_exact/tables.rs")), + ("dred_exact/weights.rs", include_str!("dred_exact/weights.rs")), + ("dred_exact/lower.rs", include_str!("dred_exact/lower.rs")), + ("dred_exact/scenario.rs", include_str!("dred_exact/scenario.rs")), + ("avatar_face.rs", include_str!("avatar_face.rs")), +]; + +/// The lines that may name a float, each a conversion from a float source by contract. +/// `int_entropy`'s two take a float model's scale table and a float hyper-decoder's σ +/// (codecs that stay `NeuralFloat`); each reads the binary32 bits and forms the +/// fixed-point value in integers. +const BOUNDARIES: &[(&str, &str)] = &[ + ("int_entropy.rs", "pub fn fixed_scale_edges(scale_table: &[f32], frac: u32) -> Vec {"), + ("int_entropy.rs", "pub fn quantize_scale(sigma: f32, frac: u32) -> i64 {"), +]; + +/// The lines that may call a serde_json parser, each with its reason. `int_synth`'s is +/// the direct decode kept for the differential tests (`legacy`, built only for tests and +/// under the test-only `legacy_int`); a sink's decode reads `int_synth.json` through the +/// operator set. `avatar_face`'s is the conformance verifier's (§9) read of an object's +/// envelope: its manifest, the container's JSON (SPEC §2), read before any face is +/// decoded; the faces themselves (§7) read no JSON. +const JSON_BOUNDARIES: &[(&str, &str)] = &[ + ("int_synth.rs", "let spec: SynthSpec = serde_json::from_slice(spec_json)"), + ("avatar_face.rs", "let m: Value = serde_json::from_slice(b.get(8..8 + n)?).ok()?;"), +]; + +/// `src` with every comment, string literal and character literal blanked (newlines +/// kept, so line numbers hold): what is left is code. +fn code_of(src: &str) -> String { + let b = src.as_bytes(); + let mut out = String::with_capacity(src.len()); + let blank = |out: &mut String, s: &str| out.extend(s.chars().map(|c| if c == '\n' { '\n' } else { ' ' })); + let ident = |i: usize| i > 0 && (b[i - 1].is_ascii_alphanumeric() || b[i - 1] == b'_'); + let mut i = 0; + while i < b.len() { + let rest = &src[i..]; + let end = if rest.starts_with("//") { + i + rest.find('\n').unwrap_or(rest.len()) + } else if rest.starts_with("/*") { + let (mut j, mut depth) = (i, 0); + loop { + if src[j..].starts_with("/*") { + depth += 1; + j += 2; + } else if src[j..].starts_with("*/") { + depth -= 1; + j += 2; + if depth == 0 { + break j; + } + } else { + j += 1; + } + } + } else if let Some(q) = raw_string_open(rest).filter(|_| !ident(i)) { + let close = format!("\"{}", "#".repeat(rest[..q].matches('#').count())); + i + q + 1 + rest[q + 1..].find(&close).expect("a raw string closes") + close.len() + } else if b[i] == b'"' || (!ident(i) && rest.starts_with("b\"")) { + let mut j = i + rest.find('"').unwrap() + 1; + while b[j] != b'"' { + j += if b[j] == b'\\' { 2 } else { 1 }; + } + j + 1 + } else if b[i] == b'\'' { + // A character literal ('x', '\n', '\u{..}'), not a lifetime ('a). + let mut cs = rest.char_indices().skip(1); + match (cs.next(), cs.next()) { + // The escaped character is at 2 (and may be a quote); the close follows. + (Some((_, '\\')), _) => i + 3 + rest[3..].find('\'').unwrap() + 1, + (Some(_), Some((k, '\''))) => i + k + 1, + _ => i + 1, + } + } else { + let c = rest.chars().next().unwrap(); + out.push(c); + i += c.len_utf8(); + continue; + }; + if b[i] == b'\'' && end == i + 1 { + out.push('\''); + } else { + blank(&mut out, &src[i..end]); + } + i = end; + } + out +} + +/// Where the opening quote of a raw string literal starting `rest` is (`r"`, `r#"`, +/// `br##"` and so on), or `None` (`r#ident`, a raw identifier, is not one). +fn raw_string_open(rest: &str) -> Option { + let after = rest.strip_prefix("br").or_else(|| rest.strip_prefix('r'))?; + let hashes = after.len() - after.trim_start_matches('#').len(); + after[hashes..].starts_with('"').then_some(rest.len() - after.len() + hashes) +} + +/// The lines of `code`, numbered from 1, without the items a `#[cfg(test)]` or +/// `#[cfg(kani)]` (alone or under `all(..)`) keeps out of a decoder's build. +fn decoder_lines(code: &str) -> Vec<(usize, &str)> { + let lines: Vec<&str> = code.lines().collect(); + let test_only = |l: &str| { + let t = l.trim(); + ["#[cfg(test)]", "#[cfg(kani)]"].contains(&t) || t.starts_with("#[cfg(all(test,") || t.starts_with("#[cfg(all(kani,") + }; + let mut out = Vec::new(); + let mut i = 0; + while i < lines.len() { + if !test_only(lines[i]) { + out.push((i + 1, lines[i])); + i += 1; + continue; + } + // Skip the attributes, then the item: to its `;`, or its braces' close. + let mut j = i; + while j < lines.len() && (lines[j].trim().starts_with("#[") || lines[j].trim().is_empty()) { + j += 1; + } + let (mut depth, mut opened) = (0i64, false); + while j < lines.len() { + let l = lines[j]; + depth += l.matches('{').count() as i64 - l.matches('}').count() as i64; + opened |= l.contains('{'); + if (!opened && l.trim_end().ends_with(';')) || (opened && depth <= 0) { + break; + } + j += 1; + } + i = j + 1; + } + out +} + +/// The floating-point tokens of one line of code. +fn float_tokens(line: &str) -> Vec<&str> { + let b = line.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while i < b.len() { + let start = i; + if b[i].is_ascii_alphabetic() || b[i] == b'_' { + while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') { + i += 1; + } + let t = &line[start..i]; + let parts: Vec<&str> = t.split('_').collect(); + let float_part = parts.iter().any(|p| matches!(*p, "f32" | "f64") || p.starts_with("f32x") || p.starts_with("f64x") || p.starts_with("float")); + let x86 = t.starts_with("_mm") && matches!(parts.last(), Some(&("pd" | "ps" | "sd" | "ss" | "ph"))); + if float_part || x86 { + out.push(t); + } + } else if b[i].is_ascii_digit() { + // A numeric literal; `x.0` (a field) is not one. + let field = start > 0 && b[start - 1] == b'.'; + while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') { + i += 1; + } + if i + 1 < b.len() && b[i] == b'.' && b[i + 1].is_ascii_digit() { + i += 1; + while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') { + i += 1; + } + } + if i < b.len() && matches!(b[i], b'+' | b'-') && matches!(b[i - 1], b'e' | b'E') { + i += 1; + while i < b.len() && b[i].is_ascii_digit() { + i += 1; + } + } + let t = &line[start..i]; + let based = t.starts_with("0x") || t.starts_with("0b") || t.starts_with("0o"); + let exp = !based && t.bytes().skip(1).zip(t.bytes().skip(2)).any(|(e, d)| matches!(e, b'e' | b'E') && (d.is_ascii_digit() || d == b'+' || d == b'-')); + if !field && !based && (t.contains('.') || exp || t.ends_with("f32") || t.ends_with("f64")) { + out.push(t); + } + } else { + i += 1; + } + } + out +} + +/// The serde_json parsers one line of code names: `from_slice`, `from_str`, +/// `from_reader` or `Deserializer` on a line that names `serde_json`. +fn json_parsers(line: &str) -> Vec<&str> { + if !line.contains("serde_json") { + return Vec::new(); + } + line.split(|c: char| !(c.is_ascii_alphanumeric() || c == '_')) + .filter(|t| matches!(*t, "from_slice" | "from_str" | "from_reader" | "Deserializer")) + .collect() +} + +/// Every floating-point token and every serde_json parser on a decoder line of `src`, as +/// `line: token`, but those on a boundary line of `file` for each. +fn findings(file: &str, src: &str) -> Vec { + let code = code_of(src); + let source: Vec<&str> = src.lines().collect(); + let on = |list: &[(&str, &str)], n: usize| list.iter().any(|&(f, l)| f == file && source[n - 1].trim() == l); + let mut out = Vec::new(); + for (n, line) in decoder_lines(&code) { + let floats = if on(BOUNDARIES, n) { Vec::new() } else { float_tokens(line) }; + let parsers = if on(JSON_BOUNDARIES, n) { Vec::new() } else { json_parsers(line) }; + for t in floats.into_iter().chain(parsers) { + out.push(format!("{file}:{n}: {t}")); + } + } + out +} + +/// Every file under `root`, as its path from `root` with `/` between the parts. +fn files_under(root: &std::path::Path) -> Vec { + let mut out = Vec::new(); + let mut dirs = vec![String::new()]; + while let Some(d) = dirs.pop() { + let dir = root.join(&d); + for e in std::fs::read_dir(&dir).unwrap_or_else(|e| panic!("{}: {e}", dir.display())) { + let e = e.expect("a directory entry"); + let name = e.file_name().into_string().expect("a UTF-8 file name"); + let rel = if d.is_empty() { name } else { format!("{d}/{name}") }; + if e.file_type().expect("a file type").is_dir() { + dirs.push(rel); + } else { + out.push(rel); + } + } + } + out.sort(); + out +} + +/// Whether `f`, a path under `src/`, is a file of the integer modules: an `int_*.rs`, or a +/// source file under `int_kernels/` or `int_payload/` but the payloads' tests. +fn integer_module(f: &str) -> bool { + let top = !f.contains('/') && f.starts_with("int_"); + let under = f.starts_with("int_kernels/") || (f.starts_with("int_payload/") && f != "int_payload/tests.rs"); + f.ends_with(".rs") && (top || under) +} + +/// The operator set's source files (`wai-ops/src`, every one), as `(name, text)`. +fn operator_set() -> Vec<(String, String)> { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../wai-ops/src"); + files_under(&root).into_iter().filter(|f| f.ends_with(".rs")) + .map(|f| { + let text = std::fs::read_to_string(root.join(&f)).unwrap_or_else(|e| panic!("wai-ops/src/{f}: {e}")); + (format!("wai-ops/src/{f}"), text) + }) + .collect() +} + +#[test] +fn the_integer_decode_paths_hold_no_floating_point() { + let ops = operator_set(); + assert!(ops.iter().any(|(f, _)| f == "wai-ops/src/interp.rs"), "the operator set's source is read"); + let read = SOURCES.iter().map(|&(f, src)| (f, src)).chain(ops.iter().map(|(f, src)| (f.as_str(), src.as_str()))); + let all: Vec = read.flat_map(|(f, src)| findings(f, src)).collect(); + assert!(all.is_empty(), "floating point, or a serde_json parser, on an integer decode path:\n{}", all.join("\n")); + // Every boundary is a line that is there, so none outlives its code. + for &(f, l) in BOUNDARIES.iter().chain(JSON_BOUNDARIES) { + let src = SOURCES.iter().find(|&&(n, _)| n == f).expect("a listed source").1; + assert!(src.lines().any(|s| s.trim() == l), "{f}: the boundary line is gone: {l}"); + } + // Every file of the integer modules is read, so a new one is not passed over. + let src = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"); + let unread: Vec = files_under(&src).into_iter() + .filter(|f| integer_module(f) && !SOURCES.iter().any(|&(n, _)| n == f)).collect(); + assert!(unread.is_empty(), "files of the integer modules this test does not read (list them in SOURCES): {unread:?}"); +} + +/// The reader finds what it must and passes what it must: it is the test. +#[test] +fn the_reader_finds_floats_in_code_and_only_there() { + let caught = "fn f(n: u64) -> u64 {\n (n as f64).sqrt() as u64\n}\n\ + fn g() { let x = 1.5; let y = 2e3; let z = 1u8 as f32; let w = 3f64; }\n\ + unsafe fn h() { _mm512_sqrt_pd(a); _mm_cvtepi32_ps(b); vaddq_f32(c, d); let v: float32x4_t; }\n\ + fn j(b: &[u8]) { let v: serde_json::Value = serde_json::from_slice(b).unwrap(); let w = serde_json::from_str::(\"1\"); }\n"; + let got: Vec = findings("x.rs", caught).into_iter().map(|s| s.split(": ").nth(1).unwrap().to_string()).collect(); + assert_eq!(got, ["f64", "1.5", "2e3", "f32", "3f64", "_mm512_sqrt_pd", "_mm_cvtepi32_ps", "vaddq_f32", "float32x4_t", "from_slice", "from_str"]); + let passed = "// (n as f64).sqrt()\n/* 1.5 */\nfn f() -> &'static str { let s = \"f64 1.5\"; let c = '.'; let r = r#\"f32\"#; t.0.1; 0x1e5; 1..2; s }\n\ + #[cfg(test)]\nmod tests {\n fn t() { let x = 1.0f64; }\n}\n\ + #[cfg(kani)]\nfn k() { let y: f32 = 0.5; }\n\ + fn ok<'a>(isqrt: u64, ph: usize, s: &'a str) -> u64 { let q = '\\''; let e = b'\"'; isqrt + 10u64 }\n\ + // serde_json::from_slice(b)\n\ + fn v(j: &serde_json::Value, s: &str) -> u32 { let t = \"serde_json::from_str\"; u32::from_str_radix(s, 16).unwrap() }\n\ + #[cfg(test)]\nfn parse() { let v: serde_json::Value = serde_json::from_slice(b\"1\").unwrap(); }\n"; + assert_eq!(findings("x.rs", passed), Vec::::new()); +} diff --git a/wai/wai-rs/src/int_entropy.rs b/wai/wai-rs/src/int_entropy.rs index 39f32377b..4dc2c28ee 100644 --- a/wai/wai-rs/src/int_entropy.rs +++ b/wai/wai-rs/src/int_entropy.rs @@ -41,12 +41,19 @@ /// Precompute a CompressAI-style ascending `scale_table` as fixed-point integer /// bucket edges (Q`frac`: `frac` fractional bits, grid step `2^-frac`). Compute /// once per model; pass the result to [`det_build_index`]. +/// +/// The table is binary32 by contract (it comes from a float model); each edge is +/// `⌊e·2^frac⌋` formed from `e`'s bits in integer arithmetic ([`floor_scaled`]), the +/// value the product in binary64 and its floor give exactly, with no floating-point +/// operation. pub fn fixed_scale_edges(scale_table: &[f32], frac: u32) -> Vec { - let s = (1u64 << frac) as f64; - scale_table.iter().map(|&e| (e as f64 * s).floor() as i64).collect() + let k = (1u64 << frac).trailing_zeros(); + scale_table.iter().map(|&e| floor_scaled(e.to_bits(), k)).collect() } -/// Snap a float `σ` to the fixed-point grid (Q`frac`), deterministic floor. +/// Snap a float `σ` to the fixed-point grid (Q`frac`), deterministic floor: `⌊σ·2^frac⌋` +/// for `σ > 0`, and 0 for a negative σ, a zero and a NaN. Formed from `σ`'s bits in +/// integer arithmetic ([`floor_scaled`]). /// /// For a codec whose σ source is still a float ONNX hyper-decoder: this removes /// drift smaller than the grid step *within* a bucket, but is **not** a hard @@ -55,8 +62,34 @@ pub fn fixed_scale_edges(scale_table: &[f32], frac: u32) -> Vec { /// σ from an integer hyper-decoder, where the integer σ is exact and this snap /// is unnecessary. pub fn quantize_scale(sigma: f32, frac: u32) -> i64 { - let s = (1u64 << frac) as f64; - ((sigma.max(0.0) as f64) * s).floor() as i64 + let k = (1u64 << frac).trailing_zeros(); + let b = sigma.to_bits(); + if b >> 31 == 1 || b & 0x7fff_ffff > 0x7f80_0000 { 0 } else { floor_scaled(b, k) } +} + +/// `⌊x·2^k⌋` for the binary32 value `x` whose bits are `bits` (`k < 64`), saturated to +/// the `i64` range as a float-to-integer conversion saturates it (an infinity to the +/// end of its sign, a NaN to 0). Integer arithmetic only: `x = ±m·2^e` with an integer +/// `m < 2^24`, so `x·2^k` is `±m` shifted, and its floor rounds a negative fraction +/// away from zero. +fn floor_scaled(bits: u32, k: u32) -> i64 { + let neg = bits >> 31 == 1; + let (ex, f) = ((bits >> 23) & 0xff, (bits & 0x7f_ffff) as i128); + if ex == 0xff { + return if f != 0 { 0 } else if neg { i64::MIN } else { i64::MAX }; + } + let (m, e) = if ex == 0 { (f, -149) } else { (f | 1 << 23, ex as i32 - 150) }; + let e = e + k as i32; + let v = if e >= 0 { + // m < 2^24 and e ≤ 104 + 63: past 2^63 the value saturates whatever its size. + let mag = if e >= 64 { if m == 0 { 0 } else { 1i128 << 64 } } else { m << e }; + if neg { -mag } else { mag } + } else { + let d = (-e) as u32; + let (q, exact) = if d >= 64 { (0, m == 0) } else { (m >> d, m & ((1i128 << d) - 1) == 0) }; + if neg { -q - if exact { 0 } else { 1 } } else { q } + }; + v.clamp(i64::MIN as i128, i64::MAX as i128) as i64 } /// Integer `build_index`: bucket a fixed-point `σ` to a scale-table index, with @@ -368,6 +401,34 @@ mod tests { } } + /// The integer conversions give what the binary64 product and its floor gave (the + /// form they replace, kept here as the reference): every exponent field, the + /// mantissa's ends and random mantissas, each sign, subnormals, zeros, infinities + /// and NaNs, at every `frac` in `0..64`. + #[test] + fn the_integer_scale_conversions_are_the_float_ones() { + let edge = |b: u32, frac: u32| (f32::from_bits(b) as f64 * (1u64 << frac) as f64).floor() as i64; + let snap = |b: u32, frac: u32| ((f32::from_bits(b).max(0.0) as f64) * (1u64 << frac) as f64).floor() as i64; + let mut seed = 0x5157_0005u64; + let mut next = || { + seed = seed.wrapping_add(0x9E37_79B9_7F4A_7C15); + let z = (seed ^ (seed >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + ((z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB) ^ (z >> 31)) as u32 + }; + let mut bits = vec![0x7fc0_0000u32, 0x7f80_0001, 0xffc0_0000]; + for ex in 0..=0xffu32 { + for f in [0u32, 1, 2, 0x40_0000, 0x7f_fffe, 0x7f_ffff, next() & 0x7f_ffff, next() & 0x7f_ffff] { + bits.extend([ex << 23 | f, 0x8000_0000 | ex << 23 | f]); + } + } + for frac in 0..64 { + for &b in &bits { + assert_eq!(fixed_scale_edges(&[f32::from_bits(b)], frac), vec![edge(b, frac)], "{b:#010x} frac {frac}"); + assert_eq!(quantize_scale(f32::from_bits(b), frac), snap(b, frac), "{b:#010x} frac {frac}"); + } + } + } + #[test] fn clamps_below_the_smallest_scale() { let st = scale_table(); diff --git a/wai/wai-rs/src/int_hyper_synth.rs b/wai/wai-rs/src/int_hyper_synth.rs index 0eae6cef2..64004e027 100644 --- a/wai/wai-rs/src/int_hyper_synth.rs +++ b/wai/wai-rs/src/int_hyper_synth.rs @@ -516,22 +516,28 @@ fn check_entropy(t: &CondEntropy) -> Result<(), &'static str> { } impl CondEntropy { - /// Parse and check the entropy tables (`cond_entropy.json`, read from a set as - /// `entropy.json`): every member present with integers in range, the lengths - /// consistent and every table well formed. Refuses rather than panics. - pub fn try_from_value(v: &Value) -> Result { - fn arr<'a>(v: &'a Value, k: &str) -> Result<&'a Vec, &'static str> { - v.get(k).and_then(Value::as_array).ok_or("an entropy member is missing or not an array") + /// Read and check the entropy tables (`cond_entropy.json`, read from a set as + /// `entropy.json`, `extensions/integer-payloads.md` §9) from the file's text. The text + /// is read strictly (RFC 8259, UTF-8, no member named twice, at most 64 levels, nothing + /// after the value) by a reader that converts no number to a float; every member must be + /// present, each value an integer as spelled (no fraction or exponent; `-0` is zero) in + /// its type's range, the lengths consistent and every table well formed. Refuses rather + /// than panics; the error says why. + pub fn read(text: &[u8]) -> Result { + use crate::strict_json::{self, Json}; + fn arr<'a>(v: &'a Json, k: &str) -> Result<&'a [Json], &'static str> { + v.get(k).and_then(Json::as_arr).ok_or("an entropy member is missing or not an array") } - fn ints>(v: &Value, k: &str) -> Result, &'static str> { + fn ints>(v: &Json, k: &str) -> Result, &'static str> { arr(v, k)?.iter().map(|x| x.as_i64().and_then(|i| T::try_from(i).ok())).collect::>() .ok_or("an entropy member holds a value out of range") } - fn rows(v: &Value, k: &str) -> Result>, &'static str> { - arr(v, k)?.iter().map(|r| r.as_array().ok_or("an entropy table is not an array") + fn rows(v: &Json, k: &str) -> Result>, &'static str> { + arr(v, k)?.iter().map(|r| r.as_arr().ok_or("an entropy table is not an array") .and_then(|r| r.iter().map(|x| x.as_i64().and_then(|i| i32::try_from(i).ok())).collect::>() .ok_or("an entropy table holds a value out of range"))).collect() } + let v = &strict_json::read(text)?; let t = CondEntropy { edges: ints(v, "scale_edges")?, cdfs: rows(v, "cdfs")?, diff --git a/wai/wai-rs/src/int_kernels.rs b/wai/wai-rs/src/int_kernels.rs index 4999162aa..045038e72 100644 --- a/wai/wai-rs/src/int_kernels.rs +++ b/wai/wai-rs/src/int_kernels.rs @@ -171,9 +171,10 @@ //! The narrow kernel sets also run the inverse GDN's norms in SIMD 64-bit lanes on x86_64 //! (each term `(γ·x²) >> gf` formed exactly from `i32` operands by `vpmuldq`, floored by //! an arithmetic shift, added in `j` order as the reference adds it), fuse the shift and -//! the addition at `gf = 16` on NEON (`ssra`), and seed each integer square root from the -//! IEEE-754 square root before correcting it in integers ([`isqrt_u64`]), which gives -//! the integer root for any seed. [`Isa::Wide`] runs the kernels as they were before the +//! the addition at `gf = 16` on NEON (`ssra`), and seed each integer square root from a +//! fixed-point reciprocal square root before correcting it ([`isqrt_u64`]), which gives +//! the integer root for any seed; on AVX-512 the roots are taken digit by digit in vector +//! lanes. No step uses floating point. [`Isa::Wide`] runs the kernels as they were before the //! narrow sets, and every set is held to the reference by the same tests ([`isas`] lists //! the sets a processor runs; the tests run each). //! @@ -1553,17 +1554,61 @@ pub(crate) fn igdn_xm(x: &[i64], c: usize, hw: usize, frac: u32, gamma: &[i64], Some(out) } -/// `⌊√n⌋`, the value of `n.isqrt()`, from a seed: the IEEE-754 square root of `n` -/// rounded to `f64` (an operation every conforming platform rounds correctly), capped at -/// the largest root a `u64` has, then corrected in integer arithmetic until `s² ≤ n < -/// (s + 1)²`. The correction alone decides the result, so it is the integer root -/// whatever the seed; the seed decides only how many steps the correction takes (at most -/// one or two from a correctly rounded root). +/// `⌊√n⌋`, the value of `n.isqrt()`, in integer arithmetic only, from a seed, then +/// corrected until `s² ≤ n < (s + 1)²` ([`isqrt_from`]). The correction alone decides the +/// result, so it is the integer root whatever the seed; the seed decides only how many +/// steps the correction takes. +/// +/// The seed is a fixed-point reciprocal square root. `n` is shifted left by an even +/// count `z` to `m ∈ [2^62, 2^64)`, so `x = m / 2^64 ∈ [1/4, 1)`, and `y ≈ 1/√x` comes +/// from [`RSQRT_SEED`] (indexed by `m`'s top nine bits, about 9 bits correct) and two +/// Newton steps `y ← y·(3 − x·y²)/2` (about 17, then 34 bits). Then `√m = m·y / 2^32`, +/// and `√n = √m / 2^(z/2)`. Every product is formed exactly in `u64` or `u128` (no step +/// can overflow: `x·y² ≈ 1`, `y ≤ 2`). The seed is within one of the root on every +/// input the tests try, so the correction takes at most one step. +/// +/// No floating-point operation is on this path: it is the inverse GDN's root on the +/// narrow kernel sets, and a decode compiled to a module with no floating point runs it. #[inline(always)] pub(crate) fn isqrt_u64(n: u64) -> u64 { - isqrt_from(n, (n as f64).sqrt() as u64) + isqrt_from(n, isqrt_seed(n)) } +/// [`isqrt_u64`]'s seed: an estimate of `⌊√n⌋` in integer arithmetic (0 for 0). +#[inline(always)] +fn isqrt_seed(n: u64) -> u64 { + if n == 0 { + return 0; + } + let z = n.leading_zeros() & !1; + let m = n << z; + // y0 ≈ 2^15/√x, from the table: m ≥ 2^62, so m >> 55 is 128..=511. + let y0 = RSQRT_SEED[(m >> 55) as usize - 128] as u64; + // y1 = y0·(3 − x·y0²)/2 at 2^30: x at 2^32 (m's top half) times y0² at 2^30 is below + // 2^64; x·y0² is within 2^-7 of 1, so the subtraction stays positive. + let p = ((m >> 32) * (y0 * y0)) >> 32; + let y1 = (y0 * ((3 << 30) - p)) >> 16; + // y2 = y1·(3 − x·y1²)/2 at 2^62: x at 2^64 (m itself) times y1² at 2^60, in u128. + let p = ((m as u128 * (y1 * y1) as u128) >> 64) as u64; + let y2 = ((y1 as u128 * ((3u64 << 60) - p) as u128) >> 29) as u64; + // √m = m·y2 / 2^(62 + 32), within one of ⌊√m⌋; shifting it by z/2 keeps that. + let r = ((m as u128 * y2 as u128) >> 94) as u64; + r >> (z >> 1) +} + +/// [`isqrt_u64`]'s seed table: for `i` in `0..384`, `⌊√(2^40 / (2·(i + 128) + 1))⌋`, +/// which is `2^15 / √x` at the middle `x = (i + 128 + 1/2) / 512` of the interval of +/// `x = m / 2^64` whose top nine bits are `i + 128`. Every entry is below `2^16`. +static RSQRT_SEED: [u16; 384] = { + let mut t = [0u16; 384]; + let mut i = 0; + while i < 384 { + t[i] = ((1u64 << 40) / (2 * (i as u64 + 128) + 1)).isqrt() as u16; + i += 1; + } + t +}; + /// `⌊√n⌋` from any seed: [`isqrt_u64`]'s correction, which alone decides the result. #[inline(always)] fn isqrt_from(n: u64, seed: u64) -> u64 { @@ -1608,8 +1653,9 @@ fn igdn_tile_narrow( } v }; - // The narrow kernel sets seed each root from the IEEE-754 square root ([`isqrt_u64`]); - // the wide set keeps the integer routine. Both give the integer root. + // The narrow kernel sets seed each root from a fixed-point reciprocal square root + // ([`isqrt_u64`]); the wide set keeps the standard library's routine. Both give the + // integer root, in integer arithmetic. let seeded = isa != Isa::Wide; // On AVX-512 the roots and the products run in vector lanes ([`x86::igdn_emit_avx512`]) // where every |x| < 2^31; the rest of each channel's positions here. @@ -2363,26 +2409,67 @@ mod tests { } } - /// The seeded integer root is `u64::isqrt` on every value class: small values, every - /// perfect square and its neighbours around the powers of two, the edge of `f64`'s - /// exact integers (2^53) and of `u64`, and random values of every magnitude. + /// `⌊√n⌋` by the binary restoring method, one digit at a time: the reference the + /// integer root is held to here, independent of its seed and of the standard library. + fn isqrt_restoring(n: u64) -> u64 { + let (mut rem, mut s, mut bit) = (n, 0u64, 1u64 << 62); + while bit != 0 { + if rem >= s + bit { + rem -= s + bit; + s = (s >> 1) + bit; + } else { + s >>= 1; + } + bit >>= 2; + } + s + } + + /// `s` is the integer root of `n` by its definition, `s² ≤ n < (s + 1)²`, in `u128`. + fn is_root(n: u64, s: u64) -> bool { + let (n, s) = (n as u128, s as u128); + s * s <= n && n < (s + 1) * (s + 1) + } + + /// The integer root over the edges of `u64`, against the definition, the restoring + /// method and `u64::isqrt`: 0, 1 and every `n` below 2^17; for roots `k` around every + /// power of two and on a stride across `0..2^32` (with `2^32 − 1`, the largest), each + /// `k² − 1`, `k²` and `k² + 1`; `u64::MAX` and its neighbours; and random values of + /// every magnitude. The seed is within one of the root on all of them, so the + /// correction takes at most one step. #[test] - fn the_seeded_root_is_the_integer_root() { + fn the_integer_root_is_exact_on_the_edges_of_u64() { let mut r = testkit::Rng(0x5157_0001); - let check = |n: u64| assert_eq!(isqrt_u64(n), n.isqrt(), "{n}"); - for n in 0..100_000u64 { + let check = |n: u64| { + let s = isqrt_u64(n); + assert!(is_root(n, s), "n {n}: {s} is not its root"); + assert_eq!(s, isqrt_restoring(n), "n {n}"); + assert_eq!(s, n.isqrt(), "n {n}"); + assert!(isqrt_seed(n).abs_diff(s) <= 1, "n {n}: the seed {} is more than one from {s}", isqrt_seed(n)); + }; + for n in 0..1u64 << 17 { check(n); } - for e in 0..32u32 { - for d in [-2i64, -1, 0, 1, 2] { - let s = (1u64 << e).wrapping_add_signed(d); - let q = s.wrapping_mul(s); - for dd in [-1i64, 0, 1] { - check(q.wrapping_add_signed(dd)); + let around = |k: u64| { + let q = k as i128 * k as i128; + for d in [-1i128, 0, 1] { + if let Ok(n) = u64::try_from(q + d) { + check(n); + } + } + }; + for e in 0..=32u32 { + for d in -3i64..=3 { + if let Some(k) = (1u64 << e).checked_add_signed(d).filter(|&k| k <= u32::MAX as u64) { + around(k); } } } - for n in [(1u64 << 53) - 1, 1 << 53, (1 << 53) + 1, u64::MAX, u64::MAX - 1, (u32::MAX as u64) * (u32::MAX as u64), (u32::MAX as u64) * (u32::MAX as u64) - 1] { + for k in (0..=u32::MAX as u64).step_by(4_093) { + around(k); + } + around(u32::MAX as u64); + for n in [u64::MAX, u64::MAX - 1, u64::MAX - 2, 1 << 63, (1 << 63) - 1, (1 << 62) - 1, 1 << 62] { check(n); } for _ in 0..200_000 { @@ -2390,9 +2477,7 @@ mod tests { check(r.next() >> sh); } // The correction decides the result whatever the seed: seeds below and above the - // root by up to a thousand, and the largest, each corrected to the root. (A - // correctly rounded square root seeds every `u64` within one of its root, so these - // are the seeds a platform whose square root were not would give.) + // root by up to a thousand, and the largest, each corrected to the root. for _ in 0..20_000 { let n = r.next() >> r.below(64) as u32; let root = n.isqrt(); @@ -2406,6 +2491,28 @@ mod tests { } } + /// Every root boundary of `u64`, the values where `⌊√n⌋` changes: for every `k` in + /// `1..2^32`, the root of `k² − 1` is `k − 1` and the root of `k²` is `k`. Some 8.6 + /// billion roots: run it alone, in release (`cargo test --release --lib -- + /// --ignored int_kernels::tests::every_root_boundary_of_u64`). + #[test] + #[ignore = "exhaustive over 2^32 roots; run alone in release"] + fn every_root_boundary_of_u64() { + let workers = std::thread::available_parallelism().map_or(1, |n| n.get()) as u64; + let span = (1u64 << 32) / workers + 1; + std::thread::scope(|sc| { + for w in 0..workers { + sc.spawn(move || { + for k in (w * span).max(1)..((w + 1) * span).min(1 << 32) { + let q = k * k; + assert_eq!(isqrt_u64(q), k, "{q}"); + assert_eq!(isqrt_u64(q - 1), k - 1, "{}", q - 1); + } + }); + } + }); + } + /// Every SIMD form of the inverse GDN's norms against the scalar sum written out, /// norm by norm, before any root is taken (a root can hide a norm that is a unit off): /// NEON at any gf and its fused form at gf = 16; AVX2 with non-negative and with signed diff --git a/wai/wai-rs/src/int_kernels/x86.rs b/wai/wai-rs/src/int_kernels/x86.rs index fb68f103c..82cb59a7f 100644 --- a/wai/wai-rs/src/int_kernels/x86.rs +++ b/wai/wai-rs/src/int_kernels/x86.rs @@ -409,8 +409,9 @@ unsafe fn igdn_avx2(x2: &[i32], np: usize, gt: &[i32], beta: /// The inverse GDN's outputs from its norms on AVX-512 (F and DQ): `dst[p] = (x[p] · /// ⌊√max(0, norm[p])⌋) >> frac` for `p` in a leading multiple of 8 (returned). Each root is -/// seeded from the IEEE-754 square root and corrected, then held to `s² ≤ n < (s + 1)²` in -/// integers; a step whose roots do not all meet it is left to the caller (none is +/// taken digit by digit in integer lanes (the binary restoring method: 32 steps, one per +/// digit of a root of `n < 2^63`), with no floating point, then held to `s² ≤ n < (s + +/// 1)²` in integers; a step whose roots do not all meet it is left to the caller (none is /// expected). The product is `vpmullq`, exact because every `|x| < 2^31` and `s < 2^32`. /// /// # Safety @@ -428,16 +429,19 @@ pub(super) unsafe fn igdn_emit_avx512(norm: &[i64], x: &[i64], frac: u32, n: usi unsafe { while p + 8 <= n { let nv = _mm512_max_epi64(_mm512_loadu_si512(norm.as_ptr().add(p) as *const __m512i), zero); - let mut s = _mm512_min_epu64(_mm512_cvttpd_epu64(_mm512_sqrt_pd(_mm512_cvtepi64_pd(nv))), cap); - for _ in 0..2 { - let over = _mm512_cmpgt_epu64_mask(_mm512_mullo_epi64(s, s), nv); - s = _mm512_mask_sub_epi64(s, over, s, one); - } - for _ in 0..2 { - let s1 = _mm512_add_epi64(s, one); - let room = _mm512_cmplt_epu64_mask(s, cap); - let under = _mm512_mask_cmple_epu64_mask(room, _mm512_mullo_epi64(s1, s1), nv); - s = _mm512_mask_add_epi64(s, under, s, one); + // ⌊√n⌋ digit by digit: `bit` runs over the powers of four from 2^62 down; where + // the remainder holds `s + bit`, it is subtracted and the digit is set. + // `s + bit` stays below 2^64 (before each step `s ≤ 2√(n·bit)`, with n < 2^63 + // and `bit ≤ 2^60` once `s` is nonzero), and the remainder never goes negative. + let mut rem = nv; + let mut s = zero; + for d in 0..32 { + let bit = _mm512_set1_epi64(1i64 << (62 - 2 * d)); + let t = _mm512_add_epi64(s, bit); + let take = _mm512_cmpge_epu64_mask(rem, t); + rem = _mm512_mask_sub_epi64(rem, take, rem, t); + s = _mm512_srli_epi64::<1>(s); + s = _mm512_mask_add_epi64(s, take, s, bit); } // s² ≤ n, and (s + 1)² > n wherever s + 1 has a square in u64 let s1 = _mm512_add_epi64(s, one); diff --git a/wai/wai-rs/src/int_mlicpp.rs b/wai/wai-rs/src/int_mlicpp.rs index a65d23a9f..8e84eca14 100644 --- a/wai/wai-rs/src/int_mlicpp.rs +++ b/wai/wai-rs/src/int_mlicpp.rs @@ -66,7 +66,8 @@ //! reconstruction: magic `WMB1`, then four sections, each a `u32` length and its //! bytes, in order `h_s`, `entropy`, `g_s` (integer networks in the compact `WHM1` //! form, [`IntModel::from_bytes`]) and the entropy tables (JSON with the fields of -//! [`CondEntropy`], plus `afrac`). +//! [`CondEntropy`], plus `afrac`, read as [`CondEntropy::read`] reads them: strictly, and +//! by a reader that converts no number to a float). //! //! ## Totality //! @@ -87,11 +88,10 @@ //! or an error for any payload, and a build with overflow checks and one without return //! the same one. Pure integer, no I/O. -use serde_json::Value; - use crate::int_entropy::{dequantize, det_build_index}; use crate::int_hyper_synth::{net_shape, CondEntropy, IntModel}; use crate::rans::RansDecoder; +use crate::strict_json::Json; /// Payload magic. pub const PAYLOAD_MAGIC: [u8; 4] = *b"WIM1"; @@ -197,8 +197,8 @@ impl IntMlicModel { let h_s = net(section("h_s")?, "h_s")?; let entropy = net(section("entropy")?, "entropy")?; let g_s = net(section("g_s")?, "g_s")?; - let tables_json: Value = serde_json::from_slice(section("tables")?) - .map_err(|e| model_err(format!("tables: {e}")))?; + // Read strictly, by a reader that converts no number to a float (§14, §9). + let tables_json = crate::strict_json::read(section("tables")?).map_err(|e| model_err(format!("tables: {e}")))?; if p != bytes.len() { return Err(model_err("trailing bytes after the tables section")); } @@ -233,19 +233,20 @@ pub fn check_cdf(cdf: &[i32], len: usize) -> Result<(), IntMlicError> { /// Read the tables section: `afrac`, `scale_edges`, `cdfs`, `cdf_lengths`, `offsets`, /// `z_cdfs`, `z_cdf_lengths`, `z_offsets`, `z_median_q` — the fields of /// `int_hyper_synth`'s `cond_entropy.json`. Refuses (does not panic on) a missing -/// field, a non-integer, or a value out of its type's range. -fn parse_tables(v: &Value) -> Result<(CondEntropy, u32), IntMlicError> { - fn arr<'a>(v: &'a Value, k: &str) -> Result<&'a Vec, IntMlicError> { - v.get(k).and_then(Value::as_array).ok_or_else(|| model_err(format!("tables: `{k}` is not an array"))) +/// field, a non-integer (a fraction or an exponent; `-0` is zero), or a value out of its +/// type's range. +fn parse_tables(v: &Json) -> Result<(CondEntropy, u32), IntMlicError> { + fn arr<'a>(v: &'a Json, k: &str) -> Result<&'a [Json], IntMlicError> { + v.get(k).and_then(Json::as_arr).ok_or_else(|| model_err(format!("tables: `{k}` is not an array"))) } - fn int>(x: &Value, k: &str) -> Result { + fn int>(x: &Json, k: &str) -> Result { x.as_i64().and_then(|i| T::try_from(i).ok()).ok_or_else(|| model_err(format!("tables: `{k}` holds a value out of range"))) } - fn ints>(v: &Value, k: &str) -> Result, IntMlicError> { + fn ints>(v: &Json, k: &str) -> Result, IntMlicError> { arr(v, k)?.iter().map(|x| int(x, k)).collect() } - fn rows(v: &Value, k: &str) -> Result>, IntMlicError> { - arr(v, k)?.iter().map(|r| r.as_array().ok_or_else(|| model_err(format!("tables: `{k}` row is not an array"))) + fn rows(v: &Json, k: &str) -> Result>, IntMlicError> { + arr(v, k)?.iter().map(|r| r.as_arr().ok_or_else(|| model_err(format!("tables: `{k}` row is not an array"))) .and_then(|r| r.iter().map(|x| int(x, k)).collect())).collect() } let afrac = v.get("afrac").ok_or_else(|| model_err("tables: no `afrac`")) @@ -1016,4 +1017,40 @@ mod tests { } assert!(IntMlicModel::from_bundle(&b2).is_err()); } + + /// The tables section is read as a set's `entropy.json` is (integer-payloads §14 and + /// §9), by a reader that converts no number to a float: `-0` is zero, a fraction or an + /// exponent is not an integer, and a member named twice or anything after the value + /// refuses the bundle. + #[test] + fn a_bundles_tables_are_read_strictly() { + let mut g = Rng(0x1A7E_4C07); + let m = model(&mut g); + let (pl, _) = encode(&m, &mut g); + let t = &m.tables; + let text = serde_json::to_string(&serde_json::json!({ + "afrac": AF, "scale_edges": t.edges, "cdfs": t.cdfs, "cdf_lengths": t.cdf_lengths, "offsets": t.offsets, + "z_cdfs": t.z_cdfs, "z_cdf_lengths": t.z_cdf_lengths, "z_offsets": t.z_offsets, "z_median_q": t.z_median_q, + })) + .unwrap(); + let bundle = |tables: &str| { + let mut b = BUNDLE_MAGIC.to_vec(); + for part in [to_whm1(&m.h_s), to_whm1(&m.entropy), to_whm1(&m.g_s), tables.as_bytes().to_vec()] { + b.extend((part.len() as u32).to_le_bytes()); + b.extend(part); + } + IntMlicModel::from_bundle(&b) + }; + let want = decode(&m, &pl).unwrap(); + assert_eq!(decode(&bundle(&text).unwrap(), &pl).unwrap(), want); + // Every table starts at 0: the first, spelled otherwise. + assert_eq!(text.matches("[[0,").count(), 2, "the latent and hyper-latent tables, each from 0"); + let first = |spelled: &str| text.replacen("[[0,", &format!("[[{spelled},"), 1); + assert_eq!(decode(&bundle(&first("-0")).expect("-0 is zero"), &pl).unwrap(), want); + for not_integer in ["0.0", "-0.0", "0e0"] { + assert!(bundle(&first(not_integer)).is_err(), "{not_integer} is not an integer"); + } + assert!(bundle(&text.replacen('{', &format!("{{\"afrac\":{AF},"), 1)).is_err(), "a member named twice"); + assert!(bundle(&format!("{text} 0")).is_err(), "bytes after the value"); + } } diff --git a/wai/wai-rs/src/int_ops.rs b/wai/wai-rs/src/int_ops.rs index ccb11e7a0..ed6443b1d 100644 --- a/wai/wai-rs/src/int_ops.rs +++ b/wai/wai-rs/src/int_ops.rs @@ -751,7 +751,7 @@ mod equivalence { let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../wai-rs/tests/vectors/int_hyper"); let read = |f: &str| std::fs::read(dir.join(f)).unwrap(); let (g, h) = (IntModel::from_bytes(&read("tiny_g_s.bin")).unwrap(), IntModel::from_bytes(&read("tiny_h_s.bin")).unwrap()); - let ent = CondEntropy::try_from_value(&serde_json::from_slice(&read("tiny_entropy.json")).unwrap()).unwrap(); + let ent = CondEntropy::read(&read("tiny_entropy.json")).unwrap(); let p = read("tiny_bitstream.wih1"); let k = crate::int_wire::parse_wih1(&p).unwrap(); let st = CondStream { diff --git a/wai/wai-rs/src/int_payload/mod.rs b/wai/wai-rs/src/int_payload/mod.rs index 9042e9907..9e317b160 100644 --- a/wai/wai-rs/src/int_payload/mod.rs +++ b/wai/wai-rs/src/int_payload/mod.rs @@ -32,7 +32,7 @@ pub use crate::int_wire as wire; #[cfg(test)] mod tests; -pub use wire::{Refusal, WireError}; +pub use wire::{Binary32, Refusal, WireError}; use crate::codecs::{ CAP_AUDIO_INT_CODEC, CAP_NEURAL_INT_HYPER, CAP_NEURAL_INT_MLICPP, CAP_NEURAL_INT_SYNTH, CAP_SPLAT_INT_CODEC, @@ -390,8 +390,9 @@ pub enum IntDecoded { Pcm { samples: Vec, sample_rate: u32, channels: u32 }, /// `wai.splat.int_codec`: the `[n_attr, s, s]` attribute tensor (the /// splat-equivalence digest's buffer), the cloud's Gaussian count, and each - /// attribute's presentation range `(lo, hi)`, which the digest does not cover. - Attr { bytes: Vec, n_attr: u32, s: u32, n_gaussians: u32, ranges: Vec<(f32, f32)> }, + /// attribute's presentation range `(lo, hi)` as binary32 bits ([`Binary32`]), which + /// the digest does not cover. + Attr { bytes: Vec, n_attr: u32, s: u32, n_gaussians: u32, ranges: Vec<(Binary32, Binary32)> }, } /// Decode `payload` through the integer capability `cap` against `params`. @@ -698,9 +699,8 @@ fn int_hyper_models( }; let g_s = net("g_s.whm")?; let h_s = net("h_s.whm")?; - let ent: serde_json::Value = serde_json::from_slice(params.member(cap, "entropy.json")?) - .map_err(|e| model(format!("entropy.json: {e}")))?; - let ent = CondEntropy::try_from_value(&ent).map_err(|e| model(format!("entropy.json: {e}")))?; + // The text is read strictly, and no float parser reads it (§9). + let ent = CondEntropy::read(params.member(cap, "entropy.json")?).map_err(|e| model(format!("entropy.json: {e}")))?; Ok((g_s, h_s, ent)) } @@ -890,26 +890,21 @@ pub fn wia1_from_json(bs: &serde_json::Value, sr: u32) -> Result, IntPay Ok(out) } -/// Convert the splat export's JSON bitstream (`splat.json`) to `WIS1`, with its -/// per-attribute presentation ranges. The export writes each range as a binary32 -/// value; a range that is not exactly one is refused, so the payload carries what the -/// JSON states. `splat_int` reads JSON numbers correctly rounded -/// (`serde_json/float_roundtrip`), so a decimal that denotes a binary32 value reads as -/// exactly that value, and no tolerance is needed. +/// Convert the splat export's JSON bitstream (`splat.json`, its text) to `WIS1`, with +/// its per-attribute presentation ranges. The export writes each range as a binary32 +/// value; a range that does not denote exactly one is refused, so the payload carries +/// what the JSON states. The text is read by [`crate::int_splat::SplatJson::read`], in +/// integer arithmetic: a decimal is the binary32 value it denotes on every machine +/// (`int_splat::binary32_of`), with no float parser to round it and no tolerance. #[cfg(feature = "splat_int")] -pub fn wis1_from_json(bs: &serde_json::Value) -> Result, IntPayloadError> { - let j = crate::int_splat::SplatJson::from_value(bs).map_err(json)?; +pub fn wis1_from_json(text: &[u8]) -> Result, IntPayloadError> { + let j = crate::int_splat::SplatJson::read(text).map_err(json)?; if j.cdfs.len() != j.c || j.offsets.len() != j.c || j.median_q.len() != j.c { return Err(json("the tables, offsets and medians are not one per channel (c)")); } for t in &j.cdfs { table_len(t.len())?; } - let exact = |x: f64| -> Result { - let f = x as f32; - if f as f64 == x { Ok(f) } else { Err(json("a range is not exactly a binary32 value")) } - }; - let ranges = j.ranges.iter().map(|&(lo, hi)| Ok((exact(lo)?, exact(hi)?))).collect::, IntPayloadError>>()?; let v = wire::Wis1 { afrac: u8::try_from(j.afrac).map_err(|_| json("afrac exceeds a byte"))?, c: to_u32(j.c, "c")?, @@ -918,7 +913,7 @@ pub fn wis1_from_json(bs: &serde_json::Value) -> Result, IntPayloadError s: to_u32(j.s, "s")?, attr: to_u32(j.n_attr, "attr")?, n_gaussians: to_u32(j.n_gaussians, "n_gaussians")?, - ranges, + ranges: j.ranges.clone(), cdfs: j.cdfs.clone(), offsets: j.offsets.clone(), median_q: j.median_q.clone(), diff --git a/wai/wai-rs/src/int_payload/tests.rs b/wai/wai-rs/src/int_payload/tests.rs index 884531c7f..9c59dada3 100644 --- a/wai/wai-rs/src/int_payload/tests.rs +++ b/wai/wai-rs/src/int_payload/tests.rs @@ -154,7 +154,7 @@ fn wih1_json_equivalence() { assert_eq!(format!("{:016x}", fnv1a(&rgb)), jload("int_hyper/tiny.json")["rgb_hash"].as_str().unwrap()); // The checked entropy reader accepts the committed tables and gives what the // unchecked one gives. - let checked = CondEntropy::try_from_value(&jload("int_hyper/tiny_entropy.json")).unwrap(); + let checked = CondEntropy::read(&read("int_hyper/tiny_entropy.json")).unwrap(); assert_eq!((checked.edges, checked.cdfs, checked.z_median_q), (ent.edges, ent.cdfs, ent.z_median_q)); } @@ -194,15 +194,22 @@ fn wia1_json_equivalence() { fn wis1_json_equivalence() { use crate::int_hyper_synth::{fnv1a, IntModel}; use crate::int_splat::{decode, SplatJson}; - let js = jload("int_splat/splat.json"); + let (text, js) = (read("int_splat/splat.json"), jload("int_splat/splat.json")); let payload = read("int_splat/splat.wis1"); - assert_eq!(wis1_from_json(&js).unwrap(), payload, "the converter writes the committed payload"); - let (v, j) = (wire::parse_wis1(&payload).unwrap(), SplatJson::from_value(&js).unwrap()); + assert_eq!(wis1_from_json(&text).unwrap(), payload, "the converter writes the committed payload"); + let (v, j) = (wire::parse_wis1(&payload).unwrap(), SplatJson::read(&text).unwrap()); assert_eq!((v.afrac as u32, v.c as usize, v.sh as usize, v.sw as usize, v.s as usize, v.attr as usize, v.n_gaussians as usize), (j.afrac, j.c, j.sh, j.sw, j.s, j.n_attr, j.n_gaussians)); assert_eq!((&v.cdfs, &v.offsets, &v.median_q, v.rans), (&j.cdfs, &j.offsets, &j.median_q, j.rans.as_slice())); - for (&(lo, hi), &(jl, jh)) in v.ranges.iter().zip(&j.ranges) { - assert_eq!((lo, hi), (jl as f32, jh as f32), "the ranges are the JSON's binary32 values"); + assert_eq!(v.ranges, j.ranges, "the ranges are the JSON's binary32 values"); + // And they are what a correctly rounded reader (std's) makes of each spelling. + for (k, key) in ["lo", "hi"].into_iter().enumerate() { + for (i, n) in spellings(&text, key).iter().enumerate() { + let x: f64 = n.parse().unwrap(); + assert_eq!((x as f32) as f64, x, "{key}[{i}] {n} denotes a binary32 value"); + let r = j.ranges[i]; + assert_eq!(if k == 0 { r.0 } else { r.1 }, wire::Binary32((x as f32).to_bits()), "{key}[{i}] {n}"); + } } let gs = read("int_splat/splat_gs.bin"); let params = unpinned(crate::codecs::CAP_SPLAT_INT_CODEC, HeldBytes::File(&gs)); @@ -211,7 +218,7 @@ fn wis1_json_equivalence() { other => panic!("splat: {other:?}"), }); assert_eq!(ranges, v.ranges); - assert_eq!(bytes, decode(&IntModel::from_bytes(&gs).unwrap(), &js).unwrap().attr, "the binary and JSON forms decode alike"); + assert_eq!(bytes, decode(&IntModel::from_bytes(&gs).unwrap(), &text).unwrap().attr, "the binary and JSON forms decode alike"); // Anchored below to output recorded before this revision: the sidecar and the // export's FNV-1a (`attr_hash`). assert_eq!(sha(&bytes), jload("int_splat/sha256.json")["splat"]["sha256"].as_str().unwrap()); @@ -571,18 +578,62 @@ fn entropy_tables_obey_the_table_length_rule() { t[n - 1] = 65_536; t }; - let ent = |n: usize| serde_json::json!({ + let ent = |n: usize| serde_json::to_vec(&serde_json::json!({ "scale_edges": [1], "cdfs": [table(n)], "cdf_lengths": [n], "offsets": [0], "z_cdfs": [table(n)], "z_cdf_lengths": [n], "z_offsets": [0], "z_median_q": [0] - }); + })).unwrap(); for n in [2, 4098] { - assert!(CondEntropy::try_from_value(&ent(n)).is_err(), "a table of {n} entries is refused"); + assert!(CondEntropy::read(&ent(n)).is_err(), "a table of {n} entries is refused"); } for n in [3, 4097] { - assert!(CondEntropy::try_from_value(&ent(n)).is_ok(), "a table of {n} entries is read"); + assert!(CondEntropy::read(&ent(n)).is_ok(), "a table of {n} entries is read"); } } +/// A set's `entropy.json` is read as §9 says, by a reader that converts no number to a +/// float: an integer is a number with no fraction and no exponent, and `-0` is zero; a +/// member named twice, anything after the value, or a value more than 64 levels deep +/// refuses the set (`model_mismatch`); a member the decode does not read is ignored. +/// serde_json, which read the tables before, read `-0` as a float (and refused the set), +/// kept the last of two members, and read deeper values. +#[test] +fn a_sets_entropy_json_is_read_strictly() { + let payload = read("int_hyper/tiny_bitstream.wih1"); + let text = String::from_utf8(read("int_hyper/tiny_entropy.json")).unwrap(); + let decode = |entropy: &str| -> Result, &'static str> { + let mut set = int_hyper_set(); + set.insert("entropy.json".into(), entropy.as_bytes().to_vec()); + let files = view(&set); + let params = unpinned(crate::codecs::CAP_NEURAL_INT_HYPER, HeldBytes::Set { listing: b"", files: &files }); + match decode_int(crate::codecs::CAP_NEURAL_INT_HYPER, &payload, ¶ms) { + Ok(IntDecoded::Image { rgb, .. }) => Ok(rgb), + Ok(other) => panic!("not an image: {other:?}"), + Err(e) => Err(e.code()), + } + }; + let want = decode(&text).expect("the committed tables decode"); + // Every table starts at 0: the first, spelled otherwise. + let first = |spelled: &str| { + assert_eq!(text.matches("[[0, ").count(), 2, "the latent and hyper-latent tables, each from 0"); + text.replacen("[[0, ", &format!("[[{spelled}, "), 1) + }; + assert_eq!(decode(&first("-0")), Ok(want.clone()), "-0 is the integer zero"); + for not_integer in ["0.0", "-0.0", "0e0", "0E+0"] { + assert_eq!(decode(&first(not_integer)), Err("model_mismatch"), "{not_integer} is not an integer"); + } + // A member named twice is refused, whichever of the two a reader would keep. + let (head, rest) = text.split_once('{').unwrap(); + let with = |members: &str| format!("{head}{{{members}, {rest}"); + assert_eq!(decode(&with(r#""offsets": [0, 0, 0, 0]"#)), Err("model_mismatch"), "a member named twice"); + assert_eq!(decode(&with(r#""note": "ignored", "extra": [1.5, -0, 1e300]"#)), Ok(want.clone()), "members not read are ignored"); + assert_eq!(decode(&format!("{text} {{}}")), Err("model_mismatch"), "bytes after the value"); + assert_eq!(decode(&format!("{text} \n")), Ok(want.clone()), "whitespace after the value"); + // The top object is level 1; a value inside n arrays in its member is at level n + 2. + let nested = |n: usize| with(&format!(r#""deep": {}0{}"#, "[".repeat(n), "]".repeat(n))); + assert_eq!(decode(&nested(62)), Ok(want.clone()), "a value 64 levels deep"); + assert_eq!(decode(&nested(63)), Err("model_mismatch"), "a value 65 levels deep"); +} + /// A network is planned in 64-bit arithmetic, layer by layer, so a size past the cap is /// `too_large` however far past it is, on every target. Planning in `usize` refused a /// transposed convolution of stride 2^31 on a side of 3 as malformed on a 32-bit sink @@ -613,27 +664,275 @@ fn network_plans_refuse_alike_on_every_target() { } } +/// `WIS1`'s range rule, read from the bits in integer arithmetic, is IEEE 754's: each +/// finite, and `lo ≤ hi` with `−0` equal to `+0`. Held to the float comparison on every +/// pair of bit-pattern classes (zeros, subnormals, normals, the greatest, infinities, quiet +/// and signalling NaNs, each sign), on random pairs, and on near pairs (a few units apart, +/// or of opposite sign); the parser refuses exactly the pairs the rule does. +#[test] +fn the_wis1_range_rule_in_integers_is_the_float_comparison() { + let committed = read("int_splat/splat.wis1"); + let classes = [ + 0u32, 0x8000_0000, 1, 0x8000_0001, 0x007f_ffff, 0x807f_ffff, 0x0080_0000, 0x3f80_0000, 0xbf80_0000, 0x4b80_0001, + 0xcb80_0001, 0x7f7f_ffff, 0xff7f_ffff, 0x7f80_0000, 0xff80_0000, 0x7fc0_0000, 0xffc0_0000, 0x7f80_0001, 0xff80_0001, + ]; + let mut r = crate::int_kernels::testkit::Rng(0x5157_0004); + let mut pairs: Vec<(u32, u32, bool)> = classes.iter().flat_map(|&a| classes.iter().map(move |&b| (a, b, true))).collect(); + for i in 0..60_000 { + let a = r.next() as u32; + let b = match i % 3 { + 0 => r.next() as u32, + 1 => a.wrapping_add(r.below(5) as u32).wrapping_sub(2), + _ => a ^ 0x8000_0000, + }; + pairs.push((a, b, i % 30 == 0)); + } + for (a, b, parse) in pairs { + let (fa, fb) = (f32::from_bits(a), f32::from_bits(b)); + let want = fa.is_finite() && fb.is_finite() && fa <= fb; + let (x, y) = (wire::Binary32(a), wire::Binary32(b)); + assert_eq!(x.is_finite() && y.is_finite() && x.finite_le(y), want, "{a:#010x} {b:#010x}"); + if parse { + let mut p = committed.clone(); + p[32..36].copy_from_slice(&a.to_le_bytes()); + p[36..40].copy_from_slice(&b.to_le_bytes()); + let got = wire::parse_wis1(&p).map(|_| ()).map_err(|e| e.code()); + assert_eq!(got, if want { Ok(()) } else { Err("field_range") }, "{a:#010x} {b:#010x}"); + } + } +} + +/// The spellings of the numbers in the splat JSON's top-level array `key`, as written. +#[cfg(feature = "splat_int")] +fn spellings(text: &[u8], key: &str) -> Vec { + match crate::strict_json::read(text).unwrap().get(key) { + Some(crate::strict_json::Json::Arr(a)) => a.iter().map(|v| match v { + crate::strict_json::Json::Num(n) => n.clone(), + other => panic!("{key}: {other:?}"), + }).collect(), + other => panic!("{key}: {other:?}"), + } +} + +/// `text` (a splat JSON) with its top-level member `key`'s value replaced by `value`, +/// spelled as given: the rest of the text, every range among it, as written. +#[cfg(feature = "splat_int")] +fn edit(text: &str, key: &str, value: &str) -> String { + let at = text.find(&format!("\"{key}\": ")).unwrap_or_else(|| panic!("no member {key}")) + key.len() + 4; + let mut depth = 0i32; + let end = at + text[at..].char_indices().find(|&(_, c)| { + match c { + '[' | '{' => depth += 1, + ']' | '}' => depth -= 1, + _ => {} + } + depth < 0 || (depth == 0 && c == ',') + }).expect("the member's value ends").0; + format!("{}{}{}", &text[..at], value, &text[end..]) +} + +/// The committed splat JSON's text with its member `key`'s value replaced ([`edit`]). +#[cfg(feature = "splat_int")] +fn splat_with(key: &str, value: &str) -> Vec { + edit(&String::from_utf8(read("int_splat/splat.json")).unwrap(), key, value).into_bytes() +} + +/// The committed splat JSON with element `i` of the top-level array `key` spelled `n`. +#[cfg(feature = "splat_int")] +fn splat_with_element(key: &str, i: usize, n: &str) -> Vec { + let mut v = spellings(&read("int_splat/splat.json"), key); + v[i] = n.to_string(); + splat_with(key, &format!("[{}]", v.join(", "))) +} + +/// The committed splat JSON with the first attribute's range `(n, n)`: `lo` and `hi` +/// both spelled `n`, so the range holds whatever value `n` is. +#[cfg(feature = "splat_int")] +fn splat_with_range0(n: &str) -> Vec { + let committed = read("int_splat/splat.json"); + let mut text = String::from_utf8(committed.clone()).unwrap(); + for key in ["lo", "hi"] { + let mut v = spellings(&committed, key); + v[0] = n.to_string(); + text = edit(&text, key, &format!("[{}]", v.join(", "))); + } + text.into_bytes() +} + /// The splat converter carries each presentation range exactly: a JSON decimal that -/// denotes a binary32 value converts to it, and one a single f64 unit away from it, -/// written as its own shortest decimal, is refused. `splat_int` reads numbers -/// correctly rounded, so no tolerance stands in for the reader. +/// denotes a binary32 value converts to it, and one a single binary64 unit away from it, +/// written as its own shortest decimal, is refused. The JSON is read in integer +/// arithmetic, so no float parser stands in for the reader, and no tolerance either. #[cfg(feature = "splat_int")] #[test] fn splat_ranges_are_carried_exactly() { - let text = String::from_utf8(read("int_splat/splat.json")).unwrap(); - let js: serde_json::Value = serde_json::from_str(&text).unwrap(); - let lo0 = js["lo"][0].as_f64().unwrap(); - assert_eq!((lo0 as f32) as f64, lo0, "the committed range is a binary32 value, read exactly"); - assert!(wis1_from_json(&js).is_ok()); + let lo0 = spellings(&read("int_splat/splat.json"), "lo")[0].parse::().unwrap(); + assert_eq!((lo0 as f32) as f64, lo0, "the committed range is a binary32 value"); + assert!(wis1_from_json(&splat_with_element("lo", 0, &format!("{lo0:?}"))).is_ok()); for step in [1i64, -1] { - let off = f64::from_bits(((lo0 as f32) as f64).to_bits().wrapping_add_signed(step)); - let mut bad: serde_json::Value = serde_json::from_str(&text).unwrap(); - bad["lo"][0] = serde_json::from_str(&format!("{off:?}")).unwrap(); - assert_eq!(bad["lo"][0].as_f64(), Some(off), "the reader is correctly rounded"); - assert_eq!(wis1_from_json(&bad).unwrap_err().code(), "field_range", "{off:?}"); + let off = f64::from_bits(lo0.to_bits().wrapping_add_signed(step)); + assert_eq!(wis1_from_json(&splat_with_element("lo", 0, &format!("{off:?}"))).unwrap_err().code(), "field_range", "{off:?}"); } } +/// A range that differs from another in its last digit decodes differently, and each to +/// the value binary64 rounding gives it, at the closest case there is: `1 + 2^-53`, the +/// midpoint between `1` and the next binary64 value, rounds to `1` (ties to even), and +/// is the binary32 value `1`; with its last digit 6 it is past the midpoint, rounds to +/// `1 + 2^-52`, which is no binary32 value, and is refused; with 4 it is `1` again. So +/// too below `1` (`1 − 2^-54`, half the unit below a power of two), and, with a tail of +/// 1300 zeros and a 1 (past the digits the reader keeps), past the midpoint. The +/// integers `2^24` and `2^24 + 2` are adjacent binary32 values; `2^24 + 1` between them +/// is none. Spellings one last digit apart that are both values decode to two adjacent +/// values: `4194304.0` and `4194304.5` (the step above `2^22` is a half), while +/// `4194304.4` and `4194304.6` are no binary32 value; and `8.388608e6` and `8.388609e6` +/// (`2^23` and the value after it). Each accepted range is the one the payload's integer +/// decode gives. +#[cfg(feature = "splat_int")] +#[test] +fn a_range_one_digit_apart_decodes_to_its_own_value() { + use crate::int_splat::SplatJson; + let gs = read("int_splat/splat_gs.bin"); + let params = unpinned(crate::codecs::CAP_SPLAT_INT_CODEC, HeldBytes::File(&gs)); + const UP: &str = "1.00000000000000011102230246251565404236316680908203125"; // 1 + 2^-53 + const DOWN: &str = "0.999999999999999944488848768742172978818416595458984375"; // 1 - 2^-54 + let one = wire::Binary32(1f32.to_bits()); + let tail = |n: &str, t: &str| format!("{n}{}{t}", "0".repeat(1300)); + let last = |n: &str, d: char| format!("{}{d}", &n[..n.len() - 1]); + let cases: Vec<(String, Option)> = vec![ + (UP.into(), Some(one)), + (last(UP, '6'), None), + (last(UP, '4'), Some(one)), + (DOWN.into(), Some(one)), + (last(DOWN, '4'), None), + (last(DOWN, '6'), Some(one)), + (tail(UP, "1"), None), + (tail(UP, ""), Some(one)), + (tail(DOWN, "1"), Some(one)), + ("16777216".into(), Some(wire::Binary32(0x4b80_0000))), + ("16777218".into(), Some(wire::Binary32(0x4b80_0001))), + ("16777217".into(), None), + ("4194304.0".into(), Some(wire::Binary32(0x4a80_0000))), + ("4194304.5".into(), Some(wire::Binary32(0x4a80_0001))), + ("4194304.4".into(), None), + ("4194304.6".into(), None), + ("8.388608e6".into(), Some(wire::Binary32(0x4b00_0000))), + ("8.388609e6".into(), Some(wire::Binary32(0x4b00_0001))), + ]; + for (n, want) in &cases { + let text = splat_with_range0(n); + match want { + Some(b) => { + assert_eq!(SplatJson::read(&text).unwrap().ranges[0], (*b, *b), "{n}"); + let p = wis1_from_json(&text).unwrap(); + assert_eq!(wire::parse_wis1(&p).unwrap().ranges[0], (*b, *b), "{n}: the payload carries it"); + assert_eq!(p[32..36], b.0.to_le_bytes(), "{n}"); + match decode_int(crate::codecs::CAP_SPLAT_INT_CODEC, &p, ¶ms) { + Ok(IntDecoded::Attr { ranges, .. }) => assert_eq!(ranges[0], (*b, *b), "{n}: the decode presents it"), + other => panic!("{n}: {other:?}"), + } + } + None => { + assert_eq!(SplatJson::read(&text).unwrap_err(), "lo holds a value that is not a binary32 value", "{n}"); + assert_eq!(wis1_from_json(&text).unwrap_err().code(), "field_range", "{n}"); + } + } + } +} + +/// The reader of a range against std's correctly rounded `f64` parse, on decimals: the +/// shortest spellings of binary32 values (each read as itself), the same values spelled +/// with 1 to 20 significant digits (mostly no binary32 value), random digit strings at +/// exponents from -60 to 45, and the edges: zeros of either sign, magnitudes that round +/// to zero in binary64 and the first that does not, the least subnormal, the greatest +/// binary32 value and the decimals just past it, and exponents far out of range. +#[cfg(feature = "splat_int")] +#[test] +fn a_decimal_range_is_the_binary64_value_it_denotes() { + use crate::int_splat::binary32_of; + let reference = |n: &str| -> Option { + let x: f64 = n.parse().unwrap(); + (x.is_finite() && (x as f32) as f64 == x).then(|| (x as f32).to_bits()) + }; + let check = |n: &str| assert_eq!(binary32_of(n).map(|b| b.0), reference(n), "{n}"); + let mut r = crate::int_kernels::testkit::Rng(0x5157_0003); + for _ in 0..4_000 { + let b = r.next() as u32; + if b & 0x7f80_0000 == 0x7f80_0000 { + continue; + } + let v = f32::from_bits(b); + check(&format!("{:?}", v as f64)); + check(&format!("{:e}", v as f64)); + for d in 1..=20 { + check(&format!("{:.*e}", d - 1, v as f64)); + } + } + for _ in 0..4_000 { + let digits: String = (0..1 + r.below(25)).map(|_| char::from(b'0' + r.below(10) as u8)).collect(); + let e = r.below(106) as i64 - 60; + check(&format!("{}{}.{}0e{e}", if r.below(2) == 0 { "" } else { "-" }, &digits[..1], &digits[1..])); + } + for n in ["0.0", "-0.0", "0e5", "-0E-5", "0.000", "1e-400", "-1e-400", "2.4703282292062327e-324", "2.4703282292062328e-324", + "-2.4703282292062328e-324", "4.9406564584124654e-324", "1.401298464324817e-45", "-1.401298464324817e-45", + "1.4012984643248169e-45", "1.1754942106924411e-38", "3.4028234663852886e38", "-3.4028234663852886E+38", + "3.4028235e38", "3.4028234663852889e38", "3.402823466385289e38", "1e38", "1e39", "1.7976931348623157e308", + "1e309", "1e99999999999999999999", "1e-99999999999999999999", "0.5", "2.5", "0.1", "0.30000000000000004"] { + check(n); + } +} + +/// An integer range is the integer it spells, and must be exactly a binary32 value: no +/// rounding at any size (`2^64 + 1`, past `u64`, is refused as `2^60 + 1` is), the +/// greatest binary32 value is one, and the integers past it are not (one more, the +/// midpoint to `2^128`, and `2^128`), and `-0` is the integer zero, so `+0` (as the +/// independent converter reads it), where the decimal `-0.0` keeps its sign. +#[cfg(feature = "splat_int")] +#[test] +fn an_integer_range_is_its_integer() { + use crate::int_splat::binary32_of; + let bits = |n: &str| binary32_of(n).map(|b| b.0); + for (n, want) in [ + ("0", Some(0)), ("-0", Some(0)), ("-0.0", Some(0x8000_0000)), ("0.0", Some(0)), ("1", Some(0x3f80_0000)), ("-1", Some(0xbf80_0000)), + ("1152921504606846976", Some(0x5d80_0000)), ("1152921504606846977", None), ("-1152921504606846977", None), + ("18446744073709551616", Some(0x5f80_0000)), ("18446744073709551617", None), ("-18446744073709551617", None), + ("340282346638528859811704183484516925440", Some(0x7f7f_ffff)), ("-340282346638528859811704183484516925440", Some(0xff7f_ffff)), + ("340282346638528859811704183484516925441", None), ("340282356779733661637539395458142568448", None), + ("340282366920938463463374607431768211456", None), + ("16777217", None), ("16777218", Some(0x4b80_0001)), + ] { + assert_eq!(bits(n), want, "{n}"); + } +} + +/// The splat JSON is read strictly and its integers as integers: a member named twice, +/// or bytes after the value, refuse the text (a lenient parser takes the last of two +/// members); a count or table entry spelled with a fraction or an exponent is refused; +/// and `-0`, the integer zero spelled with a sign, reads as `0`. +#[cfg(feature = "splat_int")] +#[test] +fn the_splat_json_is_read_strictly_and_its_integers_as_integers() { + use crate::int_splat::SplatJson; + let text = read("int_splat/splat.json"); + let committed = SplatJson::read(&text).unwrap(); + let mut twice = text.clone(); + twice.splice(1..1, b"\"c\": 32, ".iter().copied()); + assert!(SplatJson::read(&twice).is_err(), "a member named twice"); + let mut after = text.clone(); + after.extend_from_slice(b" {}"); + assert!(SplatJson::read(&after).is_err(), "bytes after the value"); + assert_eq!(SplatJson::read(&splat_with("c", "32.0")).unwrap_err(), "c"); + assert_eq!(SplatJson::read(&splat_with("afrac", "12e0")).unwrap_err(), "afrac"); + let mut m = spellings(&text, "median_q"); + assert_eq!(m[0], "0"); + m[0] = "-0".into(); + let neg_zero = splat_with("median_q", &format!("[{}]", m.join(", "))); + assert_eq!(SplatJson::read(&neg_zero).unwrap().median_q, committed.median_q); + assert_eq!(wis1_from_json(&neg_zero).unwrap(), read("int_splat/splat.wis1"), "-0 is the median 0"); + m[0] = "0.0".into(); + assert_eq!(SplatJson::read(&splat_with("median_q", &format!("[{}]", m.join(", ")))).unwrap_err(), "median_q"); +} + /// The converters check the JSON's channel count against its tables: a medians list, /// a table list or an offsets list one short is refused (`field_range`), never written /// out of step with the header. @@ -649,9 +948,10 @@ fn converters_check_the_channel_count() { } let splat = jload("int_splat/splat.json"); for key in ["median_q", "cdfs", "offsets"] { - let mut j = splat.clone(); - j[key].as_array_mut().unwrap().pop(); - let e = wis1_from_json(&j).unwrap_err(); + // The member alone is rewritten: an integer list, written back exactly. + let mut v = splat[key].clone(); + v.as_array_mut().unwrap().pop(); + let e = wis1_from_json(&splat_with(key, &v.to_string())).unwrap_err(); assert_eq!((e.code(), e.detail().contains("one per channel")), ("field_range", true), "splat {key}: {e}"); } } @@ -749,16 +1049,10 @@ fn a_one_frame_clip_has_one_header() { #[cfg(feature = "splat_int")] #[test] fn splat_integer_ranges_are_carried_exactly() { - let text = String::from_utf8(read("int_splat/splat.json")).unwrap(); - let with = |n: &str| { - let mut j: serde_json::Value = serde_json::from_str(&text).unwrap(); - let big: serde_json::Value = serde_json::from_str(n).unwrap(); - j["lo"][0] = big.clone(); - j["hi"][0] = big; - wis1_from_json(&j) - }; + let with = |n: &str| wis1_from_json(&splat_with_range0(n)); let p = with("1152921504606846976").unwrap(); - assert_eq!(wire::parse_wis1(&p).unwrap().ranges[0], ((1u64 << 60) as f32, (1u64 << 60) as f32)); + let two60 = wire::Binary32(((1u64 << 60) as f32).to_bits()); + assert_eq!(wire::parse_wis1(&p).unwrap().ranges[0], (two60, two60)); assert_eq!(with("1152921504606846977").unwrap_err().code(), "field_range"); assert_eq!(with("-1152921504606846977").unwrap_err().code(), "field_range"); } diff --git a/wai/wai-rs/src/int_payload/wire.rs b/wai/wai-rs/src/int_payload/wire.rs index 099af499a..3374c6ab5 100644 --- a/wai/wai-rs/src/int_payload/wire.rs +++ b/wai/wai-rs/src/int_payload/wire.rs @@ -671,6 +671,31 @@ pub fn write_wia1(v: &Wia1<'_>) -> Vec { // ───────────────────────────── WIS1 ───────────────────────────── +/// An IEEE 754 binary32 value held as its bits, as `WIS1` carries a presentation range +/// ([integer-payloads] §6). A parser reads, checks and writes it in integer arithmetic, +/// so no floating-point operation is on a payload's decode path; a sink that presents +/// the range takes its value from the bits (`f32::from_bits`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct Binary32(pub u32); + +impl Binary32 { + /// Finite: the exponent field is not all ones (neither an infinity nor a NaN). + pub fn is_finite(self) -> bool { + self.0 & 0x7f80_0000 != 0x7f80_0000 + } + + /// `self ≤ other` as IEEE 754 compares two finite values, `−0` equal to `+0`: by the + /// magnitude's bits, negated for a negative value (a finite magnitude's bits are + /// below `2^31 − 2^23`, so the negation stays in `i32`). + pub fn finite_le(self, other: Binary32) -> bool { + let key = |b: Binary32| { + let m = (b.0 & 0x7fff_ffff) as i32; + if b.0 >> 31 == 1 { -m } else { m } + }; + key(self) <= key(other) + } +} + /// A parsed `WIS1` payload ([integer-payloads] §6). #[derive(Debug, Clone, PartialEq)] pub struct Wis1<'a> { @@ -685,9 +710,9 @@ pub struct Wis1<'a> { pub attr: u32, /// Gaussians in the cloud: the first `n_gaussians` cells of each plane. pub n_gaussians: u32, - /// Per attribute, the range `(lo, hi)` its `u8` plane is presented over. - /// Presentation only: outside the splat-equivalence digest. - pub ranges: Vec<(f32, f32)>, + /// Per attribute, the range `(lo, hi)` its `u8` plane is presented over, as + /// binary32 bits. Presentation only: outside the splat-equivalence digest. + pub ranges: Vec<(Binary32, Binary32)>, /// Per channel, its table, its symbol offset and its median. pub cdfs: Vec>, pub offsets: Vec, @@ -728,9 +753,9 @@ pub fn parse_wis1(p: &[u8]) -> Result, WireError> { let raw = cur.take(attr as u64 * 8, "the attribute ranges")?; let mut ranges = Vec::with_capacity(attr as usize); for r in raw.chunks_exact(8) { - let lo = f32::from_bits(u32::from_le_bytes([r[0], r[1], r[2], r[3]])); - let hi = f32::from_bits(u32::from_le_bytes([r[4], r[5], r[6], r[7]])); - if !lo.is_finite() || !hi.is_finite() || lo > hi { + let lo = Binary32(u32::from_le_bytes([r[0], r[1], r[2], r[3]])); + let hi = Binary32(u32::from_le_bytes([r[4], r[5], r[6], r[7]])); + if !(lo.is_finite() && hi.is_finite() && lo.finite_le(hi)) { return Err(err(Refusal::FieldRange, "an attribute range is not finite, or its lo exceeds its hi")); } ranges.push((lo, hi)); @@ -765,8 +790,8 @@ pub fn write_wis1(v: &Wis1<'_>) -> Vec { put_u32(&mut o, d); } for &(lo, hi) in &v.ranges { - put_u32(&mut o, lo.to_bits()); - put_u32(&mut o, hi.to_bits()); + put_u32(&mut o, lo.0); + put_u32(&mut o, hi.0); } for cdf in &v.cdfs { put_u16(&mut o, cdf.len() as u16); diff --git a/wai/wai-rs/src/int_splat.rs b/wai/wai-rs/src/int_splat.rs index 435ae7b17..fcc452def 100644 --- a/wai/wai-rs/src/int_splat.rs +++ b/wai/wai-rs/src/int_splat.rs @@ -10,10 +10,12 @@ //! (`extensions/integer-operators.md` §13.2; [`crate::int_ops`]). The float rasterizer is //! sink-supplied presentation; the *attributes* are the conformance contract. -use serde_json::Value; +use std::cmp::Ordering; use crate::int_hyper_synth::{fnv1a, refused, IntModel, Refused, RefusalKind, MAX_AFRAC, MAX_TENSOR}; +use crate::int_wire::Binary32; use crate::rans::RansDecoder; +use crate::strict_json::{self, Json}; /// A decoded splat: the `[attr · s · s]` `u8` attribute image (channel-major), to be /// un-normalized + un-sorted into Gaussians by the presentation layer. @@ -53,6 +55,12 @@ pub struct SplatStream<'a> { /// the per-attribute ranges `lo`, `hi`. A converter input and a debugging aid; the /// payload is `WIS1`. Its `afrac` is read (the old JSON decoder ignored it) and must /// be the model's. +/// +/// It is read from its text ([`SplatJson::read`]) with no floating point: every count, +/// table entry, offset and median is an integer spelling read as an integer, and each +/// range is the binary32 value its spelling denotes ([`binary32_of`]), found in exact +/// integer arithmetic. So the same text reads to the same values on every machine and +/// in every build, whatever a general JSON parser would round a decimal to. #[derive(Debug, Clone)] pub struct SplatJson { pub afrac: u32, @@ -65,41 +73,40 @@ pub struct SplatJson { pub cdfs: Vec>, pub offsets: Vec, pub median_q: Vec, - /// Per attribute, `(lo, hi)` as the JSON carries them; an integer an `f64` does not - /// hold exactly is refused. - pub ranges: Vec<(f64, f64)>, + /// Per attribute, `(lo, hi)`: the binary32 values the JSON's `lo` and `hi` denote. + pub ranges: Vec<(Binary32, Binary32)>, pub rans: Vec, } impl SplatJson { - /// Read the tool's JSON bitstream. - pub fn from_value(bs: &Value) -> Result { - let uz = |k: &'static str| bs[k].as_u64().and_then(|v| usize::try_from(v).ok()).ok_or(k); + /// Read the tool's JSON bitstream from its text: one JSON text (RFC 8259, read + /// strictly: no member named twice, nothing after the value), whose integers are + /// spelled as integers (an optional `-`, then digits; `-0` is zero), and whose `lo` + /// and `hi` each hold one binary32 value per attribute ([`binary32_of`]). The error + /// names the member that fails. + pub fn read(text: &[u8]) -> Result { + let bs = strict_json::read(text).map_err(|_| "the JSON form is not one strict JSON text")?; + let get = |k: &'static str| bs.get(k).ok_or(k); + let uz = |k: &'static str| get(k)?.as_u64().and_then(|v| usize::try_from(v).ok()).ok_or(k); let (c, sh, sw, s, n_attr) = (uz("c")?, uz("sh")?, uz("sw")?, uz("s")?, uz("attr")?); let n_gaussians = uz("n_gaussians")?; - let afrac = bs["afrac"].as_u64().and_then(|v| u32::try_from(v).ok()).ok_or("afrac")?; - let offsets = bs["offsets"].as_array().ok_or("offsets")?.iter() - .map(|n| n.as_i64().and_then(|v| i32::try_from(v).ok())).collect::>>().ok_or("offsets")?; - let median_q = bs["median_q"].as_array().ok_or("median_q")?.iter() - .map(Value::as_i64).collect::>>().ok_or("median_q")?; - let cdfs = bs["cdfs"].as_array().ok_or("cdfs")?.iter() - .map(|row| row.as_array().and_then(|r| r.iter().map(|n| n.as_i64().and_then(|v| i32::try_from(v).ok())).collect::>>())) - .collect::>>().ok_or("cdfs holds a value outside i32")?; - // A range as the JSON states it: an integer only where an f64 holds it exactly - // (so 2^60 + 1 is refused rather than rounded to 2^60), a decimal as read. - let exact = |v: &Value| -> Option { - let int = v.as_i64().map(i128::from).or(v.as_u64().map(i128::from)); - match int { - Some(i) => Some(i as f64).filter(|&f| f as i128 == i), - None => v.as_f64(), - } + let afrac = get("afrac")?.as_u64().and_then(|v| u32::try_from(v).ok()).ok_or("afrac")?; + let i32s = |v: &Json| v.as_arr()?.iter().map(|n| n.as_i64().and_then(|v| i32::try_from(v).ok())).collect::>>(); + let offsets = i32s(get("offsets")?).ok_or("offsets")?; + let median_q = get("median_q")?.as_arr().and_then(|a| a.iter().map(Json::as_i64).collect::>>()).ok_or("median_q")?; + let cdfs = get("cdfs")?.as_arr().and_then(|a| a.iter().map(i32s).collect::>>()).ok_or("cdfs holds a value outside i32")?; + let ranges = |k: &'static str| -> Result, &'static str> { + let not = if k == "lo" { "lo holds a value that is not a binary32 value" } else { "hi holds a value that is not a binary32 value" }; + get(k)?.as_arr().ok_or(k)?.iter().map(|v| match v { + Json::Num(n) => binary32_of(n).ok_or(not), + _ => Err(k), + }).collect() }; - let f = |k: &'static str| bs[k].as_array().ok_or(k)?.iter().map(exact).collect::>>().ok_or(k); - let (lo, hi) = (f("lo")?, f("hi")?); + let (lo, hi) = (ranges("lo")?, ranges("hi")?); if lo.len() != n_attr || hi.len() != n_attr { return Err("lo and hi are not one per attribute"); } - let rans = b64_decode(bs["rans_b64"].as_str().ok_or("rans_b64")?)?; + let rans = b64_decode(get("rans_b64")?.as_str().ok_or("rans_b64")?)?; Ok(SplatJson { afrac, c, sh, sw, s, n_attr, n_gaussians, cdfs, offsets, median_q, ranges: lo.into_iter().zip(hi).collect(), rans }) } @@ -309,11 +316,12 @@ pub(crate) fn splat_plan(gs: &IntModel, st: &SplatStream<'_>) -> Result<(u64, u1 Ok(((8 * (latent + p.live)).max(8 * out + out), p.macs)) } -/// Decode the tool's JSON bitstream (`splat.json`) → the integer attribute tensor: -/// [`SplatJson`] then [`decode_typed`]. A debugging and converter path; the payload is -/// `WIS1`. `gs` is the WHM1 model (`splat_gs.bin`). Byte-identical on every machine. -pub fn decode(gs: &IntModel, bs: &Value) -> Result { - decode_typed(gs, &SplatJson::from_value(bs)?.stream()).map_err(|r| r.detail) +/// Decode the tool's JSON bitstream (`splat.json`, its text) → the integer attribute +/// tensor: [`SplatJson::read`] then [`decode_typed`]. A debugging and converter path; the +/// payload is `WIS1`. `gs` is the WHM1 model (`splat_gs.bin`). Byte-identical on every +/// machine. +pub fn decode(gs: &IntModel, json: &[u8]) -> Result { + decode_typed(gs, &SplatJson::read(json)?.stream()).map_err(|r| r.detail) } /// Dependency-free base64 decode (standard alphabet) — the rANS bytes ride in JSON as @@ -344,3 +352,250 @@ fn b64_decode(s: &str) -> Result, &'static str> { } Ok(out) } + +/// The most significant digits of a decimal [`binary32_of`] keeps. Every bound it +/// compares a decimal with is a multiple of `2^-1075`, so of `10^-1075`; a decimal below +/// `10^39` cut to this many digits has its last kept digit at `10^-1161` or below, so the +/// digits dropped (a nonzero tail, which it notes) cannot change any comparison. +const KEPT_DIGITS: usize = 1200; + +/// The binary32 value the JSON number spelled `n` denotes, read as the export tool and +/// the independent converter (`int-payload-conformance/int_payload_verify.py`) read it, +/// or `None` when it denotes none: +/// +/// - an integer spelling (an optional `-`, then digits: no fraction and no exponent) is +/// that integer, which must be a binary32 value exactly: `2^60` is, `2^60 + 1` is not +/// (a binary64 would round it to `2^60`); `-0` is the integer zero, so `+0`; +/// - any other spelling denotes the binary64 value nearest it (ties to even; a magnitude +/// that rounds to zero is a zero of the spelling's sign, and one past the binary64 +/// range is no value), and that must be a binary32 value: the export writes each range +/// as the shortest decimal of a binary32 value widened to binary64. +/// +/// Integer arithmetic throughout, so no machine and no build reads a spelling to another +/// value. The decimal `m·10^e` is compared exactly with binary values `a·2^b`: the largest +/// binary32 value not above its magnitude is found by bisection over the bit patterns +/// (ordered as their values are), and the decimal is that value, or the next, where it +/// lies in the closed interval of reals that round to it in binary64. +pub fn binary32_of(n: &str) -> Option { + let (neg, rest) = match n.strip_prefix('-') { + Some(r) => (true, r), + None => (false, n), + }; + let (mant, exp) = match rest.find(['e', 'E']) { + Some(i) => (&rest[..i], Some(&rest[i + 1..])), + None => (rest, None), + }; + let (int, frac) = match mant.find('.') { + Some(i) => (&mant[..i], Some(&mant[i + 1..])), + None => (mant, None), + }; + let integer = exp.is_none() && frac.is_none(); + let frac = frac.unwrap_or(""); + let digit = |b: u8| b.is_ascii_digit(); + if int.is_empty() || !int.bytes().all(digit) || !frac.bytes().all(digit) { + return None; + } + // The exponent, saturated far past any magnitude that can matter. + let mut e: i64 = 0; + if let Some(x) = exp { + let (eneg, ed) = match x.as_bytes().first() { + Some(b'-') => (true, &x[1..]), + Some(b'+') => (false, &x[1..]), + _ => (false, x), + }; + if ed.is_empty() || !ed.bytes().all(digit) { + return None; + } + for b in ed.bytes() { + e = (e * 10 + (b - b'0') as i64).min(1 << 40); + } + if eneg { + e = -e; + } + } + let sign = if neg { 0x8000_0000 } else { 0 }; + let digits: Vec = int.bytes().chain(frac.bytes()).map(|b| b - b'0').collect(); + let Some(first) = digits.iter().position(|&d| d != 0) else { + // Zero. The integer zero has no sign; a decimal zero keeps its spelling's. + return Some(Binary32(if integer { 0 } else { sign })); + }; + let last = digits.iter().rposition(|&d| d != 0).expect("a nonzero digit"); + let sig = &digits[first..=last]; + // |value| = sig · 10^e10, and 10^(q − 1) ≤ |value| < 10^q. + let e10 = e - frac.len() as i64 + (digits.len() - 1 - last) as i64; + let q = e10 + sig.len() as i64; + if q >= 40 { + // At least 10^39: past the greatest binary32 value's interval (or the integer is + // past it). + return None; + } + if q <= -324 { + // Below 10^-324, so below 2^-1075: a decimal (never an integer) that rounds to + // zero in binary64. + return Some(Binary32(sign)); + } + let kept = &sig[..sig.len().min(KEPT_DIGITS)]; + let d = Decimal::new(kept, e10 + (sig.len() - kept.len()) as i64, kept.len() < sig.len()); + // A positive binary32 bit pattern's value, `a·2^b`. + let value = |bits: u32| -> (u64, i32) { + let (x, f) = (bits >> 23, (bits & 0x7f_ffff) as u64); + if x == 0 { (f, -149) } else { (f | 1 << 23, x as i32 - 150) } + }; + // The largest pattern whose value is not above |d|, below the infinity's pattern: lo. + let (mut lo, mut hi) = (0u32, 0x7f80_0000u32); + while hi - lo > 1 { + let mid = lo + (hi - lo) / 2; + let (a, b) = value(mid); + if d.order(a, b) == Ordering::Less { + hi = mid; + } else { + lo = mid; + } + } + if integer { + // The integer itself must be the value. + let (a, b) = value(lo); + return (lo != 0 && d.order(a, b) == Ordering::Equal).then_some(Binary32(sign | lo)); + } + // The closed interval of reals binary64 rounds to the nonzero binary32 value `bits` + // (ties go to it: its binary64 significand ends in 29 zero bits, so is even, and its + // neighbours' are odd). It is normal in binary64, `m·2^31 · 2^(x − 31)` with `m` in + // `[2^23, 2^24)`, whose half unit above is `2·2^(x − 31)`, and below the same, or half + // that at a power of two. + let bounds = |bits: u32| -> ((u64, i32), (u64, i32)) { + let (mut m, mut x) = value(bits); + while m < 1 << 23 { + m <<= 1; + x -= 1; + } + let below = if m == 1 << 23 { 1 } else { 2 }; + (((m << 31) - below, x - 31), ((m << 31) + 2, x - 31)) + }; + if lo == 0 { + // Below the least subnormal: zero where binary64 rounds the decimal to zero, at + // most 2^-1075 (the tie goes to zero, whose significand is even). + if d.order(1, -1075) != Ordering::Greater { + return Some(Binary32(sign)); + } + } else { + let (_, (a, b)) = bounds(lo); + if d.order(a, b) != Ordering::Greater { + return Some(Binary32(sign | lo)); + } + } + if hi < 0x7f80_0000 { + let ((a, b), _) = bounds(hi); + if d.order(a, b) != Ordering::Less { + return Some(Binary32(sign | hi)); + } + } + None +} + +/// A positive decimal `m·10^e`, held as `m·10^max(e, 0)` over `10^max(−e, 0)`, and +/// whether digits past `m` were dropped (then it lies above `m·10^e`, by less than one +/// unit of `m`'s last digit). +struct Decimal { + num: Nat, + den: Nat, + sticky: bool, +} + +impl Decimal { + fn new(digits: &[u8], e: i64, sticky: bool) -> Decimal { + let mut num = Nat(Vec::new()); + for chunk in digits.chunks(9) { + let v = chunk.iter().fold(0u32, |a, &d| a * 10 + d as u32); + num.mul_add_small(10u32.pow(chunk.len() as u32), v); + } + let mut den = Nat(vec![1]); + if e >= 0 { num.mul_pow10(e as u64) } else { den.mul_pow10(e.unsigned_abs()) } + Decimal { num, den, sticky } + } + + /// The decimal's order against `a·2^b`. Exact: `a·2^b` with `b ≥ -1075` is a multiple + /// of the last kept digit's unit (see [`KEPT_DIGITS`]), so a dropped tail decides + /// only a tie. + fn order(&self, a: u64, b: i32) -> Ordering { + let (mut l, mut r) = (self.num.clone(), self.den.mul_u64(a)); + if b >= 0 { r.shl(b as u64) } else { l.shl(b.unsigned_abs() as u64) } + match l.order(&r) { + Ordering::Equal if self.sticky => Ordering::Greater, + o => o, + } + } +} + +/// A non-negative integer, as little-endian base-2^32 limbs with no zero limb on top: the +/// exact arithmetic [`binary32_of`] compares in. +#[derive(Clone)] +struct Nat(Vec); + +impl Nat { + /// `self·k + add`. + fn mul_add_small(&mut self, k: u32, add: u32) { + let mut carry = add as u64; + for l in &mut self.0 { + let v = *l as u64 * k as u64 + carry; + *l = v as u32; + carry = v >> 32; + } + if carry != 0 { + self.0.push(carry as u32); + } + } + + fn mul_pow10(&mut self, mut k: u64) { + while k > 0 { + let s = k.min(9); + self.mul_add_small(10u32.pow(s as u32), 0); + k -= s; + } + } + + fn mul_u64(&self, a: u64) -> Nat { + let mut out = vec![0u32; self.0.len() + 2]; + for (j, w) in [a as u32, (a >> 32) as u32].into_iter().enumerate() { + let mut carry = 0u64; + for (i, &l) in self.0.iter().enumerate() { + let v = l as u64 * w as u64 + out[i + j] as u64 + carry; + out[i + j] = v as u32; + carry = v >> 32; + } + let mut k = self.0.len() + j; + while carry != 0 { + let v = out[k] as u64 + carry; + out[k] = v as u32; + carry = v >> 32; + k += 1; + } + } + while out.last() == Some(&0) { + out.pop(); + } + Nat(out) + } + + fn shl(&mut self, bits: u64) { + if self.0.is_empty() { + return; + } + let b = (bits % 32) as u32; + if b > 0 { + let mut carry = 0u32; + for l in &mut self.0 { + let v = ((*l as u64) << b) | carry as u64; + *l = v as u32; + carry = (v >> 32) as u32; + } + if carry != 0 { + self.0.push(carry); + } + } + self.0.splice(0..0, std::iter::repeat_n(0, (bits / 32) as usize)); + } + + fn order(&self, o: &Nat) -> Ordering { + self.0.len().cmp(&o.0.len()).then_with(|| self.0.iter().rev().cmp(o.0.iter().rev())) + } +} diff --git a/wai/wai-rs/src/lib.rs b/wai/wai-rs/src/lib.rs index 124e5ed4b..b48a4d072 100644 --- a/wai/wai-rs/src/lib.rs +++ b/wai/wai-rs/src/lib.rs @@ -234,6 +234,12 @@ pub mod int_wire; #[cfg(feature = "neural_int")] pub mod int_payload; +// A test of the sources: no floating point and no serde_json parser on the integer decode +// paths (the modules above, the entropy coder below and the operator set), outside their +// tests and model-checker harnesses. +#[cfg(all(test, feature = "neural_int"))] +mod float_free; + #[cfg(feature = "neural")] pub mod neural; // The neural C FFI uses the `ffi` feature's shared buffer and error types, so it @@ -834,9 +840,11 @@ pub mod prior_wire; pub mod scene_map; // The strict JSON reader the two mappings above share, the fleet's reader of a relay's session -// summaries, and the reader of a `wai.video.int_motion` receipt's JSON. Rust-only. +// summaries, the reader of a `wai.video.int_motion` receipt's JSON, the reader of the integer +// decoders' parameter-set JSON (`entropy.json`, a `WMB1` bundle's tables) and the splat +// export's JSON reader (`int_splat`, which reads its numbers in integer arithmetic). Rust-only. /// cbindgen:ignore -#[cfg(any(feature = "prior_wire", feature = "scene_map", feature = "conform_result", feature = "fleet", feature = "video_receipt", feature = "channel_lineup"))] +#[cfg(any(feature = "prior_wire", feature = "scene_map", feature = "conform_result", feature = "fleet", feature = "video_receipt", feature = "channel_lineup", feature = "neural_int"))] pub(crate) mod strict_json; // Conformance results (extensions/conformance-results.md): `conform-result/1`, the signed outcome // of a conformance run, which `wai conform` writes and `wai verify` checks; and its corpus. diff --git a/wai/wai-rs/src/splat_exact.rs b/wai/wai-rs/src/splat_exact.rs index b563d1c9a..b5050bb85 100644 --- a/wai/wai-rs/src/splat_exact.rs +++ b/wai/wai-rs/src/splat_exact.rs @@ -725,7 +725,7 @@ fn fit_table(sym: &[i32]) -> (Vec, i32) { /// A `WIS1` payload (integer-payloads §6) that the identity set decodes to `attr` /// (`planes` planes of `s × s` bytes), with these ranges and Gaussian count. pub fn encode_identity(attr: &[u8], planes: u32, s: u32, n_gaussians: u32, ranges: &[(f32, f32)]) -> Vec { - use crate::int_payload::wire::{write_wis1, Wis1}; + use crate::int_payload::wire::{write_wis1, Binary32, Wis1}; let cells = (s * s) as usize; assert_eq!(attr.len(), planes as usize * cells); let mut cdfs = Vec::new(); @@ -751,7 +751,7 @@ pub fn encode_identity(attr: &[u8], planes: u32, s: u32, n_gaussians: u32, range s, attr: planes, n_gaussians, - ranges: ranges.to_vec(), + ranges: ranges.iter().map(|&(lo, hi)| (Binary32(lo.to_bits()), Binary32(hi.to_bits()))).collect(), cdfs, offsets, median_q: vec![0; planes as usize], @@ -1172,7 +1172,10 @@ pub fn decode(c: &Carried, held: &[Vec]) -> Result { let held = [HeldBytes::File(&set)]; let params = resolve(CAPABILITY, Some(&prior), &held, None).map_err(|_| Reason::SetMismatch)?; match decode_int(CAPABILITY, &env.payload, ¶ms) { - Ok(IntDecoded::Attr { bytes, n_attr, s, n_gaussians, ranges }) => Ok(Planes { bytes, planes: n_attr, s, n_gaussians, ranges }), + Ok(IntDecoded::Attr { bytes, n_attr, s, n_gaussians, ranges }) => { + let ranges = ranges.iter().map(|&(lo, hi)| (f32::from_bits(lo.0), f32::from_bits(hi.0))).collect(); + Ok(Planes { bytes, planes: n_attr, s, n_gaussians, ranges }) + } _ => Err(Reason::DecodeRefused), } } diff --git a/wai/wai-rs/src/strict_json.rs b/wai/wai-rs/src/strict_json.rs index fe3e309e0..4d49c98c0 100644 --- a/wai/wai-rs/src/strict_json.rs +++ b/wai/wai-rs/src/strict_json.rs @@ -1,6 +1,11 @@ //! A strict reader of JSON text (RFC 8259) for the documents the tensor-file and scene //! mappings read: `extensions/prior-carriage.md` §9 (a safetensors header) and -//! `extensions/scene-carriage.md` §3 (a glTF document). +//! `extensions/scene-carriage.md` §3 (a glTF document); for the integer decoders' parameter +//! sets, `entropy.json` and a `WMB1` bundle's tables (`extensions/integer-payloads.md` §9 and +//! §14), whose integers are read from their spellings; and for the splat export's JSON +//! bitstream, whose numbers [`crate::int_splat`] reads in integer arithmetic. No number is +//! converted to a float on the way in, so the integer decode paths read their JSON with no +//! floating point. //! //! It keeps what a lenient parser loses: members in the order they were written, a //! member named twice (refused), and each number's spelling (a number is never rounded on @@ -40,7 +45,7 @@ impl Json { _ => None, } } - #[cfg_attr(not(any(feature = "prior_wire", feature = "scene_map")), allow(dead_code))] + #[cfg_attr(not(any(feature = "prior_wire", feature = "scene_map", feature = "neural_int")), allow(dead_code))] pub fn as_arr(&self) -> Option<&[Json]> { match self { Json::Arr(a) => Some(a), @@ -56,12 +61,25 @@ impl Json { } /// A number spelled as a plain non-negative decimal integer (no sign, fraction or /// exponent, no leading zero) that fits `u64`. + #[cfg_attr( + not(any(feature = "prior_wire", feature = "scene_map", feature = "conform_result", feature = "fleet", feature = "video_receipt", feature = "channel_lineup", feature = "splat_int")), + allow(dead_code) + )] pub fn as_u64(&self) -> Option { match self { Json::Num(s) if is_plain_uint(s) => s.parse().ok(), _ => None, } } + /// A number spelled as a decimal integer (an optional `-`, then a plain non-negative + /// integer: no fraction or exponent) that fits `i64`. `-0` is zero. + #[cfg_attr(not(feature = "neural_int"), allow(dead_code))] + pub fn as_i64(&self) -> Option { + match self { + Json::Num(s) if is_plain_uint(s.strip_prefix('-').unwrap_or(s)) => s.parse().ok(), + _ => None, + } + } } fn is_plain_uint(s: &str) -> bool { @@ -326,6 +344,11 @@ mod tests { } assert_eq!(Json::Num("12".into()).as_u64(), Some(12)); assert_eq!(Json::Num("012".into()).as_u64(), None); + for (n, want) in [("-12", Some(-12)), ("-0", Some(0)), ("0", Some(0)), ("-9223372036854775808", Some(i64::MIN)), + ("9223372036854775807", Some(i64::MAX)), ("9223372036854775808", None), ("-012", None), ("1.0", None), + ("1e2", None), ("--1", None), ("-", None)] { + assert_eq!(Json::Num(n.into()).as_i64(), want, "{n}"); + } assert_eq!(quote("a\"\u{1}\n"), "\"a\\\"\\u0001\\n\""); let deep = "[".repeat(MAX_DEPTH + 2) + &"]".repeat(MAX_DEPTH + 2); assert!(read(deep.as_bytes()).is_err()); diff --git a/wai/wai-rs/tests/int_hyper_ops.rs b/wai/wai-rs/tests/int_hyper_ops.rs index 7f303acbb..da2167c9c 100644 --- a/wai/wai-rs/tests/int_hyper_ops.rs +++ b/wai/wai-rs/tests/int_hyper_ops.rs @@ -105,7 +105,7 @@ fn every_committed_set_lowers_to_its_committed_graphs_and_verdict() { let members = vec![("entropy.json", ent.clone()), ("g_s.whm", g_s.clone()), ("h_s.whm", h_s.clone())]; assert_eq!(wai_ops::generate::sets::set_digest(&members), s.digest, "{name}: the set's digest"); // The entropy tables are the stage operators' data, not lowered; the set's must load. - hyper::CondEntropy::try_from_value(&serde_json::from_slice(&ent).unwrap()).unwrap_or_else(|e| panic!("{name}: entropy.json: {e}")); + hyper::CondEntropy::read(&ent).unwrap_or_else(|e| panic!("{name}: entropy.json: {e}")); let lowered = wai_ops::lower::int_hyper(&h_s, &g_s).unwrap_or_else(|r| panic!("{name}: does not lower: {r:?}")); let roles: Vec<&str> = lowered.roles.iter().map(|(r, _)| *r).collect(); assert_eq!(roles, ["h_s", "g_s"], "{name}"); @@ -189,7 +189,7 @@ fn stream<'a>(k: &'a wai::int_wire::Wih1<'a>) -> hyper::CondStream<'a> { /// A committed set's three members, parsed: `g_s`, `h_s` and the entropy tables. fn set_of(v: &Value) -> (IntModel, IntModel, hyper::CondEntropy) { let m = |k: &str| read(v["set"][k].as_str().unwrap()); - let ent = hyper::CondEntropy::try_from_value(&serde_json::from_slice(&m("entropy.json")).unwrap()).unwrap(); + let ent = hyper::CondEntropy::read(&m("entropy.json")).unwrap(); (IntModel::from_bytes(&m("g_s.whm")).unwrap(), IntModel::from_bytes(&m("h_s.whm")).unwrap(), ent) } @@ -309,7 +309,7 @@ impl Owned { fn tiny(f: impl FnOnce(&mut IntModel, &mut IntModel, &mut hyper::CondEntropy)) -> (IntModel, IntModel, hyper::CondEntropy) { let mut g = IntModel::from_bytes(&read(&format!("{DIR}/tiny_g_s.bin"))).unwrap(); let mut h = IntModel::from_bytes(&read(&format!("{DIR}/tiny_h_s.bin"))).unwrap(); - let mut e = hyper::CondEntropy::try_from_value(&json(&format!("{DIR}/tiny_entropy.json"))).unwrap(); + let mut e = hyper::CondEntropy::read(&read(&format!("{DIR}/tiny_entropy.json"))).unwrap(); f(&mut g, &mut h, &mut e); (g, h, e) } @@ -353,7 +353,7 @@ fn every_committed_refusal_fixture_keeps_its_code() { let k = wai::int_wire::parse_wih1(&nested).unwrap_or_else(|e| panic!("{name}: a decode fixture's keyframe parses: {e:?}")); let g = IntModel::from_bytes(&set["g_s.whm"]).unwrap(); let h = IntModel::from_bytes(&set["h_s.whm"]).unwrap(); - let e = hyper::CondEntropy::try_from_value(&serde_json::from_slice(&set["entropy.json"]).unwrap()).unwrap(); + let e = hyper::CondEntropy::read(&set["entropy.json"]).unwrap(); let ops = hyper::ops::decode(&g, &h, &e, &stream(&k)).map(|d| d.0).map_err(code_of); let direct = hyper::legacy::decode_conditional_checked(&g, &h, &e, &stream(&k)).map(|d| d.0).map_err(code_of); assert!(ops == direct, "{name}: the two decodes of the keyframe differ: {:?} / {:?}", ops.as_ref().err(), direct.as_ref().err()); diff --git a/wai/wai-rs/tests/int_kernels_vectors.rs b/wai/wai-rs/tests/int_kernels_vectors.rs index f6d3b58af..c8a976eb8 100644 --- a/wai/wai-rs/tests/int_kernels_vectors.rs +++ b/wai/wai-rs/tests/int_kernels_vectors.rs @@ -89,7 +89,8 @@ fn int_splat_cloud() { let d = vroot().join("int_splat"); let gs = IntModel::from_bytes(&std::fs::read(d.join("splat_gs.bin")).unwrap()).unwrap(); let bs = jload(&d.join("splat.json")); - let attr = same_under_every_policy("int_splat", || decode(&gs, &bs).unwrap().attr); + let text = std::fs::read(d.join("splat.json")).unwrap(); + let attr = same_under_every_policy("int_splat", || decode(&gs, &text).unwrap().attr); let hash = wai::int_hyper_synth::fnv1a(&attr); assert_eq!(format!("{hash:016x}"), bs["attr_hash"].as_str().unwrap()); sha256_sidecar::one(&d, "splat", "DecodedSplat::attr: the decoded attribute tensor's bytes in decoder output order", &attr); diff --git a/wai/wai-rs/tests/int_payload_cost.rs b/wai/wai-rs/tests/int_payload_cost.rs index b7feb6b12..4aa28c6f2 100644 --- a/wai/wai-rs/tests/int_payload_cost.rs +++ b/wai/wai-rs/tests/int_payload_cost.rs @@ -140,9 +140,7 @@ fn the_declared_cost_bounds_what_the_decode_holds() { // The conditional codec's set: its tables, and its two networks parsed, then // lowered and loaded (`int_hyper_synth::ops::load`). c::CAP_NEURAL_INT_HYPER | c::CAP_VIDEO_INT_HYPER => { - let e: serde_json::Value = serde_json::from_slice(&hyp["entropy.json"]).unwrap(); - let t = wai::int_hyper_synth::CondEntropy::try_from_value(&e).unwrap(); - drop(e); + let t = wai::int_hyper_synth::CondEntropy::read(&hyp["entropy.json"]).unwrap(); let (g, h) = (wai::int_hyper_synth::IntModel::from_bytes(&hyp["g_s.whm"]).unwrap(), wai::int_hyper_synth::IntModel::from_bytes(&hyp["h_s.whm"]).unwrap()); let n = wai::int_hyper_synth::ops::load(&g, &h).unwrap(); @@ -280,7 +278,7 @@ fn the_declared_cost_bounds_what_the_decode_holds() { z_rans: k.z_rans, rans: k.rans, }; let (g, h) = (wai::int_hyper_synth::IntModel::from_bytes(&hyp["g_s.whm"]).unwrap(), wai::int_hyper_synth::IntModel::from_bytes(&hyp["h_s.whm"]).unwrap()); - let ent = wai::int_hyper_synth::CondEntropy::try_from_value(&serde_json::from_slice(&hyp["entropy.json"]).unwrap()).unwrap(); + let ent = wai::int_hyper_synth::CondEntropy::read(&hyp["entropy.json"]).unwrap(); let nets = wai::int_hyper_synth::ops::load(&g, &h).unwrap(); let (r, peak) = with_policy(Policy::reference(), || peak_during(|| wai::int_hyper_synth::ops::decode_loaded(&nets, g.afrac, &ent, &st))); r.unwrap(); diff --git a/wai/wai-rs/tests/int_splat.rs b/wai/wai-rs/tests/int_splat.rs index b0a336e26..ffce1421b 100644 --- a/wai/wai-rs/tests/int_splat.rs +++ b/wai/wai-rs/tests/int_splat.rs @@ -21,10 +21,10 @@ fn decodes_the_learned_splat_cloud_attribute_for_attribute() { // ML runtime — to the SAME [14,S,S] attribute tensor the Python integer reference // produced. A single disagreeing attribute would diverge this FNV-1a. let gs = IntModel::from_bytes(&std::fs::read(dir().join("splat_gs.bin")).unwrap()).expect("WHM1 g_s"); - let bs: serde_json::Value = - serde_json::from_slice(&std::fs::read(dir().join("splat.json")).unwrap()).unwrap(); + let text = std::fs::read(dir().join("splat.json")).unwrap(); + let bs: serde_json::Value = serde_json::from_slice(&text).unwrap(); - let clip = decode(&gs, &bs).expect("decode"); + let clip = decode(&gs, &text).expect("decode"); assert_eq!(clip.n_attr, bs["attr"].as_u64().unwrap() as usize); assert_eq!(clip.attr.len(), clip.n_attr * clip.s * clip.s, "attribute tensor size"); assert_eq!(clip.s * clip.s, bs["n_gaussians"].as_u64().unwrap() as usize, "N gaussians = S²"); diff --git a/wai/wai-rs/tests/int_splat_ops.rs b/wai/wai-rs/tests/int_splat_ops.rs index 4c777c270..22bdaffe6 100644 --- a/wai/wai-rs/tests/int_splat_ops.rs +++ b/wai/wai-rs/tests/int_splat_ops.rs @@ -153,7 +153,7 @@ fn every_committed_vector_decodes_through_the_lowering_to_its_golden() { let p = read(payload); let v = wai::int_wire::parse_wis1(&p).unwrap_or_else(|e| panic!("{name}: {e:?}")); let bs = json(from); - let j = SplatJson::from_value(&bs).unwrap(); + let j = SplatJson::read(&read(from)).unwrap(); let want = json(sidecar)[name]["sha256"].as_str().unwrap().to_string(); for (form, st) in [("WIS1", stream(&v)), ("JSON", j.stream())] { let ops = int_splat::ops::decode_typed(&gs, &st).unwrap_or_else(|r| panic!("{name} {form}: {r}")); @@ -349,7 +349,7 @@ fn every_kernel_policy_decodes_through_the_lowering_alike() { let x0 = wai_ops::lower::whm1(&read(SET), true).unwrap().tensors[0].hi; let at = |t: i64| cloud(move |o| o.median_q[1] = t - ((o.offsets[1] as i64) << o.afrac)); let mut clouds: Vec<(String, Owned)> = vec![("WIS1".into(), Owned::committed())]; - let j = SplatJson::from_value(&json("wai-rs/tests/vectors/int_splat/splat.json")).unwrap(); + let j = SplatJson::read(&read("wai-rs/tests/vectors/int_splat/splat.json")).unwrap(); clouds.push(("JSON".into(), Owned { afrac: j.afrac, c: j.c, sh: j.sh, sw: j.sw, s: j.s, n_attr: j.n_attr, cdfs: j.cdfs.clone(), offsets: j.offsets.clone(), median_q: j.median_q.clone(), rans: j.rans.clone(), diff --git a/wai/wai-rs/tests/ops_gpu.rs b/wai/wai-rs/tests/ops_gpu.rs index 192580dd9..660a8441c 100644 --- a/wai/wai-rs/tests/ops_gpu.rs +++ b/wai/wai-rs/tests/ops_gpu.rs @@ -88,8 +88,8 @@ fn bytes_of(d: &IntDecoded) -> Vec { b.extend_from_slice(&v.to_le_bytes()); } for (lo, hi) in ranges { - b.extend_from_slice(&lo.to_bits().to_le_bytes()); - b.extend_from_slice(&hi.to_bits().to_le_bytes()); + b.extend_from_slice(&lo.0.to_le_bytes()); + b.extend_from_slice(&hi.0.to_le_bytes()); } b.extend_from_slice(bytes); } diff --git a/wai/wai-web/src/int_splat.rs b/wai/wai-web/src/int_splat.rs index a1e6b462a..3a25559dc 100644 --- a/wai/wai-web/src/int_splat.rs +++ b/wai/wai-web/src/int_splat.rs @@ -14,14 +14,15 @@ use wai::int_hyper_synth::IntModel; /// A decoded splat: the `[n_attr · s · s]` `u8` attribute tensor (channel-major) — /// un-normalized to Gaussians (pos/scale/rot/color/opacity) on the JS side using the -/// per-attribute ranges the payload carries (`ranges`). +/// per-attribute ranges the payload carries (`ranges`). The ranges are held as their +/// binary32 bits, as the decode reads them, and become numbers only in the getter. #[wasm_bindgen] pub struct DecodedSplat { attr: Vec, n_attr: u32, s: u32, n_gaussians: u32, - ranges: Vec, + ranges: Vec, } #[wasm_bindgen] @@ -47,7 +48,7 @@ impl DecodedSplat { /// Presentation only: the splat-equivalence digest covers `attr`, not these. #[wasm_bindgen(getter)] pub fn ranges(&self) -> Box<[f32]> { - self.ranges.clone().into_boxed_slice() + self.ranges.iter().map(|&b| f32::from_bits(b)).collect() } } @@ -60,7 +61,7 @@ impl DecodedSplat { pub fn decode_wis1(payload: &[u8], g_s: &[u8]) -> Result { match crate::int_payload::decode("wai.splat.int_codec", payload, &[("g_s.whm", g_s)]).map_err(|e| JsError::new(&e))? { wai::int_payload::IntDecoded::Attr { bytes, n_attr, s, n_gaussians, ranges } => Ok(DecodedSplat { - attr: bytes, n_attr, s, n_gaussians, ranges: ranges.iter().flat_map(|&(lo, hi)| [lo, hi]).collect(), + attr: bytes, n_attr, s, n_gaussians, ranges: ranges.iter().flat_map(|&(lo, hi)| [lo.0, hi.0]).collect(), }), _ => Err(JsError::new("unsupported_capability: not an attribute tensor")), } @@ -69,16 +70,16 @@ pub fn decode_wis1(payload: &[u8], g_s: &[u8]) -> Result /// Decode a learned-splat model (`splat_gs.bin`, WHM1) + the tool's JSON bitstream /// (`splat.json`) to the integer attribute tensor in the browser. For debugging and /// conversion only: the payload `wai.splat.int_codec` carries is `WIS1` -/// ([`decode_wis1`]). Byte-identical to the native reference. +/// ([`decode_wis1`]). Byte-identical to the native reference. The JSON is read as the +/// native reader reads it, every number in integer arithmetic, so each range is the +/// binary32 value its decimal denotes (one that denotes none is refused). #[wasm_bindgen(js_name = decodeIntSplat)] pub fn decode_int_splat(gs_bin: &[u8], bitstream_json: &[u8]) -> Result { let gs = IntModel::from_bytes(gs_bin).map_err(JsError::new)?; - let bs: serde_json::Value = - serde_json::from_slice(bitstream_json).map_err(|e| JsError::new(&e.to_string()))?; - let j = wai::int_splat::SplatJson::from_value(&bs).map_err(JsError::new)?; + let j = wai::int_splat::SplatJson::read(bitstream_json).map_err(JsError::new)?; let d = wai::int_splat::decode_typed(&gs, &j.stream()).map_err(|r| JsError::new(r.detail))?; Ok(DecodedSplat { attr: d.attr, n_attr: d.n_attr as u32, s: d.s as u32, n_gaussians: j.n_gaussians as u32, - ranges: j.ranges.iter().flat_map(|&(lo, hi)| [lo as f32, hi as f32]).collect(), + ranges: j.ranges.iter().flat_map(|&(lo, hi)| [lo.0, hi.0]).collect(), }) } diff --git a/wai/wai-web/src/neural_int.rs b/wai/wai-web/src/neural_int.rs index a550de1fb..71122a0e0 100644 --- a/wai/wai-web/src/neural_int.rs +++ b/wai/wai-web/src/neural_int.rs @@ -116,17 +116,17 @@ fn cond_decode( Ok(DecodedImage { rgb, width: width as u32, height: height as u32 }) } -/// The debug JSON decode, through the checked readers (`CondEntropy::try_from_value`, -/// `CondBitstream::try_from_value`) and `decode_conditional_checked`, so a JSON the -/// export tool did not write is refused, never a panic: the pure-Rust half the native -/// tests run. +/// The debug JSON decode, through the checked readers (`CondEntropy::read`, which reads +/// the tables as a set's decode does, and `CondBitstream::try_from_value`) and +/// `decode_conditional_checked`, so a JSON the export tool did not write is refused, never +/// a panic: the pure-Rust half the native tests run. pub(crate) fn cond_decode_checked( g_s: &wai::int_hyper_synth::IntModel, h_s: &wai::int_hyper_synth::IntModel, entropy_json: &[u8], bitstream_json: &[u8], ) -> Result<(Vec, usize, usize), String> { use wai::int_hyper_synth::{decode_conditional_checked, CondBitstream, CondEntropy}; - let parse = |b: &[u8]| serde_json::from_slice::(b).map_err(|e| e.to_string()); - let ent = CondEntropy::try_from_value(&parse(entropy_json)?).map_err(|e| format!("model_mismatch: {e}"))?; - let bs = CondBitstream::try_from_value(&parse(bitstream_json)?).map_err(|k| format!("field_range: `{k}`"))?; + let ent = CondEntropy::read(entropy_json).map_err(|e| format!("model_mismatch: {e}"))?; + let bs = serde_json::from_slice::(bitstream_json).map_err(|e| e.to_string())?; + let bs = CondBitstream::try_from_value(&bs).map_err(|k| format!("field_range: `{k}`"))?; decode_conditional_checked(g_s, h_s, &ent, &bs.stream()).map_err(|r| r.to_string()) } -- 2.54.0 (Apple Git-157)