wai: the integer decode paths hold no floating point in their source, and read no JSON with a float parser #635
32 changed files with 1512 additions and 229 deletions
|
|
@ -40,4 +40,16 @@ audio_int = []
|
|||
# the goldens below decode `WIV1` and `WIR1` through on every leg of the matrix as well:
|
||||
# the arithmetic a 32-bit fabric runs, held to the same digests. On by default.
|
||||
i32_forms = ["wai-ops/i32_forms"]
|
||||
default = ["splat_int", "audio_int", "i32_forms"]
|
||||
# wai-rs's features that the included strict JSON reader (`strict_json.rs`, which the splat
|
||||
# decoder reads its JSON form with) names for its other users there; declared so the source's
|
||||
# `cfg` names known features. Never enabled here.
|
||||
prior_wire = []
|
||||
scene_map = []
|
||||
conform_result = []
|
||||
fleet = []
|
||||
video_receipt = []
|
||||
channel_lineup = []
|
||||
# wai-rs's feature for the integer decoders this crate includes, which the strict JSON reader
|
||||
# names for them (they read their parameter sets' JSON with it): on by default, as they are.
|
||||
neural_int = []
|
||||
default = ["neural_int", "splat_int", "audio_int", "i32_forms"]
|
||||
|
|
|
|||
|
|
@ -100,7 +100,8 @@ fn splat_wis1_decodes_to_its_attribute_golden() {
|
|||
assert_eq!(jload("int_splat/sha256.json")["splat"]["sha256"].as_str(), Some(SPLAT_ATTR_SHA256), "the sidecar");
|
||||
|
||||
let bs = jload("int_splat/splat.json");
|
||||
let j = SplatJson::from_value(&bs).expect("the JSON form reads");
|
||||
let j = SplatJson::read(&read("int_splat/splat.json")).expect("the JSON form reads");
|
||||
assert_eq!(j.ranges, v.ranges, "the JSON form's ranges are the payload's, read in integers");
|
||||
let from_json = both_kernels("the splat JSON form", || decode_splat(&gs, &j.stream()).expect("the JSON form decodes").attr);
|
||||
assert_eq!(from_json, attr, "the JSON form and the payload decode alike");
|
||||
let fnv = crate::int_hyper_synth::fnv1a(&attr);
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ fn assert_clip(frames: &[Vec<u8>], key: &str) -> String {
|
|||
fn keyframe(k: &Wih1<'_>) -> (Vec<u8>, usize, usize) {
|
||||
let g_s = IntModel::from_bytes(&read("int_hyper/tiny_g_s.bin")).expect("g_s");
|
||||
let h_s = IntModel::from_bytes(&read("int_hyper/tiny_h_s.bin")).expect("h_s");
|
||||
let ent = CondEntropy::try_from_value(&jload("int_hyper/tiny_entropy.json")).expect("entropy tables");
|
||||
let ent = CondEntropy::read(&read("int_hyper/tiny_entropy.json")).expect("entropy tables");
|
||||
let s = CondStream {
|
||||
m: k.m as usize, sh: k.sh as usize, sw: k.sw as usize, n_z: k.n_z as usize, zh: k.zh as usize, zw: k.zw as usize,
|
||||
z_rans: k.z_rans, rans: k.rans,
|
||||
|
|
@ -123,7 +123,7 @@ fn wih1_payload_is_byte_identical() {
|
|||
fn keyframe_through_the_lowering(k: &Wih1<'_>) -> (Vec<u8>, usize, usize) {
|
||||
let g_s = IntModel::from_bytes(&read("int_hyper/tiny_g_s.bin")).expect("g_s");
|
||||
let h_s = IntModel::from_bytes(&read("int_hyper/tiny_h_s.bin")).expect("h_s");
|
||||
let ent = CondEntropy::try_from_value(&jload("int_hyper/tiny_entropy.json")).expect("entropy tables");
|
||||
let ent = CondEntropy::read(&read("int_hyper/tiny_entropy.json")).expect("entropy tables");
|
||||
let s = CondStream {
|
||||
m: k.m as usize, sh: k.sh as usize, sw: k.sw as usize, n_z: k.n_z as usize, zh: k.zh as usize, zw: k.zw as usize,
|
||||
z_rans: k.z_rans, rans: k.rans,
|
||||
|
|
|
|||
|
|
@ -84,6 +84,11 @@ pub mod int_mlicpp;
|
|||
pub mod int_audio;
|
||||
#[path = "../../wai-rs/src/int_splat.rs"]
|
||||
pub mod int_splat;
|
||||
// The strict JSON reader the conditional and MLIC++ decoders read their entropy tables with,
|
||||
// and the splat decoder its JSON form, every number in integer arithmetic (so the matrix reads
|
||||
// the same values on every platform). Self-contained.
|
||||
#[path = "../../wai-rs/src/strict_json.rs"]
|
||||
pub(crate) mod strict_json;
|
||||
|
||||
// The wire parser of the integer payloads SPEC §5 pins (WIV1, WIH1, WIA1, WIS1).
|
||||
// Self-contained — std's Vec only, nothing from the crate — so the matrix parses the
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
### The integer decode paths hold no floating point in their source, and read no JSON with a float parser
|
||||
|
||||
The integer square root of the inverse GDN, the splat codec's ranges and JSON form, and `int_entropy`'s scale
|
||||
conversions no longer use floating point; the decode reads a parameter set's JSON with the strict reader, which
|
||||
converts no number to a float; and a test of the sources keeps the integer decode paths, and the operator set they
|
||||
run their networks through, free of floating point and of a float parser. No payload, corpus or decoded output
|
||||
changes: the integer corpora regenerate byte for byte, on arm64 macOS and x86_64 Linux. The test reads source, not a
|
||||
compiled decode.
|
||||
|
||||
- **The inverse GDN's root.** The narrow kernel sets seeded each integer square root from the IEEE-754 square root
|
||||
(`(n as f64).sqrt()`), and the AVX-512 kernel took eight at a time with a vector square root; the integer
|
||||
correction after each decided the result. The seed is now a fixed-point reciprocal square root (a table on the
|
||||
norm's top bits and two Newton steps, formed exactly in `u64` and `u128`), within one of the root, and the
|
||||
AVX-512 roots are taken digit by digit in integer lanes. The roots are the same integers. The test holds the root
|
||||
to its definition, an independent digit-by-digit root and `u64::isqrt` on the edges of `u64` (0, 1, every
|
||||
`k² − 1`, `k²` and `k² + 1` around the powers of two and across the range, `u64::MAX`), and an ignored test
|
||||
checks every boundary `k² − 1`, `k²` for `k` below `2^32`.
|
||||
- **`WIS1` ranges** are read, checked and written as binary32 bits (`int_payload::wire::Binary32`): finite, and
|
||||
`lo ≤ hi` as IEEE 754 compares two finite values (`−0` equal to `+0`), in integer arithmetic. The rule
|
||||
(integer-payloads §6) is unchanged; a test holds it to the float comparison on every class of bit pattern.
|
||||
- **The splat JSON form** (`splat.json`) is read from its text with the strict JSON reader: its integers as integer
|
||||
spellings, and each range by `int_splat::binary32_of` in exact integer arithmetic (integer-payloads §10, which
|
||||
now says how a number denotes a binary32 value). A number spelled as an integer is that integer, any other
|
||||
spelling denotes the binary64 value nearest it, and either must be a binary32 value; this is how the independent
|
||||
converter, `int_payload_verify.py`, reads a number. The reader before went through serde_json, whose float
|
||||
reading is correctly rounded only with its `float_roundtrip` feature (which `splat_int` turned on for the whole
|
||||
crate, and the byte-equality crate never had), and it disagreed with the independent converter in two places:
|
||||
it kept the sign of the integer `-0` (zero, so `+0`), and it rounded an integer past `u64` (refused unless it is
|
||||
a binary32 value). A range one digit from the binary64 tie between `1` and the next binary64 value now reads as
|
||||
binary64 rounds it on each side of the tie, with a test, which also holds two spellings one last digit apart
|
||||
(`4194304.0` and `4194304.5`, `8.388608e6` and `8.388609e6`) to the two adjacent values the payload's decode
|
||||
presents. The JSON decode refuses a range that is not a binary32
|
||||
value, as the converter always did, and the text is read strictly (a member named twice, or bytes after the
|
||||
value, refuse it). `splat_int` no longer turns on `float_roundtrip`; `splat_exact`, which reads an asset's
|
||||
`POSITION` bounds as JSON numbers, does.
|
||||
- **`int_entropy::fixed_scale_edges` and `quantize_scale`** form `⌊x·2^frac⌋` from the binary32 bits in integers,
|
||||
the values the binary64 product and its floor gave, which a test checks at every exponent and every `frac`.
|
||||
- **The parameter sets' JSON** (`entropy.json` of `wai.neural.int_hyper` and `wai.video.int_hyper`, integer-payloads
|
||||
§9, and a `WMB1` bundle's tables, §14) is read by the strict JSON reader, each integer from its spelling. The
|
||||
general JSON reader the crate depends on (serde_json) read it before: its number parser is binary64 code, which ran
|
||||
on a fraction, an exponent or an integer past `u64` before the integer reads refused them, so a decoder built into
|
||||
a module with no floating point could not hold it. §9 now says how the text is read, and two readings change: `-0`
|
||||
is zero, as §9's integer rule already gave (the general reader read it as a float, and the set was refused), and a
|
||||
member named twice, or a value more than 64 levels deep, refuses the set (`model_mismatch`), where the general
|
||||
reader kept the last of two members and read deeper values. The committed sets read as before, with tests of each
|
||||
reading through `decode_int` and through a `WMB1` bundle.
|
||||
- **A test of the sources** (`float_free`) fails on a floating-point type, literal or vector intrinsic in any module
|
||||
on the integer decode paths, outside its tests and model-checker harnesses: the kernels, the operator-set bridge,
|
||||
the payloads, the integer codecs, the rANS coder and its bucketing, the strict JSON reader, and the other integer
|
||||
entropy decoders and the decodes built on them (JPEG AI's mANS decoder, `WIR1`, the NNC tensor's arithmetic decoder
|
||||
and its tensor decode, `dred-int/1` but its floating-point reference and measurement, and `wai.avatar.face`); in
|
||||
every file of the operator set (`wai-ops`); and on a call of the general reader's parser in those modules, but two
|
||||
that are not decodes: the direct `int_synth` decode kept for the differential tests, and the face verifier's read
|
||||
of an envelope's manifest. Every `int_*` file, and every file under `int_kernels/` and `int_payload/`, must be
|
||||
among those it reads, so a new one fails it until it is listed. The tools' JSON forms (`from_value`, the
|
||||
`*_from_json` converters) still take a `serde_json::Value`; they are converters, not decodes. Outside these paths,
|
||||
`exact_tonemap`'s ST 2094-50 reader still forms binary64 slopes for its measurement, beside the Q30 slopes its
|
||||
integer derivation uses.
|
||||
- **Breaking:** `api:rust:wai::int_payload::IntDecoded::Attr`: its `ranges` are `(Binary32, Binary32)` pairs, the
|
||||
bits the payload carries, so the decode forms no float. A sink presenting a range takes `f32::from_bits(r.0)`.
|
||||
`Binary32` is re-exported as `int_payload::Binary32`.
|
||||
- **Breaking:** `api:rust:wai::int_payload::wire::Wis1::ranges`, and so `wire::parse_wis1`'s result and
|
||||
`wire::write_wis1`'s argument: `(Binary32, Binary32)` bits, where they were `(f32, f32)`. Build one with
|
||||
`Binary32(x.to_bits())` and read it with `f32::from_bits(b.0)`.
|
||||
- **Breaking:** `api:rust:wai::int_hyper_synth::CondEntropy::try_from_value`: replaced by `CondEntropy::read`, which
|
||||
takes the file's bytes and reads them as a set's decode does, so no float parser reads the tables first. Call it
|
||||
with `entropy.json`'s bytes in place of a parsed value.
|
||||
- **Breaking:** `api:rust:wai::int_splat::SplatJson::ranges`: the binary32 values `lo` and `hi` denote, as
|
||||
`(Binary32, Binary32)`, where they were `(f64, f64)` as a float parser read them.
|
||||
- **Breaking:** `api:rust:wai::int_splat::SplatJson::from_value`, `api:rust:wai::int_splat::decode` and
|
||||
`api:rust:wai::int_payload::wis1_from_json`: the JSON form is read from its text, so no float parser reads it
|
||||
first. Call `SplatJson::read`, `decode` and `wis1_from_json` with the file's bytes in place of a parsed value.
|
||||
|
|
@ -527,7 +527,9 @@ arithmetic is [integer-operators](integer-operators.md) §13.2 (`WHM1`) and §13
|
|||
|
||||
**`entropy.json`**, the tables of `wai.neural.int_hyper` and `wai.video.int_hyper`: a
|
||||
JSON object with these members. Other members are ignored. An integer is a JSON number
|
||||
with no fraction and no exponent.
|
||||
with no fraction and no exponent, and `-0` is zero. A file that is not a JSON text
|
||||
(RFC 8259, UTF-8, no member named twice, at most 64 levels, nothing after the value) is
|
||||
refused (`model_mismatch`). The reference decode reads the text with no float parser.
|
||||
|
||||
| member | value |
|
||||
|---|---|
|
||||
|
|
@ -568,9 +570,15 @@ implicit:
|
|||
- `WIA1`: `sr` is the export's `sample_rate` (in `audio.json`, which the bitstream
|
||||
does not carry); `lo` is the symbol offset; every channel's table must have the first
|
||||
table's length; `afrac` is carried and must be the model's.
|
||||
- `WIS1`: the ranges `lo` and `hi` must each be a binary32 value, carried exactly (a
|
||||
JSON integer included: `2^60 + 1` is refused, not rounded to `2^60`);
|
||||
`afrac` is carried and must be the model's.
|
||||
- `WIS1`: the ranges `lo` and `hi` must each be a binary32 value, carried exactly. A
|
||||
number spelled as an integer (no fraction, no exponent) is that integer: `2^60 + 1` is
|
||||
refused, not rounded to `2^60`, and `-0` is zero, so `+0`. Any other spelling denotes
|
||||
the binary64 value nearest it (ties to even; one that rounds to zero is a zero of its
|
||||
sign, and one past the binary64 range is no value), and that value must be a binary32
|
||||
value: the export writes each range as the shortest decimal of a binary32 value
|
||||
widened to binary64. The reference converter reads the JSON text in integer
|
||||
arithmetic, so no float parser decides a range. `afrac` is carried and must be the
|
||||
model's.
|
||||
|
||||
A payload and the JSON it is converted from decode to the same bytes; the conformance
|
||||
vectors check that for every committed JSON bitstream.
|
||||
|
|
@ -698,12 +706,12 @@ multiple of 4, at least 8) when the decode reaches it.
|
|||
The set is one file, a `WMB1` bundle: the magic `WMB1`, then four sections, each a
|
||||
`u32` length followed by that many bytes, in the order `h_s`, `entropy`, `g_s` (each a
|
||||
`WHM1` network, §9) and the tables. The tables are a JSON object with the members of
|
||||
`entropy.json` (§9), and `afrac`. Nothing follows the fourth section. The bundle's
|
||||
rules: the three networks' `afrac` are equal, lie in `1..=30`, and equal the tables'
|
||||
`afrac`; `1 ≤ N ≤ 65536`; and every table, over its length, follows
|
||||
[integer-operators](integer-operators.md) E1's `t2` rule (`2 ≤ n_b`, `n_b` no more than
|
||||
its array's length, `cdf[0] = 0`, `cdf[n_b − 1] = 65536`, strictly increasing), in
|
||||
place of §2's rule.
|
||||
`entropy.json` (§9), and `afrac`, read as §9 reads `entropy.json`. Nothing follows the
|
||||
fourth section. The bundle's rules: the three networks' `afrac` are equal, lie in
|
||||
`1..=30`, and equal the tables' `afrac`; `1 ≤ N ≤ 65536`; and every table, over its
|
||||
length, follows [integer-operators](integer-operators.md) E1's `t2` rule (`2 ≤ n_b`,
|
||||
`n_b` no more than its array's length, `cdf[0] = 0`, `cdf[n_b − 1] = 65536`, strictly
|
||||
increasing), in place of §2's rule.
|
||||
|
||||
The decode:
|
||||
|
||||
|
|
|
|||
|
|
@ -207,8 +207,10 @@ audio_int = ["neural_int"]
|
|||
world_int = ["neural_int"]
|
||||
# Integer-exact LEARNED Gaussian-splat ATTRIBUTE codec ("the int_hyper of 3D"):
|
||||
# learned transform over the [14,S,S] attribute image → byte-exact attribute tensor
|
||||
# (splat-equivalence, a portable cloud hash). Reuses int_hyper_synth (neural_int).
|
||||
splat_int = ["neural_int", "serde_json/float_roundtrip"]
|
||||
# (splat-equivalence, a portable cloud hash). Reuses int_hyper_synth (neural_int). Its
|
||||
# JSON form is read with the strict reader, every number in integer arithmetic, so it
|
||||
# needs nothing of serde_json's float parsing.
|
||||
splat_int = ["neural_int"]
|
||||
neural = ["neural_int", "dep:ort", "dep:ndarray", "dep:zstd-safe"]
|
||||
# The perceptual / generative-semantic path: CLIP-similarity conformance,
|
||||
# regenerate-at-sink. Reuses the ONNX runtime for the reference CLIP embedder;
|
||||
|
|
@ -478,8 +480,9 @@ robustness = ["synthesis_mark"]
|
|||
# the commitment to the frames at canonical poses; a sink's view statement; a view's class under
|
||||
# synthesis's lattice; and a float renderer of the same model, for measurement only. Reuses
|
||||
# splat_int (the decode) and synthesis (origin claims). SHA-256, BLAKE3 and Ed25519 only; no new
|
||||
# dependency.
|
||||
splat_exact = ["splat_int", "synthesis"]
|
||||
# dependency. An asset's POSITION min and max are read as JSON numbers and compared exactly with
|
||||
# the decode's, so serde_json reads them correctly rounded (`float_roundtrip`).
|
||||
splat_exact = ["splat_int", "synthesis", "serde_json/float_roundtrip"]
|
||||
# Faces from keypoints (extensions/avatars.md §6 to §13): `wai.avatar.face` — a keypoint track
|
||||
# (WFK1 segments) and an identity prior named by SHA-256 (WFI1) generate a talking head that is
|
||||
# the same bytes on every sink: the driver's integer warp and keypoint maps, then the generator's
|
||||
|
|
|
|||
|
|
@ -11,8 +11,6 @@ use wai::int_splat::decode;
|
|||
fn main() {
|
||||
let dir = std::env::args().nth(1).expect("usage: int_splat_decode <dir>");
|
||||
let gs = IntModel::from_bytes(&std::fs::read(format!("{dir}/splat_gs.bin")).unwrap()).unwrap();
|
||||
let bs: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(format!("{dir}/splat.json")).unwrap()).unwrap();
|
||||
let clip = decode(&gs, &bs).expect("decode");
|
||||
let clip = decode(&gs, &std::fs::read(format!("{dir}/splat.json")).unwrap()).expect("decode");
|
||||
println!("{} gaussians × {} attrs ({}²) attr_hash {:016x}", clip.s * clip.s, clip.n_attr, clip.s, clip.attr_hash());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -387,13 +387,13 @@ fn vector_cases(cases: &mut Vec<Case>) {
|
|||
{
|
||||
let d = format!("{VECTORS}/int_splat");
|
||||
let gs = IntModel::from_bytes(&std::fs::read(format!("{d}/splat_gs.bin")).unwrap()).unwrap();
|
||||
let bs = read_json(&format!("{d}/splat.json"));
|
||||
let n = bs["n_gaussians"].as_u64().unwrap();
|
||||
let text = std::fs::read(format!("{d}/splat.json")).unwrap();
|
||||
let n = read_json(&format!("{d}/splat.json"))["n_gaussians"].as_u64().unwrap();
|
||||
cases.push(Case {
|
||||
name: format!("vector · int_splat trained attribute decode ({n} Gaussians, committed)"),
|
||||
per_run: (n as f64 / 1e3, "kGaussians"),
|
||||
macs: None,
|
||||
run: Box::new(move || Out::B(wai::int_splat::decode(&gs, &bs).unwrap().attr)),
|
||||
run: Box::new(move || Out::B(wai::int_splat::decode(&gs, &text).unwrap().attr)),
|
||||
});
|
||||
}
|
||||
// int_world: trained transition + synthesis per generated frame.
|
||||
|
|
|
|||
|
|
@ -128,7 +128,7 @@ fn vectors(t: &mut Tree) -> Vec<Vector> {
|
|||
set: vec![("model.wam", format!("{V}/int_audio/audio_model.bin"))],
|
||||
sidecar: format!("{V}/int_audio/sha256.json"),
|
||||
});
|
||||
let wis1 = wis1_from_json(&t.json(&format!("{V}/int_splat/splat.json"))).expect("splat WIS1");
|
||||
let wis1 = wis1_from_json(&t.read(&format!("{V}/int_splat/splat.json"))).expect("splat WIS1");
|
||||
t.put(&format!("{V}/int_splat/splat.wis1"), wis1);
|
||||
out.push(Vector {
|
||||
name: "splat",
|
||||
|
|
@ -316,8 +316,8 @@ fn fixtures(t: &mut Tree) -> Vec<Fixture> {
|
|||
Some(wire::write_wis1(&v))
|
||||
};
|
||||
fx(t, "wis1_json_form", SPL, None, Some(format!("{V}/int_splat/splat.json")), "bad_magic", "parse");
|
||||
fx(t, "wis1_range_order", SPL, wis(&|v| v.ranges[0] = (1.0, -1.0)), None, "field_range", "parse");
|
||||
fx(t, "wis1_range_nan", SPL, wis(&|v| v.ranges[0].0 = f32::NAN), None, "field_range", "parse");
|
||||
fx(t, "wis1_range_order", SPL, wis(&|v| v.ranges[0] = (wire::Binary32(1f32.to_bits()), wire::Binary32((-1f32).to_bits()))), None, "field_range", "parse");
|
||||
fx(t, "wis1_range_nan", SPL, wis(&|v| v.ranges[0].0 = wire::Binary32(f32::NAN.to_bits())), None, "field_range", "parse");
|
||||
fx(t, "wis1_n_gaussians", SPL, wis(&|v| v.n_gaussians = v.s * v.s + 1), None, "field_range", "parse");
|
||||
fx(t, "wis1_cdf_len", SPL, edit(&sp, &|b| b[32 + 14 * 8..32 + 14 * 8 + 2].copy_from_slice(&2u16.to_le_bytes())), None, "field_range", "parse");
|
||||
fx(t, "wis1_trailing", SPL, edit(&sp, &|b| b.push(0)), None, "trailing", "parse");
|
||||
|
|
|
|||
366
wai/wai-rs/src/float_free.rs
Normal file
366
wai/wai-rs/src/float_free.rs
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
//! No floating point on the integer decode paths (a test of the sources).
|
||||
//!
|
||||
//! The integer capabilities decode to the same bytes on every machine because no step of
|
||||
//! their decode rounds: every value is an integer, and so is every operation on it. A
|
||||
//! floating-point step on such a path is exact only by an argument someone must make and
|
||||
//! keep, and a decoder compiled to a module that admits no floating point cannot hold one
|
||||
//! at all. So these paths hold none in their source, and this test reads the source of
|
||||
//! every module on them, and of the operator set they run their networks through
|
||||
//! (`wai-ops`, every file), outside tests and model-checker harnesses, for a
|
||||
//! floating-point type, literal or vector intrinsic: `f32` or `f64` (alone or in a name
|
||||
//! such as `vaddq_f32` or `f32x4`), a numeric literal with a fraction or an exponent, or an
|
||||
//! x86 intrinsic on packed or scalar floats (`_mm*_pd`, `_ps`, `_sd`, `_ss`, `_ph`).
|
||||
//!
|
||||
//! The one place a float may be named is a conversion from a float source by contract,
|
||||
//! listed in [`BOUNDARIES`] with its reason: it reads its argument's bits, and every step
|
||||
//! after is integer.
|
||||
//!
|
||||
//! It reads source, not a dependency's code, so a dependency could bring floating point
|
||||
//! back unseen. The one that did is serde_json: its number parser is binary64 code, which
|
||||
//! runs on a fraction, an exponent or an integer past `u64` before an integer read refuses
|
||||
//! it. So the modules read here call no serde_json parser either (but on the lines in
|
||||
//! [`JSON_BOUNDARIES`]): the parameter sets' JSON (`entropy.json`, a `WMB1` bundle's
|
||||
//! tables) is read by [`crate::strict_json`], which is read here, and `int_synth.json` by
|
||||
//! the operator set's own reader. The tools' JSON forms (`from_value` and the
|
||||
//! `*_from_json` converters, over a `serde_json::Value`) are converters and debugging aids,
|
||||
//! not decodes: their callers parse the text. Beyond these modules and the operator set, a
|
||||
//! decode reaches the capability table's lookups (`codecs`), the pin checks and their
|
||||
//! SHA-256 (`pin`, whose listing is text) and `std`, none of which reads JSON; this test
|
||||
//! does not read them.
|
||||
//!
|
||||
//! The test fails when a file of the integer modules (`int_*.rs`, and the files under
|
||||
//! `int_kernels/` and `int_payload/` but its tests) is not in [`SOURCES`], so a new one is
|
||||
//! read from the start.
|
||||
//!
|
||||
//! Outside these paths, recorded and not read: `exact_tonemap`'s ST 2094-50 reader
|
||||
//! (`read_metadata_set`) forms `Alternate::m_f64`, the slopes in binary64 (a tangent, and a
|
||||
//! rational's quotient), for §6's measurement alone, beside the Q30 slopes its integer
|
||||
//! derivation uses. The derivation's output does not depend on them, but its reader cannot
|
||||
//! be built into a module with no floating point until they are formed apart from it.
|
||||
|
||||
/// The modules on the integer decode paths: the kernels, the operator-set bridge, the
|
||||
/// payload formats and their decode, the integer codecs, the entropy coder and its
|
||||
/// bucketing, and the strict reader the decoders read their parameter sets' JSON with
|
||||
/// (and the splat codec its JSON form); and the other integer entropy decoders and the
|
||||
/// integer decodes built on them: JPEG AI's mANS decoder, the residual refinement
|
||||
/// (`WIR1`) and its step, the NNC tensor's arithmetic decoder and its tensor decode,
|
||||
/// DRED's range decoder, payload, tables, weights, lowering and stream (all of
|
||||
/// `dred-int/1` but its floating-point reference and its measurement), and the faces
|
||||
/// from keypoints.
|
||||
const SOURCES: &[(&str, &str)] = &[
|
||||
("int_kernels.rs", include_str!("int_kernels.rs")),
|
||||
("int_kernels/narrow.rs", include_str!("int_kernels/narrow.rs")),
|
||||
("int_kernels/x86.rs", include_str!("int_kernels/x86.rs")),
|
||||
("int_ops.rs", include_str!("int_ops.rs")),
|
||||
("int_payload/wire.rs", include_str!("int_payload/wire.rs")),
|
||||
("int_payload/mod.rs", include_str!("int_payload/mod.rs")),
|
||||
("int_synth.rs", include_str!("int_synth.rs")),
|
||||
("int_hyper_synth.rs", include_str!("int_hyper_synth.rs")),
|
||||
("int_mlicpp.rs", include_str!("int_mlicpp.rs")),
|
||||
("int_video.rs", include_str!("int_video.rs")),
|
||||
("int_video_enc.rs", include_str!("int_video_enc.rs")),
|
||||
("int_audio.rs", include_str!("int_audio.rs")),
|
||||
("int_splat.rs", include_str!("int_splat.rs")),
|
||||
("int_world.rs", include_str!("int_world.rs")),
|
||||
("int_sr.rs", include_str!("int_sr.rs")),
|
||||
("int_set_cache.rs", include_str!("int_set_cache.rs")),
|
||||
("int_transform.rs", include_str!("int_transform.rs")),
|
||||
("int_transformer.rs", include_str!("int_transformer.rs")),
|
||||
("int_entropy.rs", include_str!("int_entropy.rs")),
|
||||
("rans.rs", include_str!("rans.rs")),
|
||||
("strict_json.rs", include_str!("strict_json.rs")),
|
||||
("jpegai_mans.rs", include_str!("jpegai_mans.rs")),
|
||||
("staged/wir1.rs", include_str!("staged/wir1.rs")),
|
||||
("staged/wir1_arith.rs", include_str!("staged/wir1_arith.rs")),
|
||||
("prior_wire/cabac.rs", include_str!("prior_wire/cabac.rs")),
|
||||
("prior_wire/nnc.rs", include_str!("prior_wire/nnc.rs")),
|
||||
("dred_exact/ec.rs", include_str!("dred_exact/ec.rs")),
|
||||
("dred_exact/packet.rs", include_str!("dred_exact/packet.rs")),
|
||||
("dred_exact/payload.rs", include_str!("dred_exact/payload.rs")),
|
||||
("dred_exact/tables.rs", include_str!("dred_exact/tables.rs")),
|
||||
("dred_exact/weights.rs", include_str!("dred_exact/weights.rs")),
|
||||
("dred_exact/lower.rs", include_str!("dred_exact/lower.rs")),
|
||||
("dred_exact/scenario.rs", include_str!("dred_exact/scenario.rs")),
|
||||
("avatar_face.rs", include_str!("avatar_face.rs")),
|
||||
];
|
||||
|
||||
/// The lines that may name a float, each a conversion from a float source by contract.
|
||||
/// `int_entropy`'s two take a float model's scale table and a float hyper-decoder's σ
|
||||
/// (codecs that stay `NeuralFloat`); each reads the binary32 bits and forms the
|
||||
/// fixed-point value in integers.
|
||||
const BOUNDARIES: &[(&str, &str)] = &[
|
||||
("int_entropy.rs", "pub fn fixed_scale_edges(scale_table: &[f32], frac: u32) -> Vec<i64> {"),
|
||||
("int_entropy.rs", "pub fn quantize_scale(sigma: f32, frac: u32) -> i64 {"),
|
||||
];
|
||||
|
||||
/// The lines that may call a serde_json parser, each with its reason. `int_synth`'s is
|
||||
/// the direct decode kept for the differential tests (`legacy`, built only for tests and
|
||||
/// under the test-only `legacy_int`); a sink's decode reads `int_synth.json` through the
|
||||
/// operator set. `avatar_face`'s is the conformance verifier's (§9) read of an object's
|
||||
/// envelope: its manifest, the container's JSON (SPEC §2), read before any face is
|
||||
/// decoded; the faces themselves (§7) read no JSON.
|
||||
const JSON_BOUNDARIES: &[(&str, &str)] = &[
|
||||
("int_synth.rs", "let spec: SynthSpec = serde_json::from_slice(spec_json)"),
|
||||
("avatar_face.rs", "let m: Value = serde_json::from_slice(b.get(8..8 + n)?).ok()?;"),
|
||||
];
|
||||
|
||||
/// `src` with every comment, string literal and character literal blanked (newlines
|
||||
/// kept, so line numbers hold): what is left is code.
|
||||
fn code_of(src: &str) -> String {
|
||||
let b = src.as_bytes();
|
||||
let mut out = String::with_capacity(src.len());
|
||||
let blank = |out: &mut String, s: &str| out.extend(s.chars().map(|c| if c == '\n' { '\n' } else { ' ' }));
|
||||
let ident = |i: usize| i > 0 && (b[i - 1].is_ascii_alphanumeric() || b[i - 1] == b'_');
|
||||
let mut i = 0;
|
||||
while i < b.len() {
|
||||
let rest = &src[i..];
|
||||
let end = if rest.starts_with("//") {
|
||||
i + rest.find('\n').unwrap_or(rest.len())
|
||||
} else if rest.starts_with("/*") {
|
||||
let (mut j, mut depth) = (i, 0);
|
||||
loop {
|
||||
if src[j..].starts_with("/*") {
|
||||
depth += 1;
|
||||
j += 2;
|
||||
} else if src[j..].starts_with("*/") {
|
||||
depth -= 1;
|
||||
j += 2;
|
||||
if depth == 0 {
|
||||
break j;
|
||||
}
|
||||
} else {
|
||||
j += 1;
|
||||
}
|
||||
}
|
||||
} else if let Some(q) = raw_string_open(rest).filter(|_| !ident(i)) {
|
||||
let close = format!("\"{}", "#".repeat(rest[..q].matches('#').count()));
|
||||
i + q + 1 + rest[q + 1..].find(&close).expect("a raw string closes") + close.len()
|
||||
} else if b[i] == b'"' || (!ident(i) && rest.starts_with("b\"")) {
|
||||
let mut j = i + rest.find('"').unwrap() + 1;
|
||||
while b[j] != b'"' {
|
||||
j += if b[j] == b'\\' { 2 } else { 1 };
|
||||
}
|
||||
j + 1
|
||||
} else if b[i] == b'\'' {
|
||||
// A character literal ('x', '\n', '\u{..}'), not a lifetime ('a).
|
||||
let mut cs = rest.char_indices().skip(1);
|
||||
match (cs.next(), cs.next()) {
|
||||
// The escaped character is at 2 (and may be a quote); the close follows.
|
||||
(Some((_, '\\')), _) => i + 3 + rest[3..].find('\'').unwrap() + 1,
|
||||
(Some(_), Some((k, '\''))) => i + k + 1,
|
||||
_ => i + 1,
|
||||
}
|
||||
} else {
|
||||
let c = rest.chars().next().unwrap();
|
||||
out.push(c);
|
||||
i += c.len_utf8();
|
||||
continue;
|
||||
};
|
||||
if b[i] == b'\'' && end == i + 1 {
|
||||
out.push('\'');
|
||||
} else {
|
||||
blank(&mut out, &src[i..end]);
|
||||
}
|
||||
i = end;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Where the opening quote of a raw string literal starting `rest` is (`r"`, `r#"`,
|
||||
/// `br##"` and so on), or `None` (`r#ident`, a raw identifier, is not one).
|
||||
fn raw_string_open(rest: &str) -> Option<usize> {
|
||||
let after = rest.strip_prefix("br").or_else(|| rest.strip_prefix('r'))?;
|
||||
let hashes = after.len() - after.trim_start_matches('#').len();
|
||||
after[hashes..].starts_with('"').then_some(rest.len() - after.len() + hashes)
|
||||
}
|
||||
|
||||
/// The lines of `code`, numbered from 1, without the items a `#[cfg(test)]` or
|
||||
/// `#[cfg(kani)]` (alone or under `all(..)`) keeps out of a decoder's build.
|
||||
fn decoder_lines(code: &str) -> Vec<(usize, &str)> {
|
||||
let lines: Vec<&str> = code.lines().collect();
|
||||
let test_only = |l: &str| {
|
||||
let t = l.trim();
|
||||
["#[cfg(test)]", "#[cfg(kani)]"].contains(&t) || t.starts_with("#[cfg(all(test,") || t.starts_with("#[cfg(all(kani,")
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
let mut i = 0;
|
||||
while i < lines.len() {
|
||||
if !test_only(lines[i]) {
|
||||
out.push((i + 1, lines[i]));
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
// Skip the attributes, then the item: to its `;`, or its braces' close.
|
||||
let mut j = i;
|
||||
while j < lines.len() && (lines[j].trim().starts_with("#[") || lines[j].trim().is_empty()) {
|
||||
j += 1;
|
||||
}
|
||||
let (mut depth, mut opened) = (0i64, false);
|
||||
while j < lines.len() {
|
||||
let l = lines[j];
|
||||
depth += l.matches('{').count() as i64 - l.matches('}').count() as i64;
|
||||
opened |= l.contains('{');
|
||||
if (!opened && l.trim_end().ends_with(';')) || (opened && depth <= 0) {
|
||||
break;
|
||||
}
|
||||
j += 1;
|
||||
}
|
||||
i = j + 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The floating-point tokens of one line of code.
|
||||
fn float_tokens(line: &str) -> Vec<&str> {
|
||||
let b = line.as_bytes();
|
||||
let mut out = Vec::new();
|
||||
let mut i = 0;
|
||||
while i < b.len() {
|
||||
let start = i;
|
||||
if b[i].is_ascii_alphabetic() || b[i] == b'_' {
|
||||
while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') {
|
||||
i += 1;
|
||||
}
|
||||
let t = &line[start..i];
|
||||
let parts: Vec<&str> = t.split('_').collect();
|
||||
let float_part = parts.iter().any(|p| matches!(*p, "f32" | "f64") || p.starts_with("f32x") || p.starts_with("f64x") || p.starts_with("float"));
|
||||
let x86 = t.starts_with("_mm") && matches!(parts.last(), Some(&("pd" | "ps" | "sd" | "ss" | "ph")));
|
||||
if float_part || x86 {
|
||||
out.push(t);
|
||||
}
|
||||
} else if b[i].is_ascii_digit() {
|
||||
// A numeric literal; `x.0` (a field) is not one.
|
||||
let field = start > 0 && b[start - 1] == b'.';
|
||||
while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') {
|
||||
i += 1;
|
||||
}
|
||||
if i + 1 < b.len() && b[i] == b'.' && b[i + 1].is_ascii_digit() {
|
||||
i += 1;
|
||||
while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_') {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
if i < b.len() && matches!(b[i], b'+' | b'-') && matches!(b[i - 1], b'e' | b'E') {
|
||||
i += 1;
|
||||
while i < b.len() && b[i].is_ascii_digit() {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
let t = &line[start..i];
|
||||
let based = t.starts_with("0x") || t.starts_with("0b") || t.starts_with("0o");
|
||||
let exp = !based && t.bytes().skip(1).zip(t.bytes().skip(2)).any(|(e, d)| matches!(e, b'e' | b'E') && (d.is_ascii_digit() || d == b'+' || d == b'-'));
|
||||
if !field && !based && (t.contains('.') || exp || t.ends_with("f32") || t.ends_with("f64")) {
|
||||
out.push(t);
|
||||
}
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The serde_json parsers one line of code names: `from_slice`, `from_str`,
|
||||
/// `from_reader` or `Deserializer` on a line that names `serde_json`.
|
||||
fn json_parsers(line: &str) -> Vec<&str> {
|
||||
if !line.contains("serde_json") {
|
||||
return Vec::new();
|
||||
}
|
||||
line.split(|c: char| !(c.is_ascii_alphanumeric() || c == '_'))
|
||||
.filter(|t| matches!(*t, "from_slice" | "from_str" | "from_reader" | "Deserializer"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Every floating-point token and every serde_json parser on a decoder line of `src`, as
|
||||
/// `line: token`, but those on a boundary line of `file` for each.
|
||||
fn findings(file: &str, src: &str) -> Vec<String> {
|
||||
let code = code_of(src);
|
||||
let source: Vec<&str> = src.lines().collect();
|
||||
let on = |list: &[(&str, &str)], n: usize| list.iter().any(|&(f, l)| f == file && source[n - 1].trim() == l);
|
||||
let mut out = Vec::new();
|
||||
for (n, line) in decoder_lines(&code) {
|
||||
let floats = if on(BOUNDARIES, n) { Vec::new() } else { float_tokens(line) };
|
||||
let parsers = if on(JSON_BOUNDARIES, n) { Vec::new() } else { json_parsers(line) };
|
||||
for t in floats.into_iter().chain(parsers) {
|
||||
out.push(format!("{file}:{n}: {t}"));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Every file under `root`, as its path from `root` with `/` between the parts.
|
||||
fn files_under(root: &std::path::Path) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
let mut dirs = vec![String::new()];
|
||||
while let Some(d) = dirs.pop() {
|
||||
let dir = root.join(&d);
|
||||
for e in std::fs::read_dir(&dir).unwrap_or_else(|e| panic!("{}: {e}", dir.display())) {
|
||||
let e = e.expect("a directory entry");
|
||||
let name = e.file_name().into_string().expect("a UTF-8 file name");
|
||||
let rel = if d.is_empty() { name } else { format!("{d}/{name}") };
|
||||
if e.file_type().expect("a file type").is_dir() {
|
||||
dirs.push(rel);
|
||||
} else {
|
||||
out.push(rel);
|
||||
}
|
||||
}
|
||||
}
|
||||
out.sort();
|
||||
out
|
||||
}
|
||||
|
||||
/// Whether `f`, a path under `src/`, is a file of the integer modules: an `int_*.rs`, or a
|
||||
/// source file under `int_kernels/` or `int_payload/` but the payloads' tests.
|
||||
fn integer_module(f: &str) -> bool {
|
||||
let top = !f.contains('/') && f.starts_with("int_");
|
||||
let under = f.starts_with("int_kernels/") || (f.starts_with("int_payload/") && f != "int_payload/tests.rs");
|
||||
f.ends_with(".rs") && (top || under)
|
||||
}
|
||||
|
||||
/// The operator set's source files (`wai-ops/src`, every one), as `(name, text)`.
|
||||
fn operator_set() -> Vec<(String, String)> {
|
||||
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../wai-ops/src");
|
||||
files_under(&root).into_iter().filter(|f| f.ends_with(".rs"))
|
||||
.map(|f| {
|
||||
let text = std::fs::read_to_string(root.join(&f)).unwrap_or_else(|e| panic!("wai-ops/src/{f}: {e}"));
|
||||
(format!("wai-ops/src/{f}"), text)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_integer_decode_paths_hold_no_floating_point() {
|
||||
let ops = operator_set();
|
||||
assert!(ops.iter().any(|(f, _)| f == "wai-ops/src/interp.rs"), "the operator set's source is read");
|
||||
let read = SOURCES.iter().map(|&(f, src)| (f, src)).chain(ops.iter().map(|(f, src)| (f.as_str(), src.as_str())));
|
||||
let all: Vec<String> = read.flat_map(|(f, src)| findings(f, src)).collect();
|
||||
assert!(all.is_empty(), "floating point, or a serde_json parser, on an integer decode path:\n{}", all.join("\n"));
|
||||
// Every boundary is a line that is there, so none outlives its code.
|
||||
for &(f, l) in BOUNDARIES.iter().chain(JSON_BOUNDARIES) {
|
||||
let src = SOURCES.iter().find(|&&(n, _)| n == f).expect("a listed source").1;
|
||||
assert!(src.lines().any(|s| s.trim() == l), "{f}: the boundary line is gone: {l}");
|
||||
}
|
||||
// Every file of the integer modules is read, so a new one is not passed over.
|
||||
let src = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
|
||||
let unread: Vec<String> = files_under(&src).into_iter()
|
||||
.filter(|f| integer_module(f) && !SOURCES.iter().any(|&(n, _)| n == f)).collect();
|
||||
assert!(unread.is_empty(), "files of the integer modules this test does not read (list them in SOURCES): {unread:?}");
|
||||
}
|
||||
|
||||
/// The reader finds what it must and passes what it must: it is the test.
|
||||
#[test]
|
||||
fn the_reader_finds_floats_in_code_and_only_there() {
|
||||
let caught = "fn f(n: u64) -> u64 {\n (n as f64).sqrt() as u64\n}\n\
|
||||
fn g() { let x = 1.5; let y = 2e3; let z = 1u8 as f32; let w = 3f64; }\n\
|
||||
unsafe fn h() { _mm512_sqrt_pd(a); _mm_cvtepi32_ps(b); vaddq_f32(c, d); let v: float32x4_t; }\n\
|
||||
fn j(b: &[u8]) { let v: serde_json::Value = serde_json::from_slice(b).unwrap(); let w = serde_json::from_str::<u8>(\"1\"); }\n";
|
||||
let got: Vec<String> = findings("x.rs", caught).into_iter().map(|s| s.split(": ").nth(1).unwrap().to_string()).collect();
|
||||
assert_eq!(got, ["f64", "1.5", "2e3", "f32", "3f64", "_mm512_sqrt_pd", "_mm_cvtepi32_ps", "vaddq_f32", "float32x4_t", "from_slice", "from_str"]);
|
||||
let passed = "// (n as f64).sqrt()\n/* 1.5 */\nfn f() -> &'static str { let s = \"f64 1.5\"; let c = '.'; let r = r#\"f32\"#; t.0.1; 0x1e5; 1..2; s }\n\
|
||||
#[cfg(test)]\nmod tests {\n fn t() { let x = 1.0f64; }\n}\n\
|
||||
#[cfg(kani)]\nfn k() { let y: f32 = 0.5; }\n\
|
||||
fn ok<'a>(isqrt: u64, ph: usize, s: &'a str) -> u64 { let q = '\\''; let e = b'\"'; isqrt + 10u64 }\n\
|
||||
// serde_json::from_slice(b)\n\
|
||||
fn v(j: &serde_json::Value, s: &str) -> u32 { let t = \"serde_json::from_str\"; u32::from_str_radix(s, 16).unwrap() }\n\
|
||||
#[cfg(test)]\nfn parse() { let v: serde_json::Value = serde_json::from_slice(b\"1\").unwrap(); }\n";
|
||||
assert_eq!(findings("x.rs", passed), Vec::<String>::new());
|
||||
}
|
||||
|
|
@ -41,12 +41,19 @@
|
|||
/// Precompute a CompressAI-style ascending `scale_table` as fixed-point integer
|
||||
/// bucket edges (Q`frac`: `frac` fractional bits, grid step `2^-frac`). Compute
|
||||
/// once per model; pass the result to [`det_build_index`].
|
||||
///
|
||||
/// The table is binary32 by contract (it comes from a float model); each edge is
|
||||
/// `⌊e·2^frac⌋` formed from `e`'s bits in integer arithmetic ([`floor_scaled`]), the
|
||||
/// value the product in binary64 and its floor give exactly, with no floating-point
|
||||
/// operation.
|
||||
pub fn fixed_scale_edges(scale_table: &[f32], frac: u32) -> Vec<i64> {
|
||||
let s = (1u64 << frac) as f64;
|
||||
scale_table.iter().map(|&e| (e as f64 * s).floor() as i64).collect()
|
||||
let k = (1u64 << frac).trailing_zeros();
|
||||
scale_table.iter().map(|&e| floor_scaled(e.to_bits(), k)).collect()
|
||||
}
|
||||
|
||||
/// Snap a float `σ` to the fixed-point grid (Q`frac`), deterministic floor.
|
||||
/// Snap a float `σ` to the fixed-point grid (Q`frac`), deterministic floor: `⌊σ·2^frac⌋`
|
||||
/// for `σ > 0`, and 0 for a negative σ, a zero and a NaN. Formed from `σ`'s bits in
|
||||
/// integer arithmetic ([`floor_scaled`]).
|
||||
///
|
||||
/// For a codec whose σ source is still a float ONNX hyper-decoder: this removes
|
||||
/// drift smaller than the grid step *within* a bucket, but is **not** a hard
|
||||
|
|
@ -55,8 +62,34 @@ pub fn fixed_scale_edges(scale_table: &[f32], frac: u32) -> Vec<i64> {
|
|||
/// σ from an integer hyper-decoder, where the integer σ is exact and this snap
|
||||
/// is unnecessary.
|
||||
pub fn quantize_scale(sigma: f32, frac: u32) -> i64 {
|
||||
let s = (1u64 << frac) as f64;
|
||||
((sigma.max(0.0) as f64) * s).floor() as i64
|
||||
let k = (1u64 << frac).trailing_zeros();
|
||||
let b = sigma.to_bits();
|
||||
if b >> 31 == 1 || b & 0x7fff_ffff > 0x7f80_0000 { 0 } else { floor_scaled(b, k) }
|
||||
}
|
||||
|
||||
/// `⌊x·2^k⌋` for the binary32 value `x` whose bits are `bits` (`k < 64`), saturated to
|
||||
/// the `i64` range as a float-to-integer conversion saturates it (an infinity to the
|
||||
/// end of its sign, a NaN to 0). Integer arithmetic only: `x = ±m·2^e` with an integer
|
||||
/// `m < 2^24`, so `x·2^k` is `±m` shifted, and its floor rounds a negative fraction
|
||||
/// away from zero.
|
||||
fn floor_scaled(bits: u32, k: u32) -> i64 {
|
||||
let neg = bits >> 31 == 1;
|
||||
let (ex, f) = ((bits >> 23) & 0xff, (bits & 0x7f_ffff) as i128);
|
||||
if ex == 0xff {
|
||||
return if f != 0 { 0 } else if neg { i64::MIN } else { i64::MAX };
|
||||
}
|
||||
let (m, e) = if ex == 0 { (f, -149) } else { (f | 1 << 23, ex as i32 - 150) };
|
||||
let e = e + k as i32;
|
||||
let v = if e >= 0 {
|
||||
// m < 2^24 and e ≤ 104 + 63: past 2^63 the value saturates whatever its size.
|
||||
let mag = if e >= 64 { if m == 0 { 0 } else { 1i128 << 64 } } else { m << e };
|
||||
if neg { -mag } else { mag }
|
||||
} else {
|
||||
let d = (-e) as u32;
|
||||
let (q, exact) = if d >= 64 { (0, m == 0) } else { (m >> d, m & ((1i128 << d) - 1) == 0) };
|
||||
if neg { -q - if exact { 0 } else { 1 } } else { q }
|
||||
};
|
||||
v.clamp(i64::MIN as i128, i64::MAX as i128) as i64
|
||||
}
|
||||
|
||||
/// Integer `build_index`: bucket a fixed-point `σ` to a scale-table index, with
|
||||
|
|
@ -368,6 +401,34 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
/// The integer conversions give what the binary64 product and its floor gave (the
|
||||
/// form they replace, kept here as the reference): every exponent field, the
|
||||
/// mantissa's ends and random mantissas, each sign, subnormals, zeros, infinities
|
||||
/// and NaNs, at every `frac` in `0..64`.
|
||||
#[test]
|
||||
fn the_integer_scale_conversions_are_the_float_ones() {
|
||||
let edge = |b: u32, frac: u32| (f32::from_bits(b) as f64 * (1u64 << frac) as f64).floor() as i64;
|
||||
let snap = |b: u32, frac: u32| ((f32::from_bits(b).max(0.0) as f64) * (1u64 << frac) as f64).floor() as i64;
|
||||
let mut seed = 0x5157_0005u64;
|
||||
let mut next = || {
|
||||
seed = seed.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let z = (seed ^ (seed >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
((z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB) ^ (z >> 31)) as u32
|
||||
};
|
||||
let mut bits = vec![0x7fc0_0000u32, 0x7f80_0001, 0xffc0_0000];
|
||||
for ex in 0..=0xffu32 {
|
||||
for f in [0u32, 1, 2, 0x40_0000, 0x7f_fffe, 0x7f_ffff, next() & 0x7f_ffff, next() & 0x7f_ffff] {
|
||||
bits.extend([ex << 23 | f, 0x8000_0000 | ex << 23 | f]);
|
||||
}
|
||||
}
|
||||
for frac in 0..64 {
|
||||
for &b in &bits {
|
||||
assert_eq!(fixed_scale_edges(&[f32::from_bits(b)], frac), vec![edge(b, frac)], "{b:#010x} frac {frac}");
|
||||
assert_eq!(quantize_scale(f32::from_bits(b), frac), snap(b, frac), "{b:#010x} frac {frac}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clamps_below_the_smallest_scale() {
|
||||
let st = scale_table();
|
||||
|
|
|
|||
|
|
@ -516,22 +516,28 @@ fn check_entropy(t: &CondEntropy) -> Result<(), &'static str> {
|
|||
}
|
||||
|
||||
impl CondEntropy {
|
||||
/// Parse and check the entropy tables (`cond_entropy.json`, read from a set as
|
||||
/// `entropy.json`): every member present with integers in range, the lengths
|
||||
/// consistent and every table well formed. Refuses rather than panics.
|
||||
pub fn try_from_value(v: &Value) -> Result<Self, &'static str> {
|
||||
fn arr<'a>(v: &'a Value, k: &str) -> Result<&'a Vec<Value>, &'static str> {
|
||||
v.get(k).and_then(Value::as_array).ok_or("an entropy member is missing or not an array")
|
||||
/// Read and check the entropy tables (`cond_entropy.json`, read from a set as
|
||||
/// `entropy.json`, `extensions/integer-payloads.md` §9) from the file's text. The text
|
||||
/// is read strictly (RFC 8259, UTF-8, no member named twice, at most 64 levels, nothing
|
||||
/// after the value) by a reader that converts no number to a float; every member must be
|
||||
/// present, each value an integer as spelled (no fraction or exponent; `-0` is zero) in
|
||||
/// its type's range, the lengths consistent and every table well formed. Refuses rather
|
||||
/// than panics; the error says why.
|
||||
pub fn read(text: &[u8]) -> Result<Self, String> {
|
||||
use crate::strict_json::{self, Json};
|
||||
fn arr<'a>(v: &'a Json, k: &str) -> Result<&'a [Json], &'static str> {
|
||||
v.get(k).and_then(Json::as_arr).ok_or("an entropy member is missing or not an array")
|
||||
}
|
||||
fn ints<T: TryFrom<i64>>(v: &Value, k: &str) -> Result<Vec<T>, &'static str> {
|
||||
fn ints<T: TryFrom<i64>>(v: &Json, k: &str) -> Result<Vec<T>, &'static str> {
|
||||
arr(v, k)?.iter().map(|x| x.as_i64().and_then(|i| T::try_from(i).ok())).collect::<Option<_>>()
|
||||
.ok_or("an entropy member holds a value out of range")
|
||||
}
|
||||
fn rows(v: &Value, k: &str) -> Result<Vec<Vec<i32>>, &'static str> {
|
||||
arr(v, k)?.iter().map(|r| r.as_array().ok_or("an entropy table is not an array")
|
||||
fn rows(v: &Json, k: &str) -> Result<Vec<Vec<i32>>, &'static str> {
|
||||
arr(v, k)?.iter().map(|r| r.as_arr().ok_or("an entropy table is not an array")
|
||||
.and_then(|r| r.iter().map(|x| x.as_i64().and_then(|i| i32::try_from(i).ok())).collect::<Option<_>>()
|
||||
.ok_or("an entropy table holds a value out of range"))).collect()
|
||||
}
|
||||
let v = &strict_json::read(text)?;
|
||||
let t = CondEntropy {
|
||||
edges: ints(v, "scale_edges")?,
|
||||
cdfs: rows(v, "cdfs")?,
|
||||
|
|
|
|||
|
|
@ -171,9 +171,10 @@
|
|||
//! The narrow kernel sets also run the inverse GDN's norms in SIMD 64-bit lanes on x86_64
|
||||
//! (each term `(γ·x²) >> gf` formed exactly from `i32` operands by `vpmuldq`, floored by
|
||||
//! an arithmetic shift, added in `j` order as the reference adds it), fuse the shift and
|
||||
//! the addition at `gf = 16` on NEON (`ssra`), and seed each integer square root from the
|
||||
//! IEEE-754 square root before correcting it in integers ([`isqrt_u64`]), which gives
|
||||
//! the integer root for any seed. [`Isa::Wide`] runs the kernels as they were before the
|
||||
//! the addition at `gf = 16` on NEON (`ssra`), and seed each integer square root from a
|
||||
//! fixed-point reciprocal square root before correcting it ([`isqrt_u64`]), which gives
|
||||
//! the integer root for any seed; on AVX-512 the roots are taken digit by digit in vector
|
||||
//! lanes. No step uses floating point. [`Isa::Wide`] runs the kernels as they were before the
|
||||
//! narrow sets, and every set is held to the reference by the same tests ([`isas`] lists
|
||||
//! the sets a processor runs; the tests run each).
|
||||
//!
|
||||
|
|
@ -1553,17 +1554,61 @@ pub(crate) fn igdn_xm(x: &[i64], c: usize, hw: usize, frac: u32, gamma: &[i64],
|
|||
Some(out)
|
||||
}
|
||||
|
||||
/// `⌊√n⌋`, the value of `n.isqrt()`, from a seed: the IEEE-754 square root of `n`
|
||||
/// rounded to `f64` (an operation every conforming platform rounds correctly), capped at
|
||||
/// the largest root a `u64` has, then corrected in integer arithmetic until `s² ≤ n <
|
||||
/// (s + 1)²`. The correction alone decides the result, so it is the integer root
|
||||
/// whatever the seed; the seed decides only how many steps the correction takes (at most
|
||||
/// one or two from a correctly rounded root).
|
||||
/// `⌊√n⌋`, the value of `n.isqrt()`, in integer arithmetic only, from a seed, then
|
||||
/// corrected until `s² ≤ n < (s + 1)²` ([`isqrt_from`]). The correction alone decides the
|
||||
/// result, so it is the integer root whatever the seed; the seed decides only how many
|
||||
/// steps the correction takes.
|
||||
///
|
||||
/// The seed is a fixed-point reciprocal square root. `n` is shifted left by an even
|
||||
/// count `z` to `m ∈ [2^62, 2^64)`, so `x = m / 2^64 ∈ [1/4, 1)`, and `y ≈ 1/√x` comes
|
||||
/// from [`RSQRT_SEED`] (indexed by `m`'s top nine bits, about 9 bits correct) and two
|
||||
/// Newton steps `y ← y·(3 − x·y²)/2` (about 17, then 34 bits). Then `√m = m·y / 2^32`,
|
||||
/// and `√n = √m / 2^(z/2)`. Every product is formed exactly in `u64` or `u128` (no step
|
||||
/// can overflow: `x·y² ≈ 1`, `y ≤ 2`). The seed is within one of the root on every
|
||||
/// input the tests try, so the correction takes at most one step.
|
||||
///
|
||||
/// No floating-point operation is on this path: it is the inverse GDN's root on the
|
||||
/// narrow kernel sets, and a decode compiled to a module with no floating point runs it.
|
||||
#[inline(always)]
|
||||
pub(crate) fn isqrt_u64(n: u64) -> u64 {
|
||||
isqrt_from(n, (n as f64).sqrt() as u64)
|
||||
isqrt_from(n, isqrt_seed(n))
|
||||
}
|
||||
|
||||
/// [`isqrt_u64`]'s seed: an estimate of `⌊√n⌋` in integer arithmetic (0 for 0).
|
||||
#[inline(always)]
|
||||
fn isqrt_seed(n: u64) -> u64 {
|
||||
if n == 0 {
|
||||
return 0;
|
||||
}
|
||||
let z = n.leading_zeros() & !1;
|
||||
let m = n << z;
|
||||
// y0 ≈ 2^15/√x, from the table: m ≥ 2^62, so m >> 55 is 128..=511.
|
||||
let y0 = RSQRT_SEED[(m >> 55) as usize - 128] as u64;
|
||||
// y1 = y0·(3 − x·y0²)/2 at 2^30: x at 2^32 (m's top half) times y0² at 2^30 is below
|
||||
// 2^64; x·y0² is within 2^-7 of 1, so the subtraction stays positive.
|
||||
let p = ((m >> 32) * (y0 * y0)) >> 32;
|
||||
let y1 = (y0 * ((3 << 30) - p)) >> 16;
|
||||
// y2 = y1·(3 − x·y1²)/2 at 2^62: x at 2^64 (m itself) times y1² at 2^60, in u128.
|
||||
let p = ((m as u128 * (y1 * y1) as u128) >> 64) as u64;
|
||||
let y2 = ((y1 as u128 * ((3u64 << 60) - p) as u128) >> 29) as u64;
|
||||
// √m = m·y2 / 2^(62 + 32), within one of ⌊√m⌋; shifting it by z/2 keeps that.
|
||||
let r = ((m as u128 * y2 as u128) >> 94) as u64;
|
||||
r >> (z >> 1)
|
||||
}
|
||||
|
||||
/// [`isqrt_u64`]'s seed table: for `i` in `0..384`, `⌊√(2^40 / (2·(i + 128) + 1))⌋`,
|
||||
/// which is `2^15 / √x` at the middle `x = (i + 128 + 1/2) / 512` of the interval of
|
||||
/// `x = m / 2^64` whose top nine bits are `i + 128`. Every entry is below `2^16`.
|
||||
static RSQRT_SEED: [u16; 384] = {
|
||||
let mut t = [0u16; 384];
|
||||
let mut i = 0;
|
||||
while i < 384 {
|
||||
t[i] = ((1u64 << 40) / (2 * (i as u64 + 128) + 1)).isqrt() as u16;
|
||||
i += 1;
|
||||
}
|
||||
t
|
||||
};
|
||||
|
||||
/// `⌊√n⌋` from any seed: [`isqrt_u64`]'s correction, which alone decides the result.
|
||||
#[inline(always)]
|
||||
fn isqrt_from(n: u64, seed: u64) -> u64 {
|
||||
|
|
@ -1608,8 +1653,9 @@ fn igdn_tile_narrow(
|
|||
}
|
||||
v
|
||||
};
|
||||
// The narrow kernel sets seed each root from the IEEE-754 square root ([`isqrt_u64`]);
|
||||
// the wide set keeps the integer routine. Both give the integer root.
|
||||
// The narrow kernel sets seed each root from a fixed-point reciprocal square root
|
||||
// ([`isqrt_u64`]); the wide set keeps the standard library's routine. Both give the
|
||||
// integer root, in integer arithmetic.
|
||||
let seeded = isa != Isa::Wide;
|
||||
// On AVX-512 the roots and the products run in vector lanes ([`x86::igdn_emit_avx512`])
|
||||
// where every |x| < 2^31; the rest of each channel's positions here.
|
||||
|
|
@ -2363,26 +2409,67 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
/// The seeded integer root is `u64::isqrt` on every value class: small values, every
|
||||
/// perfect square and its neighbours around the powers of two, the edge of `f64`'s
|
||||
/// exact integers (2^53) and of `u64`, and random values of every magnitude.
|
||||
/// `⌊√n⌋` by the binary restoring method, one digit at a time: the reference the
|
||||
/// integer root is held to here, independent of its seed and of the standard library.
|
||||
fn isqrt_restoring(n: u64) -> u64 {
|
||||
let (mut rem, mut s, mut bit) = (n, 0u64, 1u64 << 62);
|
||||
while bit != 0 {
|
||||
if rem >= s + bit {
|
||||
rem -= s + bit;
|
||||
s = (s >> 1) + bit;
|
||||
} else {
|
||||
s >>= 1;
|
||||
}
|
||||
bit >>= 2;
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
/// `s` is the integer root of `n` by its definition, `s² ≤ n < (s + 1)²`, in `u128`.
|
||||
fn is_root(n: u64, s: u64) -> bool {
|
||||
let (n, s) = (n as u128, s as u128);
|
||||
s * s <= n && n < (s + 1) * (s + 1)
|
||||
}
|
||||
|
||||
/// The integer root over the edges of `u64`, against the definition, the restoring
|
||||
/// method and `u64::isqrt`: 0, 1 and every `n` below 2^17; for roots `k` around every
|
||||
/// power of two and on a stride across `0..2^32` (with `2^32 − 1`, the largest), each
|
||||
/// `k² − 1`, `k²` and `k² + 1`; `u64::MAX` and its neighbours; and random values of
|
||||
/// every magnitude. The seed is within one of the root on all of them, so the
|
||||
/// correction takes at most one step.
|
||||
#[test]
|
||||
fn the_seeded_root_is_the_integer_root() {
|
||||
fn the_integer_root_is_exact_on_the_edges_of_u64() {
|
||||
let mut r = testkit::Rng(0x5157_0001);
|
||||
let check = |n: u64| assert_eq!(isqrt_u64(n), n.isqrt(), "{n}");
|
||||
for n in 0..100_000u64 {
|
||||
let check = |n: u64| {
|
||||
let s = isqrt_u64(n);
|
||||
assert!(is_root(n, s), "n {n}: {s} is not its root");
|
||||
assert_eq!(s, isqrt_restoring(n), "n {n}");
|
||||
assert_eq!(s, n.isqrt(), "n {n}");
|
||||
assert!(isqrt_seed(n).abs_diff(s) <= 1, "n {n}: the seed {} is more than one from {s}", isqrt_seed(n));
|
||||
};
|
||||
for n in 0..1u64 << 17 {
|
||||
check(n);
|
||||
}
|
||||
for e in 0..32u32 {
|
||||
for d in [-2i64, -1, 0, 1, 2] {
|
||||
let s = (1u64 << e).wrapping_add_signed(d);
|
||||
let q = s.wrapping_mul(s);
|
||||
for dd in [-1i64, 0, 1] {
|
||||
check(q.wrapping_add_signed(dd));
|
||||
let around = |k: u64| {
|
||||
let q = k as i128 * k as i128;
|
||||
for d in [-1i128, 0, 1] {
|
||||
if let Ok(n) = u64::try_from(q + d) {
|
||||
check(n);
|
||||
}
|
||||
}
|
||||
};
|
||||
for e in 0..=32u32 {
|
||||
for d in -3i64..=3 {
|
||||
if let Some(k) = (1u64 << e).checked_add_signed(d).filter(|&k| k <= u32::MAX as u64) {
|
||||
around(k);
|
||||
}
|
||||
}
|
||||
}
|
||||
for n in [(1u64 << 53) - 1, 1 << 53, (1 << 53) + 1, u64::MAX, u64::MAX - 1, (u32::MAX as u64) * (u32::MAX as u64), (u32::MAX as u64) * (u32::MAX as u64) - 1] {
|
||||
for k in (0..=u32::MAX as u64).step_by(4_093) {
|
||||
around(k);
|
||||
}
|
||||
around(u32::MAX as u64);
|
||||
for n in [u64::MAX, u64::MAX - 1, u64::MAX - 2, 1 << 63, (1 << 63) - 1, (1 << 62) - 1, 1 << 62] {
|
||||
check(n);
|
||||
}
|
||||
for _ in 0..200_000 {
|
||||
|
|
@ -2390,9 +2477,7 @@ mod tests {
|
|||
check(r.next() >> sh);
|
||||
}
|
||||
// The correction decides the result whatever the seed: seeds below and above the
|
||||
// root by up to a thousand, and the largest, each corrected to the root. (A
|
||||
// correctly rounded square root seeds every `u64` within one of its root, so these
|
||||
// are the seeds a platform whose square root were not would give.)
|
||||
// root by up to a thousand, and the largest, each corrected to the root.
|
||||
for _ in 0..20_000 {
|
||||
let n = r.next() >> r.below(64) as u32;
|
||||
let root = n.isqrt();
|
||||
|
|
@ -2406,6 +2491,28 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
/// Every root boundary of `u64`, the values where `⌊√n⌋` changes: for every `k` in
|
||||
/// `1..2^32`, the root of `k² − 1` is `k − 1` and the root of `k²` is `k`. Some 8.6
|
||||
/// billion roots: run it alone, in release (`cargo test --release --lib --
|
||||
/// --ignored int_kernels::tests::every_root_boundary_of_u64`).
|
||||
#[test]
|
||||
#[ignore = "exhaustive over 2^32 roots; run alone in release"]
|
||||
fn every_root_boundary_of_u64() {
|
||||
let workers = std::thread::available_parallelism().map_or(1, |n| n.get()) as u64;
|
||||
let span = (1u64 << 32) / workers + 1;
|
||||
std::thread::scope(|sc| {
|
||||
for w in 0..workers {
|
||||
sc.spawn(move || {
|
||||
for k in (w * span).max(1)..((w + 1) * span).min(1 << 32) {
|
||||
let q = k * k;
|
||||
assert_eq!(isqrt_u64(q), k, "{q}");
|
||||
assert_eq!(isqrt_u64(q - 1), k - 1, "{}", q - 1);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Every SIMD form of the inverse GDN's norms against the scalar sum written out,
|
||||
/// norm by norm, before any root is taken (a root can hide a norm that is a unit off):
|
||||
/// NEON at any gf and its fused form at gf = 16; AVX2 with non-negative and with signed
|
||||
|
|
|
|||
|
|
@ -409,8 +409,9 @@ unsafe fn igdn_avx2<const NONNEG: bool>(x2: &[i32], np: usize, gt: &[i32], beta:
|
|||
|
||||
/// The inverse GDN's outputs from its norms on AVX-512 (F and DQ): `dst[p] = (x[p] ·
|
||||
/// ⌊√max(0, norm[p])⌋) >> frac` for `p` in a leading multiple of 8 (returned). Each root is
|
||||
/// seeded from the IEEE-754 square root and corrected, then held to `s² ≤ n < (s + 1)²` in
|
||||
/// integers; a step whose roots do not all meet it is left to the caller (none is
|
||||
/// taken digit by digit in integer lanes (the binary restoring method: 32 steps, one per
|
||||
/// digit of a root of `n < 2^63`), with no floating point, then held to `s² ≤ n < (s +
|
||||
/// 1)²` in integers; a step whose roots do not all meet it is left to the caller (none is
|
||||
/// expected). The product is `vpmullq`, exact because every `|x| < 2^31` and `s < 2^32`.
|
||||
///
|
||||
/// # Safety
|
||||
|
|
@ -428,16 +429,19 @@ pub(super) unsafe fn igdn_emit_avx512(norm: &[i64], x: &[i64], frac: u32, n: usi
|
|||
unsafe {
|
||||
while p + 8 <= n {
|
||||
let nv = _mm512_max_epi64(_mm512_loadu_si512(norm.as_ptr().add(p) as *const __m512i), zero);
|
||||
let mut s = _mm512_min_epu64(_mm512_cvttpd_epu64(_mm512_sqrt_pd(_mm512_cvtepi64_pd(nv))), cap);
|
||||
for _ in 0..2 {
|
||||
let over = _mm512_cmpgt_epu64_mask(_mm512_mullo_epi64(s, s), nv);
|
||||
s = _mm512_mask_sub_epi64(s, over, s, one);
|
||||
}
|
||||
for _ in 0..2 {
|
||||
let s1 = _mm512_add_epi64(s, one);
|
||||
let room = _mm512_cmplt_epu64_mask(s, cap);
|
||||
let under = _mm512_mask_cmple_epu64_mask(room, _mm512_mullo_epi64(s1, s1), nv);
|
||||
s = _mm512_mask_add_epi64(s, under, s, one);
|
||||
// ⌊√n⌋ digit by digit: `bit` runs over the powers of four from 2^62 down; where
|
||||
// the remainder holds `s + bit`, it is subtracted and the digit is set.
|
||||
// `s + bit` stays below 2^64 (before each step `s ≤ 2√(n·bit)`, with n < 2^63
|
||||
// and `bit ≤ 2^60` once `s` is nonzero), and the remainder never goes negative.
|
||||
let mut rem = nv;
|
||||
let mut s = zero;
|
||||
for d in 0..32 {
|
||||
let bit = _mm512_set1_epi64(1i64 << (62 - 2 * d));
|
||||
let t = _mm512_add_epi64(s, bit);
|
||||
let take = _mm512_cmpge_epu64_mask(rem, t);
|
||||
rem = _mm512_mask_sub_epi64(rem, take, rem, t);
|
||||
s = _mm512_srli_epi64::<1>(s);
|
||||
s = _mm512_mask_add_epi64(s, take, s, bit);
|
||||
}
|
||||
// s² ≤ n, and (s + 1)² > n wherever s + 1 has a square in u64
|
||||
let s1 = _mm512_add_epi64(s, one);
|
||||
|
|
|
|||
|
|
@ -66,7 +66,8 @@
|
|||
//! reconstruction: magic `WMB1`, then four sections, each a `u32` length and its
|
||||
//! bytes, in order `h_s`, `entropy`, `g_s` (integer networks in the compact `WHM1`
|
||||
//! form, [`IntModel::from_bytes`]) and the entropy tables (JSON with the fields of
|
||||
//! [`CondEntropy`], plus `afrac`).
|
||||
//! [`CondEntropy`], plus `afrac`, read as [`CondEntropy::read`] reads them: strictly, and
|
||||
//! by a reader that converts no number to a float).
|
||||
//!
|
||||
//! ## Totality
|
||||
//!
|
||||
|
|
@ -87,11 +88,10 @@
|
|||
//! or an error for any payload, and a build with overflow checks and one without return
|
||||
//! the same one. Pure integer, no I/O.
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::int_entropy::{dequantize, det_build_index};
|
||||
use crate::int_hyper_synth::{net_shape, CondEntropy, IntModel};
|
||||
use crate::rans::RansDecoder;
|
||||
use crate::strict_json::Json;
|
||||
|
||||
/// Payload magic.
|
||||
pub const PAYLOAD_MAGIC: [u8; 4] = *b"WIM1";
|
||||
|
|
@ -197,8 +197,8 @@ impl IntMlicModel {
|
|||
let h_s = net(section("h_s")?, "h_s")?;
|
||||
let entropy = net(section("entropy")?, "entropy")?;
|
||||
let g_s = net(section("g_s")?, "g_s")?;
|
||||
let tables_json: Value = serde_json::from_slice(section("tables")?)
|
||||
.map_err(|e| model_err(format!("tables: {e}")))?;
|
||||
// Read strictly, by a reader that converts no number to a float (§14, §9).
|
||||
let tables_json = crate::strict_json::read(section("tables")?).map_err(|e| model_err(format!("tables: {e}")))?;
|
||||
if p != bytes.len() {
|
||||
return Err(model_err("trailing bytes after the tables section"));
|
||||
}
|
||||
|
|
@ -233,19 +233,20 @@ pub fn check_cdf(cdf: &[i32], len: usize) -> Result<(), IntMlicError> {
|
|||
/// Read the tables section: `afrac`, `scale_edges`, `cdfs`, `cdf_lengths`, `offsets`,
|
||||
/// `z_cdfs`, `z_cdf_lengths`, `z_offsets`, `z_median_q` — the fields of
|
||||
/// `int_hyper_synth`'s `cond_entropy.json`. Refuses (does not panic on) a missing
|
||||
/// field, a non-integer, or a value out of its type's range.
|
||||
fn parse_tables(v: &Value) -> Result<(CondEntropy, u32), IntMlicError> {
|
||||
fn arr<'a>(v: &'a Value, k: &str) -> Result<&'a Vec<Value>, IntMlicError> {
|
||||
v.get(k).and_then(Value::as_array).ok_or_else(|| model_err(format!("tables: `{k}` is not an array")))
|
||||
/// field, a non-integer (a fraction or an exponent; `-0` is zero), or a value out of its
|
||||
/// type's range.
|
||||
fn parse_tables(v: &Json) -> Result<(CondEntropy, u32), IntMlicError> {
|
||||
fn arr<'a>(v: &'a Json, k: &str) -> Result<&'a [Json], IntMlicError> {
|
||||
v.get(k).and_then(Json::as_arr).ok_or_else(|| model_err(format!("tables: `{k}` is not an array")))
|
||||
}
|
||||
fn int<T: TryFrom<i64>>(x: &Value, k: &str) -> Result<T, IntMlicError> {
|
||||
fn int<T: TryFrom<i64>>(x: &Json, k: &str) -> Result<T, IntMlicError> {
|
||||
x.as_i64().and_then(|i| T::try_from(i).ok()).ok_or_else(|| model_err(format!("tables: `{k}` holds a value out of range")))
|
||||
}
|
||||
fn ints<T: TryFrom<i64>>(v: &Value, k: &str) -> Result<Vec<T>, IntMlicError> {
|
||||
fn ints<T: TryFrom<i64>>(v: &Json, k: &str) -> Result<Vec<T>, IntMlicError> {
|
||||
arr(v, k)?.iter().map(|x| int(x, k)).collect()
|
||||
}
|
||||
fn rows(v: &Value, k: &str) -> Result<Vec<Vec<i32>>, IntMlicError> {
|
||||
arr(v, k)?.iter().map(|r| r.as_array().ok_or_else(|| model_err(format!("tables: `{k}` row is not an array")))
|
||||
fn rows(v: &Json, k: &str) -> Result<Vec<Vec<i32>>, IntMlicError> {
|
||||
arr(v, k)?.iter().map(|r| r.as_arr().ok_or_else(|| model_err(format!("tables: `{k}` row is not an array")))
|
||||
.and_then(|r| r.iter().map(|x| int(x, k)).collect())).collect()
|
||||
}
|
||||
let afrac = v.get("afrac").ok_or_else(|| model_err("tables: no `afrac`"))
|
||||
|
|
@ -1016,4 +1017,40 @@ mod tests {
|
|||
}
|
||||
assert!(IntMlicModel::from_bundle(&b2).is_err());
|
||||
}
|
||||
|
||||
/// The tables section is read as a set's `entropy.json` is (integer-payloads §14 and
|
||||
/// §9), by a reader that converts no number to a float: `-0` is zero, a fraction or an
|
||||
/// exponent is not an integer, and a member named twice or anything after the value
|
||||
/// refuses the bundle.
|
||||
#[test]
|
||||
fn a_bundles_tables_are_read_strictly() {
|
||||
let mut g = Rng(0x1A7E_4C07);
|
||||
let m = model(&mut g);
|
||||
let (pl, _) = encode(&m, &mut g);
|
||||
let t = &m.tables;
|
||||
let text = serde_json::to_string(&serde_json::json!({
|
||||
"afrac": AF, "scale_edges": t.edges, "cdfs": t.cdfs, "cdf_lengths": t.cdf_lengths, "offsets": t.offsets,
|
||||
"z_cdfs": t.z_cdfs, "z_cdf_lengths": t.z_cdf_lengths, "z_offsets": t.z_offsets, "z_median_q": t.z_median_q,
|
||||
}))
|
||||
.unwrap();
|
||||
let bundle = |tables: &str| {
|
||||
let mut b = BUNDLE_MAGIC.to_vec();
|
||||
for part in [to_whm1(&m.h_s), to_whm1(&m.entropy), to_whm1(&m.g_s), tables.as_bytes().to_vec()] {
|
||||
b.extend((part.len() as u32).to_le_bytes());
|
||||
b.extend(part);
|
||||
}
|
||||
IntMlicModel::from_bundle(&b)
|
||||
};
|
||||
let want = decode(&m, &pl).unwrap();
|
||||
assert_eq!(decode(&bundle(&text).unwrap(), &pl).unwrap(), want);
|
||||
// Every table starts at 0: the first, spelled otherwise.
|
||||
assert_eq!(text.matches("[[0,").count(), 2, "the latent and hyper-latent tables, each from 0");
|
||||
let first = |spelled: &str| text.replacen("[[0,", &format!("[[{spelled},"), 1);
|
||||
assert_eq!(decode(&bundle(&first("-0")).expect("-0 is zero"), &pl).unwrap(), want);
|
||||
for not_integer in ["0.0", "-0.0", "0e0"] {
|
||||
assert!(bundle(&first(not_integer)).is_err(), "{not_integer} is not an integer");
|
||||
}
|
||||
assert!(bundle(&text.replacen('{', &format!("{{\"afrac\":{AF},"), 1)).is_err(), "a member named twice");
|
||||
assert!(bundle(&format!("{text} 0")).is_err(), "bytes after the value");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -751,7 +751,7 @@ mod equivalence {
|
|||
let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../wai-rs/tests/vectors/int_hyper");
|
||||
let read = |f: &str| std::fs::read(dir.join(f)).unwrap();
|
||||
let (g, h) = (IntModel::from_bytes(&read("tiny_g_s.bin")).unwrap(), IntModel::from_bytes(&read("tiny_h_s.bin")).unwrap());
|
||||
let ent = CondEntropy::try_from_value(&serde_json::from_slice(&read("tiny_entropy.json")).unwrap()).unwrap();
|
||||
let ent = CondEntropy::read(&read("tiny_entropy.json")).unwrap();
|
||||
let p = read("tiny_bitstream.wih1");
|
||||
let k = crate::int_wire::parse_wih1(&p).unwrap();
|
||||
let st = CondStream {
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ pub use crate::int_wire as wire;
|
|||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub use wire::{Refusal, WireError};
|
||||
pub use wire::{Binary32, Refusal, WireError};
|
||||
|
||||
use crate::codecs::{
|
||||
CAP_AUDIO_INT_CODEC, CAP_NEURAL_INT_HYPER, CAP_NEURAL_INT_MLICPP, CAP_NEURAL_INT_SYNTH, CAP_SPLAT_INT_CODEC,
|
||||
|
|
@ -390,8 +390,9 @@ pub enum IntDecoded {
|
|||
Pcm { samples: Vec<i16>, sample_rate: u32, channels: u32 },
|
||||
/// `wai.splat.int_codec`: the `[n_attr, s, s]` attribute tensor (the
|
||||
/// splat-equivalence digest's buffer), the cloud's Gaussian count, and each
|
||||
/// attribute's presentation range `(lo, hi)`, which the digest does not cover.
|
||||
Attr { bytes: Vec<u8>, n_attr: u32, s: u32, n_gaussians: u32, ranges: Vec<(f32, f32)> },
|
||||
/// attribute's presentation range `(lo, hi)` as binary32 bits ([`Binary32`]), which
|
||||
/// the digest does not cover.
|
||||
Attr { bytes: Vec<u8>, n_attr: u32, s: u32, n_gaussians: u32, ranges: Vec<(Binary32, Binary32)> },
|
||||
}
|
||||
|
||||
/// Decode `payload` through the integer capability `cap` against `params`.
|
||||
|
|
@ -698,9 +699,8 @@ fn int_hyper_models(
|
|||
};
|
||||
let g_s = net("g_s.whm")?;
|
||||
let h_s = net("h_s.whm")?;
|
||||
let ent: serde_json::Value = serde_json::from_slice(params.member(cap, "entropy.json")?)
|
||||
.map_err(|e| model(format!("entropy.json: {e}")))?;
|
||||
let ent = CondEntropy::try_from_value(&ent).map_err(|e| model(format!("entropy.json: {e}")))?;
|
||||
// The text is read strictly, and no float parser reads it (§9).
|
||||
let ent = CondEntropy::read(params.member(cap, "entropy.json")?).map_err(|e| model(format!("entropy.json: {e}")))?;
|
||||
Ok((g_s, h_s, ent))
|
||||
}
|
||||
|
||||
|
|
@ -890,26 +890,21 @@ pub fn wia1_from_json(bs: &serde_json::Value, sr: u32) -> Result<Vec<u8>, IntPay
|
|||
Ok(out)
|
||||
}
|
||||
|
||||
/// Convert the splat export's JSON bitstream (`splat.json`) to `WIS1`, with its
|
||||
/// per-attribute presentation ranges. The export writes each range as a binary32
|
||||
/// value; a range that is not exactly one is refused, so the payload carries what the
|
||||
/// JSON states. `splat_int` reads JSON numbers correctly rounded
|
||||
/// (`serde_json/float_roundtrip`), so a decimal that denotes a binary32 value reads as
|
||||
/// exactly that value, and no tolerance is needed.
|
||||
/// Convert the splat export's JSON bitstream (`splat.json`, its text) to `WIS1`, with
|
||||
/// its per-attribute presentation ranges. The export writes each range as a binary32
|
||||
/// value; a range that does not denote exactly one is refused, so the payload carries
|
||||
/// what the JSON states. The text is read by [`crate::int_splat::SplatJson::read`], in
|
||||
/// integer arithmetic: a decimal is the binary32 value it denotes on every machine
|
||||
/// (`int_splat::binary32_of`), with no float parser to round it and no tolerance.
|
||||
#[cfg(feature = "splat_int")]
|
||||
pub fn wis1_from_json(bs: &serde_json::Value) -> Result<Vec<u8>, IntPayloadError> {
|
||||
let j = crate::int_splat::SplatJson::from_value(bs).map_err(json)?;
|
||||
pub fn wis1_from_json(text: &[u8]) -> Result<Vec<u8>, IntPayloadError> {
|
||||
let j = crate::int_splat::SplatJson::read(text).map_err(json)?;
|
||||
if j.cdfs.len() != j.c || j.offsets.len() != j.c || j.median_q.len() != j.c {
|
||||
return Err(json("the tables, offsets and medians are not one per channel (c)"));
|
||||
}
|
||||
for t in &j.cdfs {
|
||||
table_len(t.len())?;
|
||||
}
|
||||
let exact = |x: f64| -> Result<f32, IntPayloadError> {
|
||||
let f = x as f32;
|
||||
if f as f64 == x { Ok(f) } else { Err(json("a range is not exactly a binary32 value")) }
|
||||
};
|
||||
let ranges = j.ranges.iter().map(|&(lo, hi)| Ok((exact(lo)?, exact(hi)?))).collect::<Result<Vec<_>, IntPayloadError>>()?;
|
||||
let v = wire::Wis1 {
|
||||
afrac: u8::try_from(j.afrac).map_err(|_| json("afrac exceeds a byte"))?,
|
||||
c: to_u32(j.c, "c")?,
|
||||
|
|
@ -918,7 +913,7 @@ pub fn wis1_from_json(bs: &serde_json::Value) -> Result<Vec<u8>, IntPayloadError
|
|||
s: to_u32(j.s, "s")?,
|
||||
attr: to_u32(j.n_attr, "attr")?,
|
||||
n_gaussians: to_u32(j.n_gaussians, "n_gaussians")?,
|
||||
ranges,
|
||||
ranges: j.ranges.clone(),
|
||||
cdfs: j.cdfs.clone(),
|
||||
offsets: j.offsets.clone(),
|
||||
median_q: j.median_q.clone(),
|
||||
|
|
|
|||
|
|
@ -154,7 +154,7 @@ fn wih1_json_equivalence() {
|
|||
assert_eq!(format!("{:016x}", fnv1a(&rgb)), jload("int_hyper/tiny.json")["rgb_hash"].as_str().unwrap());
|
||||
// The checked entropy reader accepts the committed tables and gives what the
|
||||
// unchecked one gives.
|
||||
let checked = CondEntropy::try_from_value(&jload("int_hyper/tiny_entropy.json")).unwrap();
|
||||
let checked = CondEntropy::read(&read("int_hyper/tiny_entropy.json")).unwrap();
|
||||
assert_eq!((checked.edges, checked.cdfs, checked.z_median_q), (ent.edges, ent.cdfs, ent.z_median_q));
|
||||
}
|
||||
|
||||
|
|
@ -194,15 +194,22 @@ fn wia1_json_equivalence() {
|
|||
fn wis1_json_equivalence() {
|
||||
use crate::int_hyper_synth::{fnv1a, IntModel};
|
||||
use crate::int_splat::{decode, SplatJson};
|
||||
let js = jload("int_splat/splat.json");
|
||||
let (text, js) = (read("int_splat/splat.json"), jload("int_splat/splat.json"));
|
||||
let payload = read("int_splat/splat.wis1");
|
||||
assert_eq!(wis1_from_json(&js).unwrap(), payload, "the converter writes the committed payload");
|
||||
let (v, j) = (wire::parse_wis1(&payload).unwrap(), SplatJson::from_value(&js).unwrap());
|
||||
assert_eq!(wis1_from_json(&text).unwrap(), payload, "the converter writes the committed payload");
|
||||
let (v, j) = (wire::parse_wis1(&payload).unwrap(), SplatJson::read(&text).unwrap());
|
||||
assert_eq!((v.afrac as u32, v.c as usize, v.sh as usize, v.sw as usize, v.s as usize, v.attr as usize, v.n_gaussians as usize),
|
||||
(j.afrac, j.c, j.sh, j.sw, j.s, j.n_attr, j.n_gaussians));
|
||||
assert_eq!((&v.cdfs, &v.offsets, &v.median_q, v.rans), (&j.cdfs, &j.offsets, &j.median_q, j.rans.as_slice()));
|
||||
for (&(lo, hi), &(jl, jh)) in v.ranges.iter().zip(&j.ranges) {
|
||||
assert_eq!((lo, hi), (jl as f32, jh as f32), "the ranges are the JSON's binary32 values");
|
||||
assert_eq!(v.ranges, j.ranges, "the ranges are the JSON's binary32 values");
|
||||
// And they are what a correctly rounded reader (std's) makes of each spelling.
|
||||
for (k, key) in ["lo", "hi"].into_iter().enumerate() {
|
||||
for (i, n) in spellings(&text, key).iter().enumerate() {
|
||||
let x: f64 = n.parse().unwrap();
|
||||
assert_eq!((x as f32) as f64, x, "{key}[{i}] {n} denotes a binary32 value");
|
||||
let r = j.ranges[i];
|
||||
assert_eq!(if k == 0 { r.0 } else { r.1 }, wire::Binary32((x as f32).to_bits()), "{key}[{i}] {n}");
|
||||
}
|
||||
}
|
||||
let gs = read("int_splat/splat_gs.bin");
|
||||
let params = unpinned(crate::codecs::CAP_SPLAT_INT_CODEC, HeldBytes::File(&gs));
|
||||
|
|
@ -211,7 +218,7 @@ fn wis1_json_equivalence() {
|
|||
other => panic!("splat: {other:?}"),
|
||||
});
|
||||
assert_eq!(ranges, v.ranges);
|
||||
assert_eq!(bytes, decode(&IntModel::from_bytes(&gs).unwrap(), &js).unwrap().attr, "the binary and JSON forms decode alike");
|
||||
assert_eq!(bytes, decode(&IntModel::from_bytes(&gs).unwrap(), &text).unwrap().attr, "the binary and JSON forms decode alike");
|
||||
// Anchored below to output recorded before this revision: the sidecar and the
|
||||
// export's FNV-1a (`attr_hash`).
|
||||
assert_eq!(sha(&bytes), jload("int_splat/sha256.json")["splat"]["sha256"].as_str().unwrap());
|
||||
|
|
@ -571,18 +578,62 @@ fn entropy_tables_obey_the_table_length_rule() {
|
|||
t[n - 1] = 65_536;
|
||||
t
|
||||
};
|
||||
let ent = |n: usize| serde_json::json!({
|
||||
let ent = |n: usize| serde_json::to_vec(&serde_json::json!({
|
||||
"scale_edges": [1], "cdfs": [table(n)], "cdf_lengths": [n], "offsets": [0],
|
||||
"z_cdfs": [table(n)], "z_cdf_lengths": [n], "z_offsets": [0], "z_median_q": [0]
|
||||
});
|
||||
})).unwrap();
|
||||
for n in [2, 4098] {
|
||||
assert!(CondEntropy::try_from_value(&ent(n)).is_err(), "a table of {n} entries is refused");
|
||||
assert!(CondEntropy::read(&ent(n)).is_err(), "a table of {n} entries is refused");
|
||||
}
|
||||
for n in [3, 4097] {
|
||||
assert!(CondEntropy::try_from_value(&ent(n)).is_ok(), "a table of {n} entries is read");
|
||||
assert!(CondEntropy::read(&ent(n)).is_ok(), "a table of {n} entries is read");
|
||||
}
|
||||
}
|
||||
|
||||
/// A set's `entropy.json` is read as §9 says, by a reader that converts no number to a
|
||||
/// float: an integer is a number with no fraction and no exponent, and `-0` is zero; a
|
||||
/// member named twice, anything after the value, or a value more than 64 levels deep
|
||||
/// refuses the set (`model_mismatch`); a member the decode does not read is ignored.
|
||||
/// serde_json, which read the tables before, read `-0` as a float (and refused the set),
|
||||
/// kept the last of two members, and read deeper values.
|
||||
#[test]
|
||||
fn a_sets_entropy_json_is_read_strictly() {
|
||||
let payload = read("int_hyper/tiny_bitstream.wih1");
|
||||
let text = String::from_utf8(read("int_hyper/tiny_entropy.json")).unwrap();
|
||||
let decode = |entropy: &str| -> Result<Vec<u8>, &'static str> {
|
||||
let mut set = int_hyper_set();
|
||||
set.insert("entropy.json".into(), entropy.as_bytes().to_vec());
|
||||
let files = view(&set);
|
||||
let params = unpinned(crate::codecs::CAP_NEURAL_INT_HYPER, HeldBytes::Set { listing: b"", files: &files });
|
||||
match decode_int(crate::codecs::CAP_NEURAL_INT_HYPER, &payload, ¶ms) {
|
||||
Ok(IntDecoded::Image { rgb, .. }) => Ok(rgb),
|
||||
Ok(other) => panic!("not an image: {other:?}"),
|
||||
Err(e) => Err(e.code()),
|
||||
}
|
||||
};
|
||||
let want = decode(&text).expect("the committed tables decode");
|
||||
// Every table starts at 0: the first, spelled otherwise.
|
||||
let first = |spelled: &str| {
|
||||
assert_eq!(text.matches("[[0, ").count(), 2, "the latent and hyper-latent tables, each from 0");
|
||||
text.replacen("[[0, ", &format!("[[{spelled}, "), 1)
|
||||
};
|
||||
assert_eq!(decode(&first("-0")), Ok(want.clone()), "-0 is the integer zero");
|
||||
for not_integer in ["0.0", "-0.0", "0e0", "0E+0"] {
|
||||
assert_eq!(decode(&first(not_integer)), Err("model_mismatch"), "{not_integer} is not an integer");
|
||||
}
|
||||
// A member named twice is refused, whichever of the two a reader would keep.
|
||||
let (head, rest) = text.split_once('{').unwrap();
|
||||
let with = |members: &str| format!("{head}{{{members}, {rest}");
|
||||
assert_eq!(decode(&with(r#""offsets": [0, 0, 0, 0]"#)), Err("model_mismatch"), "a member named twice");
|
||||
assert_eq!(decode(&with(r#""note": "ignored", "extra": [1.5, -0, 1e300]"#)), Ok(want.clone()), "members not read are ignored");
|
||||
assert_eq!(decode(&format!("{text} {{}}")), Err("model_mismatch"), "bytes after the value");
|
||||
assert_eq!(decode(&format!("{text} \n")), Ok(want.clone()), "whitespace after the value");
|
||||
// The top object is level 1; a value inside n arrays in its member is at level n + 2.
|
||||
let nested = |n: usize| with(&format!(r#""deep": {}0{}"#, "[".repeat(n), "]".repeat(n)));
|
||||
assert_eq!(decode(&nested(62)), Ok(want.clone()), "a value 64 levels deep");
|
||||
assert_eq!(decode(&nested(63)), Err("model_mismatch"), "a value 65 levels deep");
|
||||
}
|
||||
|
||||
/// A network is planned in 64-bit arithmetic, layer by layer, so a size past the cap is
|
||||
/// `too_large` however far past it is, on every target. Planning in `usize` refused a
|
||||
/// transposed convolution of stride 2^31 on a side of 3 as malformed on a 32-bit sink
|
||||
|
|
@ -613,27 +664,275 @@ fn network_plans_refuse_alike_on_every_target() {
|
|||
}
|
||||
}
|
||||
|
||||
/// `WIS1`'s range rule, read from the bits in integer arithmetic, is IEEE 754's: each
|
||||
/// finite, and `lo ≤ hi` with `−0` equal to `+0`. Held to the float comparison on every
|
||||
/// pair of bit-pattern classes (zeros, subnormals, normals, the greatest, infinities, quiet
|
||||
/// and signalling NaNs, each sign), on random pairs, and on near pairs (a few units apart,
|
||||
/// or of opposite sign); the parser refuses exactly the pairs the rule does.
|
||||
#[test]
|
||||
fn the_wis1_range_rule_in_integers_is_the_float_comparison() {
|
||||
let committed = read("int_splat/splat.wis1");
|
||||
let classes = [
|
||||
0u32, 0x8000_0000, 1, 0x8000_0001, 0x007f_ffff, 0x807f_ffff, 0x0080_0000, 0x3f80_0000, 0xbf80_0000, 0x4b80_0001,
|
||||
0xcb80_0001, 0x7f7f_ffff, 0xff7f_ffff, 0x7f80_0000, 0xff80_0000, 0x7fc0_0000, 0xffc0_0000, 0x7f80_0001, 0xff80_0001,
|
||||
];
|
||||
let mut r = crate::int_kernels::testkit::Rng(0x5157_0004);
|
||||
let mut pairs: Vec<(u32, u32, bool)> = classes.iter().flat_map(|&a| classes.iter().map(move |&b| (a, b, true))).collect();
|
||||
for i in 0..60_000 {
|
||||
let a = r.next() as u32;
|
||||
let b = match i % 3 {
|
||||
0 => r.next() as u32,
|
||||
1 => a.wrapping_add(r.below(5) as u32).wrapping_sub(2),
|
||||
_ => a ^ 0x8000_0000,
|
||||
};
|
||||
pairs.push((a, b, i % 30 == 0));
|
||||
}
|
||||
for (a, b, parse) in pairs {
|
||||
let (fa, fb) = (f32::from_bits(a), f32::from_bits(b));
|
||||
let want = fa.is_finite() && fb.is_finite() && fa <= fb;
|
||||
let (x, y) = (wire::Binary32(a), wire::Binary32(b));
|
||||
assert_eq!(x.is_finite() && y.is_finite() && x.finite_le(y), want, "{a:#010x} {b:#010x}");
|
||||
if parse {
|
||||
let mut p = committed.clone();
|
||||
p[32..36].copy_from_slice(&a.to_le_bytes());
|
||||
p[36..40].copy_from_slice(&b.to_le_bytes());
|
||||
let got = wire::parse_wis1(&p).map(|_| ()).map_err(|e| e.code());
|
||||
assert_eq!(got, if want { Ok(()) } else { Err("field_range") }, "{a:#010x} {b:#010x}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The spellings of the numbers in the splat JSON's top-level array `key`, as written.
|
||||
#[cfg(feature = "splat_int")]
|
||||
fn spellings(text: &[u8], key: &str) -> Vec<String> {
|
||||
match crate::strict_json::read(text).unwrap().get(key) {
|
||||
Some(crate::strict_json::Json::Arr(a)) => a.iter().map(|v| match v {
|
||||
crate::strict_json::Json::Num(n) => n.clone(),
|
||||
other => panic!("{key}: {other:?}"),
|
||||
}).collect(),
|
||||
other => panic!("{key}: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// `text` (a splat JSON) with its top-level member `key`'s value replaced by `value`,
|
||||
/// spelled as given: the rest of the text, every range among it, as written.
|
||||
#[cfg(feature = "splat_int")]
|
||||
fn edit(text: &str, key: &str, value: &str) -> String {
|
||||
let at = text.find(&format!("\"{key}\": ")).unwrap_or_else(|| panic!("no member {key}")) + key.len() + 4;
|
||||
let mut depth = 0i32;
|
||||
let end = at + text[at..].char_indices().find(|&(_, c)| {
|
||||
match c {
|
||||
'[' | '{' => depth += 1,
|
||||
']' | '}' => depth -= 1,
|
||||
_ => {}
|
||||
}
|
||||
depth < 0 || (depth == 0 && c == ',')
|
||||
}).expect("the member's value ends").0;
|
||||
format!("{}{}{}", &text[..at], value, &text[end..])
|
||||
}
|
||||
|
||||
/// The committed splat JSON's text with its member `key`'s value replaced ([`edit`]).
|
||||
#[cfg(feature = "splat_int")]
|
||||
fn splat_with(key: &str, value: &str) -> Vec<u8> {
|
||||
edit(&String::from_utf8(read("int_splat/splat.json")).unwrap(), key, value).into_bytes()
|
||||
}
|
||||
|
||||
/// The committed splat JSON with element `i` of the top-level array `key` spelled `n`.
|
||||
#[cfg(feature = "splat_int")]
|
||||
fn splat_with_element(key: &str, i: usize, n: &str) -> Vec<u8> {
|
||||
let mut v = spellings(&read("int_splat/splat.json"), key);
|
||||
v[i] = n.to_string();
|
||||
splat_with(key, &format!("[{}]", v.join(", ")))
|
||||
}
|
||||
|
||||
/// The committed splat JSON with the first attribute's range `(n, n)`: `lo` and `hi`
|
||||
/// both spelled `n`, so the range holds whatever value `n` is.
|
||||
#[cfg(feature = "splat_int")]
|
||||
fn splat_with_range0(n: &str) -> Vec<u8> {
|
||||
let committed = read("int_splat/splat.json");
|
||||
let mut text = String::from_utf8(committed.clone()).unwrap();
|
||||
for key in ["lo", "hi"] {
|
||||
let mut v = spellings(&committed, key);
|
||||
v[0] = n.to_string();
|
||||
text = edit(&text, key, &format!("[{}]", v.join(", ")));
|
||||
}
|
||||
text.into_bytes()
|
||||
}
|
||||
|
||||
/// The splat converter carries each presentation range exactly: a JSON decimal that
|
||||
/// denotes a binary32 value converts to it, and one a single f64 unit away from it,
|
||||
/// written as its own shortest decimal, is refused. `splat_int` reads numbers
|
||||
/// correctly rounded, so no tolerance stands in for the reader.
|
||||
/// denotes a binary32 value converts to it, and one a single binary64 unit away from it,
|
||||
/// written as its own shortest decimal, is refused. The JSON is read in integer
|
||||
/// arithmetic, so no float parser stands in for the reader, and no tolerance either.
|
||||
#[cfg(feature = "splat_int")]
|
||||
#[test]
|
||||
fn splat_ranges_are_carried_exactly() {
|
||||
let text = String::from_utf8(read("int_splat/splat.json")).unwrap();
|
||||
let js: serde_json::Value = serde_json::from_str(&text).unwrap();
|
||||
let lo0 = js["lo"][0].as_f64().unwrap();
|
||||
assert_eq!((lo0 as f32) as f64, lo0, "the committed range is a binary32 value, read exactly");
|
||||
assert!(wis1_from_json(&js).is_ok());
|
||||
let lo0 = spellings(&read("int_splat/splat.json"), "lo")[0].parse::<f64>().unwrap();
|
||||
assert_eq!((lo0 as f32) as f64, lo0, "the committed range is a binary32 value");
|
||||
assert!(wis1_from_json(&splat_with_element("lo", 0, &format!("{lo0:?}"))).is_ok());
|
||||
for step in [1i64, -1] {
|
||||
let off = f64::from_bits(((lo0 as f32) as f64).to_bits().wrapping_add_signed(step));
|
||||
let mut bad: serde_json::Value = serde_json::from_str(&text).unwrap();
|
||||
bad["lo"][0] = serde_json::from_str(&format!("{off:?}")).unwrap();
|
||||
assert_eq!(bad["lo"][0].as_f64(), Some(off), "the reader is correctly rounded");
|
||||
assert_eq!(wis1_from_json(&bad).unwrap_err().code(), "field_range", "{off:?}");
|
||||
let off = f64::from_bits(lo0.to_bits().wrapping_add_signed(step));
|
||||
assert_eq!(wis1_from_json(&splat_with_element("lo", 0, &format!("{off:?}"))).unwrap_err().code(), "field_range", "{off:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A range that differs from another in its last digit decodes differently, and each to
|
||||
/// the value binary64 rounding gives it, at the closest case there is: `1 + 2^-53`, the
|
||||
/// midpoint between `1` and the next binary64 value, rounds to `1` (ties to even), and
|
||||
/// is the binary32 value `1`; with its last digit 6 it is past the midpoint, rounds to
|
||||
/// `1 + 2^-52`, which is no binary32 value, and is refused; with 4 it is `1` again. So
|
||||
/// too below `1` (`1 − 2^-54`, half the unit below a power of two), and, with a tail of
|
||||
/// 1300 zeros and a 1 (past the digits the reader keeps), past the midpoint. The
|
||||
/// integers `2^24` and `2^24 + 2` are adjacent binary32 values; `2^24 + 1` between them
|
||||
/// is none. Spellings one last digit apart that are both values decode to two adjacent
|
||||
/// values: `4194304.0` and `4194304.5` (the step above `2^22` is a half), while
|
||||
/// `4194304.4` and `4194304.6` are no binary32 value; and `8.388608e6` and `8.388609e6`
|
||||
/// (`2^23` and the value after it). Each accepted range is the one the payload's integer
|
||||
/// decode gives.
|
||||
#[cfg(feature = "splat_int")]
|
||||
#[test]
|
||||
fn a_range_one_digit_apart_decodes_to_its_own_value() {
|
||||
use crate::int_splat::SplatJson;
|
||||
let gs = read("int_splat/splat_gs.bin");
|
||||
let params = unpinned(crate::codecs::CAP_SPLAT_INT_CODEC, HeldBytes::File(&gs));
|
||||
const UP: &str = "1.00000000000000011102230246251565404236316680908203125"; // 1 + 2^-53
|
||||
const DOWN: &str = "0.999999999999999944488848768742172978818416595458984375"; // 1 - 2^-54
|
||||
let one = wire::Binary32(1f32.to_bits());
|
||||
let tail = |n: &str, t: &str| format!("{n}{}{t}", "0".repeat(1300));
|
||||
let last = |n: &str, d: char| format!("{}{d}", &n[..n.len() - 1]);
|
||||
let cases: Vec<(String, Option<wire::Binary32>)> = vec![
|
||||
(UP.into(), Some(one)),
|
||||
(last(UP, '6'), None),
|
||||
(last(UP, '4'), Some(one)),
|
||||
(DOWN.into(), Some(one)),
|
||||
(last(DOWN, '4'), None),
|
||||
(last(DOWN, '6'), Some(one)),
|
||||
(tail(UP, "1"), None),
|
||||
(tail(UP, ""), Some(one)),
|
||||
(tail(DOWN, "1"), Some(one)),
|
||||
("16777216".into(), Some(wire::Binary32(0x4b80_0000))),
|
||||
("16777218".into(), Some(wire::Binary32(0x4b80_0001))),
|
||||
("16777217".into(), None),
|
||||
("4194304.0".into(), Some(wire::Binary32(0x4a80_0000))),
|
||||
("4194304.5".into(), Some(wire::Binary32(0x4a80_0001))),
|
||||
("4194304.4".into(), None),
|
||||
("4194304.6".into(), None),
|
||||
("8.388608e6".into(), Some(wire::Binary32(0x4b00_0000))),
|
||||
("8.388609e6".into(), Some(wire::Binary32(0x4b00_0001))),
|
||||
];
|
||||
for (n, want) in &cases {
|
||||
let text = splat_with_range0(n);
|
||||
match want {
|
||||
Some(b) => {
|
||||
assert_eq!(SplatJson::read(&text).unwrap().ranges[0], (*b, *b), "{n}");
|
||||
let p = wis1_from_json(&text).unwrap();
|
||||
assert_eq!(wire::parse_wis1(&p).unwrap().ranges[0], (*b, *b), "{n}: the payload carries it");
|
||||
assert_eq!(p[32..36], b.0.to_le_bytes(), "{n}");
|
||||
match decode_int(crate::codecs::CAP_SPLAT_INT_CODEC, &p, ¶ms) {
|
||||
Ok(IntDecoded::Attr { ranges, .. }) => assert_eq!(ranges[0], (*b, *b), "{n}: the decode presents it"),
|
||||
other => panic!("{n}: {other:?}"),
|
||||
}
|
||||
}
|
||||
None => {
|
||||
assert_eq!(SplatJson::read(&text).unwrap_err(), "lo holds a value that is not a binary32 value", "{n}");
|
||||
assert_eq!(wis1_from_json(&text).unwrap_err().code(), "field_range", "{n}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The reader of a range against std's correctly rounded `f64` parse, on decimals: the
|
||||
/// shortest spellings of binary32 values (each read as itself), the same values spelled
|
||||
/// with 1 to 20 significant digits (mostly no binary32 value), random digit strings at
|
||||
/// exponents from -60 to 45, and the edges: zeros of either sign, magnitudes that round
|
||||
/// to zero in binary64 and the first that does not, the least subnormal, the greatest
|
||||
/// binary32 value and the decimals just past it, and exponents far out of range.
|
||||
#[cfg(feature = "splat_int")]
|
||||
#[test]
|
||||
fn a_decimal_range_is_the_binary64_value_it_denotes() {
|
||||
use crate::int_splat::binary32_of;
|
||||
let reference = |n: &str| -> Option<u32> {
|
||||
let x: f64 = n.parse().unwrap();
|
||||
(x.is_finite() && (x as f32) as f64 == x).then(|| (x as f32).to_bits())
|
||||
};
|
||||
let check = |n: &str| assert_eq!(binary32_of(n).map(|b| b.0), reference(n), "{n}");
|
||||
let mut r = crate::int_kernels::testkit::Rng(0x5157_0003);
|
||||
for _ in 0..4_000 {
|
||||
let b = r.next() as u32;
|
||||
if b & 0x7f80_0000 == 0x7f80_0000 {
|
||||
continue;
|
||||
}
|
||||
let v = f32::from_bits(b);
|
||||
check(&format!("{:?}", v as f64));
|
||||
check(&format!("{:e}", v as f64));
|
||||
for d in 1..=20 {
|
||||
check(&format!("{:.*e}", d - 1, v as f64));
|
||||
}
|
||||
}
|
||||
for _ in 0..4_000 {
|
||||
let digits: String = (0..1 + r.below(25)).map(|_| char::from(b'0' + r.below(10) as u8)).collect();
|
||||
let e = r.below(106) as i64 - 60;
|
||||
check(&format!("{}{}.{}0e{e}", if r.below(2) == 0 { "" } else { "-" }, &digits[..1], &digits[1..]));
|
||||
}
|
||||
for n in ["0.0", "-0.0", "0e5", "-0E-5", "0.000", "1e-400", "-1e-400", "2.4703282292062327e-324", "2.4703282292062328e-324",
|
||||
"-2.4703282292062328e-324", "4.9406564584124654e-324", "1.401298464324817e-45", "-1.401298464324817e-45",
|
||||
"1.4012984643248169e-45", "1.1754942106924411e-38", "3.4028234663852886e38", "-3.4028234663852886E+38",
|
||||
"3.4028235e38", "3.4028234663852889e38", "3.402823466385289e38", "1e38", "1e39", "1.7976931348623157e308",
|
||||
"1e309", "1e99999999999999999999", "1e-99999999999999999999", "0.5", "2.5", "0.1", "0.30000000000000004"] {
|
||||
check(n);
|
||||
}
|
||||
}
|
||||
|
||||
/// An integer range is the integer it spells, and must be exactly a binary32 value: no
|
||||
/// rounding at any size (`2^64 + 1`, past `u64`, is refused as `2^60 + 1` is), the
|
||||
/// greatest binary32 value is one, and the integers past it are not (one more, the
|
||||
/// midpoint to `2^128`, and `2^128`), and `-0` is the integer zero, so `+0` (as the
|
||||
/// independent converter reads it), where the decimal `-0.0` keeps its sign.
|
||||
#[cfg(feature = "splat_int")]
|
||||
#[test]
|
||||
fn an_integer_range_is_its_integer() {
|
||||
use crate::int_splat::binary32_of;
|
||||
let bits = |n: &str| binary32_of(n).map(|b| b.0);
|
||||
for (n, want) in [
|
||||
("0", Some(0)), ("-0", Some(0)), ("-0.0", Some(0x8000_0000)), ("0.0", Some(0)), ("1", Some(0x3f80_0000)), ("-1", Some(0xbf80_0000)),
|
||||
("1152921504606846976", Some(0x5d80_0000)), ("1152921504606846977", None), ("-1152921504606846977", None),
|
||||
("18446744073709551616", Some(0x5f80_0000)), ("18446744073709551617", None), ("-18446744073709551617", None),
|
||||
("340282346638528859811704183484516925440", Some(0x7f7f_ffff)), ("-340282346638528859811704183484516925440", Some(0xff7f_ffff)),
|
||||
("340282346638528859811704183484516925441", None), ("340282356779733661637539395458142568448", None),
|
||||
("340282366920938463463374607431768211456", None),
|
||||
("16777217", None), ("16777218", Some(0x4b80_0001)),
|
||||
] {
|
||||
assert_eq!(bits(n), want, "{n}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The splat JSON is read strictly and its integers as integers: a member named twice,
|
||||
/// or bytes after the value, refuse the text (a lenient parser takes the last of two
|
||||
/// members); a count or table entry spelled with a fraction or an exponent is refused;
|
||||
/// and `-0`, the integer zero spelled with a sign, reads as `0`.
|
||||
#[cfg(feature = "splat_int")]
|
||||
#[test]
|
||||
fn the_splat_json_is_read_strictly_and_its_integers_as_integers() {
|
||||
use crate::int_splat::SplatJson;
|
||||
let text = read("int_splat/splat.json");
|
||||
let committed = SplatJson::read(&text).unwrap();
|
||||
let mut twice = text.clone();
|
||||
twice.splice(1..1, b"\"c\": 32, ".iter().copied());
|
||||
assert!(SplatJson::read(&twice).is_err(), "a member named twice");
|
||||
let mut after = text.clone();
|
||||
after.extend_from_slice(b" {}");
|
||||
assert!(SplatJson::read(&after).is_err(), "bytes after the value");
|
||||
assert_eq!(SplatJson::read(&splat_with("c", "32.0")).unwrap_err(), "c");
|
||||
assert_eq!(SplatJson::read(&splat_with("afrac", "12e0")).unwrap_err(), "afrac");
|
||||
let mut m = spellings(&text, "median_q");
|
||||
assert_eq!(m[0], "0");
|
||||
m[0] = "-0".into();
|
||||
let neg_zero = splat_with("median_q", &format!("[{}]", m.join(", ")));
|
||||
assert_eq!(SplatJson::read(&neg_zero).unwrap().median_q, committed.median_q);
|
||||
assert_eq!(wis1_from_json(&neg_zero).unwrap(), read("int_splat/splat.wis1"), "-0 is the median 0");
|
||||
m[0] = "0.0".into();
|
||||
assert_eq!(SplatJson::read(&splat_with("median_q", &format!("[{}]", m.join(", ")))).unwrap_err(), "median_q");
|
||||
}
|
||||
|
||||
/// The converters check the JSON's channel count against its tables: a medians list,
|
||||
/// a table list or an offsets list one short is refused (`field_range`), never written
|
||||
/// out of step with the header.
|
||||
|
|
@ -649,9 +948,10 @@ fn converters_check_the_channel_count() {
|
|||
}
|
||||
let splat = jload("int_splat/splat.json");
|
||||
for key in ["median_q", "cdfs", "offsets"] {
|
||||
let mut j = splat.clone();
|
||||
j[key].as_array_mut().unwrap().pop();
|
||||
let e = wis1_from_json(&j).unwrap_err();
|
||||
// The member alone is rewritten: an integer list, written back exactly.
|
||||
let mut v = splat[key].clone();
|
||||
v.as_array_mut().unwrap().pop();
|
||||
let e = wis1_from_json(&splat_with(key, &v.to_string())).unwrap_err();
|
||||
assert_eq!((e.code(), e.detail().contains("one per channel")), ("field_range", true), "splat {key}: {e}");
|
||||
}
|
||||
}
|
||||
|
|
@ -749,16 +1049,10 @@ fn a_one_frame_clip_has_one_header() {
|
|||
#[cfg(feature = "splat_int")]
|
||||
#[test]
|
||||
fn splat_integer_ranges_are_carried_exactly() {
|
||||
let text = String::from_utf8(read("int_splat/splat.json")).unwrap();
|
||||
let with = |n: &str| {
|
||||
let mut j: serde_json::Value = serde_json::from_str(&text).unwrap();
|
||||
let big: serde_json::Value = serde_json::from_str(n).unwrap();
|
||||
j["lo"][0] = big.clone();
|
||||
j["hi"][0] = big;
|
||||
wis1_from_json(&j)
|
||||
};
|
||||
let with = |n: &str| wis1_from_json(&splat_with_range0(n));
|
||||
let p = with("1152921504606846976").unwrap();
|
||||
assert_eq!(wire::parse_wis1(&p).unwrap().ranges[0], ((1u64 << 60) as f32, (1u64 << 60) as f32));
|
||||
let two60 = wire::Binary32(((1u64 << 60) as f32).to_bits());
|
||||
assert_eq!(wire::parse_wis1(&p).unwrap().ranges[0], (two60, two60));
|
||||
assert_eq!(with("1152921504606846977").unwrap_err().code(), "field_range");
|
||||
assert_eq!(with("-1152921504606846977").unwrap_err().code(), "field_range");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -671,6 +671,31 @@ pub fn write_wia1(v: &Wia1<'_>) -> Vec<u8> {
|
|||
|
||||
// ───────────────────────────── WIS1 ─────────────────────────────
|
||||
|
||||
/// An IEEE 754 binary32 value held as its bits, as `WIS1` carries a presentation range
|
||||
/// ([integer-payloads] §6). A parser reads, checks and writes it in integer arithmetic,
|
||||
/// so no floating-point operation is on a payload's decode path; a sink that presents
|
||||
/// the range takes its value from the bits (`f32::from_bits`).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct Binary32(pub u32);
|
||||
|
||||
impl Binary32 {
|
||||
/// Finite: the exponent field is not all ones (neither an infinity nor a NaN).
|
||||
pub fn is_finite(self) -> bool {
|
||||
self.0 & 0x7f80_0000 != 0x7f80_0000
|
||||
}
|
||||
|
||||
/// `self ≤ other` as IEEE 754 compares two finite values, `−0` equal to `+0`: by the
|
||||
/// magnitude's bits, negated for a negative value (a finite magnitude's bits are
|
||||
/// below `2^31 − 2^23`, so the negation stays in `i32`).
|
||||
pub fn finite_le(self, other: Binary32) -> bool {
|
||||
let key = |b: Binary32| {
|
||||
let m = (b.0 & 0x7fff_ffff) as i32;
|
||||
if b.0 >> 31 == 1 { -m } else { m }
|
||||
};
|
||||
key(self) <= key(other)
|
||||
}
|
||||
}
|
||||
|
||||
/// A parsed `WIS1` payload ([integer-payloads] §6).
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Wis1<'a> {
|
||||
|
|
@ -685,9 +710,9 @@ pub struct Wis1<'a> {
|
|||
pub attr: u32,
|
||||
/// Gaussians in the cloud: the first `n_gaussians` cells of each plane.
|
||||
pub n_gaussians: u32,
|
||||
/// Per attribute, the range `(lo, hi)` its `u8` plane is presented over.
|
||||
/// Presentation only: outside the splat-equivalence digest.
|
||||
pub ranges: Vec<(f32, f32)>,
|
||||
/// Per attribute, the range `(lo, hi)` its `u8` plane is presented over, as
|
||||
/// binary32 bits. Presentation only: outside the splat-equivalence digest.
|
||||
pub ranges: Vec<(Binary32, Binary32)>,
|
||||
/// Per channel, its table, its symbol offset and its median.
|
||||
pub cdfs: Vec<Vec<i32>>,
|
||||
pub offsets: Vec<i32>,
|
||||
|
|
@ -728,9 +753,9 @@ pub fn parse_wis1(p: &[u8]) -> Result<Wis1<'_>, WireError> {
|
|||
let raw = cur.take(attr as u64 * 8, "the attribute ranges")?;
|
||||
let mut ranges = Vec::with_capacity(attr as usize);
|
||||
for r in raw.chunks_exact(8) {
|
||||
let lo = f32::from_bits(u32::from_le_bytes([r[0], r[1], r[2], r[3]]));
|
||||
let hi = f32::from_bits(u32::from_le_bytes([r[4], r[5], r[6], r[7]]));
|
||||
if !lo.is_finite() || !hi.is_finite() || lo > hi {
|
||||
let lo = Binary32(u32::from_le_bytes([r[0], r[1], r[2], r[3]]));
|
||||
let hi = Binary32(u32::from_le_bytes([r[4], r[5], r[6], r[7]]));
|
||||
if !(lo.is_finite() && hi.is_finite() && lo.finite_le(hi)) {
|
||||
return Err(err(Refusal::FieldRange, "an attribute range is not finite, or its lo exceeds its hi"));
|
||||
}
|
||||
ranges.push((lo, hi));
|
||||
|
|
@ -765,8 +790,8 @@ pub fn write_wis1(v: &Wis1<'_>) -> Vec<u8> {
|
|||
put_u32(&mut o, d);
|
||||
}
|
||||
for &(lo, hi) in &v.ranges {
|
||||
put_u32(&mut o, lo.to_bits());
|
||||
put_u32(&mut o, hi.to_bits());
|
||||
put_u32(&mut o, lo.0);
|
||||
put_u32(&mut o, hi.0);
|
||||
}
|
||||
for cdf in &v.cdfs {
|
||||
put_u16(&mut o, cdf.len() as u16);
|
||||
|
|
|
|||
|
|
@ -10,10 +10,12 @@
|
|||
//! (`extensions/integer-operators.md` §13.2; [`crate::int_ops`]). The float rasterizer is
|
||||
//! sink-supplied presentation; the *attributes* are the conformance contract.
|
||||
|
||||
use serde_json::Value;
|
||||
use std::cmp::Ordering;
|
||||
|
||||
use crate::int_hyper_synth::{fnv1a, refused, IntModel, Refused, RefusalKind, MAX_AFRAC, MAX_TENSOR};
|
||||
use crate::int_wire::Binary32;
|
||||
use crate::rans::RansDecoder;
|
||||
use crate::strict_json::{self, Json};
|
||||
|
||||
/// A decoded splat: the `[attr · s · s]` `u8` attribute image (channel-major), to be
|
||||
/// un-normalized + un-sorted into Gaussians by the presentation layer.
|
||||
|
|
@ -53,6 +55,12 @@ pub struct SplatStream<'a> {
|
|||
/// the per-attribute ranges `lo`, `hi`. A converter input and a debugging aid; the
|
||||
/// payload is `WIS1`. Its `afrac` is read (the old JSON decoder ignored it) and must
|
||||
/// be the model's.
|
||||
///
|
||||
/// It is read from its text ([`SplatJson::read`]) with no floating point: every count,
|
||||
/// table entry, offset and median is an integer spelling read as an integer, and each
|
||||
/// range is the binary32 value its spelling denotes ([`binary32_of`]), found in exact
|
||||
/// integer arithmetic. So the same text reads to the same values on every machine and
|
||||
/// in every build, whatever a general JSON parser would round a decimal to.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SplatJson {
|
||||
pub afrac: u32,
|
||||
|
|
@ -65,41 +73,40 @@ pub struct SplatJson {
|
|||
pub cdfs: Vec<Vec<i32>>,
|
||||
pub offsets: Vec<i32>,
|
||||
pub median_q: Vec<i64>,
|
||||
/// Per attribute, `(lo, hi)` as the JSON carries them; an integer an `f64` does not
|
||||
/// hold exactly is refused.
|
||||
pub ranges: Vec<(f64, f64)>,
|
||||
/// Per attribute, `(lo, hi)`: the binary32 values the JSON's `lo` and `hi` denote.
|
||||
pub ranges: Vec<(Binary32, Binary32)>,
|
||||
pub rans: Vec<u8>,
|
||||
}
|
||||
|
||||
impl SplatJson {
|
||||
/// Read the tool's JSON bitstream.
|
||||
pub fn from_value(bs: &Value) -> Result<Self, &'static str> {
|
||||
let uz = |k: &'static str| bs[k].as_u64().and_then(|v| usize::try_from(v).ok()).ok_or(k);
|
||||
/// Read the tool's JSON bitstream from its text: one JSON text (RFC 8259, read
|
||||
/// strictly: no member named twice, nothing after the value), whose integers are
|
||||
/// spelled as integers (an optional `-`, then digits; `-0` is zero), and whose `lo`
|
||||
/// and `hi` each hold one binary32 value per attribute ([`binary32_of`]). The error
|
||||
/// names the member that fails.
|
||||
pub fn read(text: &[u8]) -> Result<Self, &'static str> {
|
||||
let bs = strict_json::read(text).map_err(|_| "the JSON form is not one strict JSON text")?;
|
||||
let get = |k: &'static str| bs.get(k).ok_or(k);
|
||||
let uz = |k: &'static str| get(k)?.as_u64().and_then(|v| usize::try_from(v).ok()).ok_or(k);
|
||||
let (c, sh, sw, s, n_attr) = (uz("c")?, uz("sh")?, uz("sw")?, uz("s")?, uz("attr")?);
|
||||
let n_gaussians = uz("n_gaussians")?;
|
||||
let afrac = bs["afrac"].as_u64().and_then(|v| u32::try_from(v).ok()).ok_or("afrac")?;
|
||||
let offsets = bs["offsets"].as_array().ok_or("offsets")?.iter()
|
||||
.map(|n| n.as_i64().and_then(|v| i32::try_from(v).ok())).collect::<Option<Vec<i32>>>().ok_or("offsets")?;
|
||||
let median_q = bs["median_q"].as_array().ok_or("median_q")?.iter()
|
||||
.map(Value::as_i64).collect::<Option<Vec<i64>>>().ok_or("median_q")?;
|
||||
let cdfs = bs["cdfs"].as_array().ok_or("cdfs")?.iter()
|
||||
.map(|row| row.as_array().and_then(|r| r.iter().map(|n| n.as_i64().and_then(|v| i32::try_from(v).ok())).collect::<Option<Vec<i32>>>()))
|
||||
.collect::<Option<Vec<_>>>().ok_or("cdfs holds a value outside i32")?;
|
||||
// A range as the JSON states it: an integer only where an f64 holds it exactly
|
||||
// (so 2^60 + 1 is refused rather than rounded to 2^60), a decimal as read.
|
||||
let exact = |v: &Value| -> Option<f64> {
|
||||
let int = v.as_i64().map(i128::from).or(v.as_u64().map(i128::from));
|
||||
match int {
|
||||
Some(i) => Some(i as f64).filter(|&f| f as i128 == i),
|
||||
None => v.as_f64(),
|
||||
}
|
||||
let afrac = get("afrac")?.as_u64().and_then(|v| u32::try_from(v).ok()).ok_or("afrac")?;
|
||||
let i32s = |v: &Json| v.as_arr()?.iter().map(|n| n.as_i64().and_then(|v| i32::try_from(v).ok())).collect::<Option<Vec<i32>>>();
|
||||
let offsets = i32s(get("offsets")?).ok_or("offsets")?;
|
||||
let median_q = get("median_q")?.as_arr().and_then(|a| a.iter().map(Json::as_i64).collect::<Option<Vec<i64>>>()).ok_or("median_q")?;
|
||||
let cdfs = get("cdfs")?.as_arr().and_then(|a| a.iter().map(i32s).collect::<Option<Vec<_>>>()).ok_or("cdfs holds a value outside i32")?;
|
||||
let ranges = |k: &'static str| -> Result<Vec<Binary32>, &'static str> {
|
||||
let not = if k == "lo" { "lo holds a value that is not a binary32 value" } else { "hi holds a value that is not a binary32 value" };
|
||||
get(k)?.as_arr().ok_or(k)?.iter().map(|v| match v {
|
||||
Json::Num(n) => binary32_of(n).ok_or(not),
|
||||
_ => Err(k),
|
||||
}).collect()
|
||||
};
|
||||
let f = |k: &'static str| bs[k].as_array().ok_or(k)?.iter().map(exact).collect::<Option<Vec<f64>>>().ok_or(k);
|
||||
let (lo, hi) = (f("lo")?, f("hi")?);
|
||||
let (lo, hi) = (ranges("lo")?, ranges("hi")?);
|
||||
if lo.len() != n_attr || hi.len() != n_attr {
|
||||
return Err("lo and hi are not one per attribute");
|
||||
}
|
||||
let rans = b64_decode(bs["rans_b64"].as_str().ok_or("rans_b64")?)?;
|
||||
let rans = b64_decode(get("rans_b64")?.as_str().ok_or("rans_b64")?)?;
|
||||
Ok(SplatJson { afrac, c, sh, sw, s, n_attr, n_gaussians, cdfs, offsets, median_q, ranges: lo.into_iter().zip(hi).collect(), rans })
|
||||
}
|
||||
|
||||
|
|
@ -309,11 +316,12 @@ pub(crate) fn splat_plan(gs: &IntModel, st: &SplatStream<'_>) -> Result<(u64, u1
|
|||
Ok(((8 * (latent + p.live)).max(8 * out + out), p.macs))
|
||||
}
|
||||
|
||||
/// Decode the tool's JSON bitstream (`splat.json`) → the integer attribute tensor:
|
||||
/// [`SplatJson`] then [`decode_typed`]. A debugging and converter path; the payload is
|
||||
/// `WIS1`. `gs` is the WHM1 model (`splat_gs.bin`). Byte-identical on every machine.
|
||||
pub fn decode(gs: &IntModel, bs: &Value) -> Result<DecodedSplat, &'static str> {
|
||||
decode_typed(gs, &SplatJson::from_value(bs)?.stream()).map_err(|r| r.detail)
|
||||
/// Decode the tool's JSON bitstream (`splat.json`, its text) → the integer attribute
|
||||
/// tensor: [`SplatJson::read`] then [`decode_typed`]. A debugging and converter path; the
|
||||
/// payload is `WIS1`. `gs` is the WHM1 model (`splat_gs.bin`). Byte-identical on every
|
||||
/// machine.
|
||||
pub fn decode(gs: &IntModel, json: &[u8]) -> Result<DecodedSplat, &'static str> {
|
||||
decode_typed(gs, &SplatJson::read(json)?.stream()).map_err(|r| r.detail)
|
||||
}
|
||||
|
||||
/// Dependency-free base64 decode (standard alphabet) — the rANS bytes ride in JSON as
|
||||
|
|
@ -344,3 +352,250 @@ fn b64_decode(s: &str) -> Result<Vec<u8>, &'static str> {
|
|||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// The most significant digits of a decimal [`binary32_of`] keeps. Every bound it
|
||||
/// compares a decimal with is a multiple of `2^-1075`, so of `10^-1075`; a decimal below
|
||||
/// `10^39` cut to this many digits has its last kept digit at `10^-1161` or below, so the
|
||||
/// digits dropped (a nonzero tail, which it notes) cannot change any comparison.
|
||||
const KEPT_DIGITS: usize = 1200;
|
||||
|
||||
/// The binary32 value the JSON number spelled `n` denotes, read as the export tool and
|
||||
/// the independent converter (`int-payload-conformance/int_payload_verify.py`) read it,
|
||||
/// or `None` when it denotes none:
|
||||
///
|
||||
/// - an integer spelling (an optional `-`, then digits: no fraction and no exponent) is
|
||||
/// that integer, which must be a binary32 value exactly: `2^60` is, `2^60 + 1` is not
|
||||
/// (a binary64 would round it to `2^60`); `-0` is the integer zero, so `+0`;
|
||||
/// - any other spelling denotes the binary64 value nearest it (ties to even; a magnitude
|
||||
/// that rounds to zero is a zero of the spelling's sign, and one past the binary64
|
||||
/// range is no value), and that must be a binary32 value: the export writes each range
|
||||
/// as the shortest decimal of a binary32 value widened to binary64.
|
||||
///
|
||||
/// Integer arithmetic throughout, so no machine and no build reads a spelling to another
|
||||
/// value. The decimal `m·10^e` is compared exactly with binary values `a·2^b`: the largest
|
||||
/// binary32 value not above its magnitude is found by bisection over the bit patterns
|
||||
/// (ordered as their values are), and the decimal is that value, or the next, where it
|
||||
/// lies in the closed interval of reals that round to it in binary64.
|
||||
pub fn binary32_of(n: &str) -> Option<Binary32> {
|
||||
let (neg, rest) = match n.strip_prefix('-') {
|
||||
Some(r) => (true, r),
|
||||
None => (false, n),
|
||||
};
|
||||
let (mant, exp) = match rest.find(['e', 'E']) {
|
||||
Some(i) => (&rest[..i], Some(&rest[i + 1..])),
|
||||
None => (rest, None),
|
||||
};
|
||||
let (int, frac) = match mant.find('.') {
|
||||
Some(i) => (&mant[..i], Some(&mant[i + 1..])),
|
||||
None => (mant, None),
|
||||
};
|
||||
let integer = exp.is_none() && frac.is_none();
|
||||
let frac = frac.unwrap_or("");
|
||||
let digit = |b: u8| b.is_ascii_digit();
|
||||
if int.is_empty() || !int.bytes().all(digit) || !frac.bytes().all(digit) {
|
||||
return None;
|
||||
}
|
||||
// The exponent, saturated far past any magnitude that can matter.
|
||||
let mut e: i64 = 0;
|
||||
if let Some(x) = exp {
|
||||
let (eneg, ed) = match x.as_bytes().first() {
|
||||
Some(b'-') => (true, &x[1..]),
|
||||
Some(b'+') => (false, &x[1..]),
|
||||
_ => (false, x),
|
||||
};
|
||||
if ed.is_empty() || !ed.bytes().all(digit) {
|
||||
return None;
|
||||
}
|
||||
for b in ed.bytes() {
|
||||
e = (e * 10 + (b - b'0') as i64).min(1 << 40);
|
||||
}
|
||||
if eneg {
|
||||
e = -e;
|
||||
}
|
||||
}
|
||||
let sign = if neg { 0x8000_0000 } else { 0 };
|
||||
let digits: Vec<u8> = int.bytes().chain(frac.bytes()).map(|b| b - b'0').collect();
|
||||
let Some(first) = digits.iter().position(|&d| d != 0) else {
|
||||
// Zero. The integer zero has no sign; a decimal zero keeps its spelling's.
|
||||
return Some(Binary32(if integer { 0 } else { sign }));
|
||||
};
|
||||
let last = digits.iter().rposition(|&d| d != 0).expect("a nonzero digit");
|
||||
let sig = &digits[first..=last];
|
||||
// |value| = sig · 10^e10, and 10^(q − 1) ≤ |value| < 10^q.
|
||||
let e10 = e - frac.len() as i64 + (digits.len() - 1 - last) as i64;
|
||||
let q = e10 + sig.len() as i64;
|
||||
if q >= 40 {
|
||||
// At least 10^39: past the greatest binary32 value's interval (or the integer is
|
||||
// past it).
|
||||
return None;
|
||||
}
|
||||
if q <= -324 {
|
||||
// Below 10^-324, so below 2^-1075: a decimal (never an integer) that rounds to
|
||||
// zero in binary64.
|
||||
return Some(Binary32(sign));
|
||||
}
|
||||
let kept = &sig[..sig.len().min(KEPT_DIGITS)];
|
||||
let d = Decimal::new(kept, e10 + (sig.len() - kept.len()) as i64, kept.len() < sig.len());
|
||||
// A positive binary32 bit pattern's value, `a·2^b`.
|
||||
let value = |bits: u32| -> (u64, i32) {
|
||||
let (x, f) = (bits >> 23, (bits & 0x7f_ffff) as u64);
|
||||
if x == 0 { (f, -149) } else { (f | 1 << 23, x as i32 - 150) }
|
||||
};
|
||||
// The largest pattern whose value is not above |d|, below the infinity's pattern: lo.
|
||||
let (mut lo, mut hi) = (0u32, 0x7f80_0000u32);
|
||||
while hi - lo > 1 {
|
||||
let mid = lo + (hi - lo) / 2;
|
||||
let (a, b) = value(mid);
|
||||
if d.order(a, b) == Ordering::Less {
|
||||
hi = mid;
|
||||
} else {
|
||||
lo = mid;
|
||||
}
|
||||
}
|
||||
if integer {
|
||||
// The integer itself must be the value.
|
||||
let (a, b) = value(lo);
|
||||
return (lo != 0 && d.order(a, b) == Ordering::Equal).then_some(Binary32(sign | lo));
|
||||
}
|
||||
// The closed interval of reals binary64 rounds to the nonzero binary32 value `bits`
|
||||
// (ties go to it: its binary64 significand ends in 29 zero bits, so is even, and its
|
||||
// neighbours' are odd). It is normal in binary64, `m·2^31 · 2^(x − 31)` with `m` in
|
||||
// `[2^23, 2^24)`, whose half unit above is `2·2^(x − 31)`, and below the same, or half
|
||||
// that at a power of two.
|
||||
let bounds = |bits: u32| -> ((u64, i32), (u64, i32)) {
|
||||
let (mut m, mut x) = value(bits);
|
||||
while m < 1 << 23 {
|
||||
m <<= 1;
|
||||
x -= 1;
|
||||
}
|
||||
let below = if m == 1 << 23 { 1 } else { 2 };
|
||||
(((m << 31) - below, x - 31), ((m << 31) + 2, x - 31))
|
||||
};
|
||||
if lo == 0 {
|
||||
// Below the least subnormal: zero where binary64 rounds the decimal to zero, at
|
||||
// most 2^-1075 (the tie goes to zero, whose significand is even).
|
||||
if d.order(1, -1075) != Ordering::Greater {
|
||||
return Some(Binary32(sign));
|
||||
}
|
||||
} else {
|
||||
let (_, (a, b)) = bounds(lo);
|
||||
if d.order(a, b) != Ordering::Greater {
|
||||
return Some(Binary32(sign | lo));
|
||||
}
|
||||
}
|
||||
if hi < 0x7f80_0000 {
|
||||
let ((a, b), _) = bounds(hi);
|
||||
if d.order(a, b) != Ordering::Less {
|
||||
return Some(Binary32(sign | hi));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// A positive decimal `m·10^e`, held as `m·10^max(e, 0)` over `10^max(−e, 0)`, and
|
||||
/// whether digits past `m` were dropped (then it lies above `m·10^e`, by less than one
|
||||
/// unit of `m`'s last digit).
|
||||
struct Decimal {
|
||||
num: Nat,
|
||||
den: Nat,
|
||||
sticky: bool,
|
||||
}
|
||||
|
||||
impl Decimal {
|
||||
fn new(digits: &[u8], e: i64, sticky: bool) -> Decimal {
|
||||
let mut num = Nat(Vec::new());
|
||||
for chunk in digits.chunks(9) {
|
||||
let v = chunk.iter().fold(0u32, |a, &d| a * 10 + d as u32);
|
||||
num.mul_add_small(10u32.pow(chunk.len() as u32), v);
|
||||
}
|
||||
let mut den = Nat(vec![1]);
|
||||
if e >= 0 { num.mul_pow10(e as u64) } else { den.mul_pow10(e.unsigned_abs()) }
|
||||
Decimal { num, den, sticky }
|
||||
}
|
||||
|
||||
/// The decimal's order against `a·2^b`. Exact: `a·2^b` with `b ≥ -1075` is a multiple
|
||||
/// of the last kept digit's unit (see [`KEPT_DIGITS`]), so a dropped tail decides
|
||||
/// only a tie.
|
||||
fn order(&self, a: u64, b: i32) -> Ordering {
|
||||
let (mut l, mut r) = (self.num.clone(), self.den.mul_u64(a));
|
||||
if b >= 0 { r.shl(b as u64) } else { l.shl(b.unsigned_abs() as u64) }
|
||||
match l.order(&r) {
|
||||
Ordering::Equal if self.sticky => Ordering::Greater,
|
||||
o => o,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A non-negative integer, as little-endian base-2^32 limbs with no zero limb on top: the
|
||||
/// exact arithmetic [`binary32_of`] compares in.
|
||||
#[derive(Clone)]
|
||||
struct Nat(Vec<u32>);
|
||||
|
||||
impl Nat {
|
||||
/// `self·k + add`.
|
||||
fn mul_add_small(&mut self, k: u32, add: u32) {
|
||||
let mut carry = add as u64;
|
||||
for l in &mut self.0 {
|
||||
let v = *l as u64 * k as u64 + carry;
|
||||
*l = v as u32;
|
||||
carry = v >> 32;
|
||||
}
|
||||
if carry != 0 {
|
||||
self.0.push(carry as u32);
|
||||
}
|
||||
}
|
||||
|
||||
fn mul_pow10(&mut self, mut k: u64) {
|
||||
while k > 0 {
|
||||
let s = k.min(9);
|
||||
self.mul_add_small(10u32.pow(s as u32), 0);
|
||||
k -= s;
|
||||
}
|
||||
}
|
||||
|
||||
fn mul_u64(&self, a: u64) -> Nat {
|
||||
let mut out = vec![0u32; self.0.len() + 2];
|
||||
for (j, w) in [a as u32, (a >> 32) as u32].into_iter().enumerate() {
|
||||
let mut carry = 0u64;
|
||||
for (i, &l) in self.0.iter().enumerate() {
|
||||
let v = l as u64 * w as u64 + out[i + j] as u64 + carry;
|
||||
out[i + j] = v as u32;
|
||||
carry = v >> 32;
|
||||
}
|
||||
let mut k = self.0.len() + j;
|
||||
while carry != 0 {
|
||||
let v = out[k] as u64 + carry;
|
||||
out[k] = v as u32;
|
||||
carry = v >> 32;
|
||||
k += 1;
|
||||
}
|
||||
}
|
||||
while out.last() == Some(&0) {
|
||||
out.pop();
|
||||
}
|
||||
Nat(out)
|
||||
}
|
||||
|
||||
fn shl(&mut self, bits: u64) {
|
||||
if self.0.is_empty() {
|
||||
return;
|
||||
}
|
||||
let b = (bits % 32) as u32;
|
||||
if b > 0 {
|
||||
let mut carry = 0u32;
|
||||
for l in &mut self.0 {
|
||||
let v = ((*l as u64) << b) | carry as u64;
|
||||
*l = v as u32;
|
||||
carry = (v >> 32) as u32;
|
||||
}
|
||||
if carry != 0 {
|
||||
self.0.push(carry);
|
||||
}
|
||||
}
|
||||
self.0.splice(0..0, std::iter::repeat_n(0, (bits / 32) as usize));
|
||||
}
|
||||
|
||||
fn order(&self, o: &Nat) -> Ordering {
|
||||
self.0.len().cmp(&o.0.len()).then_with(|| self.0.iter().rev().cmp(o.0.iter().rev()))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -234,6 +234,12 @@ pub mod int_wire;
|
|||
#[cfg(feature = "neural_int")]
|
||||
pub mod int_payload;
|
||||
|
||||
// A test of the sources: no floating point and no serde_json parser on the integer decode
|
||||
// paths (the modules above, the entropy coder below and the operator set), outside their
|
||||
// tests and model-checker harnesses.
|
||||
#[cfg(all(test, feature = "neural_int"))]
|
||||
mod float_free;
|
||||
|
||||
#[cfg(feature = "neural")]
|
||||
pub mod neural;
|
||||
// The neural C FFI uses the `ffi` feature's shared buffer and error types, so it
|
||||
|
|
@ -834,9 +840,11 @@ pub mod prior_wire;
|
|||
pub mod scene_map;
|
||||
|
||||
// The strict JSON reader the two mappings above share, the fleet's reader of a relay's session
|
||||
// summaries, and the reader of a `wai.video.int_motion` receipt's JSON. Rust-only.
|
||||
// summaries, the reader of a `wai.video.int_motion` receipt's JSON, the reader of the integer
|
||||
// decoders' parameter-set JSON (`entropy.json`, a `WMB1` bundle's tables) and the splat
|
||||
// export's JSON reader (`int_splat`, which reads its numbers in integer arithmetic). Rust-only.
|
||||
/// cbindgen:ignore
|
||||
#[cfg(any(feature = "prior_wire", feature = "scene_map", feature = "conform_result", feature = "fleet", feature = "video_receipt", feature = "channel_lineup"))]
|
||||
#[cfg(any(feature = "prior_wire", feature = "scene_map", feature = "conform_result", feature = "fleet", feature = "video_receipt", feature = "channel_lineup", feature = "neural_int"))]
|
||||
pub(crate) mod strict_json;
|
||||
// Conformance results (extensions/conformance-results.md): `conform-result/1`, the signed outcome
|
||||
// of a conformance run, which `wai conform` writes and `wai verify` checks; and its corpus.
|
||||
|
|
|
|||
|
|
@ -725,7 +725,7 @@ fn fit_table(sym: &[i32]) -> (Vec<i32>, i32) {
|
|||
/// A `WIS1` payload (integer-payloads §6) that the identity set decodes to `attr`
|
||||
/// (`planes` planes of `s × s` bytes), with these ranges and Gaussian count.
|
||||
pub fn encode_identity(attr: &[u8], planes: u32, s: u32, n_gaussians: u32, ranges: &[(f32, f32)]) -> Vec<u8> {
|
||||
use crate::int_payload::wire::{write_wis1, Wis1};
|
||||
use crate::int_payload::wire::{write_wis1, Binary32, Wis1};
|
||||
let cells = (s * s) as usize;
|
||||
assert_eq!(attr.len(), planes as usize * cells);
|
||||
let mut cdfs = Vec::new();
|
||||
|
|
@ -751,7 +751,7 @@ pub fn encode_identity(attr: &[u8], planes: u32, s: u32, n_gaussians: u32, range
|
|||
s,
|
||||
attr: planes,
|
||||
n_gaussians,
|
||||
ranges: ranges.to_vec(),
|
||||
ranges: ranges.iter().map(|&(lo, hi)| (Binary32(lo.to_bits()), Binary32(hi.to_bits()))).collect(),
|
||||
cdfs,
|
||||
offsets,
|
||||
median_q: vec![0; planes as usize],
|
||||
|
|
@ -1172,7 +1172,10 @@ pub fn decode(c: &Carried, held: &[Vec<u8>]) -> Result<Planes, Reason> {
|
|||
let held = [HeldBytes::File(&set)];
|
||||
let params = resolve(CAPABILITY, Some(&prior), &held, None).map_err(|_| Reason::SetMismatch)?;
|
||||
match decode_int(CAPABILITY, &env.payload, ¶ms) {
|
||||
Ok(IntDecoded::Attr { bytes, n_attr, s, n_gaussians, ranges }) => Ok(Planes { bytes, planes: n_attr, s, n_gaussians, ranges }),
|
||||
Ok(IntDecoded::Attr { bytes, n_attr, s, n_gaussians, ranges }) => {
|
||||
let ranges = ranges.iter().map(|&(lo, hi)| (f32::from_bits(lo.0), f32::from_bits(hi.0))).collect();
|
||||
Ok(Planes { bytes, planes: n_attr, s, n_gaussians, ranges })
|
||||
}
|
||||
_ => Err(Reason::DecodeRefused),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
//! A strict reader of JSON text (RFC 8259) for the documents the tensor-file and scene
|
||||
//! mappings read: `extensions/prior-carriage.md` §9 (a safetensors header) and
|
||||
//! `extensions/scene-carriage.md` §3 (a glTF document).
|
||||
//! `extensions/scene-carriage.md` §3 (a glTF document); for the integer decoders' parameter
|
||||
//! sets, `entropy.json` and a `WMB1` bundle's tables (`extensions/integer-payloads.md` §9 and
|
||||
//! §14), whose integers are read from their spellings; and for the splat export's JSON
|
||||
//! bitstream, whose numbers [`crate::int_splat`] reads in integer arithmetic. No number is
|
||||
//! converted to a float on the way in, so the integer decode paths read their JSON with no
|
||||
//! floating point.
|
||||
//!
|
||||
//! It keeps what a lenient parser loses: members in the order they were written, a
|
||||
//! member named twice (refused), and each number's spelling (a number is never rounded on
|
||||
|
|
@ -40,7 +45,7 @@ impl Json {
|
|||
_ => None,
|
||||
}
|
||||
}
|
||||
#[cfg_attr(not(any(feature = "prior_wire", feature = "scene_map")), allow(dead_code))]
|
||||
#[cfg_attr(not(any(feature = "prior_wire", feature = "scene_map", feature = "neural_int")), allow(dead_code))]
|
||||
pub fn as_arr(&self) -> Option<&[Json]> {
|
||||
match self {
|
||||
Json::Arr(a) => Some(a),
|
||||
|
|
@ -56,12 +61,25 @@ impl Json {
|
|||
}
|
||||
/// A number spelled as a plain non-negative decimal integer (no sign, fraction or
|
||||
/// exponent, no leading zero) that fits `u64`.
|
||||
#[cfg_attr(
|
||||
not(any(feature = "prior_wire", feature = "scene_map", feature = "conform_result", feature = "fleet", feature = "video_receipt", feature = "channel_lineup", feature = "splat_int")),
|
||||
allow(dead_code)
|
||||
)]
|
||||
pub fn as_u64(&self) -> Option<u64> {
|
||||
match self {
|
||||
Json::Num(s) if is_plain_uint(s) => s.parse().ok(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
/// A number spelled as a decimal integer (an optional `-`, then a plain non-negative
|
||||
/// integer: no fraction or exponent) that fits `i64`. `-0` is zero.
|
||||
#[cfg_attr(not(feature = "neural_int"), allow(dead_code))]
|
||||
pub fn as_i64(&self) -> Option<i64> {
|
||||
match self {
|
||||
Json::Num(s) if is_plain_uint(s.strip_prefix('-').unwrap_or(s)) => s.parse().ok(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_plain_uint(s: &str) -> bool {
|
||||
|
|
@ -326,6 +344,11 @@ mod tests {
|
|||
}
|
||||
assert_eq!(Json::Num("12".into()).as_u64(), Some(12));
|
||||
assert_eq!(Json::Num("012".into()).as_u64(), None);
|
||||
for (n, want) in [("-12", Some(-12)), ("-0", Some(0)), ("0", Some(0)), ("-9223372036854775808", Some(i64::MIN)),
|
||||
("9223372036854775807", Some(i64::MAX)), ("9223372036854775808", None), ("-012", None), ("1.0", None),
|
||||
("1e2", None), ("--1", None), ("-", None)] {
|
||||
assert_eq!(Json::Num(n.into()).as_i64(), want, "{n}");
|
||||
}
|
||||
assert_eq!(quote("a\"\u{1}\n"), "\"a\\\"\\u0001\\n\"");
|
||||
let deep = "[".repeat(MAX_DEPTH + 2) + &"]".repeat(MAX_DEPTH + 2);
|
||||
assert!(read(deep.as_bytes()).is_err());
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ fn every_committed_set_lowers_to_its_committed_graphs_and_verdict() {
|
|||
let members = vec![("entropy.json", ent.clone()), ("g_s.whm", g_s.clone()), ("h_s.whm", h_s.clone())];
|
||||
assert_eq!(wai_ops::generate::sets::set_digest(&members), s.digest, "{name}: the set's digest");
|
||||
// The entropy tables are the stage operators' data, not lowered; the set's must load.
|
||||
hyper::CondEntropy::try_from_value(&serde_json::from_slice(&ent).unwrap()).unwrap_or_else(|e| panic!("{name}: entropy.json: {e}"));
|
||||
hyper::CondEntropy::read(&ent).unwrap_or_else(|e| panic!("{name}: entropy.json: {e}"));
|
||||
let lowered = wai_ops::lower::int_hyper(&h_s, &g_s).unwrap_or_else(|r| panic!("{name}: does not lower: {r:?}"));
|
||||
let roles: Vec<&str> = lowered.roles.iter().map(|(r, _)| *r).collect();
|
||||
assert_eq!(roles, ["h_s", "g_s"], "{name}");
|
||||
|
|
@ -189,7 +189,7 @@ fn stream<'a>(k: &'a wai::int_wire::Wih1<'a>) -> hyper::CondStream<'a> {
|
|||
/// A committed set's three members, parsed: `g_s`, `h_s` and the entropy tables.
|
||||
fn set_of(v: &Value) -> (IntModel, IntModel, hyper::CondEntropy) {
|
||||
let m = |k: &str| read(v["set"][k].as_str().unwrap());
|
||||
let ent = hyper::CondEntropy::try_from_value(&serde_json::from_slice(&m("entropy.json")).unwrap()).unwrap();
|
||||
let ent = hyper::CondEntropy::read(&m("entropy.json")).unwrap();
|
||||
(IntModel::from_bytes(&m("g_s.whm")).unwrap(), IntModel::from_bytes(&m("h_s.whm")).unwrap(), ent)
|
||||
}
|
||||
|
||||
|
|
@ -309,7 +309,7 @@ impl Owned {
|
|||
fn tiny(f: impl FnOnce(&mut IntModel, &mut IntModel, &mut hyper::CondEntropy)) -> (IntModel, IntModel, hyper::CondEntropy) {
|
||||
let mut g = IntModel::from_bytes(&read(&format!("{DIR}/tiny_g_s.bin"))).unwrap();
|
||||
let mut h = IntModel::from_bytes(&read(&format!("{DIR}/tiny_h_s.bin"))).unwrap();
|
||||
let mut e = hyper::CondEntropy::try_from_value(&json(&format!("{DIR}/tiny_entropy.json"))).unwrap();
|
||||
let mut e = hyper::CondEntropy::read(&read(&format!("{DIR}/tiny_entropy.json"))).unwrap();
|
||||
f(&mut g, &mut h, &mut e);
|
||||
(g, h, e)
|
||||
}
|
||||
|
|
@ -353,7 +353,7 @@ fn every_committed_refusal_fixture_keeps_its_code() {
|
|||
let k = wai::int_wire::parse_wih1(&nested).unwrap_or_else(|e| panic!("{name}: a decode fixture's keyframe parses: {e:?}"));
|
||||
let g = IntModel::from_bytes(&set["g_s.whm"]).unwrap();
|
||||
let h = IntModel::from_bytes(&set["h_s.whm"]).unwrap();
|
||||
let e = hyper::CondEntropy::try_from_value(&serde_json::from_slice(&set["entropy.json"]).unwrap()).unwrap();
|
||||
let e = hyper::CondEntropy::read(&set["entropy.json"]).unwrap();
|
||||
let ops = hyper::ops::decode(&g, &h, &e, &stream(&k)).map(|d| d.0).map_err(code_of);
|
||||
let direct = hyper::legacy::decode_conditional_checked(&g, &h, &e, &stream(&k)).map(|d| d.0).map_err(code_of);
|
||||
assert!(ops == direct, "{name}: the two decodes of the keyframe differ: {:?} / {:?}", ops.as_ref().err(), direct.as_ref().err());
|
||||
|
|
|
|||
|
|
@ -89,7 +89,8 @@ fn int_splat_cloud() {
|
|||
let d = vroot().join("int_splat");
|
||||
let gs = IntModel::from_bytes(&std::fs::read(d.join("splat_gs.bin")).unwrap()).unwrap();
|
||||
let bs = jload(&d.join("splat.json"));
|
||||
let attr = same_under_every_policy("int_splat", || decode(&gs, &bs).unwrap().attr);
|
||||
let text = std::fs::read(d.join("splat.json")).unwrap();
|
||||
let attr = same_under_every_policy("int_splat", || decode(&gs, &text).unwrap().attr);
|
||||
let hash = wai::int_hyper_synth::fnv1a(&attr);
|
||||
assert_eq!(format!("{hash:016x}"), bs["attr_hash"].as_str().unwrap());
|
||||
sha256_sidecar::one(&d, "splat", "DecodedSplat::attr: the decoded attribute tensor's bytes in decoder output order", &attr);
|
||||
|
|
|
|||
|
|
@ -140,9 +140,7 @@ fn the_declared_cost_bounds_what_the_decode_holds() {
|
|||
// The conditional codec's set: its tables, and its two networks parsed, then
|
||||
// lowered and loaded (`int_hyper_synth::ops::load`).
|
||||
c::CAP_NEURAL_INT_HYPER | c::CAP_VIDEO_INT_HYPER => {
|
||||
let e: serde_json::Value = serde_json::from_slice(&hyp["entropy.json"]).unwrap();
|
||||
let t = wai::int_hyper_synth::CondEntropy::try_from_value(&e).unwrap();
|
||||
drop(e);
|
||||
let t = wai::int_hyper_synth::CondEntropy::read(&hyp["entropy.json"]).unwrap();
|
||||
let (g, h) = (wai::int_hyper_synth::IntModel::from_bytes(&hyp["g_s.whm"]).unwrap(),
|
||||
wai::int_hyper_synth::IntModel::from_bytes(&hyp["h_s.whm"]).unwrap());
|
||||
let n = wai::int_hyper_synth::ops::load(&g, &h).unwrap();
|
||||
|
|
@ -280,7 +278,7 @@ fn the_declared_cost_bounds_what_the_decode_holds() {
|
|||
z_rans: k.z_rans, rans: k.rans,
|
||||
};
|
||||
let (g, h) = (wai::int_hyper_synth::IntModel::from_bytes(&hyp["g_s.whm"]).unwrap(), wai::int_hyper_synth::IntModel::from_bytes(&hyp["h_s.whm"]).unwrap());
|
||||
let ent = wai::int_hyper_synth::CondEntropy::try_from_value(&serde_json::from_slice(&hyp["entropy.json"]).unwrap()).unwrap();
|
||||
let ent = wai::int_hyper_synth::CondEntropy::read(&hyp["entropy.json"]).unwrap();
|
||||
let nets = wai::int_hyper_synth::ops::load(&g, &h).unwrap();
|
||||
let (r, peak) = with_policy(Policy::reference(), || peak_during(|| wai::int_hyper_synth::ops::decode_loaded(&nets, g.afrac, &ent, &st)));
|
||||
r.unwrap();
|
||||
|
|
|
|||
|
|
@ -21,10 +21,10 @@ fn decodes_the_learned_splat_cloud_attribute_for_attribute() {
|
|||
// ML runtime — to the SAME [14,S,S] attribute tensor the Python integer reference
|
||||
// produced. A single disagreeing attribute would diverge this FNV-1a.
|
||||
let gs = IntModel::from_bytes(&std::fs::read(dir().join("splat_gs.bin")).unwrap()).expect("WHM1 g_s");
|
||||
let bs: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(dir().join("splat.json")).unwrap()).unwrap();
|
||||
let text = std::fs::read(dir().join("splat.json")).unwrap();
|
||||
let bs: serde_json::Value = serde_json::from_slice(&text).unwrap();
|
||||
|
||||
let clip = decode(&gs, &bs).expect("decode");
|
||||
let clip = decode(&gs, &text).expect("decode");
|
||||
assert_eq!(clip.n_attr, bs["attr"].as_u64().unwrap() as usize);
|
||||
assert_eq!(clip.attr.len(), clip.n_attr * clip.s * clip.s, "attribute tensor size");
|
||||
assert_eq!(clip.s * clip.s, bs["n_gaussians"].as_u64().unwrap() as usize, "N gaussians = S²");
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ fn every_committed_vector_decodes_through_the_lowering_to_its_golden() {
|
|||
let p = read(payload);
|
||||
let v = wai::int_wire::parse_wis1(&p).unwrap_or_else(|e| panic!("{name}: {e:?}"));
|
||||
let bs = json(from);
|
||||
let j = SplatJson::from_value(&bs).unwrap();
|
||||
let j = SplatJson::read(&read(from)).unwrap();
|
||||
let want = json(sidecar)[name]["sha256"].as_str().unwrap().to_string();
|
||||
for (form, st) in [("WIS1", stream(&v)), ("JSON", j.stream())] {
|
||||
let ops = int_splat::ops::decode_typed(&gs, &st).unwrap_or_else(|r| panic!("{name} {form}: {r}"));
|
||||
|
|
@ -349,7 +349,7 @@ fn every_kernel_policy_decodes_through_the_lowering_alike() {
|
|||
let x0 = wai_ops::lower::whm1(&read(SET), true).unwrap().tensors[0].hi;
|
||||
let at = |t: i64| cloud(move |o| o.median_q[1] = t - ((o.offsets[1] as i64) << o.afrac));
|
||||
let mut clouds: Vec<(String, Owned)> = vec![("WIS1".into(), Owned::committed())];
|
||||
let j = SplatJson::from_value(&json("wai-rs/tests/vectors/int_splat/splat.json")).unwrap();
|
||||
let j = SplatJson::read(&read("wai-rs/tests/vectors/int_splat/splat.json")).unwrap();
|
||||
clouds.push(("JSON".into(), Owned {
|
||||
afrac: j.afrac, c: j.c, sh: j.sh, sw: j.sw, s: j.s, n_attr: j.n_attr,
|
||||
cdfs: j.cdfs.clone(), offsets: j.offsets.clone(), median_q: j.median_q.clone(), rans: j.rans.clone(),
|
||||
|
|
|
|||
|
|
@ -88,8 +88,8 @@ fn bytes_of(d: &IntDecoded) -> Vec<u8> {
|
|||
b.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
for (lo, hi) in ranges {
|
||||
b.extend_from_slice(&lo.to_bits().to_le_bytes());
|
||||
b.extend_from_slice(&hi.to_bits().to_le_bytes());
|
||||
b.extend_from_slice(&lo.0.to_le_bytes());
|
||||
b.extend_from_slice(&hi.0.to_le_bytes());
|
||||
}
|
||||
b.extend_from_slice(bytes);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,14 +14,15 @@ use wai::int_hyper_synth::IntModel;
|
|||
|
||||
/// A decoded splat: the `[n_attr · s · s]` `u8` attribute tensor (channel-major) —
|
||||
/// un-normalized to Gaussians (pos/scale/rot/color/opacity) on the JS side using the
|
||||
/// per-attribute ranges the payload carries (`ranges`).
|
||||
/// per-attribute ranges the payload carries (`ranges`). The ranges are held as their
|
||||
/// binary32 bits, as the decode reads them, and become numbers only in the getter.
|
||||
#[wasm_bindgen]
|
||||
pub struct DecodedSplat {
|
||||
attr: Vec<u8>,
|
||||
n_attr: u32,
|
||||
s: u32,
|
||||
n_gaussians: u32,
|
||||
ranges: Vec<f32>,
|
||||
ranges: Vec<u32>,
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
|
|
@ -47,7 +48,7 @@ impl DecodedSplat {
|
|||
/// Presentation only: the splat-equivalence digest covers `attr`, not these.
|
||||
#[wasm_bindgen(getter)]
|
||||
pub fn ranges(&self) -> Box<[f32]> {
|
||||
self.ranges.clone().into_boxed_slice()
|
||||
self.ranges.iter().map(|&b| f32::from_bits(b)).collect()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -60,7 +61,7 @@ impl DecodedSplat {
|
|||
pub fn decode_wis1(payload: &[u8], g_s: &[u8]) -> Result<DecodedSplat, JsError> {
|
||||
match crate::int_payload::decode("wai.splat.int_codec", payload, &[("g_s.whm", g_s)]).map_err(|e| JsError::new(&e))? {
|
||||
wai::int_payload::IntDecoded::Attr { bytes, n_attr, s, n_gaussians, ranges } => Ok(DecodedSplat {
|
||||
attr: bytes, n_attr, s, n_gaussians, ranges: ranges.iter().flat_map(|&(lo, hi)| [lo, hi]).collect(),
|
||||
attr: bytes, n_attr, s, n_gaussians, ranges: ranges.iter().flat_map(|&(lo, hi)| [lo.0, hi.0]).collect(),
|
||||
}),
|
||||
_ => Err(JsError::new("unsupported_capability: not an attribute tensor")),
|
||||
}
|
||||
|
|
@ -69,16 +70,16 @@ pub fn decode_wis1(payload: &[u8], g_s: &[u8]) -> Result<DecodedSplat, JsError>
|
|||
/// Decode a learned-splat model (`splat_gs.bin`, WHM1) + the tool's JSON bitstream
|
||||
/// (`splat.json`) to the integer attribute tensor in the browser. For debugging and
|
||||
/// conversion only: the payload `wai.splat.int_codec` carries is `WIS1`
|
||||
/// ([`decode_wis1`]). Byte-identical to the native reference.
|
||||
/// ([`decode_wis1`]). Byte-identical to the native reference. The JSON is read as the
|
||||
/// native reader reads it, every number in integer arithmetic, so each range is the
|
||||
/// binary32 value its decimal denotes (one that denotes none is refused).
|
||||
#[wasm_bindgen(js_name = decodeIntSplat)]
|
||||
pub fn decode_int_splat(gs_bin: &[u8], bitstream_json: &[u8]) -> Result<DecodedSplat, JsError> {
|
||||
let gs = IntModel::from_bytes(gs_bin).map_err(JsError::new)?;
|
||||
let bs: serde_json::Value =
|
||||
serde_json::from_slice(bitstream_json).map_err(|e| JsError::new(&e.to_string()))?;
|
||||
let j = wai::int_splat::SplatJson::from_value(&bs).map_err(JsError::new)?;
|
||||
let j = wai::int_splat::SplatJson::read(bitstream_json).map_err(JsError::new)?;
|
||||
let d = wai::int_splat::decode_typed(&gs, &j.stream()).map_err(|r| JsError::new(r.detail))?;
|
||||
Ok(DecodedSplat {
|
||||
attr: d.attr, n_attr: d.n_attr as u32, s: d.s as u32, n_gaussians: j.n_gaussians as u32,
|
||||
ranges: j.ranges.iter().flat_map(|&(lo, hi)| [lo as f32, hi as f32]).collect(),
|
||||
ranges: j.ranges.iter().flat_map(|&(lo, hi)| [lo.0, hi.0]).collect(),
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,17 +116,17 @@ fn cond_decode(
|
|||
Ok(DecodedImage { rgb, width: width as u32, height: height as u32 })
|
||||
}
|
||||
|
||||
/// The debug JSON decode, through the checked readers (`CondEntropy::try_from_value`,
|
||||
/// `CondBitstream::try_from_value`) and `decode_conditional_checked`, so a JSON the
|
||||
/// export tool did not write is refused, never a panic: the pure-Rust half the native
|
||||
/// tests run.
|
||||
/// The debug JSON decode, through the checked readers (`CondEntropy::read`, which reads
|
||||
/// the tables as a set's decode does, and `CondBitstream::try_from_value`) and
|
||||
/// `decode_conditional_checked`, so a JSON the export tool did not write is refused, never
|
||||
/// a panic: the pure-Rust half the native tests run.
|
||||
pub(crate) fn cond_decode_checked(
|
||||
g_s: &wai::int_hyper_synth::IntModel, h_s: &wai::int_hyper_synth::IntModel, entropy_json: &[u8], bitstream_json: &[u8],
|
||||
) -> Result<(Vec<u8>, usize, usize), String> {
|
||||
use wai::int_hyper_synth::{decode_conditional_checked, CondBitstream, CondEntropy};
|
||||
let parse = |b: &[u8]| serde_json::from_slice::<serde_json::Value>(b).map_err(|e| e.to_string());
|
||||
let ent = CondEntropy::try_from_value(&parse(entropy_json)?).map_err(|e| format!("model_mismatch: {e}"))?;
|
||||
let bs = CondBitstream::try_from_value(&parse(bitstream_json)?).map_err(|k| format!("field_range: `{k}`"))?;
|
||||
let ent = CondEntropy::read(entropy_json).map_err(|e| format!("model_mismatch: {e}"))?;
|
||||
let bs = serde_json::from_slice::<serde_json::Value>(bitstream_json).map_err(|e| e.to_string())?;
|
||||
let bs = CondBitstream::try_from_value(&bs).map_err(|k| format!("field_range: `{k}`"))?;
|
||||
decode_conditional_checked(g_s, h_s, &ent, &bs.stream()).map_err(|r| r.to_string())
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue