From eb36fa03bc9d2654dbcfafd34be4e9a5972ab8c7 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Sun, 11 Oct 2026 22:20:30 -0400 Subject: [PATCH 01/10] =?UTF-8?q?wai:=20WIP,=20count=20a=20manifest's=20va?= =?UTF-8?q?lues=20before=20parsing=20it=20(SPEC=20=C2=A72=20reader=20limit?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check_manifest_text: one pass refusing a member named twice and more than 2^22 values (manifest_too_many_values); the readers and writers use it. The memory probe is scratch and will become the measured test. Co-Authored-By: Claude Opus 5.5 --- wai/wai-rs/src/container.rs | 106 +++++++++++++--- wai/wai-rs/src/http_binding/head.rs | 6 +- wai/wai-rs/tests/manifest_parse_memory.rs | 140 ++++++++++++++++++++++ wai/wai-web/src/lib.rs | 7 +- 4 files changed, 235 insertions(+), 24 deletions(-) create mode 100644 wai/wai-rs/tests/manifest_parse_memory.rs diff --git a/wai/wai-rs/src/container.rs b/wai/wai-rs/src/container.rs index afc5363ae..bd6e0146b 100644 --- a/wai/wai-rs/src/container.rs +++ b/wai/wai-rs/src/container.rs @@ -41,12 +41,22 @@ pub const MAGIC_V2: &[u8; 4] = b"WAI2"; // v1.1 multi-rendition /// `WIR1` layer may cover (staged-delivery Appendix A.3, `n_units ≤ 2^20`), about 90 /// bytes a run. The reference readers refuse a longer one before they read it, and the /// reference writers refuse to write one, so `man_len` never asks a reader to read -/// more. It bounds the bytes read, not the memory a parse takes: the reference readers -/// build a JSON tree, which can take some 40 times the manifest's length. The -/// manifest's nesting has its own limit: the JSON reader refuses 128 nested arrays and -/// objects and takes 127 (`check_unique_members`). +/// more. It bounds the bytes read; [`MANIFEST_MAX_VALUES`] bounds what a parse builds +/// from them. The manifest's nesting has its own limit: the JSON reader refuses 128 +/// nested arrays and objects and takes 127 ([`check_manifest_text`]). pub const MANIFEST_MAX_LEN: usize = 1 << 27; +/// The most JSON values a manifest may hold that a reader must accept (SPEC §2, *Reader +/// limits*): 2^22, counting the manifest object and every value nested in it, at any +/// depth; a member's name is not a value. It is sized to the same layer as +/// [`MANIFEST_MAX_LEN`]: a `prev` that names a different producer for each of 2^20 units +/// is three values a run (the run, its `producer` and its `count`), 3·2^20 and a few more +/// in all. The reference readers count a manifest's values before they parse it +/// ([`check_manifest_text`]) and refuse it as soon as the count passes the limit +/// (`manifest_too_many_values`), so the JSON tree a parse builds holds at most this many +/// values, whatever the manifest's length. +pub const MANIFEST_MAX_VALUES: usize = 1 << 22; + /// SPEC §2, *Reader limits*: `Err` with the code `manifest_too_large` when a manifest /// of `len` bytes is longer than [`MANIFEST_MAX_LEN`]. pub fn check_manifest_len(len: u64) -> Result<(), String> { @@ -75,7 +85,10 @@ impl WriteLimits { /// A manifest's JSON as the writers emit it (no insignificant whitespace, members in /// order), refused with `manifest_too_large` as soon as it would pass `cap` bytes -/// ([`MANIFEST_MAX_LEN`] in the writers), so a writer never builds more of it than that. +/// ([`MANIFEST_MAX_LEN`] in the writers), so a writer never builds more of it than that; +/// then read as a reader reads it ([`check_manifest_text`]), so a writer never emits a +/// manifest past [`MANIFEST_MAX_VALUES`] (`manifest_too_many_values`), nested past 127, or +/// naming a member twice. fn manifest_json(m: &T, cap: usize) -> std::io::Result> { struct Capped(Vec, usize); impl std::io::Write for Capped { @@ -93,6 +106,7 @@ fn manifest_json(m: &T, cap: usize) -> std::io::Result> { } let mut w = Capped(Vec::new(), cap); serde_json::to_writer(&mut w, m).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + check_manifest_text(&w.0).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; Ok(w.0) } @@ -1299,7 +1313,7 @@ impl Wai { check_manifest_len(ml as u64) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; need(8, ml, "manifest body")?; - check_unique_members(&bytes[8..8 + ml]) + check_manifest_text(&bytes[8..8 + ml]) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; let m: Manifest = serde_json::from_slice(&bytes[8..8 + ml]) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; @@ -1531,18 +1545,52 @@ impl Wai { /// (SPEC §2: member names are unique within an object). `serde_json` keeps the last of /// two equal names in a `Value` and refuses them only in a struct's own fields, so a /// manifest is read once more by this check, which refuses any repeat. Names compare -/// after escapes are read, so `"a"` and `"\u0061"` are one name. The text's own JSON -/// errors are reported as they are. +/// after escapes are read, so `"a"` and `"a"` are one name. The text's own JSON +/// errors are reported as they are. A manifest is read by [`check_manifest_text`], +/// which makes this check and counts the manifest's values; this one counts nothing. pub fn check_unique_members(text: &[u8]) -> Result<(), serde_json::Error> { + scan_json(text, usize::MAX).map(|_| ()) +} + +/// The checks a reader makes on a manifest's text before it builds anything from it +/// (SPEC §2, *Reader limits*): it is JSON, it names no member twice in one object +/// ([`check_unique_members`]), it nests at most 127 deep, and it holds at most +/// [`MANIFEST_MAX_VALUES`] values, refused with `manifest_too_many_values` as soon as the +/// count passes the limit, so a manifest past it is read only that far. The pass builds +/// no value: it holds the member names of each object open on the path to the value it +/// reads, borrowed from `text` unless a name carries an escape. Returns the number of +/// values. +pub fn check_manifest_text(text: &[u8]) -> Result { + scan_json(text, MANIFEST_MAX_VALUES) +} + +/// One pass over the JSON text `text`: refuses a member named twice in one object, and +/// more than `max_values` values; returns how many values it holds. +fn scan_json(text: &[u8], max_values: usize) -> Result { use serde::de::{DeserializeSeed, Deserializer, Error as _, MapAccess, SeqAccess, Visitor}; - struct Unique; - impl<'de> DeserializeSeed<'de> for Unique { + use std::borrow::Cow; + use std::cell::Cell; + /// One value, counted before it is read. + #[derive(Clone, Copy)] + struct Scan<'c> { + values: &'c Cell, + max: usize, + } + impl<'de> DeserializeSeed<'de> for Scan<'_> { type Value = (); fn deserialize>(self, d: D) -> Result<(), D::Error> { - d.deserialize_any(Unique) + let n = self.values.get() + 1; + if n > self.max { + return Err(D::Error::custom(format_args!( + "manifest_too_many_values: the manifest holds more than {} values, the most a reader need accept (SPEC §2)", + self.max + ))); + } + self.values.set(n); + d.deserialize_any(self) } } - impl<'de> Visitor<'de> for Unique { + impl<'de> Visitor<'de> for Scan<'_> { type Value = (); fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("a JSON value") @@ -1566,23 +1614,45 @@ pub fn check_unique_members(text: &[u8]) -> Result<(), serde_json::Error> { Ok(()) } fn visit_seq>(self, mut seq: A) -> Result<(), A::Error> { - while seq.next_element_seed(Unique)?.is_some() {} + while seq.next_element_seed(self)?.is_some() {} Ok(()) } fn visit_map>(self, mut map: A) -> Result<(), A::Error> { let mut seen = std::collections::HashSet::new(); - while let Some(k) = map.next_key::()? { + while let Some(k) = map.next_key_seed(Name)? { if !seen.insert(k) { return Err(A::Error::custom("an object names a member twice")); } - map.next_value_seed(Unique)?; + map.next_value_seed(self)?; } Ok(()) } } + /// A member's name, borrowed from the text when it carries no escape. + struct Name; + impl<'de> DeserializeSeed<'de> for Name { + type Value = Cow<'de, str>; + fn deserialize>(self, d: D) -> Result, D::Error> { + d.deserialize_str(Name) + } + } + impl<'de> Visitor<'de> for Name { + type Value = Cow<'de, str>; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("a member name") + } + fn visit_borrowed_str(self, s: &'de str) -> Result, E> { + Ok(Cow::Borrowed(s)) + } + fn visit_str(self, s: &str) -> Result, E> { + Ok(Cow::Owned(s.to_owned())) + } + } + let values = Cell::new(0); let mut de = serde_json::Deserializer::from_slice(text); - Unique.deserialize(&mut de)?; - de.end() + Scan { values: &values, max: max_values }.deserialize(&mut de)?; + de.end()?; + Ok(values.get()) } /// The manifest bytes of a `WAI1` or `WAI2` envelope, exactly as they were @@ -2611,7 +2681,7 @@ impl WaiMulti { check_manifest_len(ml as u64) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; need(8, ml, "manifest body")?; - check_unique_members(&bytes[8..8 + ml]) + check_manifest_text(&bytes[8..8 + ml]) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; let manifest: ManifestV2 = serde_json::from_slice(&bytes[8..8 + ml]) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; diff --git a/wai/wai-rs/src/http_binding/head.rs b/wai/wai-rs/src/http_binding/head.rs index 83fe94929..4ef7e5157 100644 --- a/wai/wai-rs/src/http_binding/head.rs +++ b/wai/wai-rs/src/http_binding/head.rs @@ -4,7 +4,7 @@ //! component's byte range and, through each entry's `sha256`, its digest. use super::{RangeRefusal, DOMAIN_LEN}; -use crate::container::{check_unique_members, Manifest, ManifestV2}; +use crate::container::{check_manifest_text, Manifest, ManifestV2}; /// Which container an object is. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -137,7 +137,7 @@ impl Head { return Err(RangeRefusal::HeadMalformed("not a WAI1 head".into())); } let b = self.manifest_bytes(); - check_unique_members(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))?; + check_manifest_text(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))?; serde_json::from_slice(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string())) } @@ -147,7 +147,7 @@ impl Head { return Err(RangeRefusal::HeadMalformed("not a WAI2 head".into())); } let b = self.manifest_bytes(); - check_unique_members(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))?; + check_manifest_text(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))?; serde_json::from_slice(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string())) } diff --git a/wai/wai-rs/tests/manifest_parse_memory.rs b/wai/wai-rs/tests/manifest_parse_memory.rs new file mode 100644 index 000000000..627a18c96 --- /dev/null +++ b/wai/wai-rs/tests/manifest_parse_memory.rs @@ -0,0 +1,140 @@ +//! Probe (scratch): peak allocation of the manifest readers on large manifests. +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; +use std::sync::Mutex; + +struct Counting; +static NOW: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); +fn grow(n: usize) { + let now = NOW.fetch_add(n, Relaxed) + n; + PEAK.fetch_max(now, Relaxed); +} +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + let p = unsafe { System.alloc(l) }; + if !p.is_null() { + grow(l.size()); + } + p + } + unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 { + let p = unsafe { System.alloc_zeroed(l) }; + if !p.is_null() { + grow(l.size()); + } + p + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + unsafe { System.dealloc(p, l) }; + NOW.fetch_sub(l.size(), Relaxed); + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { + let q = unsafe { System.realloc(p, l, new) }; + if !q.is_null() { + grow(new); + NOW.fetch_sub(l.size(), Relaxed); + } + q + } +} +#[global_allocator] +static A: Counting = Counting; +static ONE: Mutex<()> = Mutex::new(()); + +fn peak_of(f: impl FnOnce() -> T) -> (T, usize, f64) { + let base = NOW.load(Relaxed); + PEAK.store(base, Relaxed); + let t = std::time::Instant::now(); + let r = f(); + (r, PEAK.load(Relaxed) - base, t.elapsed().as_secs_f64()) +} + +fn envelope(m: &str) -> Vec { + let mut b = b"WAI1".to_vec(); + b.extend_from_slice(&(m.len() as u32).to_le_bytes()); + b.extend_from_slice(m.as_bytes()); + b.extend_from_slice(&3u32.to_le_bytes()); + b.extend_from_slice(b"abc"); + b +} +const HEAD: &str = r#"{"wai":"1.0","media":"text","intent":"replicate","model_requirement":{"capability":"wai.text.zstd"},"conditioning":{"kind":"zstd"},"target":"#; + +fn zeros(len: usize) -> String { + let mut m = String::with_capacity(len); + m.push_str(HEAD); + m.push('['); + while m.len() + 3 < len { + m.push_str("0,"); + } + m.push('0'); + while m.len() + 2 < len { + m.push(' '); + } + m.push_str("]}"); + assert_eq!(m.len(), len); + m +} + +fn layer(n: usize) -> String { + let a = "ab".repeat(32); + let b = "cd".repeat(32); + let mut m = String::new(); + m.push_str(r#"{"wai":"1.0","media":"image","intent":"replicate","model_requirement":{"capability":"wai.image.int_refine","fallback":null},"conditioning":{"kind":"int_refine"},"refines":{"base":""#); + m.push_str(&a); + m.push_str(r#"","prev":["#); + for i in 0..n { + if i > 0 { + m.push(','); + } + m.push_str(r#"{"producer":""#); + m.push_str(if i % 2 == 0 { &a } else { &b }); + m.push_str(r#"","count":1}"#); + } + m.push_str(r#"],"layer":2},"target":null}"#); + m +} + + +const LIM: usize = 1 << 22; +const HEAD2: &str = r#"{"wai":"1.0","media":"text","intent":"replicate","model_requirement":{"capability":"wai.text.zstd"},"conditioning":{"kind":"zstd"},"target":null,"pad":"#; +// 10 values before the pad's elements: the manifest, wai, media, intent, model_requirement, +// capability, conditioning, kind, target, pad. +fn pad(n: usize, elem: impl Fn(usize, &mut String), open: &str, close: &str) -> String { + let mut m = String::from(HEAD2); + m.push_str(open); + for i in 0..n { + if i > 0 { m.push(','); } + elem(i, &mut m); + } + m.push_str(close); + m.push('}'); + m +} + +#[test] +fn probe() { + let _g = ONE.lock().unwrap(); + let n = LIM - 10; + let cases: Vec<(&str, String)> = vec![ + ("pad zeros", pad(n, |_, m| m.push('0'), "[", "]")), + ("pad members", pad(n, |i, m| { m.push('"'); m.push_str(&format!("{i:x}")); m.push_str("\":0"); }, "{", "}")), + ("pad 1-member objs", pad(n / 2, |_, m| m.push_str("{\"a\":0}"), "[", "]")), + ("pad empty objs", pad(n, |_, m| m.push_str("{}"), "[", "]")), + ("pad strings 30", pad(n, |_, m| m.push_str("\"abcdefghijklmnopqrstuvwxyzabcd\""), "[", "]")), + ("layer 2^20", layer(1 << 20)), + ("zeros 2^27 (refused)", zeros(1 << 27)), + ("pad zeros +1 (refused)", pad(n + 1, |_, m| m.push('0'), "[", "]")), + ]; + for (name, m) in cases { + let env = envelope(&m); + let ml = m.len(); + drop(m); + let (c, p0, t0) = peak_of(|| wai::container::check_manifest_text(&env[8..8 + ml]).map_err(|e| e.to_string())); + let (r, p, t) = peak_of(|| wai::container::Wai::from_bytes(&env).map(|w| w.manifest.target.is_null())); + eprintln!( + "{name}: len {ml} values {:?} scan {} B {t0:.2}s from_bytes peak {} B = {:.2}x len, {:.1} B/value {t:.2}s ok={}", + c.as_ref().map_err(|e| &e[..40]), p0, p, p as f64 / ml as f64, p as f64 / LIM as f64, r.is_ok() + ); + } +} diff --git a/wai/wai-web/src/lib.rs b/wai/wai-web/src/lib.rs index a8e4b1a14..13af9fbad 100644 --- a/wai/wai-web/src/lib.rs +++ b/wai/wai-web/src/lib.rs @@ -494,10 +494,11 @@ fn pack_refusal(m: &wai::container::Manifest, payload: &[u8]) -> Result<(), Stri } /// A manifest a writer emits as given, read as every reader reads it (SPEC §2): a -/// member named twice in one object, at any depth, is refused, so a writer never emits -/// an envelope the reader refuses. +/// member named twice in one object, at any depth, and a manifest past the value limit +/// (`manifest_too_many_values`) are refused, so a writer never emits an envelope the +/// reader refuses. fn read_manifest(json: &str) -> Result { - wai::container::check_unique_members(json.as_bytes()).map_err(|e| format!("manifest parse: {e}"))?; + wai::container::check_manifest_text(json.as_bytes()).map_err(|e| format!("manifest parse: {e}"))?; serde_json::from_str(json).map_err(|e| format!("manifest parse: {e}")) } -- 2.54.0 (Apple Git-157) From ddc695d23f0c46ac06235cc7345f066e6522f459 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Sun, 11 Oct 2026 22:57:23 -0400 Subject: [PATCH 02/10] =?UTF-8?q?wai:=20read=20a=20manifest=20within=20a?= =?UTF-8?q?=20bound=20of=20its=20values=20and=20length=20(SPEC=20=C2=A72)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit read_manifest and read_manifest_v2 count the manifest's values in the first pass (refusing past 2^22, manifest_too_many_values), then build one JSON tree at the sizes counted and move its members into the manifest types, so no member is held twice as serde's flatten holds it. Every reference reader (Wai, WaiMulti, the HTTP head) reads through them; the writers refuse what they refuse. tests/manifest_parse_memory.rs measures the peak with a counting allocator against 128 bytes a value plus one a byte; tests/manifest_reader_equivalence.rs holds the reader to serde's derived reader over every committed envelope and 1488 mutations. Co-Authored-By: Claude Opus 5.5 --- wai/wai-rs/include/wai.h | 20 +- wai/wai-rs/src/container.rs | 334 +++++++++++++++++- wai/wai-rs/src/http_binding/head.rs | 10 +- wai/wai-rs/src/http_binding/tests.rs | 16 + wai/wai-rs/tests/manifest_parse_memory.rs | 278 +++++++++++---- .../tests/manifest_reader_equivalence.rs | 171 +++++++++ 6 files changed, 737 insertions(+), 92 deletions(-) create mode 100644 wai/wai-rs/tests/manifest_reader_equivalence.rs diff --git a/wai/wai-rs/include/wai.h b/wai/wai-rs/include/wai.h index 07d1f35c3..6c6a0c46a 100644 --- a/wai/wai-rs/include/wai.h +++ b/wai/wai-rs/include/wai.h @@ -37,13 +37,25 @@ * `WIR1` layer may cover (staged-delivery Appendix A.3, `n_units ≤ 2^20`), about 90 * bytes a run. The reference readers refuse a longer one before they read it, and the * reference writers refuse to write one, so `man_len` never asks a reader to read - * more. It bounds the bytes read, not the memory a parse takes: the reference readers - * build a JSON tree, which can take some 40 times the manifest's length. The - * manifest's nesting has its own limit: the JSON reader refuses 128 nested arrays and - * objects and takes 127 (`check_unique_members`). + * more. It bounds the bytes read; [`MANIFEST_MAX_VALUES`] bounds what a parse builds + * from them. The manifest's nesting has its own limit: the JSON reader refuses 128 + * nested arrays and objects and takes 127 ([`check_manifest_text`]). */ #define MANIFEST_MAX_LEN (1 << 27) +/** + * The most JSON values a manifest may hold that a reader must accept (SPEC §2, *Reader + * limits*): 2^22, counting the manifest object and every value nested in it, at any + * depth; a member's name is not a value. It is sized to the same layer as + * [`MANIFEST_MAX_LEN`]: a `prev` that names a different producer for each of 2^20 units + * is three values a run (the run, its `producer` and its `count`), 3·2^20 and a few more + * in all. The reference readers count a manifest's values before they parse it + * ([`check_manifest_text`]) and refuse it as soon as the count passes the limit + * (`manifest_too_many_values`), so the JSON tree a parse builds holds at most this many + * values, whatever the manifest's length. + */ +#define MANIFEST_MAX_VALUES (1 << 22) + #define WAI_OK 0 #define WAI_ERR_INVALID_ARG 1 diff --git a/wai/wai-rs/src/container.rs b/wai/wai-rs/src/container.rs index bd6e0146b..b7b223f05 100644 --- a/wai/wai-rs/src/container.rs +++ b/wai/wai-rs/src/container.rs @@ -1313,9 +1313,7 @@ impl Wai { check_manifest_len(ml as u64) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; need(8, ml, "manifest body")?; - check_manifest_text(&bytes[8..8 + ml]) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - let m: Manifest = serde_json::from_slice(&bytes[8..8 + ml]) + let m = read_manifest(&bytes[8..8 + ml]) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; let p_off = 8 + ml; need(p_off, 4, "payload length")?; @@ -1549,7 +1547,7 @@ impl Wai { /// errors are reported as they are. A manifest is read by [`check_manifest_text`], /// which makes this check and counts the manifest's values; this one counts nothing. pub fn check_unique_members(text: &[u8]) -> Result<(), serde_json::Error> { - scan_json(text, usize::MAX).map(|_| ()) + scan_json(text, usize::MAX, None).map(|_| ()) } /// The checks a reader makes on a manifest's text before it builds anything from it @@ -1561,20 +1559,40 @@ pub fn check_unique_members(text: &[u8]) -> Result<(), serde_json::Error> { /// reads, borrowed from `text` unless a name carries an escape. Returns the number of /// values. pub fn check_manifest_text(text: &[u8]) -> Result { - scan_json(text, MANIFEST_MAX_VALUES) + scan_json(text, MANIFEST_MAX_VALUES, None) } /// One pass over the JSON text `text`: refuses a member named twice in one object, and -/// more than `max_values` values; returns how many values it holds. -fn scan_json(text: &[u8], max_values: usize) -> Result { +/// more than `max_values` values; returns how many values it holds. With `sizes`, pushes +/// each array's and object's number of elements or members, the containers in the order +/// they open. +fn scan_json(text: &[u8], max_values: usize, sizes: Option<&std::cell::RefCell>>) -> Result { use serde::de::{DeserializeSeed, Deserializer, Error as _, MapAccess, SeqAccess, Visitor}; use std::borrow::Cow; - use std::cell::Cell; + use std::cell::{Cell, RefCell}; /// One value, counted before it is read. #[derive(Clone, Copy)] struct Scan<'c> { values: &'c Cell, max: usize, + sizes: Option<&'c RefCell>>, + } + impl Scan<'_> { + /// A slot for the size of the container that opens now. + fn open(self) -> Option { + self.sizes.map(|s| { + let mut s = s.borrow_mut(); + s.push(0); + s.len() - 1 + }) + } + /// The container in `slot` closed with `n` elements or members (at most + /// `max_values`, so within `u32` for a manifest). + fn close(self, slot: Option, n: usize) { + if let (Some(s), Some(i)) = (self.sizes, slot) { + s.borrow_mut()[i] = u32::try_from(n).unwrap_or(u32::MAX); + } + } } impl<'de> DeserializeSeed<'de> for Scan<'_> { type Value = (); @@ -1614,10 +1632,16 @@ fn scan_json(text: &[u8], max_values: usize) -> Result Ok(()) } fn visit_seq>(self, mut seq: A) -> Result<(), A::Error> { - while seq.next_element_seed(self)?.is_some() {} + let slot = self.open(); + let mut n = 0; + while seq.next_element_seed(self)?.is_some() { + n += 1; + } + self.close(slot, n); Ok(()) } fn visit_map>(self, mut map: A) -> Result<(), A::Error> { + let slot = self.open(); let mut seen = std::collections::HashSet::new(); while let Some(k) = map.next_key_seed(Name)? { if !seen.insert(k) { @@ -1625,6 +1649,7 @@ fn scan_json(text: &[u8], max_values: usize) -> Result } map.next_value_seed(self)?; } + self.close(slot, seen.len()); Ok(()) } } @@ -1650,11 +1675,250 @@ fn scan_json(text: &[u8], max_values: usize) -> Result } let values = Cell::new(0); let mut de = serde_json::Deserializer::from_slice(text); - Scan { values: &values, max: max_values }.deserialize(&mut de)?; + Scan { values: &values, max: max_values, sizes }.deserialize(&mut de)?; de.end()?; Ok(values.get()) } +/// A `WAI1` manifest read from its text as the reference readers read it (SPEC §2): the +/// checks of [`check_manifest_text`], then the [`Manifest`] that +/// `serde_json::from_slice::` builds from the same text, built with less +/// memory. The text is read once more into one JSON tree, each array and object +/// allocated at the size the first pass counted, and the manifest's members are moved +/// out of that tree into their fields; nothing is held twice, where serde's derived +/// reader holds every member a type does not define as an intermediate copy while it +/// builds the tree it keeps. So what a parse allocates is bounded by the value limit +/// and the text's length (SPEC §2, *Reader limits*; `tests/manifest_parse_memory.rs`). +pub fn read_manifest(text: &[u8]) -> Result { + manifest_from_tree(read_tree(text)?) +} + +/// A `WAI2` manifest (SPEC §7.1) read as [`read_manifest`] reads a `WAI1`'s. A +/// `renditions` longer than the 65 535 rows a `WAI2` table holds is refused before its +/// entries are built. +pub fn read_manifest_v2(text: &[u8]) -> Result { + manifest_v2_from_tree(read_tree(text)?) +} + +/// The JSON tree of the manifest text `text`, after [`check_manifest_text`]'s checks, +/// each array and object allocated at its size. +fn read_tree(text: &[u8]) -> Result { + use serde::de::{DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; + use std::cell::{Cell, RefCell}; + let sizes = RefCell::new(Vec::new()); + scan_json(text, MANIFEST_MAX_VALUES, Some(&sizes))?; + let mut sizes = sizes.into_inner(); + sizes.shrink_to_fit(); + /// One value of the tree; `at` is the next container's index in `sizes`. + #[derive(Clone, Copy)] + struct Tree<'s> { + sizes: &'s [u32], + at: &'s Cell, + } + impl Tree<'_> { + fn next_size(self) -> usize { + let i = self.at.get(); + self.at.set(i + 1); + self.sizes.get(i).map_or(0, |&n| n as usize) + } + } + impl<'de> DeserializeSeed<'de> for Tree<'_> { + type Value = Value; + fn deserialize>(self, d: D) -> Result { + d.deserialize_any(self) + } + } + // As `serde_json`'s own `Value` visitor builds each value. + impl<'de> Visitor<'de> for Tree<'_> { + type Value = Value; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("a JSON value") + } + fn visit_bool(self, b: bool) -> Result { + Ok(Value::Bool(b)) + } + fn visit_i64(self, n: i64) -> Result { + Ok(Value::Number(n.into())) + } + fn visit_u64(self, n: u64) -> Result { + Ok(Value::Number(n.into())) + } + fn visit_f64(self, x: f64) -> Result { + Ok(serde_json::Number::from_f64(x).map_or(Value::Null, Value::Number)) + } + fn visit_str(self, s: &str) -> Result { + Ok(Value::String(s.to_owned())) + } + fn visit_string(self, s: String) -> Result { + Ok(Value::String(s)) + } + fn visit_unit(self) -> Result { + Ok(Value::Null) + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut v = Vec::with_capacity(self.next_size()); + while let Some(x) = seq.next_element_seed(self)? { + v.push(x); + } + Ok(Value::Array(v)) + } + fn visit_map>(self, mut map: A) -> Result { + let mut m = Map::with_capacity(self.next_size()); + while let Some(k) = map.next_key::()? { + let v = map.next_value_seed(self)?; + m.insert(k, v); + } + Ok(Value::Object(m)) + } + } + let at = Cell::new(0); + let mut de = serde_json::Deserializer::from_slice(text); + let v = Tree { sizes: &sizes, at: &at }.deserialize(&mut de)?; + de.end()?; + Ok(v) +} + +// The manifest types built from a tree by moving its members, each member read as +// serde's derived reader reads it: a field `T` from its value as `T` reads it, a missing +// field refused unless the type gives it a default (an `Option` is `None`, `target` is +// `null`), an `Option` whose value is `null` is `None`, and the members a type does not +// define left in order in `extra`. `tests/manifest_reader_equivalence.rs` holds the two +// readers to the same result. +type Members = Map; + +fn object_of(v: Value, what: &str) -> Result { + match v { + Value::Object(m) => Ok(m), + _ => Err(serde::de::Error::custom(format_args!("invalid type: {what} is not an object"))), + } +} + +fn required(m: &mut Members, k: &'static str) -> Result { + match m.shift_remove(k) { + Some(v) => serde_json::from_value(v), + None => Err(serde::de::Error::missing_field(k)), + } +} + +fn optional(m: &mut Members, k: &str) -> Result, serde_json::Error> { + match m.shift_remove(k) { + Some(v) => serde_json::from_value(v), + None => Ok(None), + } +} + +/// An `Option` member, moved: `null` or absent is `None`. +fn optional_tree(m: &mut Members, k: &str) -> Option { + m.shift_remove(k).filter(|v| !v.is_null()) +} + +/// An `Option` member whose `T` is one of the manifest types. +fn optional_of(m: &mut Members, k: &str, build: fn(Members) -> Result) -> Result, serde_json::Error> { + match m.shift_remove(k) { + None | Some(Value::Null) => Ok(None), + Some(v) => build(object_of(v, k)?).map(Some), + } +} + +fn required_of(m: &mut Members, k: &'static str, build: fn(Members) -> Result) -> Result { + match m.shift_remove(k) { + None => Err(serde::de::Error::missing_field(k)), + Some(v) => build(object_of(v, k)?), + } +} + +fn model_requirement_from(mut m: Members) -> Result { + Ok(ModelRequirement { + capability: required(&mut m, "capability")?, + fallback: optional(&mut m, "fallback")?, + min_sim: optional(&mut m, "min_sim")?, + sim_metric: optional(&mut m, "sim_metric")?, + extra: m, + }) +} + +fn supersedes_from(mut m: Members) -> Result { + Ok(Supersedes { sha256: required(&mut m, "sha256")?, form: optional(&mut m, "form")?, extra: m }) +} + +fn prior_from(mut m: Members) -> Result { + Ok(Prior { + id: required(&mut m, "id")?, + sha256: required(&mut m, "sha256")?, + determinism: required(&mut m, "determinism")?, + form: optional(&mut m, "form")?, + tier: optional(&mut m, "tier")?, + supersedes: optional_of(&mut m, "supersedes", supersedes_from)?, + extra: m, + }) +} + +fn conditioning_from(mut m: Members) -> Result { + Ok(Conditioning { kind: required(&mut m, "kind")?, prior: optional_of(&mut m, "prior", prior_from)?, extra: m }) +} + +fn security_from(mut m: Members) -> Result { + Ok(Security { encryption: optional(&mut m, "encryption")?, kid: optional(&mut m, "kid")?, extra: m }) +} + +fn manifest_from_tree(v: Value) -> Result { + let mut m = object_of(v, "the manifest")?; + Ok(Manifest { + wai: required(&mut m, "wai")?, + media: required(&mut m, "media")?, + intent: required(&mut m, "intent")?, + model_requirement: required_of(&mut m, "model_requirement", model_requirement_from)?, + conditioning: required_of(&mut m, "conditioning", conditioning_from)?, + security: optional_of(&mut m, "security", security_from)?, + refines: optional_tree(&mut m, "refines"), + derived_from: optional_tree(&mut m, "derived_from"), + target: m.shift_remove("target").unwrap_or(Value::Null), + extra: m, + }) +} + +fn rendition_from(mut m: Members) -> Result { + Ok(RenditionMeta { + capability: required(&mut m, "capability")?, + kind: required(&mut m, "kind")?, + id: optional(&mut m, "id")?, + sha256: optional(&mut m, "sha256")?, + prior: optional_of(&mut m, "prior", prior_from)?, + role: optional(&mut m, "role")?, + companion_of: optional(&mut m, "companion_of")?, + companion_of_id: optional(&mut m, "companion_of_id")?, + refines: optional_tree(&mut m, "refines"), + envelope_manifest: optional_tree(&mut m, "envelope_manifest"), + derived_from: optional_tree(&mut m, "derived_from"), + extra: m, + }) +} + +fn manifest_v2_from_tree(v: Value) -> Result { + let mut m = object_of(v, "the manifest")?; + let wai = required(&mut m, "wai")?; + let media = required(&mut m, "media")?; + let intent = required(&mut m, "intent")?; + let renditions = match m.shift_remove("renditions") { + None => return Err(serde::de::Error::missing_field("renditions")), + Some(Value::Array(rs)) if rs.len() > u16::MAX as usize => { + return Err(serde::de::Error::custom(format_args!( + "the manifest lists {} renditions, more than the 65535 rows a WAI2 table holds (SPEC §7.1)", + rs.len() + ))); + } + Some(Value::Array(rs)) => { + let mut out = Vec::with_capacity(rs.len()); + for r in rs { + out.push(rendition_from(object_of(r, "a rendition")?)?); + } + out + } + Some(_) => return Err(serde::de::Error::custom("invalid type: renditions is not an array")), + }; + Ok(ManifestV2 { wai, media, intent, renditions, target: m.shift_remove("target").unwrap_or(Value::Null), extra: m }) +} + /// The manifest bytes of a `WAI1` or `WAI2` envelope, exactly as they were /// written, bounds-checked as the parsers check them. A tool that hands a manifest /// on (the C FFI's unpack calls) returns these, so nothing the Rust types do not @@ -2681,9 +2945,7 @@ impl WaiMulti { check_manifest_len(ml as u64) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; need(8, ml, "manifest body")?; - check_manifest_text(&bytes[8..8 + ml]) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - let manifest: ManifestV2 = serde_json::from_slice(&bytes[8..8 + ml]) + let manifest = read_manifest_v2(&bytes[8..8 + ml]) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; let mut o = 8 + ml; @@ -3035,7 +3297,7 @@ pub fn detect_version(bytes: &[u8]) -> std::io::Result { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; /// A pin verifier that accepts every pin: for tests about selection order. @@ -4570,6 +4832,50 @@ mod tests { } } + /// A `WAI1` (or, with `v2`, a `WAI2`) envelope whose manifest holds exactly `values` + /// values: the small manifest of [`envelope_with_manifest_len`], whose `pad` is an + /// array of zeros. Its other members are 10 values in either form, `pad` among them. + pub(crate) fn envelope_with_manifest_values(v2: bool, values: usize) -> Vec { + let base = envelope_with_manifest_len(v2, 300); + let ml = u32::from_le_bytes(base[4..8].try_into().unwrap()) as usize; + let text = std::str::from_utf8(&base[8..8 + ml]).unwrap(); + let pad = text.find("\"pad\":").unwrap(); + // A reader that read one value more or less than this would fail the tests that + // use it at the limit and one past it. + let m = format!("{}\"pad\":[0{}]}}", &text[..pad], ",0".repeat(values - 11)); + [&base[..4], &(m.len() as u32).to_le_bytes(), m.as_bytes(), &base[8 + ml..]].concat() + } + + /// SPEC §2, *Reader limits*: a manifest of [`MANIFEST_MAX_VALUES`] values is read and + /// one value more is refused (`manifest_too_many_values`), by both readers, in `WAI1` + /// and `WAI2` manifests alike; the writers refuse to write one past the limit. The + /// count takes every value at any depth, the manifest object among them, and no + /// member name. + #[test] + fn a_manifest_past_the_value_limit_is_refused_and_never_written() { + assert_eq!(MANIFEST_MAX_VALUES, 4_194_304); + assert_eq!(check_manifest_text(br#"{"a":[1,{"b":null}],"c":"d"}"#).unwrap(), 6); + assert_eq!(check_manifest_text(br#"{}"#).unwrap(), 1); + let too_many = |e: String| assert!(e.starts_with("manifest_too_many_values:"), "{e}"); + for v2 in [false, true] { + let read = |b: &[u8]| if v2 { WaiMulti::from_bytes(b).map(|_| ()) } else { Wai::from_bytes(b).map(|_| ()) }; + let at = envelope_with_manifest_values(v2, MANIFEST_MAX_VALUES); + read(&at).unwrap_or_else(|e| panic!("v2={v2}: the limit is read: {e}")); + too_many(read(&envelope_with_manifest_values(v2, MANIFEST_MAX_VALUES + 1)).unwrap_err().to_string()); + } + // The writers: the envelope at the limit is written back; one zero more is not. + let at = envelope_with_manifest_values(false, MANIFEST_MAX_VALUES); + let mut w = Wai::from_bytes(&at).unwrap(); + assert_eq!(w.to_bytes().unwrap(), at); + w.manifest.extra["pad"].as_array_mut().unwrap().push(Value::from(0)); + too_many(w.to_bytes().unwrap_err().to_string()); + let at = envelope_with_manifest_values(true, MANIFEST_MAX_VALUES); + let mut w2 = WaiMulti::from_bytes(&at).unwrap(); + assert_eq!(w2.to_bytes().unwrap(), at); + w2.manifest.extra["pad"].as_array_mut().unwrap().push(Value::Null); + too_many(w2.to_bytes().unwrap_err().to_string()); + } + /// A length a container writes into a u32 field must fit it: the writers refuse a /// payload past 2^32 − 1 bytes instead of writing its length modulo 2^32. #[test] diff --git a/wai/wai-rs/src/http_binding/head.rs b/wai/wai-rs/src/http_binding/head.rs index 4ef7e5157..98d7c4e22 100644 --- a/wai/wai-rs/src/http_binding/head.rs +++ b/wai/wai-rs/src/http_binding/head.rs @@ -4,7 +4,7 @@ //! component's byte range and, through each entry's `sha256`, its digest. use super::{RangeRefusal, DOMAIN_LEN}; -use crate::container::{check_manifest_text, Manifest, ManifestV2}; +use crate::container::{read_manifest, read_manifest_v2, Manifest, ManifestV2}; /// Which container an object is. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -136,9 +136,7 @@ impl Head { if self.version != Version::Wai1 { return Err(RangeRefusal::HeadMalformed("not a WAI1 head".into())); } - let b = self.manifest_bytes(); - check_manifest_text(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))?; - serde_json::from_slice(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string())) + read_manifest(self.manifest_bytes()).map_err(|e| RangeRefusal::HeadMalformed(e.to_string())) } /// The `WAI2` manifest. @@ -146,9 +144,7 @@ impl Head { if self.version != Version::Wai2 { return Err(RangeRefusal::HeadMalformed("not a WAI2 head".into())); } - let b = self.manifest_bytes(); - check_manifest_text(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))?; - serde_json::from_slice(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string())) + read_manifest_v2(self.manifest_bytes()).map_err(|e| RangeRefusal::HeadMalformed(e.to_string())) } /// The component at `index`. diff --git a/wai/wai-rs/src/http_binding/tests.rs b/wai/wai-rs/src/http_binding/tests.rs index 00f4f8fd6..c35c36d21 100644 --- a/wai/wai-rs/src/http_binding/tests.rs +++ b/wai/wai-rs/src/http_binding/tests.rs @@ -119,6 +119,22 @@ fn head_parse_refuses_a_manifest_past_the_reader_limit() { } } +/// SPEC §2, *Reader limits*: a head whose manifest holds more than +/// `MANIFEST_MAX_VALUES` values is refused (`head_malformed`, as +/// `manifest_too_many_values`); one at the limit is read. +#[test] +fn head_parse_refuses_a_manifest_past_the_value_limit() { + let max = crate::container::MANIFEST_MAX_VALUES; + for v2 in [false, true] { + let at = crate::container::tests::envelope_with_manifest_values(v2, max); + assert!(matches!(parse_head(&at, at.len() as u64), Ok(HeadParse::Head(_))), "v2={v2}"); + let over = crate::container::tests::envelope_with_manifest_values(v2, max + 1); + let e = parse_head(&over, over.len() as u64).unwrap_err(); + assert_eq!(e.code(), "head_malformed"); + assert!(matches!(&e, RangeRefusal::HeadMalformed(m) if m.starts_with("manifest_too_many_values:")), "{e:?}"); + } +} + #[test] fn object_length_rule() { let d = appendix_d(); diff --git a/wai/wai-rs/tests/manifest_parse_memory.rs b/wai/wai-rs/tests/manifest_parse_memory.rs index 627a18c96..36950ed2c 100644 --- a/wai/wai-rs/tests/manifest_parse_memory.rs +++ b/wai/wai-rs/tests/manifest_parse_memory.rs @@ -1,15 +1,42 @@ -//! Probe (scratch): peak allocation of the manifest readers on large manifests. +//! What reading a manifest allocates (SPEC §2, *Reader limits*; SECURITY.md, *Manifest +//! parsing*). A counting global allocator records every byte the reader asks for, and +//! each case requires the peak to stay within the bound the reference states: 128 bytes +//! for each value the manifest holds, one for each byte of its text, and, for a `WAI2`, +//! one [`RenditionMeta`] for each entry it lists, with 64 KiB for fixed costs. With the +//! value limit (2^22) and the length limit (2^27) that is at most 640 MiB for a `WAI1` +//! manifest, whatever its shape. +//! +//! The cases: a manifest at the length limit made of small values, refused for its +//! values before anything is built from it (the reader that parsed such a manifest into a +//! JSON tree took some 5 GB); one at the length limit within the value limit, which is +//! read; manifests of small values in each shape at the value limit, read, and one value +//! past it, refused; the largest manifest a conforming staged layer has, a `prev` with a +//! different producer for each of 2^20 units; and a `WAI2` listing as many renditions as +//! its table can hold, read, and one more, refused. +//! +//! The allocator counts what is asked of it, so the figures are the same on every 64-bit +//! target. A reallocation is counted as the new block allocated before the old one is +//! freed, which is the most it can hold at once. One test runs at a time (`ONE`), so the +//! count is this reader's alone. + use std::alloc::{GlobalAlloc, Layout, System}; -use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; +use std::time::Instant; + +use wai::container::{ + check_manifest_text, RenditionMeta, Wai, WaiMulti, MANIFEST_MAX_LEN, MANIFEST_MAX_VALUES, +}; struct Counting; static NOW: AtomicUsize = AtomicUsize::new(0); static PEAK: AtomicUsize = AtomicUsize::new(0); + fn grow(n: usize) { let now = NOW.fetch_add(n, Relaxed) + n; PEAK.fetch_max(now, Relaxed); } + unsafe impl GlobalAlloc for Counting { unsafe fn alloc(&self, l: Layout) -> *mut u8 { let p = unsafe { System.alloc(l) }; @@ -38,52 +65,175 @@ unsafe impl GlobalAlloc for Counting { q } } + #[global_allocator] -static A: Counting = Counting; +static COUNTING: Counting = Counting; + static ONE: Mutex<()> = Mutex::new(()); +/// The most the reference may allocate reading a manifest of `len` bytes holding +/// `values` values and listing `renditions` `WAI2` entries (SPEC §2; SECURITY.md). +fn bound(len: usize, values: usize, renditions: usize) -> usize { + 128 * values + len + renditions * std::mem::size_of::() + (1 << 16) +} + +/// What `f` allocates at its peak beyond what was held when it started, with its result +/// (which the peak includes) and the seconds it took. fn peak_of(f: impl FnOnce() -> T) -> (T, usize, f64) { let base = NOW.load(Relaxed); PEAK.store(base, Relaxed); - let t = std::time::Instant::now(); + let t = Instant::now(); let r = f(); (r, PEAK.load(Relaxed) - base, t.elapsed().as_secs_f64()) } -fn envelope(m: &str) -> Vec { - let mut b = b"WAI1".to_vec(); +fn envelope(magic: &[u8; 4], m: &str, rows: usize) -> Vec { + let mut b = magic.to_vec(); b.extend_from_slice(&(m.len() as u32).to_le_bytes()); b.extend_from_slice(m.as_bytes()); - b.extend_from_slice(&3u32.to_le_bytes()); - b.extend_from_slice(b"abc"); + if magic == b"WAI2" { + b.extend_from_slice(&(rows as u16).to_le_bytes()); + for _ in 0..rows { + b.extend_from_slice(&[0; 8]); + } + } else { + b.extend_from_slice(&0u32.to_le_bytes()); + } b } -const HEAD: &str = r#"{"wai":"1.0","media":"text","intent":"replicate","model_requirement":{"capability":"wai.text.zstd"},"conditioning":{"kind":"zstd"},"target":"#; -fn zeros(len: usize) -> String { - let mut m = String::with_capacity(len); +/// A `WAI1` manifest with a member no revision defines, `pad` (a reader ignores it, SPEC +/// §8), whose value is `open`, `n` elements written by `elem`, and `close`: with `pad`, +/// 10 values and the elements'. +const HEAD: &str = r#"{"wai":"1.0","media":"text","intent":"replicate","model_requirement":{"capability":"wai.text.zstd"},"conditioning":{"kind":"zstd"},"target":null,"pad":"#; +const FIXED: usize = 10; + +fn padded(n: usize, open: &str, close: &str, mut elem: impl FnMut(usize, &mut String)) -> String { + let mut m = String::with_capacity(HEAD.len() + 2 + n * 8); m.push_str(HEAD); - m.push('['); - while m.len() + 3 < len { - m.push_str("0,"); + m.push_str(open); + for i in 0..n { + if i > 0 { + m.push(','); + } + elem(i, &mut m); } - m.push('0'); - while m.len() + 2 < len { - m.push(' '); - } - m.push_str("]}"); - assert_eq!(m.len(), len); + m.push_str(close); + m.push('}'); m } -fn layer(n: usize) -> String { - let a = "ab".repeat(32); - let b = "cd".repeat(32); - let mut m = String::new(); +/// Read `m` as a `WAI1` envelope; the peak and time, and whether it was read. +fn read_wai1(m: &str) -> (Result<(), String>, usize, f64) { + let env = envelope(b"WAI1", m, 0); + peak_of(|| Wai::from_bytes(&env).map(|_| ()).map_err(|e| e.to_string())) +} + +fn report(name: &str, len: usize, values: usize, peak: usize, secs: f64, bound: usize) { + eprintln!( + "{name}: {len} bytes, {values} values: peak {peak} bytes ({:.2} per byte, {:.1} per value) in {secs:.2} s; bound {bound}", + peak as f64 / len as f64, + peak as f64 / values as f64 + ); + assert!(peak <= bound, "{name}: the read allocated {peak} bytes, past the bound {bound}"); +} + +/// A manifest at the length limit made of small values is refused from its value count, +/// having built nothing: the first pass holds no value, and stops at the value past the +/// limit, some 8 MB into the text. +#[test] +fn a_manifest_of_small_values_at_the_length_limit_is_refused_before_it_is_built() { + let _one = ONE.lock().unwrap_or_else(|e| e.into_inner()); + // Zeros, the last of them a number of ones long enough to bring the text to exactly + // 2^27 bytes. + let k = (MANIFEST_MAX_LEN - HEAD.len() - 3) / 2; + let mut m = padded(k, "[", "]", |_, m| m.push('0')); + let short = MANIFEST_MAX_LEN - m.len(); + let last = m.len() - 3; + m.replace_range(last..last + 1, &"1".repeat(short + 1)); + assert_eq!(m.len(), MANIFEST_MAX_LEN); + let (r, peak, secs) = read_wai1(&m); + let e = r.expect_err("refused"); + assert!(e.starts_with("manifest_too_many_values:"), "{e}"); + report("2^27 bytes of zeros (refused)", m.len(), MANIFEST_MAX_VALUES + 1, peak, secs, 1 << 16); +} + +/// A manifest at the length limit holding exactly the value limit is read within the +/// bound: 2^22 values, the most a reader must accept, each a string, to 2^27 bytes. +#[test] +fn a_manifest_at_both_limits_is_read_within_the_bound() { + let _one = ONE.lock().unwrap_or_else(|e| e.into_inner()); + let n = MANIFEST_MAX_VALUES - FIXED; + // Each string `"…"` and its comma: share out what is left of 2^27 bytes. + let room = MANIFEST_MAX_LEN - HEAD.len() - 3 - (n - 1) - 2 * n; + let (q, r) = (room / n, room % n); + let m = padded(n, "[", "]", |i, m| { + m.push('"'); + m.extend(std::iter::repeat_n('s', q + usize::from(i < r))); + m.push('"'); + }); + assert_eq!(m.len(), MANIFEST_MAX_LEN); + assert_eq!(check_manifest_text(m.as_bytes()).unwrap(), MANIFEST_MAX_VALUES); + let (r, peak, secs) = read_wai1(&m); + r.expect("read"); + report("2^27 bytes, 2^22 strings", m.len(), MANIFEST_MAX_VALUES, peak, secs, bound(m.len(), MANIFEST_MAX_VALUES, 0)); +} + +/// Manifests of small values, in every shape that costs a reader the most for each +/// value, at exactly the value limit, are read within the bound; one value more is +/// refused (`manifest_too_many_values`). +#[test] +fn small_values_at_the_value_limit_are_read_within_the_bound_and_one_more_is_refused() { + let _one = ONE.lock().unwrap_or_else(|e| e.into_inner()); + let n = MANIFEST_MAX_VALUES - FIXED; + type Shape = (&'static str, usize, &'static str, &'static str, fn(usize, &mut String)); + let shapes: [Shape; 6] = [ + ("an array of zeros", n, "[", "]", |_, m| m.push('0')), + ("an array of empty arrays", n, "[", "]", |_, m| m.push_str("[]")), + ("an array of empty objects", n, "[", "]", |_, m| m.push_str("{}")), + ("an array of one-element arrays", n / 2, "[", "]", |_, m| m.push_str("[0]")), + ("an array of one-member objects", n / 2, "[", "]", |_, m| m.push_str(r#"{"a":0}"#)), + ("one object of many members", n, "{", "}", |i, m| { + m.push('"'); + m.push_str(&format!("{i:x}")); + m.push_str("\":0"); + }), + ]; + for (name, k, open, close, elem) in shapes { + // A shape of two values an element takes one zero more when `n` is odd. + let mut m = padded(k, open, close, elem); + if k != n && n % 2 == 1 { + m.insert_str(m.len() - 2, ",0"); + } + let values = check_manifest_text(m.as_bytes()).unwrap(); + assert_eq!(values, MANIFEST_MAX_VALUES, "{name}"); + let (r, peak, secs) = read_wai1(&m); + r.expect(name); + report(name, m.len(), values, peak, secs, bound(m.len(), values, 0)); + + m.insert_str(m.len() - 2, if open == "{" { r#","x":0"# } else { ",0" }); + let (r, peak, _) = read_wai1(&m); + let e = r.expect_err("one value past the limit is refused"); + assert!(e.starts_with("manifest_too_many_values:"), "{name}: {e}"); + // The first pass holds the names of an object's members to find a repeat, so the + // refusal of one object of many members holds them; it stays within the bound. + assert!(peak <= bound(m.len(), MANIFEST_MAX_VALUES, 0), "{name}: the refusal allocated {peak} bytes"); + } +} + +/// The largest manifest a conforming staged layer has (SPEC §2): a `prev` that names a +/// different producer for each of the 2^20 units a `WIR1` layer may cover, about 94 MB +/// and 3·2^20 values, is read within the bound, and its `refines` reads well formed. +#[test] +fn the_largest_conforming_layer_manifest_is_read_within_the_bound() { + let _one = ONE.lock().unwrap_or_else(|e| e.into_inner()); + let (a, b) = ("ab".repeat(32), "cd".repeat(32)); + let units = 1usize << 20; + let mut m = String::with_capacity(95 << 20); m.push_str(r#"{"wai":"1.0","media":"image","intent":"replicate","model_requirement":{"capability":"wai.image.int_refine","fallback":null},"conditioning":{"kind":"int_refine"},"refines":{"base":""#); m.push_str(&a); m.push_str(r#"","prev":["#); - for i in 0..n { + for i in 0..units { if i > 0 { m.push(','); } @@ -92,49 +242,43 @@ fn layer(n: usize) -> String { m.push_str(r#"","count":1}"#); } m.push_str(r#"],"layer":2},"target":null}"#); - m -} - - -const LIM: usize = 1 << 22; -const HEAD2: &str = r#"{"wai":"1.0","media":"text","intent":"replicate","model_requirement":{"capability":"wai.text.zstd"},"conditioning":{"kind":"zstd"},"target":null,"pad":"#; -// 10 values before the pad's elements: the manifest, wai, media, intent, model_requirement, -// capability, conditioning, kind, target, pad. -fn pad(n: usize, elem: impl Fn(usize, &mut String), open: &str, close: &str) -> String { - let mut m = String::from(HEAD2); - m.push_str(open); - for i in 0..n { - if i > 0 { m.push(','); } - elem(i, &mut m); - } - m.push_str(close); - m.push('}'); - m + let values = check_manifest_text(m.as_bytes()).unwrap(); + assert_eq!(values, 3 * units + 14); + let env = envelope(b"WAI1", &m, 0); + let (w, peak, secs) = peak_of(|| Wai::from_bytes(&env).expect("read")); + report("a layer naming 2^20 producers", m.len(), values, peak, secs, bound(m.len(), values, 0)); + let r = w.refines().expect("refines").expect("well formed"); + assert_eq!(r.units(units as u32), (0, units as u32)); + r.check_units(units as u32).expect("the runs cover the units"); } +/// A `WAI2` manifest listing the 65 535 renditions its table can hold is read within the +/// bound, which counts the entries it builds; a `renditions` of 65 536 is refused before +/// its entries are built. #[test] -fn probe() { - let _g = ONE.lock().unwrap(); - let n = LIM - 10; - let cases: Vec<(&str, String)> = vec![ - ("pad zeros", pad(n, |_, m| m.push('0'), "[", "]")), - ("pad members", pad(n, |i, m| { m.push('"'); m.push_str(&format!("{i:x}")); m.push_str("\":0"); }, "{", "}")), - ("pad 1-member objs", pad(n / 2, |_, m| m.push_str("{\"a\":0}"), "[", "]")), - ("pad empty objs", pad(n, |_, m| m.push_str("{}"), "[", "]")), - ("pad strings 30", pad(n, |_, m| m.push_str("\"abcdefghijklmnopqrstuvwxyzabcd\""), "[", "]")), - ("layer 2^20", layer(1 << 20)), - ("zeros 2^27 (refused)", zeros(1 << 27)), - ("pad zeros +1 (refused)", pad(n + 1, |_, m| m.push('0'), "[", "]")), - ]; - for (name, m) in cases { - let env = envelope(&m); - let ml = m.len(); - drop(m); - let (c, p0, t0) = peak_of(|| wai::container::check_manifest_text(&env[8..8 + ml]).map_err(|e| e.to_string())); - let (r, p, t) = peak_of(|| wai::container::Wai::from_bytes(&env).map(|w| w.manifest.target.is_null())); - eprintln!( - "{name}: len {ml} values {:?} scan {} B {t0:.2}s from_bytes peak {} B = {:.2}x len, {:.1} B/value {t:.2}s ok={}", - c.as_ref().map_err(|e| &e[..40]), p0, p, p as f64 / ml as f64, p as f64 / LIM as f64, r.is_ok() - ); - } +fn a_wai2_listing_its_tables_rows_is_read_within_the_bound() { + let _one = ONE.lock().unwrap_or_else(|e| e.into_inner()); + let head = r#"{"wai":"1.1","media":"text","intent":"replicate","renditions":["#; + let manifest = |rows: usize| { + let mut m = String::from(head); + for i in 0..rows { + if i > 0 { + m.push(','); + } + m.push_str(r#"{"capability":"wai.text.zstd","kind":"zstd"}"#); + } + m.push_str(r#"],"target":null}"#); + m + }; + let rows = u16::MAX as usize; + let m = manifest(rows); + let values = check_manifest_text(m.as_bytes()).unwrap(); + let env = envelope(b"WAI2", &m, rows); + let (w, peak, secs) = peak_of(|| WaiMulti::from_bytes(&env).map(|w| w.manifest.renditions.len())); + assert_eq!(w.expect("read"), rows); + report("a WAI2 of 65535 renditions", m.len(), values, peak, secs, bound(m.len(), values, rows)); + + let m = manifest(rows + 1); + let e = WaiMulti::from_bytes(&envelope(b"WAI2", &m, 0)).err().expect("refused").to_string(); + assert!(e.contains("more than the 65535 rows a WAI2 table holds"), "{e}"); } diff --git a/wai/wai-rs/tests/manifest_reader_equivalence.rs b/wai/wai-rs/tests/manifest_reader_equivalence.rs new file mode 100644 index 000000000..921b31d7a --- /dev/null +++ b/wai/wai-rs/tests/manifest_reader_equivalence.rs @@ -0,0 +1,171 @@ +//! The reference readers build a manifest with `container::read_manifest` and +//! `read_manifest_v2`, which move its members out of one JSON tree (SPEC §2, *Reader +//! limits*: so a parse allocates within a bound), where they used serde's derived reader +//! (`serde_json::from_slice::`). This test holds the two to the same result: +//! for the manifest of every envelope committed under `wai/`, and for each of them with +//! every member, at the manifest's top level and one level down, removed or given a +//! value of each JSON type, both readers read it or both refuse it, and what they read +//! writes back to the same JSON. + +use std::path::{Path, PathBuf}; + +use serde_json::{json, Value}; +use wai::container::{check_unique_members, manifest_bytes, read_manifest, read_manifest_v2, Manifest, ManifestV2}; + +fn envelopes(dir: &Path, out: &mut Vec) { + let Ok(rd) = std::fs::read_dir(dir) else { return }; + let mut entries: Vec<_> = rd.flatten().map(|e| e.path()).collect(); + entries.sort(); + for p in entries { + let name = p.file_name().and_then(|n| n.to_str()).unwrap_or(""); + if p.is_dir() { + if name != "target" && name != "node_modules" && !name.starts_with('.') { + envelopes(&p, out); + } + } else if name.ends_with(".wai") { + out.push(p); + } + } +} + +/// Both readers on `text`: the same verdict, and the same JSON when both read it. +fn same(text: &[u8], v2: bool, what: &str) -> bool { + let (derived, moved) = if v2 { + let d = check_unique_members(text).and_then(|_| serde_json::from_slice::(text)).map(|m| serde_json::to_string(&m).unwrap()); + let m = read_manifest_v2(text).map(|m| serde_json::to_string(&m).unwrap()); + (d, m) + } else { + let d = check_unique_members(text).and_then(|_| serde_json::from_slice::(text)).map(|m| serde_json::to_string(&m).unwrap()); + let m = read_manifest(text).map(|m| serde_json::to_string(&m).unwrap()); + (d, m) + }; + match (&derived, &moved) { + (Ok(a), Ok(b)) => assert_eq!(a, b, "{what}: the readers read different manifests"), + (Err(_), Err(_)) => {} + _ => panic!("{what}: the derived reader gives {derived:?}, the reference reader {moved:?}"), + } + moved.is_ok() +} + +/// The values a member is given: one of each JSON type, and a few of each kind the +/// manifest types read (a string, an integer, a float, `null`). +fn replacements() -> Vec { + vec![json!(null), json!(true), json!(0), json!(-1), json!(1.5), json!(18446744073709551615u64), json!("s"), json!([]), json!({}), json!([{}]), json!({"x": 1})] +} + +fn mutations(text: &[u8], v2: bool, what: &str) -> usize { + let Ok(Value::Object(top)) = serde_json::from_slice::(text) else { return 0 }; + let mut n = 0; + let mut check = |m: &serde_json::Map, why: String| { + same(serde_json::to_string(m).unwrap().as_bytes(), v2, &format!("{what}: {why}")); + n += 1; + }; + for (k, v) in &top { + let mut m = top.clone(); + m.shift_remove(k); + check(&m, format!("without {k}")); + for r in replacements() { + let mut m = top.clone(); + m.insert(k.clone(), r.clone()); + check(&m, format!("{k} = {r}")); + } + // One level down: each member of an object member, and of the first object of + // an array member (a WAI2's first rendition entry). + let (inner, at): (Option<&serde_json::Map>, Option) = match v { + Value::Object(o) => (Some(o), None), + Value::Array(a) => match a.first() { + Some(Value::Object(o)) => (Some(o), Some(0)), + _ => (None, None), + }, + _ => (None, None), + }; + let Some(inner) = inner else { continue }; + let put = |o: serde_json::Map| { + let mut m = top.clone(); + let nv = match at { + None => Value::Object(o), + Some(i) => { + let mut a = v.as_array().unwrap().clone(); + a[i] = Value::Object(o); + Value::Array(a) + } + }; + m.insert(k.clone(), nv); + m + }; + for ik in inner.keys() { + let mut o = inner.clone(); + o.shift_remove(ik); + check(&put(o), format!("{k} without {ik}")); + for r in replacements() { + let mut o = inner.clone(); + o.insert(ik.clone(), r.clone()); + check(&put(o), format!("{k}.{ik} = {r}")); + } + } + } + n +} + +#[test] +fn the_reference_reader_reads_every_committed_manifest_as_the_derived_reader_does() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join(".."); + let mut files = Vec::new(); + envelopes(&root, &mut files); + assert!(files.len() >= 400, "found only {} envelopes under {}", files.len(), root.display()); + let (mut read, mut refused, mut mutated) = (0, 0, 0); + // Mutate the manifests of each distinct set of top-level members once. + let mut shapes = std::collections::BTreeSet::new(); + for f in &files { + let b = std::fs::read(f).unwrap(); + let Ok(text) = manifest_bytes(&b) else { continue }; + let v2 = &b[..4] == b"WAI2"; + let what = f.strip_prefix(&root).unwrap_or(f).display().to_string(); + if same(text, v2, &what) { + read += 1; + } else { + refused += 1; + } + if let Ok(Value::Object(o)) = serde_json::from_slice::(text) { + let keys: Vec = o.keys().cloned().collect(); + if shapes.insert((v2, keys)) { + mutated += mutations(text, v2, &what); + } + } + } + eprintln!("{} envelopes: {read} manifests read alike, {refused} refused alike; {mutated} mutations agree", files.len()); + assert!(read >= 300, "only {read} manifests read"); + assert!(mutated >= 1000, "only {mutated} mutations"); +} + +/// The cases the derived reader treats specially, each read alike: a member whose value +/// is `null` where an `Option` is (`None`, not written back), `target` absent (`null`), +/// a member no revision defines kept in order, a float `min_sim`, and an integer past +/// `u64` as `kid`. +#[test] +fn the_derived_readers_special_cases_are_read_alike() { + let base = r#"{"wai":"1.0","media":"image","intent":"replicate","model_requirement":{"capability":"wai.image.png","fallback":null,"min_sim":0.5,"z":1,"a":[1]},"conditioning":{"kind":"png","prior":null,"q":{}},"security":null,"refines":null,"derived_from":null,"later":{"b":2},"earlier":3}"#; + for t in [ + base.to_string(), + base.replace(r#""min_sim":0.5"#, r#""min_sim":1"#), + base.replace(r#""security":null"#, r#""security":{"encryption":"none","kid":18446744073709551616}"#), + base.replace(r#""security":null"#, r#""security":{"kid":-1}"#), + base.replace(r#""conditioning":{"kind":"png","prior":null,"q":{}}"#, r#""conditioning":{"kind":"png","prior":{"id":"x","sha256":"y","determinism":"z","supersedes":null,"k":[]}}"#), + base.replace(r#""conditioning":{"kind":"png","prior":null,"q":{}}"#, r#""conditioning":{"kind":"png","prior":{"id":"x","sha256":"y","determinism":"z","supersedes":{"sha256":"w","form":null}}}"#), + base.replace(r#""conditioning":{"kind":"png","prior":null,"q":{}}"#, r#""conditioning":{"kind":"png","prior":{"id":"x","sha256":"y"}}"#), + "[]".into(), + "null".into(), + "{}".into(), + ] { + same(t.as_bytes(), false, &t); + } + let v2 = r#"{"wai":"1.1","media":"text","intent":"replicate","renditions":[{"capability":"wai.text.zstd","kind":"zstd","refines":null,"envelope_manifest":{"a":1},"x":[]}],"security":{"encryption":"none"},"z":1}"#; + for t in [ + v2.to_string(), + v2.replace(r#""renditions":[{"#, r#""renditions":[1,{"#), + v2.replace(r#""renditions":["#, r#""renditions":{"a":"#).replace(r#""x":[]}],"#, r#""x":[]}},"#), + v2.replace(r#""kind":"zstd","#, ""), + ] { + same(t.as_bytes(), true, &t); + } +} -- 2.54.0 (Apple Git-157) From be4c1f0d933b34cc62605d212cb3802a35f8ffd0 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Sun, 11 Oct 2026 23:32:44 -0400 Subject: [PATCH 03/10] =?UTF-8?q?wai:=20the=20value=20limit=20in=20SPEC=20?= =?UTF-8?q?=C2=A72,=20the=20Python=20verifiers=20and=20the=20HTTP=20corpus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SPEC §2 states the value limit (2^22, manifest_too_many_values) beside the length and nesting limits, and what the two bound together. staged_verify, claim_verify, http_verify and pack_verify refuse past it and check both edges in memory; the HTTP corpus gains parse_head cases at the limit and one past it (pad_values), verified by wai_http_vectors and http_verify.py alike. Co-Authored-By: Claude Opus 5.5 --- wai/SPEC.md | 40 ++++++++++++++------- wai/claim-conformance/claim_verify.py | 4 +-- wai/http-conformance/cases.json | 4 +++ wai/http-conformance/http_verify.py | 14 +++++++- wai/pack-conformance/pack_verify.py | 35 ++++++++++++++++-- wai/staged-conformance/staged_verify.py | 41 ++++++++++++++++++---- wai/wai-rs/src/bin/wai_cli/http_vectors.rs | 28 +++++++++++++++ 7 files changed, 140 insertions(+), 26 deletions(-) diff --git a/wai/SPEC.md b/wai/SPEC.md index a1c587a72..0c9eb3f84 100644 --- a/wai/SPEC.md +++ b/wai/SPEC.md @@ -86,20 +86,34 @@ OPTIONAL; §5 says which are RECOMMENDED. All WAI multi-byte integers are little-endian unless stated otherwise. -**Reader limits.** A reader MUST NOT refuse a manifest for its length or its nesting -while it is at most 134 217 728 bytes (`man_len` ≤ 2²⁷) and its arrays and objects nest -at most 127 deep, the manifest object itself counting as one. It MAY refuse a longer or -deeper manifest. The length limit holds the largest manifest a conforming layer has: a -`prev` ([staged-delivery](extensions/staged-delivery.md) §3) that names a different -producer for each of the 2²⁰ units a `WIR1` layer may cover (staged-delivery -Appendix A.3, `n_units` ≤ 2²⁰) is about 94 MB. A reader that refuses one past the -length limit does so from `man_len` alone, before it reads or requests the manifest -(`manifest_too_large`). The reference readers refuse both. The same limits apply to the -`WAI2` manifest (§7.1). They bound what a reader must read, not the memory its parse -takes, and move no byte of the layout above, which stays frozen (§8). Since a +**Reader limits.** A reader MUST NOT refuse a manifest for its length, its nesting or +the number of its values while it is at most 134 217 728 bytes (`man_len` ≤ 2²⁷), its +arrays and objects nest at most 127 deep, the manifest object itself counting as one, +and it holds at most 4 194 304 JSON values (2²²), counting the manifest object and every +value nested in it at any depth; a member's name is not a value. It MAY refuse a longer +or deeper manifest, or one that holds more values. The limits hold the largest manifest +a conforming layer has: a `prev` ([staged-delivery](extensions/staged-delivery.md) §3) +that names a different producer for each of the 2²⁰ units a `WIR1` layer may cover +(staged-delivery Appendix A.3, `n_units` ≤ 2²⁰) is about 94 MB, and three values a run +(the run, its `producer` and its `count`), 3·2²⁰ and a few more in all. A reader that +refuses one past the length limit does so from `man_len` alone, before it reads or +requests the manifest (`manifest_too_large`). A reader that refuses one past the value +limit does so while it counts the values, before it builds anything from the manifest +(`manifest_too_many_values`). The reference readers refuse all three. The same limits +apply to the `WAI2` manifest (§7.1), whose `renditions` lists no more entries than its +table holds. They move no byte of the layout above, which stays frozen (§8). Since a conforming encoder emits only containers every conforming sink can read (§7), a -conforming encoder writes no manifest past either limit; the reference writers refuse -one past the length limit. +conforming encoder writes no manifest past any of these limits; the reference writers +refuse one past any of them. + +Together the length and value limits bound what parsing a manifest takes, which the +length limit alone does not: a parsed value takes more memory than its text, and many +small values can take many times their length. The reference readers count a manifest's +values before they build anything from it, then build each array and object at the size +counted, and on a 64-bit target allocate at most 128 bytes for each value and one for +each byte of the manifest, with, for a `WAI2`, a rendition entry of 768 bytes for each it +lists: at most 640 MiB for a `WAI1` manifest at both limits, whatever its shape, and +48 MiB more for a `WAI2`'s entries (informative; `SECURITY.md`). An envelope delivered over HTTP is one object of the proposed media type `application/wai`, named by its receipt content hash, and fetched whole or, for a diff --git a/wai/claim-conformance/claim_verify.py b/wai/claim-conformance/claim_verify.py index 4020ab6f3..d50987c24 100644 --- a/wai/claim-conformance/claim_verify.py +++ b/wai/claim-conformance/claim_verify.py @@ -771,7 +771,7 @@ def wai2_read(data): if ml > SV.MANIFEST_MAX or 8 + ml > len(data): return None try: - m = SV.strict_json(data[8:8 + ml]) + m = SV.strict_json(data[8:8 + ml], SV.MANIFEST_MAX_VALUES) SV.check_manifest_v2(m) except (SV.NotJSON, SV.Schema): return None @@ -1965,7 +1965,7 @@ def main(argv): print(("PASS" if not bm else "FAIL") + " bracket arithmetic equals the literals pinned in wai-rs " + ",".join(bm)) fail += bool(bm) rl = SV.reader_limit_failures(lambda b: wai2_read(b) is not None, True) - print(("PASS" if not rl else "FAIL") + " wai2_read refuses a manifest past the reader limit (SPEC §2) " + "; ".join(rl)) + print(("PASS" if not rl else "FAIL") + " wai2_read refuses a manifest past the reader limits (SPEC §2) " + "; ".join(rl)) fail += bool(rl) rd = OER.check_canonical_form() # the OER reference's own fixtures, its reading of -0 among them print(("PASS" if not rd else "FAIL") + " the OER reference's canonical form and number reading hold " + ",".join(rd)) diff --git a/wai/http-conformance/cases.json b/wai/http-conformance/cases.json index 672f87603..8d330faf0 100644 --- a/wai/http-conformance/cases.json +++ b/wai/http-conformance/cases.json @@ -14,8 +14,12 @@ {"name":"head_malformed_offset","op":"parse_head","input":{"object":"OD","patch_object":[[574,"01"]]},"expect":{"error":"head_malformed"}}, {"name":"head_malformed_manifest_too_large_O5","op":"parse_head","input":{"object":"O5","prefix":8,"patch_object":[[4,"01"],[5,"00"],[6,"00"],[7,"08"]]},"expect":{"error":"head_malformed"}}, {"name":"parse_head_need_manifest_at_limit_O5","op":"parse_head","input":{"object":"O5","prefix":8,"patch_object":[[4,"00"],[5,"00"],[6,"00"],[7,"08"]]},"expect":{"need":134217740}}, +{"name":"parse_head_manifest_values_at_limit_O5","op":"parse_head","input":{"object":"O5","pad_values":4194294},"expect":{"version":"WAI1","head_len":8388753,"envelope_len":8388847,"covered_len":8388847,"components":[[8388753,94]]}}, +{"name":"head_malformed_manifest_too_many_values_O5","op":"parse_head","input":{"object":"O5","pad_values":4194295},"expect":{"error":"head_malformed"}}, {"name":"head_malformed_manifest_too_large_OD","op":"parse_head","input":{"object":"OD","prefix":8,"patch_object":[[4,"01"],[5,"00"],[6,"00"],[7,"08"]]},"expect":{"error":"head_malformed"}}, {"name":"parse_head_need_manifest_at_limit_OD","op":"parse_head","input":{"object":"OD","prefix":8,"patch_object":[[4,"00"],[5,"00"],[6,"00"],[7,"08"]]},"expect":{"need":134217738}}, +{"name":"parse_head_manifest_values_at_limit_OD","op":"parse_head","input":{"object":"OD","pad_values":4194279},"expect":{"version":"WAI2","head_len":8389160,"envelope_len":8442360,"covered_len":8389617,"components":[[8389160,3000],[8392160,50000],[8442160,200]]}}, +{"name":"head_malformed_manifest_too_many_values_OD","op":"parse_head","input":{"object":"OD","pad_values":4194280},"expect":{"error":"head_malformed"}}, {"name":"head_proof_ok_O1","op":"head_proof","input":{"object":"O1"},"expect":{"covered_len":859,"n_cv":0,"envelope_len":859}}, {"name":"head_proof_ok_O10","op":"head_proof","input":{"object":"O10"},"expect":{"covered_len":1009,"n_cv":1,"envelope_len":1825}}, {"name":"head_proof_ok_O10L1","op":"head_proof","input":{"object":"O10L1"},"expect":{"covered_len":1009,"n_cv":4,"envelope_len":12711}}, diff --git a/wai/http-conformance/http_verify.py b/wai/http-conformance/http_verify.py index a72102e23..e28253049 100644 --- a/wai/http-conformance/http_verify.py +++ b/wai/http-conformance/http_verify.py @@ -116,7 +116,8 @@ def read_manifest(b, v2): """SPEC §2, §3, §7.1: the manifest read strictly, with the members SPEC requires and their types (staged_verify's reader); `head_malformed` otherwise.""" try: - m = SV.strict_json(b) + # SPEC §2, *Reader limits* (--mutate manifest-values-uncounted drops the value limit). + m = SV.strict_json(b, None if MUTATE == "manifest-values-uncounted" else SV.MANIFEST_MAX_VALUES) (SV.check_manifest_v2 if v2 else SV.check_manifest_v1)(m) except (SV.NotJSON, SV.Schema): raise Refusal("head_malformed") @@ -3284,6 +3285,15 @@ def receipt_evaluate(d, objs, op, inp): return {"fault": str(e)} +def pad_values(b, n): + """SPEC §2, *Reader limits*: the envelope `b` with a member no revision defines added + at the end of its manifest, "pad", an array of `n` zeros, and its man_len set to the + manifest's new length; the manifest holds n + 1 values more.""" + ml = u32(b, 4) + m = b[8:8 + ml - 1] + b',"pad":[0' + b",0" * (n - 1) + b"]}" + return b[:4] + len(m).to_bytes(4, "little") + m + b[8 + ml:] + + def evaluate(d, objs, op, inp): def obj(i, patch=True): if "envelope_hex" in i: @@ -3292,6 +3302,8 @@ def evaluate(d, objs, op, inp): b = objs[i["object"]] else: b = open(os.path.join(d, i["file"]), "rb").read() + if "pad_values" in i: + b = pad_values(b, i["pad_values"]) return apply_patch(b, i.get("patch_object")) if patch else b try: diff --git a/wai/pack-conformance/pack_verify.py b/wai/pack-conformance/pack_verify.py index edb463710..3e637f56d 100644 --- a/wai/pack-conformance/pack_verify.py +++ b/wai/pack-conformance/pack_verify.py @@ -90,6 +90,9 @@ HV.load_registry() MAX_INTEGER = (1 << 53) - 1 # The longest manifest a reader must accept (SPEC §2, *Reader limits*). MANIFEST_MAX = 1 << 27 +# The most JSON values a manifest may hold that a reader must accept (SPEC §2, *Reader +# limits*): the manifest and every value in it, at any depth, and no member name. +MANIFEST_MAX_VALUES = 1 << 22 U32 = (1 << 32) - 1 # The frames of each piece an encode vector codes: the first ENCODE_FRAMES. ENCODE_FRAMES = 12 @@ -1414,6 +1417,19 @@ def read_json(b, minus_zero=False): return None +def json_values(v): + """The JSON values `v` holds: itself and every value in it, at any depth.""" + n, stack = 0, [v] + while stack: + x = stack.pop() + n += 1 + if isinstance(x, dict): + stack.extend(x.values()) + elif isinstance(x, list): + stack.extend(x) + return n + + def opt_str(o, k): """An OPTIONAL string member: absent, null, or a string.""" return k not in o or o[k] is None or isinstance(o[k], str) @@ -1476,6 +1492,8 @@ def read_envelope(b): if ml > MANIFEST_MAX or 8 + ml > len(b): return None man = read_json(b[8:8 + ml]) + if man is not None and json_values(man) > MANIFEST_MAX_VALUES: + return None # §6.3: "does not parse" is the typed parse with SPEC §3's members. if not manifest_typed(man, b[:4] == b"WAI2"): return None @@ -1504,9 +1522,11 @@ def read_envelope(b): def check_reader_limit(rep): """SPEC §2, *Reader limits*, checked in memory, since no file in the corpus is past - 128 MiB: read_envelope reads a WAI1 and a WAI2 whose manifests are 300 bytes, and - refuses each with its manifest padded to MANIFEST_MAX + 1 bytes by a member no - revision defines, which a reader ignores (SPEC §8), so only the limit refuses it.""" + 128 MiB or holds 4 Mi values: read_envelope reads a WAI1 and a WAI2 whose manifests + are 300 bytes, and refuses each with its manifest padded to MANIFEST_MAX + 1 bytes by + a member no revision defines, which a reader ignores (SPEC §8), so only the limit + refuses it; it reads each with that member an array of zeros that brings the manifest + to MANIFEST_MAX_VALUES values, and refuses each with one zero more.""" for magic, head, rest in ( (b"WAI1", b'{"wai":"1.0","media":"text","intent":"replicate","model_requirement":' b'{"capability":"wai.text.zstd"},"conditioning":{"kind":"zstd"},"pad":"', @@ -1520,6 +1540,15 @@ def check_reader_limit(rep): rep.check(read_envelope(env(MANIFEST_MAX + 1)) is None, f"reader limit: a {magic.decode()} manifest of {MANIFEST_MAX + 1} bytes, past the limit, is refused") + def valued(n): + fixed = json_values(read_json(head[:-1] + b"[]}")) + m = head[:-1] + b"[0" + b",0" * (n - fixed - 1) + b"]}" + return b"".join([magic, len(m).to_bytes(4, "little"), m, rest, b"abc"]) + rep.check(read_envelope(valued(MANIFEST_MAX_VALUES)) is not None, + f"reader limit: a {magic.decode()} manifest of {MANIFEST_MAX_VALUES} values is read") + rep.check(read_envelope(valued(MANIFEST_MAX_VALUES + 1)) is None, + f"reader limit: a {magic.decode()} manifest of {MANIFEST_MAX_VALUES + 1} values, past the limit, is refused") + WIR1_KIND = {"wai.image.int_refine": "rgb8", "wai.video.int_refine": "rgb8", "wai.audio.int_refine": "pcm", "wai.splat.int_refine": "tensor"} diff --git a/wai/staged-conformance/staged_verify.py b/wai/staged-conformance/staged_verify.py index fa14dc34a..fc15fda07 100644 --- a/wai/staged-conformance/staged_verify.py +++ b/wai/staged-conformance/staged_verify.py @@ -70,13 +70,19 @@ DEPTH_LIMIT = 127 # reader refuses a longer one from its length field alone (`manifest_too_large`). MANIFEST_MAX = 1 << 27 +# The most JSON values a manifest may hold that a reader must accept (SPEC §2, *Reader +# limits*), counting the manifest and every value in it, at any depth, and no member +# name; the reference reader refuses one more (`manifest_too_many_values`). +MANIFEST_MAX_VALUES = 1 << 22 + class NotJSON(ValueError): pass -def strict_json(b): - """The JSON value of the bytes `b`, read strictly, or NotJSON.""" +def strict_json(b, max_values=None): + """The JSON value of the bytes `b`, read strictly, or NotJSON. With `max_values`, a + text holding more values than that is refused (`manifest_too_many_values`).""" if b[:3] == b"\xef\xbb\xbf": raise NotJSON("a byte-order mark") try: @@ -121,8 +127,12 @@ def strict_json(b): except (ValueError, RecursionError) as e: raise NotJSON(str(e)) stack = [(v, 1)] + values = 0 while stack: x, depth = stack.pop() + values += 1 + if max_values is not None and values > max_values: + raise NotJSON("manifest_too_many_values") if isinstance(x, (dict, list)): if depth > DEPTH_LIMIT: raise NotJSON("nested past the depth limit") @@ -261,7 +271,7 @@ def read_wai1(b): raise NotWai1("truncated manifest") text = b[8:8 + ml] try: - m = strict_json(text) + m = strict_json(text, MANIFEST_MAX_VALUES) check_manifest_v1(m) except (NotJSON, Schema) as e: raise NotWai1(str(e)) @@ -1180,7 +1190,7 @@ def read_wai2(b): if 8 + ml > len(b): raise NotWai2("truncated manifest") try: - m = strict_json(b[8:8 + ml]) + m = strict_json(b[8:8 + ml], MANIFEST_MAX_VALUES) check_manifest_v2(m) except (NotJSON, Schema) as e: raise NotWai2(str(e)) @@ -1226,17 +1236,34 @@ def envelope_with_manifest_len(v2, n): b'"}', rest, b"abc"]) +def envelope_with_manifest_values(v2, n): + """The envelope of envelope_with_manifest_len(v2, 300) with its manifest holding + exactly `n` values: its `pad` an array of zeros, the manifest's other members 10 + values, `pad` among them.""" + b = envelope_with_manifest_len(v2, 300) + ml = struct.unpack_from(" Result { } } +/// `pad_values` (SPEC §2, *Reader limits*): the envelope `b` with a member no revision +/// defines added at the end of its manifest, `"pad"`, an array of `n` zeros, and its +/// `man_len` set to the manifest's new length. The manifest then holds `n + 1` values +/// more. +fn pad_values(b: &[u8], n: usize) -> Result, String> { + let ml = wai::container::manifest_bytes(b).map_err(|e| e.to_string())?.len(); + let text = &b[8..8 + ml]; + if n == 0 || text.last() != Some(&b'}') { + return Err("pad_values needs a manifest object and at least one zero".into()); + } + let mut m = text[..ml - 1].to_vec(); + m.extend_from_slice(b",\"pad\":[0"); + m.extend_from_slice(&b",0".repeat(n - 1)); + m.extend_from_slice(b"]}"); + Ok([&b[..4], &(m.len() as u32).to_le_bytes(), &m, &b[8 + ml..]].concat()) +} + // ---------------------------------------------------------------- the objects fn read(dir: &Path, rel: &str) -> Result, String> { @@ -339,6 +356,9 @@ impl Corpus { } fn object(&self, input: &Value) -> Result, String> { let mut b = self.original(input)?; + if let Some(n) = input["pad_values"].as_u64() { + b = pad_values(&b, n as usize)?; + } patch(&mut b, &input["patch_object"])?; Ok(b) } @@ -872,6 +892,14 @@ fn cases(objs: &BTreeMap>, extra: &mut BTreeMap> json!({ "object": id, "prefix": 8, "patch_object": man_len((1 << 27) + 1) }), "head_malformed")); v.push(case(format!("parse_head_need_manifest_at_limit_{id}"), "parse_head", json!({ "object": id, "prefix": 8, "patch_object": man_len(1 << 27) }))); + // The value limit: the object's manifest with a `pad` of zeros bringing it to + // exactly 2^22 values is read; one zero more is refused. + let text = wai::container::manifest_bytes(&objs[id]).map_err(|e| e.to_string())?; + let held = wai::container::check_manifest_text(text).map_err(|e| e.to_string())?; + let at = wai::container::MANIFEST_MAX_VALUES - held - 1; + v.push(case(format!("parse_head_manifest_values_at_limit_{id}"), "parse_head", json!({ "object": id, "pad_values": at }))); + v.push(refused(format!("head_malformed_manifest_too_many_values_{id}"), "parse_head", + json!({ "object": id, "pad_values": at + 1 }), "head_malformed")); } // head_proof: every object, and every way a proof can be wrong. -- 2.54.0 (Apple Git-157) From 12252092b02d5bfbeb69a65b9e181750a71a90ac Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Sun, 11 Oct 2026 23:33:42 -0400 Subject: [PATCH 04/10] wai-http falsify: a patch and a mutation that drop the manifest value limit Co-Authored-By: Claude Opus 5.5 --- .../falsify/manifest-values-uncounted.patch | 15 +++++++++++++++ wai/http-conformance/falsify/mutations.txt | 1 + 2 files changed, 16 insertions(+) create mode 100644 wai/http-conformance/falsify/manifest-values-uncounted.patch diff --git a/wai/http-conformance/falsify/manifest-values-uncounted.patch b/wai/http-conformance/falsify/manifest-values-uncounted.patch new file mode 100644 index 000000000..e847e50ae --- /dev/null +++ b/wai/http-conformance/falsify/manifest-values-uncounted.patch @@ -0,0 +1,15 @@ +# must fail: head_malformed_manifest_too_many_values_O5 head_malformed_manifest_too_many_values_OD +# a manifest past the value limit is built, not refused (container::read_tree) +diff --git a/wai/wai-rs/src/container.rs b/wai/wai-rs/src/container.rs +index b7b223f0..a06a74fd 100644 +--- a/wai/wai-rs/src/container.rs ++++ b/wai/wai-rs/src/container.rs +@@ -1706,7 +1706,7 @@ fn read_tree(text: &[u8]) -> Result { + use serde::de::{DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; + use std::cell::{Cell, RefCell}; + let sizes = RefCell::new(Vec::new()); +- scan_json(text, MANIFEST_MAX_VALUES, Some(&sizes))?; ++ scan_json(text, usize::MAX, Some(&sizes))?; + let mut sizes = sizes.into_inner(); + sizes.shrink_to_fit(); + /// One value of the tree; `at` is the next container's index in `sizes`. diff --git a/wai/http-conformance/falsify/mutations.txt b/wai/http-conformance/falsify/mutations.txt index 67b8ada87..3c81c40af 100644 --- a/wai/http-conformance/falsify/mutations.txt +++ b/wai/http-conformance/falsify/mutations.txt @@ -30,6 +30,7 @@ key-unchecked live_edge_other_key live_edge_other_key_walked walk-unbounded live_edge_reanchored live_edge_reanchored_then_reloaded live_edge_reanchored_skipping reload-walked live_edge_unchanged_reload live_edge_reanchored_then_reloaded manifest-limit-dropped head_malformed_manifest_too_large_O5 head_malformed_manifest_too_large_OD +manifest-values-uncounted head_malformed_manifest_too_many_values_O5 head_malformed_manifest_too_many_values_OD xml-char-unchecked mpd_rep_id_not_a_char xml-attribute-twice dash_adjacency_attribute_twice dash-start-ignored dash_adjacency_period_start -- 2.54.0 (Apple Git-157) From 62f0c8117d290857e46802340bba4bc5a7de1dba Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Mon, 12 Oct 2026 20:34:58 -0400 Subject: [PATCH 05/10] wai: zstd and xz decode within a bound of their output and window wai.text.zstd and wai.text.xz decoded the whole stream into memory, so a few kilobytes could ask for any amount. Both decoders now write at most 2^27 bytes (text_decoded_too_long) into a buffer that is never sized past that, and give the decoder at most 2^27 bytes of window or dictionary (text_window_too_large), refused from the stream's declaration. A zstd frame that declares a longer content size is refused before anything is decoded. The writers refuse what a reader may refuse. Co-Authored-By: Claude Opus 5.5 --- wai/wai-rs/src/codecs/text.rs | 131 ++++++++++++++++- wai/wai-rs/tests/text_decode_bounds.rs | 195 +++++++++++++++++++++++++ wai/wai-rs/tests/text_decode_memory.rs | 108 ++++++++++++++ 3 files changed, 427 insertions(+), 7 deletions(-) create mode 100644 wai/wai-rs/tests/text_decode_bounds.rs create mode 100644 wai/wai-rs/tests/text_decode_memory.rs diff --git a/wai/wai-rs/src/codecs/text.rs b/wai/wai-rs/src/codecs/text.rs index 9d63f244f..32791339d 100644 --- a/wai/wai-rs/src/codecs/text.rs +++ b/wai/wai-rs/src/codecs/text.rs @@ -12,17 +12,112 @@ impl From for TextError { pub type Result = std::result::Result; -/// zstd at the given compression level (1..=22; 22 is "ultra", slow). +/// zstd at the given compression level (1..=22; 22 is "ultra", slow). Refuses more +/// than [`TEXT_DECODED_MAX_LEN`] bytes (`text_decoded_too_long`): a conforming encoder +/// writes no payload a conforming reader may refuse (SPEC §2, *Reader limits*). No +/// level asks for a window past [`TEXT_WINDOW_MAX_LEN`]. pub fn zstd_encode(data: &[u8], level: i32) -> Result> { + if data.len() > TEXT_DECODED_MAX_LEN { + return Err(too_long(TEXT_DECODED_MAX_LEN)); + } Ok(zstd::stream::encode_all(data, level)?) } -pub fn zstd_decode(bytes: &[u8]) -> Result> { - Ok(zstd::stream::decode_all(bytes)?) +/// The most bytes a `wai.text.zstd` or `wai.text.xz` payload may decode to that a reader +/// must accept (SPEC §2, *Reader limits*): 2^27. The decoders here write no more than +/// this, whatever the payload asks for: one that decodes to more is refused +/// (`text_decoded_too_long`) when its output would pass the limit, so the output buffer +/// never holds more. The reference writers refuse to encode more. +pub const TEXT_DECODED_MAX_LEN: usize = 1 << 27; + +/// The base-2 logarithm of [`TEXT_WINDOW_MAX_LEN`]. +pub const TEXT_WINDOW_MAX_LOG: u32 = 27; + +/// The most history a `wai.text.zstd` or `wai.text.xz` stream may ask its decoder to +/// keep that a reader must accept (SPEC §2, *Reader limits*): 2^27 bytes, as the zstd +/// frame's window size or the LZMA2 dictionary size. A stream that declares more is +/// refused (`text_window_too_large`) from the declaration, before the decoder allocates +/// it. Together with [`TEXT_DECODED_MAX_LEN`] it bounds the memory a decode takes: the +/// length limit bounds the output, and this bounds what the decoder holds beside it. +pub const TEXT_WINDOW_MAX_LEN: usize = 1 << TEXT_WINDOW_MAX_LOG; + +/// What liblzma may allocate to decode one stream: the dictionary, at most +/// [`TEXT_WINDOW_MAX_LEN`], and the decoder's own fixed state beside it. +const XZ_DECODER_MEMLIMIT: u64 = TEXT_WINDOW_MAX_LEN as u64 + (1 << 20); + +fn too_long(max_out: usize) -> TextError { + TextError(format!( + "text_decoded_too_long: the payload decodes to more than {max_out} bytes, the most this reader accepts (SPEC §2)" + )) } -/// XZ/LZMA2 at the given preset (0..=9; 6 is the default, 9 is "extreme"). +fn window_too_large() -> TextError { + TextError(format!( + "text_window_too_large: the stream asks its decoder to keep more than {TEXT_WINDOW_MAX_LEN} bytes of history, the most a reader need give it (SPEC §2)" + )) +} + +/// Reads `r` to its end into a buffer that never holds, and is never sized for, more +/// than `max_out` bytes. `Ok(None)` when the reader has more than that to give. +fn read_within(mut r: R, max_out: usize) -> std::io::Result>> { + let mut out: Vec = Vec::new(); + let mut chunk = vec![0u8; 64 * 1024]; + loop { + let n = match r.read(&mut chunk) { + Ok(0) => return Ok(Some(out)), + Ok(n) => n, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + }; + let room = max_out - out.len(); + if n > room { + return Ok(None); + } + if out.capacity() - out.len() < n { + // Grow as a `Vec` would, by about its length, but never past the limit. + out.reserve_exact(out.len().max(chunk.len()).min(room).max(n)); + } + out.extend_from_slice(&chunk[..n]); + } +} + +/// Decodes a `wai.text.zstd` payload within the reader limits of SPEC §2: +/// [`TEXT_DECODED_MAX_LEN`] bytes of output and [`TEXT_WINDOW_MAX_LEN`] bytes of window. +pub fn zstd_decode(bytes: &[u8]) -> Result> { + zstd_decode_within(bytes, TEXT_DECODED_MAX_LEN) +} + +/// [`zstd_decode`] with the output limit the caller gives, for a sink that accepts less +/// than the reader limit, and for tests. `max_out` is capped at +/// [`TEXT_DECODED_MAX_LEN`]. The window limit is not the caller's to change. +pub fn zstd_decode_within(bytes: &[u8], max_out: usize) -> Result> { + let max_out = max_out.min(TEXT_DECODED_MAX_LEN); + // A frame that declares its content size past the limit is refused from the + // declaration, before anything is decoded. A frame need not declare it, and a + // payload may hold more frames than the first, so the output is counted too. + if let Ok(Some(declared)) = zstd::zstd_safe::get_frame_content_size(bytes) { + if declared > max_out as u64 { + return Err(too_long(max_out)); + } + } + let mut dec = zstd::stream::read::Decoder::with_buffer(bytes)?; + dec.window_log_max(TEXT_WINDOW_MAX_LOG)?; + match read_within(dec, max_out) { + Ok(Some(out)) => Ok(out), + Ok(None) => Err(too_long(max_out)), + // libzstd's name for a frame whose window is past the decoder's limit. + Err(e) if e.to_string().contains("too much memory") => Err(window_too_large()), + Err(e) => Err(e.into()), + } +} + +/// XZ/LZMA2 at the given preset (0..=9; 6 is the default, 9 is "extreme"). Refuses more +/// than [`TEXT_DECODED_MAX_LEN`] bytes (`text_decoded_too_long`), as [`zstd_encode`] +/// does. No preset asks for a dictionary past [`TEXT_WINDOW_MAX_LEN`]. pub fn xz_encode(data: &[u8], level: u32) -> Result> { + if data.len() > TEXT_DECODED_MAX_LEN { + return Err(too_long(TEXT_DECODED_MAX_LEN)); + } let mut out = Vec::with_capacity(data.len() / 4); let mut enc = xz2::write::XzEncoder::new(&mut out, level); enc.write_all(data)?; @@ -30,10 +125,32 @@ pub fn xz_encode(data: &[u8], level: u32) -> Result> { Ok(out) } +/// Decodes a `wai.text.xz` payload within the reader limits of SPEC §2: +/// [`TEXT_DECODED_MAX_LEN`] bytes of output and [`TEXT_WINDOW_MAX_LEN`] bytes of +/// dictionary. pub fn xz_decode(bytes: &[u8]) -> Result> { - let mut out = Vec::new(); - xz2::read::XzDecoder::new(bytes).read_to_end(&mut out)?; - Ok(out) + xz_decode_within(bytes, TEXT_DECODED_MAX_LEN) +} + +/// [`xz_decode`] with the output limit the caller gives, for a sink that accepts less +/// than the reader limit, and for tests. `max_out` is capped at +/// [`TEXT_DECODED_MAX_LEN`]. The dictionary limit is not the caller's to change. +pub fn xz_decode_within(bytes: &[u8], max_out: usize) -> Result> { + let max_out = max_out.min(TEXT_DECODED_MAX_LEN); + // The decoder is given a memory limit, so a block that declares a dictionary past + // it is refused from its header, before the dictionary is allocated. An xz stream + // states its decoded length only at its end, so the output is counted as it comes. + let stream = xz2::stream::Stream::new_stream_decoder(XZ_DECODER_MEMLIMIT, 0)?; + let dec = xz2::read::XzDecoder::new_stream(bytes, stream); + match read_within(dec, max_out) { + Ok(Some(out)) => Ok(out), + Ok(None) => Err(too_long(max_out)), + Err(e) if matches!( + e.get_ref().and_then(|i| i.downcast_ref::()), + Some(xz2::stream::Error::MemLimit) + ) => Err(window_too_large()), + Err(e) => Err(e.into()), + } } #[cfg(test)] diff --git a/wai/wai-rs/tests/text_decode_bounds.rs b/wai/wai-rs/tests/text_decode_bounds.rs new file mode 100644 index 000000000..6094e2eec --- /dev/null +++ b/wai/wai-rs/tests/text_decode_bounds.rs @@ -0,0 +1,195 @@ +//! `wai.text.zstd` and `wai.text.xz` decode within the reader limits of SPEC §2: a +//! payload that decodes to more than the length limit, or whose stream asks its decoder +//! to keep more history than the window limit, is refused by name, and one at either +//! limit decodes. +//! +//! - The length limit, at a small limit the caller gives and at the limit the +//! specification sets, for a frame that declares its length and for one that does not. +//! - A zstd frame that declares a length past the limit is refused from the declaration: +//! the same frame cut off after its header is refused the same way. +//! - The window limit, on streams made here that declare one at the limit and one past +//! it. Each holds a byte or a line of content: what is refused is the declaration. +//! - The writers refuse what the readers may refuse. +//! +//! The memory the output buffer takes is measured in `text_decode_memory.rs`. +//! +//! Exercises the codec wrappers, so it requires the `codecs` feature. +#![cfg(feature = "codecs")] + +use std::io::Write; + +use wai::codecs::text::{ + xz_decode, xz_decode_within, xz_encode, zstd_decode, zstd_decode_within, zstd_encode, + TEXT_DECODED_MAX_LEN, TEXT_WINDOW_MAX_LEN, +}; + +fn refusal(r: Result) -> String { + r.expect_err("the payload must be refused").0 +} + +/// `n` bytes that compress well and are not all one value. +fn pattern(n: usize) -> Vec { + (0..n).map(|i| (i / 4096) as u8).collect() +} + +/// A zstd stream of `n` zero bytes whose frame declares no content size, written in +/// pieces so that making it takes no buffer of `n` bytes. +fn zstd_zeros(n: usize) -> Vec { + let piece = vec![0u8; 1 << 20]; + let mut enc = zstd::stream::write::Encoder::new(Vec::new(), 1).unwrap(); + let mut left = n; + while left > 0 { + let k = left.min(piece.len()); + enc.write_all(&piece[..k]).unwrap(); + left -= k; + } + enc.finish().unwrap() +} + +/// The same for xz. +fn xz_zeros(n: usize) -> Vec { + let piece = vec![0u8; 1 << 20]; + let mut enc = xz2::write::XzEncoder::new(Vec::new(), 0); + let mut left = n; + while left > 0 { + let k = left.min(piece.len()); + enc.write_all(&piece[..k]).unwrap(); + left -= k; + } + enc.finish().unwrap() +} + +#[test] +fn a_payload_at_the_callers_limit_decodes_and_one_byte_past_it_is_refused() { + const LIMIT: usize = 1 << 20; + let at = pattern(LIMIT); + let past = pattern(LIMIT + 1); + + // zstd, the frame declaring no content size (a streamed encode). + assert_eq!(zstd_decode_within(&zstd_encode(&at, 3).unwrap(), LIMIT).unwrap(), at); + let why = refusal(zstd_decode_within(&zstd_encode(&past, 3).unwrap(), LIMIT)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + + // zstd, the frame declaring its content size (a one-shot encode). + let declared_at = zstd::bulk::compress(&at, 3).unwrap(); + let declared_past = zstd::bulk::compress(&past, 3).unwrap(); + assert_eq!(zstd::zstd_safe::get_frame_content_size(&declared_past).ok().flatten(), Some(LIMIT as u64 + 1)); + assert_eq!(zstd_decode_within(&declared_at, LIMIT).unwrap(), at); + let why = refusal(zstd_decode_within(&declared_past, LIMIT)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + + // xz. + assert_eq!(xz_decode_within(&xz_encode(&at, 1).unwrap(), LIMIT).unwrap(), at); + let why = refusal(xz_decode_within(&xz_encode(&past, 1).unwrap(), LIMIT)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); +} + +#[test] +fn a_declared_length_past_the_limit_is_refused_from_the_declaration() { + const LIMIT: usize = 1 << 16; + let frame = zstd::bulk::compress(&pattern(LIMIT + 1), 3).unwrap(); + // The frame's first 18 bytes: its header, which is never longer, and at most a few + // bytes of its first block. Nothing can be decoded from them. + assert!(frame.len() > 18); + let header_only = &frame[..18]; + assert_eq!(zstd::zstd_safe::get_frame_content_size(header_only).ok().flatten(), Some(LIMIT as u64 + 1)); + let why = refusal(zstd_decode_within(header_only, LIMIT)); + assert!(why.starts_with("text_decoded_too_long:"), "refused for another reason: {why}"); + // The same cut, within the limit, is the truncated frame it is. + let why = refusal(zstd_decode_within(header_only, LIMIT + 1)); + assert!(!why.starts_with("text_decoded_too_long:"), "{why}"); +} + +#[test] +fn the_limit_without_an_argument_is_the_one_the_specification_sets() { + assert_eq!(TEXT_DECODED_MAX_LEN, 1 << 27); + + let out = zstd_decode(&zstd_zeros(TEXT_DECODED_MAX_LEN)).unwrap(); + assert_eq!(out.len(), TEXT_DECODED_MAX_LEN); + assert!(out.iter().all(|&b| b == 0)); + drop(out); + let past = zstd_zeros(TEXT_DECODED_MAX_LEN + 1); + let why = refusal(zstd_decode(&past)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + // A caller cannot raise it. + let why = refusal(zstd_decode_within(&past, usize::MAX)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + + let out = xz_decode(&xz_zeros(TEXT_DECODED_MAX_LEN)).unwrap(); + assert_eq!(out.len(), TEXT_DECODED_MAX_LEN); + drop(out); + let past = xz_zeros(TEXT_DECODED_MAX_LEN + 1); + let why = refusal(xz_decode(&past)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + let why = refusal(xz_decode_within(&past, usize::MAX)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); +} + +/// A zstd frame of one byte of content that declares a window of `2^window_log` bytes +/// (RFC 8878 §3.1.1): the magic number; a frame header descriptor of 0, which says a +/// window descriptor follows and nothing else does; the window descriptor, whose high +/// five bits are `window_log − 10`; and one last block, of type RLE and size 1. +fn zstd_frame_declaring_window(window_log: u32) -> Vec { + let mut f = vec![0x28, 0xB5, 0x2F, 0xFD, 0x00, ((window_log - 10) << 3) as u8]; + f.extend_from_slice(&[0x0B, 0x00, 0x00, b'A']); + f +} + +#[test] +fn a_zstd_window_at_the_limit_decodes_and_one_past_it_is_refused() { + assert_eq!(TEXT_WINDOW_MAX_LEN, 1 << 27); + assert_eq!(zstd_decode(&zstd_frame_declaring_window(27)).unwrap(), b"A"); + let why = refusal(zstd_decode(&zstd_frame_declaring_window(28))); + assert!(why.starts_with("text_window_too_large:"), "{why}"); +} + +/// CRC-32 as the xz format uses it (IEEE 802.3, reflected). +fn crc32(bytes: &[u8]) -> u32 { + let mut c = !0u32; + for &b in bytes { + c ^= b as u32; + for _ in 0..8 { + c = (c >> 1) ^ (0xEDB8_8320 & (!(c & 1)).wrapping_add(1)); + } + } + !c +} + +/// An xz stream of a line of text whose one block declares the LZMA2 dictionary size +/// coded as `code` (the xz file format, §5.3.1: 30 is 2^27 bytes, 31 is 3·2^26). Made by +/// encoding the line, then rewriting the dictionary size in the block header and the +/// header's CRC-32. The stream header is 12 bytes; the block header follows it: its +/// size, the block flags, then the filter (id 0x21, one byte of properties, which is the +/// dictionary size), padding, and the CRC-32 of all of it. +fn xz_stream_declaring_dictionary(code: u8) -> (Vec, Vec) { + let line = b"one line of text, long enough to be worth a block\n".to_vec(); + let mut s = xz_encode(&line, 0).unwrap(); + let start = 12; + let size = (s[start] as usize + 1) * 4; + let header = start..start + size; + let at = (header.start + 2..header.end - 5) + .find(|&i| s[i] == 0x21 && s[i + 1] == 0x01) + .expect("the block header names the LZMA2 filter with one byte of properties"); + s[at + 2] = code; + let crc = crc32(&s[header.start..header.end - 4]); + s[header.end - 4..header.end].copy_from_slice(&crc.to_le_bytes()); + (s, line) +} + +#[test] +fn an_xz_dictionary_at_the_limit_decodes_and_one_past_it_is_refused() { + let (at, line) = xz_stream_declaring_dictionary(30); + assert_eq!(xz_decode(&at).unwrap(), line); + let (past, _) = xz_stream_declaring_dictionary(31); + let why = refusal(xz_decode(&past)); + assert!(why.starts_with("text_window_too_large:"), "{why}"); +} + +#[test] +fn the_writers_refuse_what_a_reader_may_refuse() { + let over = vec![0u8; TEXT_DECODED_MAX_LEN + 1]; + let why = refusal(zstd_encode(&over, 1)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + let why = refusal(xz_encode(&over, 0)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); +} diff --git a/wai/wai-rs/tests/text_decode_memory.rs b/wai/wai-rs/tests/text_decode_memory.rs new file mode 100644 index 000000000..a33e3b319 --- /dev/null +++ b/wai/wai-rs/tests/text_decode_memory.rs @@ -0,0 +1,108 @@ +//! Decoding a `wai.text.zstd` or `wai.text.xz` payload that would expand past the +//! length limit takes no more output buffer than the limit (SPEC §2, *Reader limits*). +//! Measured with a global allocator that counts the bytes live and their peak. +//! +//! Each payload here is a few kilobytes and decodes to four times the limit the call is +//! given. The call must refuse it, and the bytes it allocated on the way must stay +//! within the limit and a small allowance for its read buffer. A decode that grew its +//! buffer past the limit before checking, or that decoded everything and then measured +//! it, would not. +//! +//! The counter sees what this crate allocates: the output buffer and the read buffer. +//! The decoding library's own working memory is allocated by the library and is bounded +//! by the window limit, which `text_decode_bounds.rs` tests. +//! +//! Its own test binary with a single test, so that no other test's allocations land in +//! the windows it measures. Requires the `codecs` feature. +#![cfg(feature = "codecs")] + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::io::Write; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; + +use wai::codecs::text::{xz_decode_within, zstd_decode_within}; + +/// The system allocator, counting the bytes live and their peak. +struct Count; +static CUR: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +fn grow(n: usize) { + PEAK.fetch_max(CUR.fetch_add(n, Relaxed) + n, Relaxed); +} + +unsafe impl GlobalAlloc for Count { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + grow(l.size()); + unsafe { System.alloc(l) } + } + unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 { + grow(l.size()); + unsafe { System.alloc_zeroed(l) } + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { + if new >= l.size() { + grow(new - l.size()); + } else { + CUR.fetch_sub(l.size() - new, Relaxed); + } + unsafe { System.realloc(p, l, new) } + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + CUR.fetch_sub(l.size(), Relaxed); + unsafe { System.dealloc(p, l) } + } +} + +#[global_allocator] +static ALLOC: Count = Count; + +/// The most bytes live at once while `f` runs, above what was live when it started. +fn peak_of(f: impl FnOnce() -> T) -> (T, usize) { + let base = CUR.load(Relaxed); + PEAK.store(base, Relaxed); + let r = f(); + (r, PEAK.load(Relaxed).saturating_sub(base)) +} + +const LIMIT: usize = 8 << 20; +/// The read buffer, and room for the error's text. +const ALLOWANCE: usize = 1 << 20; + +fn zeros_through(mut w: W, n: usize) -> W { + let piece = vec![0u8; 1 << 20]; + let mut left = n; + while left > 0 { + let k = left.min(piece.len()); + w.write_all(&piece[..k]).unwrap(); + left -= k; + } + w +} + +#[test] +fn a_payload_that_expands_past_the_limit_takes_no_more_buffer_than_the_limit() { + let zstd_payload = zeros_through(zstd::stream::write::Encoder::new(Vec::new(), 1).unwrap(), 4 * LIMIT) + .finish() + .unwrap(); + let xz_payload = zeros_through(xz2::write::XzEncoder::new(Vec::new(), 0), 4 * LIMIT).finish().unwrap(); + assert!(zstd_payload.len() < LIMIT / 64 && xz_payload.len() < LIMIT / 64, "the payloads are small"); + + let (r, peak) = peak_of(|| zstd_decode_within(&zstd_payload, LIMIT)); + let why = r.expect_err("zstd: the payload must be refused").0; + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + assert!(peak <= LIMIT + ALLOWANCE, "zstd: refusing took {peak} bytes of buffer, limit {LIMIT}"); + assert!(peak >= LIMIT / 2, "zstd: the refusal came before the output neared the limit ({peak} bytes): is the payload what it should be?"); + + let (r, peak) = peak_of(|| xz_decode_within(&xz_payload, LIMIT)); + let why = r.expect_err("xz: the payload must be refused").0; + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + assert!(peak <= LIMIT + ALLOWANCE, "xz: refusing took {peak} bytes of buffer, limit {LIMIT}"); + assert!(peak >= LIMIT / 2, "xz: the refusal came before the output neared the limit ({peak} bytes): is the payload what it should be?"); + + // At the limit the decode succeeds, in a buffer of the limit. + let zstd_at = zeros_through(zstd::stream::write::Encoder::new(Vec::new(), 1).unwrap(), LIMIT).finish().unwrap(); + let (r, peak) = peak_of(|| zstd_decode_within(&zstd_at, LIMIT)); + assert_eq!(r.unwrap().len(), LIMIT); + assert!(peak <= LIMIT + ALLOWANCE, "zstd: decoding to the limit took {peak} bytes of buffer, limit {LIMIT}"); +} -- 2.54.0 (Apple Git-157) From 0263942fda3bd340817b9becc63578151b83901c Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Mon, 12 Oct 2026 20:40:30 -0400 Subject: [PATCH 06/10] =?UTF-8?q?wai:=20the=20text=20decode=20limits=20in?= =?UTF-8?q?=20SPEC=20=C2=A72,=20SECURITY.md=20and=20the=20C=20header?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SPEC §2 gains the reader limits for wai.text.zstd and wai.text.xz: 2^27 decoded bytes and 2^27 bytes of window or dictionary, with their refusal reasons. SECURITY.md now states what the manifest readers and the text decoders bound and which tests measure it, in place of the two gaps it listed. Co-Authored-By: Claude Opus 5.5 --- wai/SECURITY.md | 60 ++++++++++++++++++++++++----------- wai/SPEC.md | 17 ++++++++++ wai/wai-rs/include/wai.h | 35 ++++++++++++++++++++ wai/wai-rs/src/codecs/text.rs | 8 ++--- wai/wai-rs/src/ffi_codecs.rs | 7 ++++ 5 files changed, 105 insertions(+), 22 deletions(-) diff --git a/wai/SECURITY.md b/wai/SECURITY.md index bbaedeab9..465265b84 100644 --- a/wai/SECURITY.md +++ b/wai/SECURITY.md @@ -41,6 +41,7 @@ What the readers bound: |---|---|---| | every length field (`man_len`, `payload_len`, each rendition's offset and length) | checked against the bytes in hand before anything is sliced; sums formed so they cannot wrap (`saturating_add`; the `WAI2` payload total in `u64`) | SPEC §2, §7.1 | | manifest length | at most 134 217 728 bytes (2²⁷), sized to the largest manifest a conforming staged layer has. A longer one is refused from `man_len` alone, before it is read or, over HTTP, requested (`manifest_too_large`; `head_malformed` from `parse_head`) | SPEC §2, *Reader limits* | +| manifest values | at most 4 194 304 JSON values (2²²), the manifest object and every value in it at any depth; a member's name is not one. One more is refused while the values are counted, before anything is built from the manifest (`manifest_too_many_values`; `head_malformed` from `parse_head`) | SPEC §2, *Reader limits* | | manifest nesting | at most 127 nested arrays and objects, the manifest object counting as one; 128 are refused | SPEC §2, *Reader limits* | | member names | a member named twice in one object, at any depth, is refused | SPEC §2 | | `WAI2` rendition table | at most 65 535 rows (`u16`), as many as the manifest's `renditions`, packed contiguously from offset 0 | SPEC §7.1 | @@ -52,16 +53,26 @@ it allocate more than it was given. A `WAI1` reader ignores bytes after the payl HTTP object that carries any is refused ([http-delivery](extensions/http-delivery.md) §5.1). -**Parsing a manifest takes more memory than its length.** The reference readers parse -the manifest into a JSON tree (`serde_json::Value`, for `target`, `refines`, -`derived_from` and every member this revision does not define), and a parsed value -takes more memory than its text: about seven times the length of a 94 MB `prev` that -names a producer for each of 2²⁰ units, the largest manifest a conforming layer has, -and nearly 40 times the length of a manifest of many small values, such as one long -array of zeros. A manifest at the length limit can take some 5 GB and several seconds -to parse, and a reader may not refuse it for its length (SPEC §2). The length limit -bounds what a reader must read, not what parsing it costs; the reference sets no limit -of its own on the parse. +**What parsing a manifest takes.** A parsed value takes more memory than its text, and +many small values can take many times their length, so the length limit bounds what a +reader must read and not what a parse costs. The value limit bounds the rest. The +reference readers count a manifest's values before they build anything from it +(`check_manifest_text`) and refuse it as soon as the count passes the limit; they then +build each array and object at the size counted, and move the manifest's members out of +that one tree (`read_manifest`, `read_manifest_v2`). On a 64-bit target a read allocates +at most 128 bytes for each value and one for each byte of the manifest, and for a `WAI2` +one rendition entry of 768 bytes for each it lists: at most 640 MiB for a `WAI1` +manifest at both limits, whatever its shape, and 48 MiB more for a `WAI2`'s entries. + +`tests/manifest_parse_memory.rs` measures this with an allocator that counts every byte +asked for: a manifest of small values at the length limit is refused for its values +before anything is built; one at the length limit within the value limit is read; a +manifest of each shape at the value limit is read and one value past it refused; and so +are the largest manifest a conforming staged layer has and a `WAI2` listing as many +renditions as its table holds. Each stays within the bound above. +`tests/manifest_reader_equivalence.rs` holds these readers to the result of the derived +reader they replace, for every envelope committed under `wai/` and for each with every +member removed or given a value of each JSON type. The library tests in `container.rs` and `http_binding/tests.rs` check the manifest limits at their edges: from its first eight bytes, every reader refuses a manifest one @@ -77,9 +88,10 @@ limit. The stdlib Python verifiers read envelopes too. `staged_verify.py`, `claim_verify.py`, `pack_verify.py` and `http_verify.py` refuse a manifest past the -length limit as the reference does. `http_verify.py` and `wai_http_vectors` agree on -`parse_head` at the limit in the HTTP corpus; the other three each build an envelope in -memory with a manifest one byte past it and require it refused. Those that read +length limit, or past the value limit, as the reference does. `http_verify.py` and `wai_http_vectors` agree on +`parse_head` at each limit in the HTTP corpus; the other three each build an envelope in +memory with a manifest one byte past the length limit, and one value past the value +limit, and require each refused. Those that read manifests through `staged_verify.strict_json` refuse nesting past 127; `pack_verify.py` reads up to the interpreter's recursion limit and refuses past it as unparseable. @@ -90,6 +102,21 @@ bytes and 2²⁰ frames, and a sink cost limit (`int_payload::check_cost`, `wai.neural.int_mlicpp` refuses a header whose shapes, or whose buffers, pass its cap before it allocates them (`tests/int_mlicpp_preflight.rs`). +`wai.text.zstd` and `wai.text.xz` carry streams whose formats bound neither what a +payload decodes to nor the history its decoder must keep, so the specification sets +both (SPEC §2): at most 2²⁷ decoded bytes, and at most 2²⁷ bytes of window or +dictionary. The reference decoders (`codecs/text.rs`) write their output into a buffer +that is never sized past the length limit and refuse the payload when its output would +pass it (`text_decoded_too_long`); a Zstandard frame that declares a longer content +size is refused before anything is decoded. They give the decoding library the window +limit, so a stream that declares more is refused from its header, before the library +allocates it (`text_window_too_large`). The encoders refuse input past the length limit. +`tests/text_decode_bounds.rs` checks each limit at its edge, on streams made in the test +that are at it and one step past it; `tests/text_decode_memory.rs` measures, with a +counting allocator, that refusing a payload of a few kilobytes that would expand to +four times a limit takes no more output buffer than the limit. The C entry points +`wai_text_zstd_decode` and `wai_text_xz_decode` are these decoders. + ## Native codec libraries and sandboxed decode The default `codecs` feature binds C and C++ libraries for the floor and the @@ -132,11 +159,8 @@ element beside each kernel and over every committed integer-decoder vector Known gaps: -- **Decompression output is not bounded.** `wai.text.zstd`, in the floor, and - `wai.text.xz` decode the whole stream into memory, so a small payload can expand to - any size the format allows. The image, audio and video decoders allocate what the - bitstream declares, within the libraries' own limits. The reference sets no output - limit of its own. +- **The image, audio and video decoders set no output limit of their own.** They + allocate what the bitstream declares, within the libraries' own limits. - **Fuzzing is shallow.** `tests/fuzz_smoke.rs` feeds seeded random bytes to each codec wrapper and to `Wai::from_bytes` and requires that nothing panics. It needs the `codecs` feature, which no CI workflow builds, so it runs only by hand. There is diff --git a/wai/SPEC.md b/wai/SPEC.md index 0c9eb3f84..dab3520c0 100644 --- a/wai/SPEC.md +++ b/wai/SPEC.md @@ -115,6 +115,23 @@ each byte of the manifest, with, for a `WAI2`, a rendition entry of 768 bytes fo lists: at most 640 MiB for a `WAI1` manifest at both limits, whatever its shape, and 48 MiB more for a `WAI2`'s entries (informative; `SECURITY.md`). +A `wai.text.zstd` or `wai.text.xz` payload (§5) has reader limits of its own, since +neither stream format bounds what a payload may ask of its decoder. A reader MUST NOT +refuse one for the length it decodes to, or for the history its decoder must keep, while +it decodes to at most 134 217 728 bytes (2²⁷) and every frame or block of it declares at +most 2²⁷ bytes of history: for Zstandard the frame's window size, which for a +single-segment frame is its content size (RFC 8878 §3.1.1.1); for XZ the LZMA2 +dictionary size. It MAY refuse one that decodes to more (`text_decoded_too_long`) or +declares more (`text_window_too_large`). A reader that refuses for length does so when +its output would pass the limit, in a buffer it never sizes past the limit; where a +Zstandard frame declares a content size past the limit, it does so from the declaration, +before it decodes anything. A reader that refuses for the history does so from the +declaration, before its decoder allocates it. The reference readers refuse both. A +decode then takes its output, at most the length limit, the decoder's history, at most +the other, and the decoder's fixed state. As for a manifest, a conforming encoder writes +no payload past either limit, and the reference writers refuse one; bytes longer than +the length limit are carried as more than one object. + An envelope delivered over HTTP is one object of the proposed media type `application/wai`, named by its receipt content hash, and fetched whole or, for a `WAI2`, one verified component at a time diff --git a/wai/wai-rs/include/wai.h b/wai/wai-rs/include/wai.h index 6c6a0c46a..01ce30f7d 100644 --- a/wai/wai-rs/include/wai.h +++ b/wai/wai-rs/include/wai.h @@ -30,6 +30,30 @@ #include #include +/** + * The most bytes a `wai.text.zstd` or `wai.text.xz` payload may decode to that a reader + * must accept (SPEC §2, *Reader limits*): 2^27. The decoders here write no more than + * this, whatever the payload asks for: one that decodes to more is refused + * (`text_decoded_too_long`) when its output would pass the limit, so the output buffer + * never holds more. The reference writers refuse to encode more. + */ +#define TEXT_DECODED_MAX_LEN (1 << 27) + +/** + * The base-2 logarithm of [`TEXT_WINDOW_MAX_LEN`]. + */ +#define TEXT_WINDOW_MAX_LOG 27 + +/** + * The most history a `wai.text.zstd` or `wai.text.xz` stream may ask its decoder to + * keep that a reader must accept (SPEC §2, *Reader limits*): 2^27 bytes, as the zstd + * frame's window size or the LZMA2 dictionary size. A stream that declares more is + * refused (`text_window_too_large`) from the declaration, before the decoder allocates + * it. Together with [`TEXT_DECODED_MAX_LEN`] it bounds the memory a decode takes: the + * length limit bounds the output, and this bounds what the decoder holds beside it. + */ +#define TEXT_WINDOW_MAX_LEN (1 << TEXT_WINDOW_MAX_LOG) + /** * The longest manifest, in bytes, a reader must accept (SPEC §2, *Reader limits*): * 2^27 (128 MiB). It is sized to the largest manifest a conforming layer has: a `prev` @@ -680,6 +704,13 @@ int wai_text_zstd_encode(const uint8_t *data, int32_t level, struct wai_buffer_t *out); +/** + * Decode a `wai.text.zstd` payload within the reader limits of SPEC §2: at most + * [`TEXT_DECODED_MAX_LEN`](crate::codecs::text::TEXT_DECODED_MAX_LEN) bytes of output + * and [`TEXT_WINDOW_MAX_LEN`](crate::codecs::text::TEXT_WINDOW_MAX_LEN) bytes of + * window. A payload past either is refused with `WAI_ERR_CODEC`; the message begins + * `text_decoded_too_long` or `text_window_too_large`. + */ int wai_text_zstd_decode(const uint8_t *bytes, uintptr_t len, struct wai_buffer_t *out); @@ -688,6 +719,10 @@ int wai_text_xz_encode(const uint8_t *data, uint32_t level, struct wai_buffer_t *out); +/** + * Decode a `wai.text.xz` payload within the same limits as + * [`wai_text_zstd_decode`], the window limit applying to the dictionary. + */ int wai_text_xz_decode(const uint8_t *bytes, uintptr_t len, struct wai_buffer_t *out); /** diff --git a/wai/wai-rs/src/codecs/text.rs b/wai/wai-rs/src/codecs/text.rs index 32791339d..f012a865d 100644 --- a/wai/wai-rs/src/codecs/text.rs +++ b/wai/wai-rs/src/codecs/text.rs @@ -87,8 +87,8 @@ pub fn zstd_decode(bytes: &[u8]) -> Result> { zstd_decode_within(bytes, TEXT_DECODED_MAX_LEN) } -/// [`zstd_decode`] with the output limit the caller gives, for a sink that accepts less -/// than the reader limit, and for tests. `max_out` is capped at +/// [`zstd_decode`] with the output limit the caller gives: for a caller that knows what +/// its payload may hold, and for tests. `max_out` is capped at /// [`TEXT_DECODED_MAX_LEN`]. The window limit is not the caller's to change. pub fn zstd_decode_within(bytes: &[u8], max_out: usize) -> Result> { let max_out = max_out.min(TEXT_DECODED_MAX_LEN); @@ -132,8 +132,8 @@ pub fn xz_decode(bytes: &[u8]) -> Result> { xz_decode_within(bytes, TEXT_DECODED_MAX_LEN) } -/// [`xz_decode`] with the output limit the caller gives, for a sink that accepts less -/// than the reader limit, and for tests. `max_out` is capped at +/// [`xz_decode`] with the output limit the caller gives: for a caller that knows what +/// its payload may hold, and for tests. `max_out` is capped at /// [`TEXT_DECODED_MAX_LEN`]. The dictionary limit is not the caller's to change. pub fn xz_decode_within(bytes: &[u8], max_out: usize) -> Result> { let max_out = max_out.min(TEXT_DECODED_MAX_LEN); diff --git a/wai/wai-rs/src/ffi_codecs.rs b/wai/wai-rs/src/ffi_codecs.rs index 6b10c8ca8..44bd304e9 100644 --- a/wai/wai-rs/src/ffi_codecs.rs +++ b/wai/wai-rs/src/ffi_codecs.rs @@ -269,6 +269,11 @@ pub unsafe extern "C" fn wai_text_zstd_encode( }) } +/// Decode a `wai.text.zstd` payload within the reader limits of SPEC §2: at most +/// [`TEXT_DECODED_MAX_LEN`](crate::codecs::text::TEXT_DECODED_MAX_LEN) bytes of output +/// and [`TEXT_WINDOW_MAX_LEN`](crate::codecs::text::TEXT_WINDOW_MAX_LEN) bytes of +/// window. A payload past either is refused with `WAI_ERR_CODEC`; the message begins +/// `text_decoded_too_long` or `text_window_too_large`. #[unsafe(no_mangle)] pub unsafe extern "C" fn wai_text_zstd_decode( bytes: *const u8, len: usize, out: *mut WaiBuffer, @@ -293,6 +298,8 @@ pub unsafe extern "C" fn wai_text_xz_encode( }) } +/// Decode a `wai.text.xz` payload within the same limits as +/// [`wai_text_zstd_decode`], the window limit applying to the dictionary. #[unsafe(no_mangle)] pub unsafe extern "C" fn wai_text_xz_decode( bytes: *const u8, len: usize, out: *mut WaiBuffer, -- 2.54.0 (Apple Git-157) From 86d9260ff9dd9566d54e7baf4c59cd859c09176c Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Mon, 12 Oct 2026 20:43:11 -0400 Subject: [PATCH 07/10] wai: the codec wrappers run in CI; SECURITY.md and the changelog say what is bounded wai-conformance.yml gains codec-wrappers on the macOS runner: fuzz smoke, the text decode limits and the C entry points, built with the codecs feature, which no job built before. The step fails when a test binary ran no test. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wai-conformance.yml | 39 +++++++++++++++++++++++++++ wai/CHANGELOG.md | 32 ++++++++++++++++++++++ wai/SECURITY.md | 15 ++++++----- 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/.github/workflows/wai-conformance.yml b/.github/workflows/wai-conformance.yml index de128ba15..09fce6a7d 100644 --- a/.github/workflows/wai-conformance.yml +++ b/.github/workflows/wai-conformance.yml @@ -882,6 +882,45 @@ jobs: working-directory: wai/byte-exact-conformance run: cargo test --release + # The codec wrappers are behind `codecs`, which binds C and C++ libraries that no + # container image of this fleet carries, so no other job compiles them. A feature no + # workflow enables has no CI coverage at all: without this job the wrappers' tests, + # and the limits on what a text payload may decode to (SPEC §2), pass by not running. + # + # One platform, the `macos` runner, whose host has the libraries. What the job proves + # is how the wrappers treat hostile bytes and where their limits fall, which is the + # same code on every host. A missing library fails the build step by name. + codec-wrappers: + name: codec wrappers · fuzz smoke, text decode limits, C entry points + runs-on: macos + steps: + - uses: actions/checkout@v5 + + - name: Install Rust 1.98.0 + shell: bash + run: | + export PATH="$HOME/.cargo/bin:$PATH" + rustup toolchain install 1.98.0 --profile minimal + rustup default 1.98.0 + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + # Every one of these test files is `#![cfg(feature = "codecs")]`: built without + # the feature each reports "0 passed" and exits 0. So the step names the feature, + # and then requires that each of the four binaries ran at least one test. + - name: Fuzz smoke, the text decode limits and the C entry points + working-directory: wai/wai-rs + shell: bash + run: | + set -o pipefail + cargo test --locked --features codecs \ + --test fuzz_smoke --test text_decode_bounds --test text_decode_memory --test ffi_robustness \ + 2>&1 | tee "$RUNNER_TEMP/codec-wrappers.log" + ran=$(grep -cE '^test result: ok\. [1-9][0-9]* passed' "$RUNNER_TEMP/codec-wrappers.log" || true) + if [ "$ran" -ne 4 ]; then + echo "::error::expected four test binaries that each ran tests; $ran did" + exit 1 + fi + # The C2PA emitter is gated behind an off-by-default feature, so the four-platform # job above — which builds a fixed feature list — never compiles it. A feature no # workflow enables has no CI coverage at all: the suite passes, and the tests in it diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index bc5858dcf..bf4bbaee3 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -6,6 +6,38 @@ wire format may change and such changes are recorded here. ## v0.4.0 — current +### Reader limits that bound memory: manifest values, and text decode + +Two reader limits join the manifest length limit (SPEC §2, *Reader limits*), and with +them a reader's memory is bounded by what the specification states, not by what an +envelope asks for. `SECURITY.md` listed both as known gaps; it now states the bounds +and the tests that measure them. + +- **Manifest values.** A reader need accept no manifest of more than 2²² JSON values, + the manifest object and every value in it at any depth. The reference readers count + the values before they build anything, refuse one past the limit + (`manifest_too_many_values`), and build what they read at the size counted: at most + 128 bytes a value and one a byte on a 64-bit target, where a manifest of small + values at the length limit took several gigabytes to parse. The stdlib Python + verifiers refuse the same manifests, and the HTTP corpus holds a case at the limit + and one past it. +- **Text decode.** A `wai.text.zstd` or `wai.text.xz` payload decodes to at most 2²⁷ + bytes and declares at most 2²⁷ bytes of window or dictionary; a reader may refuse + more (`text_decoded_too_long`, `text_window_too_large`). The reference decoders + write into a buffer never sized past the limit, refuse a Zstandard frame that + declares a longer content size before decoding anything, and give the decoding + library the window limit, so a declaration past it is refused from the stream's + header. The reference encoders refuse input past the length limit. The C entry + points `wai_text_zstd_decode` and `wai_text_xz_decode` are bounded the same way; the + header gains `TEXT_DECODED_MAX_LEN` and `TEXT_WINDOW_MAX_LEN`. +- **Planned break for writers.** A conforming encoder writes no manifest past the value + limit and no text payload past either text limit. No committed envelope or vector is + past any of them, and no digest changes. +- **The codec wrappers in CI.** `wai-conformance.yml` gains `codec-wrappers`, on the + macOS runner: the wrappers' fuzz smoke, the text decode limits and the C entry + points, built with `codecs`. The step fails if any of its test binaries ran no test, + which is what a build without the feature does. + ### wai-quantum 0.4.3: metered runs, placement by run, and the fabric wrapper Additive. The default features, `full`, both implementations of the WebGPU member, and diff --git a/wai/SECURITY.md b/wai/SECURITY.md index 465265b84..11205c26d 100644 --- a/wai/SECURITY.md +++ b/wai/SECURITY.md @@ -135,11 +135,12 @@ run it in a WebAssembly runtime. What does not touch them: -- CI builds, except the neural workflows: no workflow enables `codecs` or `semantic`, - so no CI build links the codec libraries. `wai-neural-gates.yml` and +- Most CI builds. One job builds `codecs`: `codec-wrappers` in `wai-conformance.yml`, + on the macOS runner, which runs the wrappers' fuzz smoke, the text decode limits and + the C entry points through the libraries. `wai-neural-gates.yml` and `wai-neural-e2e.yml` build the `neural` feature, which links ONNX Runtime and - libzstd, and run the float neural decoders through them; no other workflow's build - links any of these libraries. + libzstd, and run the float neural decoders through them. No workflow enables + `semantic`, and no other workflow's build links any of these libraries. - WAI's own decoders are Rust: the integer codecs, worlds, scores, films and the generator. Outside the C ABI (`ffi.rs`, `ffi_codecs.rs`, `ffi_neural.rs`), the only `unsafe` in `wai-rs`'s library code is in the AArch64 SIMD kernels of @@ -162,9 +163,9 @@ Known gaps: - **The image, audio and video decoders set no output limit of their own.** They allocate what the bitstream declares, within the libraries' own limits. - **Fuzzing is shallow.** `tests/fuzz_smoke.rs` feeds seeded random bytes to each - codec wrapper and to `Wai::from_bytes` and requires that nothing panics. It needs - the `codecs` feature, which no CI workflow builds, so it runs only by hand. There is - no coverage-guided fuzzing of the manifest parser, the integer payload parsers or the + codec wrapper and to `Wai::from_bytes` and requires that nothing panics. It runs in + CI on one platform (`codec-wrappers` in `wai-conformance.yml`). There is no + coverage-guided fuzzing of the manifest parser, the integer payload parsers or the codec wrappers. ## Module authorisation -- 2.54.0 (Apple Git-157) From 3ba49c32bd762c84ae0d7539bb7b39f2afdd1a11 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Tue, 13 Oct 2026 00:55:29 -0400 Subject: [PATCH 08/10] wai: the refused-layer memory check measures against the envelope's read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The check bounded a refused layer's peak by twice the envelope's length. The manifest reader now builds its tree at the size counted, within the bound SPEC §2 states, and that read is the larger part of what a refusal takes, so the old bound failed and a peak could no longer show a unit decoded after the read. The check now measures reading the same envelope and bounds what the refusal allocates beyond it. Falsified: one unit's output allocated before the refusal fails it. Co-Authored-By: Claude Opus 5.5 --- wai/wai-rs/tests/staged_resources.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/wai/wai-rs/tests/staged_resources.rs b/wai/wai-rs/tests/staged_resources.rs index 3e5991480..8cd7240b6 100644 --- a/wai/wai-rs/tests/staged_resources.rs +++ b/wai/wai-rs/tests/staged_resources.rs @@ -351,9 +351,17 @@ fn staged_sink_resources() { let chain = sink.chain(&bh).unwrap(); assert!(chain.units().iter().all(|s| s.stage == 0)); assert_eq!(chain.state_bytes(), unit_bytes); - // The refusal held the envelope as parsed and the payload's parse, and no unit's output. - assert!((refused.peak as u64) < 2 * l1.len() as u64 + c.working_bytes - c.output_bytes / 4, - "a refused layer held {} bytes", refused.peak); + // The refusal read the envelope and its payload's header, and decoded no unit. What it + // allocated is set against what reading the same envelope allocates, measured here: + // the read is the larger part, and SPEC §2 bounds it by the manifest's values and + // length. Beyond the read, the refusal allocated less than the envelope's length, the + // payload's parse and half of one unit's output (the layer covers four units). Bytes + // allocated, not the peak: a unit decoded after the read would sit under the read's + // own peak and never raise it. + let (_, read) = measure(|| Wai::from_bytes(&l1).unwrap()); + let beyond = (refused.allocated as u64).saturating_sub(read.allocated as u64); + assert!(beyond < l1.len() as u64 + c.working_bytes + c.output_bytes / 8, + "a refused layer allocated {beyond} bytes beyond reading its envelope"); // At the budget: applied; the state is the units' new output and 32 bytes a stage. let mut sink = StageSink::new(cfg(16, unit_bytes + reserve)); sink.add_base(info.clone(), units.clone()); -- 2.54.0 (Apple Git-157) From dbe084adda4c8e05bdfb5f675cfff294c410f4cb Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Tue, 13 Oct 2026 00:58:45 -0400 Subject: [PATCH 09/10] wai-conformance: run the manifest memory and reader-equivalence tests Neither test file was named by any step, so no job ran them. They join the release step of the cargo-class job on every platform leg. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wai-conformance.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wai-conformance.yml b/.github/workflows/wai-conformance.yml index 09fce6a7d..0f8049376 100644 --- a/.github/workflows/wai-conformance.yml +++ b/.github/workflows/wai-conformance.yml @@ -248,12 +248,18 @@ jobs: # instr-v0.json lists as accepted (one at each bound) made and wrapped, every object # it lists as refused refused for the reason it states, and each refusal writing # nothing; tests/generate.rs holds the same lists to a second statement of form v0. + # tests/manifest_parse_memory.rs (its own binary, a counting allocator) checks that + # reading a manifest allocates within the bound SPEC §2's reader limits give, at the + # length limit and at the value limit, and tests/manifest_reader_equivalence.rs holds + # the readers to the derived reader's result on every committed envelope. Both run in + # the release step only: the first writes manifests of 2^27 bytes, which takes a debug + # build most of a minute. - name: Fast integer kernels ≡ reference kernels (overflow checks on) working-directory: wai/wai-rs run: cargo test --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds - name: Fast integer kernels ≡ reference kernels (release, overflow checks off) working-directory: wai/wai-rs - run: cargo test --release --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds + run: cargo test --release --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds --test manifest_parse_memory --test manifest_reader_equivalence # The integer decode path builds under each of its features alone, so a sink can take # one medium without the others, and for wasm32 (a browser sink); `int_payload` arms a # medium only under its feature, and none needs `neural`. So does the staged-measurement -- 2.54.0 (Apple Git-157) From a8d2ba297ef338393816c8490e00bb0f1c47c98c Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Tue, 13 Oct 2026 01:01:58 -0400 Subject: [PATCH 10/10] wai changelog: record the integer-operator crate, the bounds pass and its certificates The two pull requests that added wai-ops, wai-verify, the operator corpus, the independent certifier and load step 8 wrote no changelog entry. This is it. Co-Authored-By: Claude Opus 5.5 --- wai/CHANGELOG.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index bf4bbaee3..00b859e3d 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -38,6 +38,38 @@ and the tests that measure them. points, built with `codecs`. The step fails if any of its test binaries ran no test, which is what a build without the feature does. +### Integer operators: the reference crate, the bounds pass and its certificates + +The crate, the tool and the corpus that +[integer-operators](extensions/integer-operators.md) (Draft) names exist. No capability +decodes through them yet: each capability's reference decoder is still the one its +payload extension names. + +- **`wai-ops`**, the reference crate, `no_std` with `alloc` and no dependencies: the + 512-bit saturating integer of §8.1, the canonical JSON writer of §8.5, the `WHG1` + parser and writer, the load-time and run-time plans, every graph operator of §5, the + stage operators E1–E13, and the lowerings of §13. +- **Load step 8.** `wai_ops::bounds` runs the bounds pass of §8.1–§8.5 and writes the + canonical certificate. The interpreter runs only a loaded graph, so nothing decodes a + parameter set its certificate does not admit. `interp::run_observed` reports every + value a run forms; `bounds::Within` holds a run to its certificate, and every golden + decode runs that way. +- **`wai-verify`**: `bounds`, `lower` and `vectors`, with `--check`, `--witness` and + `--advise`. +- **`wai/ops-conformance/`**: per-operator, stage, graph, load and lowering vectors, the + lowered graph of every committed parameter set, and each set's certificate. +- **Two certifiers.** `tools/wai_ops_check.py`, written from the extension text and + sharing no code with the crate, recomputes every vector and every certificate byte + for byte. +- **`wai_ops::limb32`**, a reference model of the 32-bit arithmetic the `acc32` profile's + two products need, checked against 64-bit arithmetic. +- **Model checking.** The operator set's harnesses are in `wai-ops/src/proofs/`, hosted + by `wai-verify` and run by `wai-formal-verification.yml`. A harness that does not yet + verify within the job's budget is kept, behind `cfg(kani_open)`, and is not run or + counted as proved. +- **`wai-ops.yml`** runs the crate's tests and `vectors verify` natively, under + wasm32-wasip2 and under wasm32 in Node, and runs the checker. + ### wai-quantum 0.4.3: metered runs, placement by run, and the fabric wrapper Additive. The default features, `full`, both implementations of the WebGPU member, and -- 2.54.0 (Apple Git-157)