diff --git a/.github/workflows/wai-conformance.yml b/.github/workflows/wai-conformance.yml index de128ba15..0f8049376 100644 --- a/.github/workflows/wai-conformance.yml +++ b/.github/workflows/wai-conformance.yml @@ -248,12 +248,18 @@ jobs: # instr-v0.json lists as accepted (one at each bound) made and wrapped, every object # it lists as refused refused for the reason it states, and each refusal writing # nothing; tests/generate.rs holds the same lists to a second statement of form v0. + # tests/manifest_parse_memory.rs (its own binary, a counting allocator) checks that + # reading a manifest allocates within the bound SPEC §2's reader limits give, at the + # length limit and at the value limit, and tests/manifest_reader_equivalence.rs holds + # the readers to the derived reader's result on every committed envelope. Both run in + # the release step only: the first writes manifests of 2^27 bytes, which takes a debug + # build most of a minute. - name: Fast integer kernels ≡ reference kernels (overflow checks on) working-directory: wai/wai-rs run: cargo test --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds - name: Fast integer kernels ≡ reference kernels (release, overflow checks off) working-directory: wai/wai-rs - run: cargo test --release --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds + run: cargo test --release --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds --test manifest_parse_memory --test manifest_reader_equivalence # The integer decode path builds under each of its features alone, so a sink can take # one medium without the others, and for wasm32 (a browser sink); `int_payload` arms a # medium only under its feature, and none needs `neural`. So does the staged-measurement @@ -882,6 +888,45 @@ jobs: working-directory: wai/byte-exact-conformance run: cargo test --release + # The codec wrappers are behind `codecs`, which binds C and C++ libraries that no + # container image of this fleet carries, so no other job compiles them. A feature no + # workflow enables has no CI coverage at all: without this job the wrappers' tests, + # and the limits on what a text payload may decode to (SPEC §2), pass by not running. + # + # One platform, the `macos` runner, whose host has the libraries. What the job proves + # is how the wrappers treat hostile bytes and where their limits fall, which is the + # same code on every host. A missing library fails the build step by name. + codec-wrappers: + name: codec wrappers · fuzz smoke, text decode limits, C entry points + runs-on: macos + steps: + - uses: actions/checkout@v5 + + - name: Install Rust 1.98.0 + shell: bash + run: | + export PATH="$HOME/.cargo/bin:$PATH" + rustup toolchain install 1.98.0 --profile minimal + rustup default 1.98.0 + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + + # Every one of these test files is `#![cfg(feature = "codecs")]`: built without + # the feature each reports "0 passed" and exits 0. So the step names the feature, + # and then requires that each of the four binaries ran at least one test. + - name: Fuzz smoke, the text decode limits and the C entry points + working-directory: wai/wai-rs + shell: bash + run: | + set -o pipefail + cargo test --locked --features codecs \ + --test fuzz_smoke --test text_decode_bounds --test text_decode_memory --test ffi_robustness \ + 2>&1 | tee "$RUNNER_TEMP/codec-wrappers.log" + ran=$(grep -cE '^test result: ok\. [1-9][0-9]* passed' "$RUNNER_TEMP/codec-wrappers.log" || true) + if [ "$ran" -ne 4 ]; then + echo "::error::expected four test binaries that each ran tests; $ran did" + exit 1 + fi + # The C2PA emitter is gated behind an off-by-default feature, so the four-platform # job above — which builds a fixed feature list — never compiles it. A feature no # workflow enables has no CI coverage at all: the suite passes, and the tests in it diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index bc5858dcf..00b859e3d 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -6,6 +6,70 @@ wire format may change and such changes are recorded here. ## v0.4.0 — current +### Reader limits that bound memory: manifest values, and text decode + +Two reader limits join the manifest length limit (SPEC §2, *Reader limits*), and with +them a reader's memory is bounded by what the specification states, not by what an +envelope asks for. `SECURITY.md` listed both as known gaps; it now states the bounds +and the tests that measure them. + +- **Manifest values.** A reader need accept no manifest of more than 2²² JSON values, + the manifest object and every value in it at any depth. The reference readers count + the values before they build anything, refuse one past the limit + (`manifest_too_many_values`), and build what they read at the size counted: at most + 128 bytes a value and one a byte on a 64-bit target, where a manifest of small + values at the length limit took several gigabytes to parse. The stdlib Python + verifiers refuse the same manifests, and the HTTP corpus holds a case at the limit + and one past it. +- **Text decode.** A `wai.text.zstd` or `wai.text.xz` payload decodes to at most 2²⁷ + bytes and declares at most 2²⁷ bytes of window or dictionary; a reader may refuse + more (`text_decoded_too_long`, `text_window_too_large`). The reference decoders + write into a buffer never sized past the limit, refuse a Zstandard frame that + declares a longer content size before decoding anything, and give the decoding + library the window limit, so a declaration past it is refused from the stream's + header. The reference encoders refuse input past the length limit. The C entry + points `wai_text_zstd_decode` and `wai_text_xz_decode` are bounded the same way; the + header gains `TEXT_DECODED_MAX_LEN` and `TEXT_WINDOW_MAX_LEN`. +- **Planned break for writers.** A conforming encoder writes no manifest past the value + limit and no text payload past either text limit. No committed envelope or vector is + past any of them, and no digest changes. +- **The codec wrappers in CI.** `wai-conformance.yml` gains `codec-wrappers`, on the + macOS runner: the wrappers' fuzz smoke, the text decode limits and the C entry + points, built with `codecs`. The step fails if any of its test binaries ran no test, + which is what a build without the feature does. + +### Integer operators: the reference crate, the bounds pass and its certificates + +The crate, the tool and the corpus that +[integer-operators](extensions/integer-operators.md) (Draft) names exist. No capability +decodes through them yet: each capability's reference decoder is still the one its +payload extension names. + +- **`wai-ops`**, the reference crate, `no_std` with `alloc` and no dependencies: the + 512-bit saturating integer of §8.1, the canonical JSON writer of §8.5, the `WHG1` + parser and writer, the load-time and run-time plans, every graph operator of §5, the + stage operators E1–E13, and the lowerings of §13. +- **Load step 8.** `wai_ops::bounds` runs the bounds pass of §8.1–§8.5 and writes the + canonical certificate. The interpreter runs only a loaded graph, so nothing decodes a + parameter set its certificate does not admit. `interp::run_observed` reports every + value a run forms; `bounds::Within` holds a run to its certificate, and every golden + decode runs that way. +- **`wai-verify`**: `bounds`, `lower` and `vectors`, with `--check`, `--witness` and + `--advise`. +- **`wai/ops-conformance/`**: per-operator, stage, graph, load and lowering vectors, the + lowered graph of every committed parameter set, and each set's certificate. +- **Two certifiers.** `tools/wai_ops_check.py`, written from the extension text and + sharing no code with the crate, recomputes every vector and every certificate byte + for byte. +- **`wai_ops::limb32`**, a reference model of the 32-bit arithmetic the `acc32` profile's + two products need, checked against 64-bit arithmetic. +- **Model checking.** The operator set's harnesses are in `wai-ops/src/proofs/`, hosted + by `wai-verify` and run by `wai-formal-verification.yml`. A harness that does not yet + verify within the job's budget is kept, behind `cfg(kani_open)`, and is not run or + counted as proved. +- **`wai-ops.yml`** runs the crate's tests and `vectors verify` natively, under + wasm32-wasip2 and under wasm32 in Node, and runs the checker. + ### wai-quantum 0.4.3: metered runs, placement by run, and the fabric wrapper Additive. The default features, `full`, both implementations of the WebGPU member, and diff --git a/wai/SECURITY.md b/wai/SECURITY.md index bbaedeab9..11205c26d 100644 --- a/wai/SECURITY.md +++ b/wai/SECURITY.md @@ -41,6 +41,7 @@ What the readers bound: |---|---|---| | every length field (`man_len`, `payload_len`, each rendition's offset and length) | checked against the bytes in hand before anything is sliced; sums formed so they cannot wrap (`saturating_add`; the `WAI2` payload total in `u64`) | SPEC §2, §7.1 | | manifest length | at most 134 217 728 bytes (2²⁷), sized to the largest manifest a conforming staged layer has. A longer one is refused from `man_len` alone, before it is read or, over HTTP, requested (`manifest_too_large`; `head_malformed` from `parse_head`) | SPEC §2, *Reader limits* | +| manifest values | at most 4 194 304 JSON values (2²²), the manifest object and every value in it at any depth; a member's name is not one. One more is refused while the values are counted, before anything is built from the manifest (`manifest_too_many_values`; `head_malformed` from `parse_head`) | SPEC §2, *Reader limits* | | manifest nesting | at most 127 nested arrays and objects, the manifest object counting as one; 128 are refused | SPEC §2, *Reader limits* | | member names | a member named twice in one object, at any depth, is refused | SPEC §2 | | `WAI2` rendition table | at most 65 535 rows (`u16`), as many as the manifest's `renditions`, packed contiguously from offset 0 | SPEC §7.1 | @@ -52,16 +53,26 @@ it allocate more than it was given. A `WAI1` reader ignores bytes after the payl HTTP object that carries any is refused ([http-delivery](extensions/http-delivery.md) §5.1). -**Parsing a manifest takes more memory than its length.** The reference readers parse -the manifest into a JSON tree (`serde_json::Value`, for `target`, `refines`, -`derived_from` and every member this revision does not define), and a parsed value -takes more memory than its text: about seven times the length of a 94 MB `prev` that -names a producer for each of 2²⁰ units, the largest manifest a conforming layer has, -and nearly 40 times the length of a manifest of many small values, such as one long -array of zeros. A manifest at the length limit can take some 5 GB and several seconds -to parse, and a reader may not refuse it for its length (SPEC §2). The length limit -bounds what a reader must read, not what parsing it costs; the reference sets no limit -of its own on the parse. +**What parsing a manifest takes.** A parsed value takes more memory than its text, and +many small values can take many times their length, so the length limit bounds what a +reader must read and not what a parse costs. The value limit bounds the rest. The +reference readers count a manifest's values before they build anything from it +(`check_manifest_text`) and refuse it as soon as the count passes the limit; they then +build each array and object at the size counted, and move the manifest's members out of +that one tree (`read_manifest`, `read_manifest_v2`). On a 64-bit target a read allocates +at most 128 bytes for each value and one for each byte of the manifest, and for a `WAI2` +one rendition entry of 768 bytes for each it lists: at most 640 MiB for a `WAI1` +manifest at both limits, whatever its shape, and 48 MiB more for a `WAI2`'s entries. + +`tests/manifest_parse_memory.rs` measures this with an allocator that counts every byte +asked for: a manifest of small values at the length limit is refused for its values +before anything is built; one at the length limit within the value limit is read; a +manifest of each shape at the value limit is read and one value past it refused; and so +are the largest manifest a conforming staged layer has and a `WAI2` listing as many +renditions as its table holds. Each stays within the bound above. +`tests/manifest_reader_equivalence.rs` holds these readers to the result of the derived +reader they replace, for every envelope committed under `wai/` and for each with every +member removed or given a value of each JSON type. The library tests in `container.rs` and `http_binding/tests.rs` check the manifest limits at their edges: from its first eight bytes, every reader refuses a manifest one @@ -77,9 +88,10 @@ limit. The stdlib Python verifiers read envelopes too. `staged_verify.py`, `claim_verify.py`, `pack_verify.py` and `http_verify.py` refuse a manifest past the -length limit as the reference does. `http_verify.py` and `wai_http_vectors` agree on -`parse_head` at the limit in the HTTP corpus; the other three each build an envelope in -memory with a manifest one byte past it and require it refused. Those that read +length limit, or past the value limit, as the reference does. `http_verify.py` and `wai_http_vectors` agree on +`parse_head` at each limit in the HTTP corpus; the other three each build an envelope in +memory with a manifest one byte past the length limit, and one value past the value +limit, and require each refused. Those that read manifests through `staged_verify.strict_json` refuse nesting past 127; `pack_verify.py` reads up to the interpreter's recursion limit and refuses past it as unparseable. @@ -90,6 +102,21 @@ bytes and 2²⁰ frames, and a sink cost limit (`int_payload::check_cost`, `wai.neural.int_mlicpp` refuses a header whose shapes, or whose buffers, pass its cap before it allocates them (`tests/int_mlicpp_preflight.rs`). +`wai.text.zstd` and `wai.text.xz` carry streams whose formats bound neither what a +payload decodes to nor the history its decoder must keep, so the specification sets +both (SPEC §2): at most 2²⁷ decoded bytes, and at most 2²⁷ bytes of window or +dictionary. The reference decoders (`codecs/text.rs`) write their output into a buffer +that is never sized past the length limit and refuse the payload when its output would +pass it (`text_decoded_too_long`); a Zstandard frame that declares a longer content +size is refused before anything is decoded. They give the decoding library the window +limit, so a stream that declares more is refused from its header, before the library +allocates it (`text_window_too_large`). The encoders refuse input past the length limit. +`tests/text_decode_bounds.rs` checks each limit at its edge, on streams made in the test +that are at it and one step past it; `tests/text_decode_memory.rs` measures, with a +counting allocator, that refusing a payload of a few kilobytes that would expand to +four times a limit takes no more output buffer than the limit. The C entry points +`wai_text_zstd_decode` and `wai_text_xz_decode` are these decoders. + ## Native codec libraries and sandboxed decode The default `codecs` feature binds C and C++ libraries for the floor and the @@ -108,11 +135,12 @@ run it in a WebAssembly runtime. What does not touch them: -- CI builds, except the neural workflows: no workflow enables `codecs` or `semantic`, - so no CI build links the codec libraries. `wai-neural-gates.yml` and +- Most CI builds. One job builds `codecs`: `codec-wrappers` in `wai-conformance.yml`, + on the macOS runner, which runs the wrappers' fuzz smoke, the text decode limits and + the C entry points through the libraries. `wai-neural-gates.yml` and `wai-neural-e2e.yml` build the `neural` feature, which links ONNX Runtime and - libzstd, and run the float neural decoders through them; no other workflow's build - links any of these libraries. + libzstd, and run the float neural decoders through them. No workflow enables + `semantic`, and no other workflow's build links any of these libraries. - WAI's own decoders are Rust: the integer codecs, worlds, scores, films and the generator. Outside the C ABI (`ffi.rs`, `ffi_codecs.rs`, `ffi_neural.rs`), the only `unsafe` in `wai-rs`'s library code is in the AArch64 SIMD kernels of @@ -132,15 +160,12 @@ element beside each kernel and over every committed integer-decoder vector Known gaps: -- **Decompression output is not bounded.** `wai.text.zstd`, in the floor, and - `wai.text.xz` decode the whole stream into memory, so a small payload can expand to - any size the format allows. The image, audio and video decoders allocate what the - bitstream declares, within the libraries' own limits. The reference sets no output - limit of its own. +- **The image, audio and video decoders set no output limit of their own.** They + allocate what the bitstream declares, within the libraries' own limits. - **Fuzzing is shallow.** `tests/fuzz_smoke.rs` feeds seeded random bytes to each - codec wrapper and to `Wai::from_bytes` and requires that nothing panics. It needs - the `codecs` feature, which no CI workflow builds, so it runs only by hand. There is - no coverage-guided fuzzing of the manifest parser, the integer payload parsers or the + codec wrapper and to `Wai::from_bytes` and requires that nothing panics. It runs in + CI on one platform (`codec-wrappers` in `wai-conformance.yml`). There is no + coverage-guided fuzzing of the manifest parser, the integer payload parsers or the codec wrappers. ## Module authorisation diff --git a/wai/SPEC.md b/wai/SPEC.md index a1c587a72..dab3520c0 100644 --- a/wai/SPEC.md +++ b/wai/SPEC.md @@ -86,20 +86,51 @@ OPTIONAL; §5 says which are RECOMMENDED. All WAI multi-byte integers are little-endian unless stated otherwise. -**Reader limits.** A reader MUST NOT refuse a manifest for its length or its nesting -while it is at most 134 217 728 bytes (`man_len` ≤ 2²⁷) and its arrays and objects nest -at most 127 deep, the manifest object itself counting as one. It MAY refuse a longer or -deeper manifest. The length limit holds the largest manifest a conforming layer has: a -`prev` ([staged-delivery](extensions/staged-delivery.md) §3) that names a different -producer for each of the 2²⁰ units a `WIR1` layer may cover (staged-delivery -Appendix A.3, `n_units` ≤ 2²⁰) is about 94 MB. A reader that refuses one past the -length limit does so from `man_len` alone, before it reads or requests the manifest -(`manifest_too_large`). The reference readers refuse both. The same limits apply to the -`WAI2` manifest (§7.1). They bound what a reader must read, not the memory its parse -takes, and move no byte of the layout above, which stays frozen (§8). Since a +**Reader limits.** A reader MUST NOT refuse a manifest for its length, its nesting or +the number of its values while it is at most 134 217 728 bytes (`man_len` ≤ 2²⁷), its +arrays and objects nest at most 127 deep, the manifest object itself counting as one, +and it holds at most 4 194 304 JSON values (2²²), counting the manifest object and every +value nested in it at any depth; a member's name is not a value. It MAY refuse a longer +or deeper manifest, or one that holds more values. The limits hold the largest manifest +a conforming layer has: a `prev` ([staged-delivery](extensions/staged-delivery.md) §3) +that names a different producer for each of the 2²⁰ units a `WIR1` layer may cover +(staged-delivery Appendix A.3, `n_units` ≤ 2²⁰) is about 94 MB, and three values a run +(the run, its `producer` and its `count`), 3·2²⁰ and a few more in all. A reader that +refuses one past the length limit does so from `man_len` alone, before it reads or +requests the manifest (`manifest_too_large`). A reader that refuses one past the value +limit does so while it counts the values, before it builds anything from the manifest +(`manifest_too_many_values`). The reference readers refuse all three. The same limits +apply to the `WAI2` manifest (§7.1), whose `renditions` lists no more entries than its +table holds. They move no byte of the layout above, which stays frozen (§8). Since a conforming encoder emits only containers every conforming sink can read (§7), a -conforming encoder writes no manifest past either limit; the reference writers refuse -one past the length limit. +conforming encoder writes no manifest past any of these limits; the reference writers +refuse one past any of them. + +Together the length and value limits bound what parsing a manifest takes, which the +length limit alone does not: a parsed value takes more memory than its text, and many +small values can take many times their length. The reference readers count a manifest's +values before they build anything from it, then build each array and object at the size +counted, and on a 64-bit target allocate at most 128 bytes for each value and one for +each byte of the manifest, with, for a `WAI2`, a rendition entry of 768 bytes for each it +lists: at most 640 MiB for a `WAI1` manifest at both limits, whatever its shape, and +48 MiB more for a `WAI2`'s entries (informative; `SECURITY.md`). + +A `wai.text.zstd` or `wai.text.xz` payload (§5) has reader limits of its own, since +neither stream format bounds what a payload may ask of its decoder. A reader MUST NOT +refuse one for the length it decodes to, or for the history its decoder must keep, while +it decodes to at most 134 217 728 bytes (2²⁷) and every frame or block of it declares at +most 2²⁷ bytes of history: for Zstandard the frame's window size, which for a +single-segment frame is its content size (RFC 8878 §3.1.1.1); for XZ the LZMA2 +dictionary size. It MAY refuse one that decodes to more (`text_decoded_too_long`) or +declares more (`text_window_too_large`). A reader that refuses for length does so when +its output would pass the limit, in a buffer it never sizes past the limit; where a +Zstandard frame declares a content size past the limit, it does so from the declaration, +before it decodes anything. A reader that refuses for the history does so from the +declaration, before its decoder allocates it. The reference readers refuse both. A +decode then takes its output, at most the length limit, the decoder's history, at most +the other, and the decoder's fixed state. As for a manifest, a conforming encoder writes +no payload past either limit, and the reference writers refuse one; bytes longer than +the length limit are carried as more than one object. An envelope delivered over HTTP is one object of the proposed media type `application/wai`, named by its receipt content hash, and fetched whole or, for a diff --git a/wai/claim-conformance/claim_verify.py b/wai/claim-conformance/claim_verify.py index 4020ab6f3..d50987c24 100644 --- a/wai/claim-conformance/claim_verify.py +++ b/wai/claim-conformance/claim_verify.py @@ -771,7 +771,7 @@ def wai2_read(data): if ml > SV.MANIFEST_MAX or 8 + ml > len(data): return None try: - m = SV.strict_json(data[8:8 + ml]) + m = SV.strict_json(data[8:8 + ml], SV.MANIFEST_MAX_VALUES) SV.check_manifest_v2(m) except (SV.NotJSON, SV.Schema): return None @@ -1965,7 +1965,7 @@ def main(argv): print(("PASS" if not bm else "FAIL") + " bracket arithmetic equals the literals pinned in wai-rs " + ",".join(bm)) fail += bool(bm) rl = SV.reader_limit_failures(lambda b: wai2_read(b) is not None, True) - print(("PASS" if not rl else "FAIL") + " wai2_read refuses a manifest past the reader limit (SPEC §2) " + "; ".join(rl)) + print(("PASS" if not rl else "FAIL") + " wai2_read refuses a manifest past the reader limits (SPEC §2) " + "; ".join(rl)) fail += bool(rl) rd = OER.check_canonical_form() # the OER reference's own fixtures, its reading of -0 among them print(("PASS" if not rd else "FAIL") + " the OER reference's canonical form and number reading hold " + ",".join(rd)) diff --git a/wai/http-conformance/cases.json b/wai/http-conformance/cases.json index 672f87603..8d330faf0 100644 --- a/wai/http-conformance/cases.json +++ b/wai/http-conformance/cases.json @@ -14,8 +14,12 @@ {"name":"head_malformed_offset","op":"parse_head","input":{"object":"OD","patch_object":[[574,"01"]]},"expect":{"error":"head_malformed"}}, {"name":"head_malformed_manifest_too_large_O5","op":"parse_head","input":{"object":"O5","prefix":8,"patch_object":[[4,"01"],[5,"00"],[6,"00"],[7,"08"]]},"expect":{"error":"head_malformed"}}, {"name":"parse_head_need_manifest_at_limit_O5","op":"parse_head","input":{"object":"O5","prefix":8,"patch_object":[[4,"00"],[5,"00"],[6,"00"],[7,"08"]]},"expect":{"need":134217740}}, +{"name":"parse_head_manifest_values_at_limit_O5","op":"parse_head","input":{"object":"O5","pad_values":4194294},"expect":{"version":"WAI1","head_len":8388753,"envelope_len":8388847,"covered_len":8388847,"components":[[8388753,94]]}}, +{"name":"head_malformed_manifest_too_many_values_O5","op":"parse_head","input":{"object":"O5","pad_values":4194295},"expect":{"error":"head_malformed"}}, {"name":"head_malformed_manifest_too_large_OD","op":"parse_head","input":{"object":"OD","prefix":8,"patch_object":[[4,"01"],[5,"00"],[6,"00"],[7,"08"]]},"expect":{"error":"head_malformed"}}, {"name":"parse_head_need_manifest_at_limit_OD","op":"parse_head","input":{"object":"OD","prefix":8,"patch_object":[[4,"00"],[5,"00"],[6,"00"],[7,"08"]]},"expect":{"need":134217738}}, +{"name":"parse_head_manifest_values_at_limit_OD","op":"parse_head","input":{"object":"OD","pad_values":4194279},"expect":{"version":"WAI2","head_len":8389160,"envelope_len":8442360,"covered_len":8389617,"components":[[8389160,3000],[8392160,50000],[8442160,200]]}}, +{"name":"head_malformed_manifest_too_many_values_OD","op":"parse_head","input":{"object":"OD","pad_values":4194280},"expect":{"error":"head_malformed"}}, {"name":"head_proof_ok_O1","op":"head_proof","input":{"object":"O1"},"expect":{"covered_len":859,"n_cv":0,"envelope_len":859}}, {"name":"head_proof_ok_O10","op":"head_proof","input":{"object":"O10"},"expect":{"covered_len":1009,"n_cv":1,"envelope_len":1825}}, {"name":"head_proof_ok_O10L1","op":"head_proof","input":{"object":"O10L1"},"expect":{"covered_len":1009,"n_cv":4,"envelope_len":12711}}, diff --git a/wai/http-conformance/falsify/manifest-values-uncounted.patch b/wai/http-conformance/falsify/manifest-values-uncounted.patch new file mode 100644 index 000000000..e847e50ae --- /dev/null +++ b/wai/http-conformance/falsify/manifest-values-uncounted.patch @@ -0,0 +1,15 @@ +# must fail: head_malformed_manifest_too_many_values_O5 head_malformed_manifest_too_many_values_OD +# a manifest past the value limit is built, not refused (container::read_tree) +diff --git a/wai/wai-rs/src/container.rs b/wai/wai-rs/src/container.rs +index b7b223f0..a06a74fd 100644 +--- a/wai/wai-rs/src/container.rs ++++ b/wai/wai-rs/src/container.rs +@@ -1706,7 +1706,7 @@ fn read_tree(text: &[u8]) -> Result { + use serde::de::{DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; + use std::cell::{Cell, RefCell}; + let sizes = RefCell::new(Vec::new()); +- scan_json(text, MANIFEST_MAX_VALUES, Some(&sizes))?; ++ scan_json(text, usize::MAX, Some(&sizes))?; + let mut sizes = sizes.into_inner(); + sizes.shrink_to_fit(); + /// One value of the tree; `at` is the next container's index in `sizes`. diff --git a/wai/http-conformance/falsify/mutations.txt b/wai/http-conformance/falsify/mutations.txt index 67b8ada87..3c81c40af 100644 --- a/wai/http-conformance/falsify/mutations.txt +++ b/wai/http-conformance/falsify/mutations.txt @@ -30,6 +30,7 @@ key-unchecked live_edge_other_key live_edge_other_key_walked walk-unbounded live_edge_reanchored live_edge_reanchored_then_reloaded live_edge_reanchored_skipping reload-walked live_edge_unchanged_reload live_edge_reanchored_then_reloaded manifest-limit-dropped head_malformed_manifest_too_large_O5 head_malformed_manifest_too_large_OD +manifest-values-uncounted head_malformed_manifest_too_many_values_O5 head_malformed_manifest_too_many_values_OD xml-char-unchecked mpd_rep_id_not_a_char xml-attribute-twice dash_adjacency_attribute_twice dash-start-ignored dash_adjacency_period_start diff --git a/wai/http-conformance/http_verify.py b/wai/http-conformance/http_verify.py index a72102e23..e28253049 100644 --- a/wai/http-conformance/http_verify.py +++ b/wai/http-conformance/http_verify.py @@ -116,7 +116,8 @@ def read_manifest(b, v2): """SPEC §2, §3, §7.1: the manifest read strictly, with the members SPEC requires and their types (staged_verify's reader); `head_malformed` otherwise.""" try: - m = SV.strict_json(b) + # SPEC §2, *Reader limits* (--mutate manifest-values-uncounted drops the value limit). + m = SV.strict_json(b, None if MUTATE == "manifest-values-uncounted" else SV.MANIFEST_MAX_VALUES) (SV.check_manifest_v2 if v2 else SV.check_manifest_v1)(m) except (SV.NotJSON, SV.Schema): raise Refusal("head_malformed") @@ -3284,6 +3285,15 @@ def receipt_evaluate(d, objs, op, inp): return {"fault": str(e)} +def pad_values(b, n): + """SPEC §2, *Reader limits*: the envelope `b` with a member no revision defines added + at the end of its manifest, "pad", an array of `n` zeros, and its man_len set to the + manifest's new length; the manifest holds n + 1 values more.""" + ml = u32(b, 4) + m = b[8:8 + ml - 1] + b',"pad":[0' + b",0" * (n - 1) + b"]}" + return b[:4] + len(m).to_bytes(4, "little") + m + b[8 + ml:] + + def evaluate(d, objs, op, inp): def obj(i, patch=True): if "envelope_hex" in i: @@ -3292,6 +3302,8 @@ def evaluate(d, objs, op, inp): b = objs[i["object"]] else: b = open(os.path.join(d, i["file"]), "rb").read() + if "pad_values" in i: + b = pad_values(b, i["pad_values"]) return apply_patch(b, i.get("patch_object")) if patch else b try: diff --git a/wai/pack-conformance/pack_verify.py b/wai/pack-conformance/pack_verify.py index edb463710..3e637f56d 100644 --- a/wai/pack-conformance/pack_verify.py +++ b/wai/pack-conformance/pack_verify.py @@ -90,6 +90,9 @@ HV.load_registry() MAX_INTEGER = (1 << 53) - 1 # The longest manifest a reader must accept (SPEC §2, *Reader limits*). MANIFEST_MAX = 1 << 27 +# The most JSON values a manifest may hold that a reader must accept (SPEC §2, *Reader +# limits*): the manifest and every value in it, at any depth, and no member name. +MANIFEST_MAX_VALUES = 1 << 22 U32 = (1 << 32) - 1 # The frames of each piece an encode vector codes: the first ENCODE_FRAMES. ENCODE_FRAMES = 12 @@ -1414,6 +1417,19 @@ def read_json(b, minus_zero=False): return None +def json_values(v): + """The JSON values `v` holds: itself and every value in it, at any depth.""" + n, stack = 0, [v] + while stack: + x = stack.pop() + n += 1 + if isinstance(x, dict): + stack.extend(x.values()) + elif isinstance(x, list): + stack.extend(x) + return n + + def opt_str(o, k): """An OPTIONAL string member: absent, null, or a string.""" return k not in o or o[k] is None or isinstance(o[k], str) @@ -1476,6 +1492,8 @@ def read_envelope(b): if ml > MANIFEST_MAX or 8 + ml > len(b): return None man = read_json(b[8:8 + ml]) + if man is not None and json_values(man) > MANIFEST_MAX_VALUES: + return None # §6.3: "does not parse" is the typed parse with SPEC §3's members. if not manifest_typed(man, b[:4] == b"WAI2"): return None @@ -1504,9 +1522,11 @@ def read_envelope(b): def check_reader_limit(rep): """SPEC §2, *Reader limits*, checked in memory, since no file in the corpus is past - 128 MiB: read_envelope reads a WAI1 and a WAI2 whose manifests are 300 bytes, and - refuses each with its manifest padded to MANIFEST_MAX + 1 bytes by a member no - revision defines, which a reader ignores (SPEC §8), so only the limit refuses it.""" + 128 MiB or holds 4 Mi values: read_envelope reads a WAI1 and a WAI2 whose manifests + are 300 bytes, and refuses each with its manifest padded to MANIFEST_MAX + 1 bytes by + a member no revision defines, which a reader ignores (SPEC §8), so only the limit + refuses it; it reads each with that member an array of zeros that brings the manifest + to MANIFEST_MAX_VALUES values, and refuses each with one zero more.""" for magic, head, rest in ( (b"WAI1", b'{"wai":"1.0","media":"text","intent":"replicate","model_requirement":' b'{"capability":"wai.text.zstd"},"conditioning":{"kind":"zstd"},"pad":"', @@ -1520,6 +1540,15 @@ def check_reader_limit(rep): rep.check(read_envelope(env(MANIFEST_MAX + 1)) is None, f"reader limit: a {magic.decode()} manifest of {MANIFEST_MAX + 1} bytes, past the limit, is refused") + def valued(n): + fixed = json_values(read_json(head[:-1] + b"[]}")) + m = head[:-1] + b"[0" + b",0" * (n - fixed - 1) + b"]}" + return b"".join([magic, len(m).to_bytes(4, "little"), m, rest, b"abc"]) + rep.check(read_envelope(valued(MANIFEST_MAX_VALUES)) is not None, + f"reader limit: a {magic.decode()} manifest of {MANIFEST_MAX_VALUES} values is read") + rep.check(read_envelope(valued(MANIFEST_MAX_VALUES + 1)) is None, + f"reader limit: a {magic.decode()} manifest of {MANIFEST_MAX_VALUES + 1} values, past the limit, is refused") + WIR1_KIND = {"wai.image.int_refine": "rgb8", "wai.video.int_refine": "rgb8", "wai.audio.int_refine": "pcm", "wai.splat.int_refine": "tensor"} diff --git a/wai/staged-conformance/staged_verify.py b/wai/staged-conformance/staged_verify.py index fa14dc34a..fc15fda07 100644 --- a/wai/staged-conformance/staged_verify.py +++ b/wai/staged-conformance/staged_verify.py @@ -70,13 +70,19 @@ DEPTH_LIMIT = 127 # reader refuses a longer one from its length field alone (`manifest_too_large`). MANIFEST_MAX = 1 << 27 +# The most JSON values a manifest may hold that a reader must accept (SPEC §2, *Reader +# limits*), counting the manifest and every value in it, at any depth, and no member +# name; the reference reader refuses one more (`manifest_too_many_values`). +MANIFEST_MAX_VALUES = 1 << 22 + class NotJSON(ValueError): pass -def strict_json(b): - """The JSON value of the bytes `b`, read strictly, or NotJSON.""" +def strict_json(b, max_values=None): + """The JSON value of the bytes `b`, read strictly, or NotJSON. With `max_values`, a + text holding more values than that is refused (`manifest_too_many_values`).""" if b[:3] == b"\xef\xbb\xbf": raise NotJSON("a byte-order mark") try: @@ -121,8 +127,12 @@ def strict_json(b): except (ValueError, RecursionError) as e: raise NotJSON(str(e)) stack = [(v, 1)] + values = 0 while stack: x, depth = stack.pop() + values += 1 + if max_values is not None and values > max_values: + raise NotJSON("manifest_too_many_values") if isinstance(x, (dict, list)): if depth > DEPTH_LIMIT: raise NotJSON("nested past the depth limit") @@ -261,7 +271,7 @@ def read_wai1(b): raise NotWai1("truncated manifest") text = b[8:8 + ml] try: - m = strict_json(text) + m = strict_json(text, MANIFEST_MAX_VALUES) check_manifest_v1(m) except (NotJSON, Schema) as e: raise NotWai1(str(e)) @@ -1180,7 +1190,7 @@ def read_wai2(b): if 8 + ml > len(b): raise NotWai2("truncated manifest") try: - m = strict_json(b[8:8 + ml]) + m = strict_json(b[8:8 + ml], MANIFEST_MAX_VALUES) check_manifest_v2(m) except (NotJSON, Schema) as e: raise NotWai2(str(e)) @@ -1226,17 +1236,34 @@ def envelope_with_manifest_len(v2, n): b'"}', rest, b"abc"]) +def envelope_with_manifest_values(v2, n): + """The envelope of envelope_with_manifest_len(v2, 300) with its manifest holding + exactly `n` values: its `pad` an array of zeros, the manifest's other members 10 + values, `pad` among them.""" + b = envelope_with_manifest_len(v2, 300) + ml = struct.unpack_from(" #include +/** + * The most bytes a `wai.text.zstd` or `wai.text.xz` payload may decode to that a reader + * must accept (SPEC §2, *Reader limits*): 2^27. The decoders here write no more than + * this, whatever the payload asks for: one that decodes to more is refused + * (`text_decoded_too_long`) when its output would pass the limit, so the output buffer + * never holds more. The reference writers refuse to encode more. + */ +#define TEXT_DECODED_MAX_LEN (1 << 27) + +/** + * The base-2 logarithm of [`TEXT_WINDOW_MAX_LEN`]. + */ +#define TEXT_WINDOW_MAX_LOG 27 + +/** + * The most history a `wai.text.zstd` or `wai.text.xz` stream may ask its decoder to + * keep that a reader must accept (SPEC §2, *Reader limits*): 2^27 bytes, as the zstd + * frame's window size or the LZMA2 dictionary size. A stream that declares more is + * refused (`text_window_too_large`) from the declaration, before the decoder allocates + * it. Together with [`TEXT_DECODED_MAX_LEN`] it bounds the memory a decode takes: the + * length limit bounds the output, and this bounds what the decoder holds beside it. + */ +#define TEXT_WINDOW_MAX_LEN (1 << TEXT_WINDOW_MAX_LOG) + /** * The longest manifest, in bytes, a reader must accept (SPEC §2, *Reader limits*): * 2^27 (128 MiB). It is sized to the largest manifest a conforming layer has: a `prev` @@ -37,13 +61,25 @@ * `WIR1` layer may cover (staged-delivery Appendix A.3, `n_units ≤ 2^20`), about 90 * bytes a run. The reference readers refuse a longer one before they read it, and the * reference writers refuse to write one, so `man_len` never asks a reader to read - * more. It bounds the bytes read, not the memory a parse takes: the reference readers - * build a JSON tree, which can take some 40 times the manifest's length. The - * manifest's nesting has its own limit: the JSON reader refuses 128 nested arrays and - * objects and takes 127 (`check_unique_members`). + * more. It bounds the bytes read; [`MANIFEST_MAX_VALUES`] bounds what a parse builds + * from them. The manifest's nesting has its own limit: the JSON reader refuses 128 + * nested arrays and objects and takes 127 ([`check_manifest_text`]). */ #define MANIFEST_MAX_LEN (1 << 27) +/** + * The most JSON values a manifest may hold that a reader must accept (SPEC §2, *Reader + * limits*): 2^22, counting the manifest object and every value nested in it, at any + * depth; a member's name is not a value. It is sized to the same layer as + * [`MANIFEST_MAX_LEN`]: a `prev` that names a different producer for each of 2^20 units + * is three values a run (the run, its `producer` and its `count`), 3·2^20 and a few more + * in all. The reference readers count a manifest's values before they parse it + * ([`check_manifest_text`]) and refuse it as soon as the count passes the limit + * (`manifest_too_many_values`), so the JSON tree a parse builds holds at most this many + * values, whatever the manifest's length. + */ +#define MANIFEST_MAX_VALUES (1 << 22) + #define WAI_OK 0 #define WAI_ERR_INVALID_ARG 1 @@ -668,6 +704,13 @@ int wai_text_zstd_encode(const uint8_t *data, int32_t level, struct wai_buffer_t *out); +/** + * Decode a `wai.text.zstd` payload within the reader limits of SPEC §2: at most + * [`TEXT_DECODED_MAX_LEN`](crate::codecs::text::TEXT_DECODED_MAX_LEN) bytes of output + * and [`TEXT_WINDOW_MAX_LEN`](crate::codecs::text::TEXT_WINDOW_MAX_LEN) bytes of + * window. A payload past either is refused with `WAI_ERR_CODEC`; the message begins + * `text_decoded_too_long` or `text_window_too_large`. + */ int wai_text_zstd_decode(const uint8_t *bytes, uintptr_t len, struct wai_buffer_t *out); @@ -676,6 +719,10 @@ int wai_text_xz_encode(const uint8_t *data, uint32_t level, struct wai_buffer_t *out); +/** + * Decode a `wai.text.xz` payload within the same limits as + * [`wai_text_zstd_decode`], the window limit applying to the dictionary. + */ int wai_text_xz_decode(const uint8_t *bytes, uintptr_t len, struct wai_buffer_t *out); /** diff --git a/wai/wai-rs/src/bin/wai_cli/http_vectors.rs b/wai/wai-rs/src/bin/wai_cli/http_vectors.rs index fed4d5604..e1ee7d1b6 100644 --- a/wai/wai-rs/src/bin/wai_cli/http_vectors.rs +++ b/wai/wai-rs/src/bin/wai_cli/http_vectors.rs @@ -99,6 +99,23 @@ pub fn main(args: &[String]) -> Result { } } +/// `pad_values` (SPEC §2, *Reader limits*): the envelope `b` with a member no revision +/// defines added at the end of its manifest, `"pad"`, an array of `n` zeros, and its +/// `man_len` set to the manifest's new length. The manifest then holds `n + 1` values +/// more. +fn pad_values(b: &[u8], n: usize) -> Result, String> { + let ml = wai::container::manifest_bytes(b).map_err(|e| e.to_string())?.len(); + let text = &b[8..8 + ml]; + if n == 0 || text.last() != Some(&b'}') { + return Err("pad_values needs a manifest object and at least one zero".into()); + } + let mut m = text[..ml - 1].to_vec(); + m.extend_from_slice(b",\"pad\":[0"); + m.extend_from_slice(&b",0".repeat(n - 1)); + m.extend_from_slice(b"]}"); + Ok([&b[..4], &(m.len() as u32).to_le_bytes(), &m, &b[8 + ml..]].concat()) +} + // ---------------------------------------------------------------- the objects fn read(dir: &Path, rel: &str) -> Result, String> { @@ -339,6 +356,9 @@ impl Corpus { } fn object(&self, input: &Value) -> Result, String> { let mut b = self.original(input)?; + if let Some(n) = input["pad_values"].as_u64() { + b = pad_values(&b, n as usize)?; + } patch(&mut b, &input["patch_object"])?; Ok(b) } @@ -872,6 +892,14 @@ fn cases(objs: &BTreeMap>, extra: &mut BTreeMap> json!({ "object": id, "prefix": 8, "patch_object": man_len((1 << 27) + 1) }), "head_malformed")); v.push(case(format!("parse_head_need_manifest_at_limit_{id}"), "parse_head", json!({ "object": id, "prefix": 8, "patch_object": man_len(1 << 27) }))); + // The value limit: the object's manifest with a `pad` of zeros bringing it to + // exactly 2^22 values is read; one zero more is refused. + let text = wai::container::manifest_bytes(&objs[id]).map_err(|e| e.to_string())?; + let held = wai::container::check_manifest_text(text).map_err(|e| e.to_string())?; + let at = wai::container::MANIFEST_MAX_VALUES - held - 1; + v.push(case(format!("parse_head_manifest_values_at_limit_{id}"), "parse_head", json!({ "object": id, "pad_values": at }))); + v.push(refused(format!("head_malformed_manifest_too_many_values_{id}"), "parse_head", + json!({ "object": id, "pad_values": at + 1 }), "head_malformed")); } // head_proof: every object, and every way a proof can be wrong. diff --git a/wai/wai-rs/src/codecs/text.rs b/wai/wai-rs/src/codecs/text.rs index 9d63f244f..f012a865d 100644 --- a/wai/wai-rs/src/codecs/text.rs +++ b/wai/wai-rs/src/codecs/text.rs @@ -12,17 +12,112 @@ impl From for TextError { pub type Result = std::result::Result; -/// zstd at the given compression level (1..=22; 22 is "ultra", slow). +/// zstd at the given compression level (1..=22; 22 is "ultra", slow). Refuses more +/// than [`TEXT_DECODED_MAX_LEN`] bytes (`text_decoded_too_long`): a conforming encoder +/// writes no payload a conforming reader may refuse (SPEC §2, *Reader limits*). No +/// level asks for a window past [`TEXT_WINDOW_MAX_LEN`]. pub fn zstd_encode(data: &[u8], level: i32) -> Result> { + if data.len() > TEXT_DECODED_MAX_LEN { + return Err(too_long(TEXT_DECODED_MAX_LEN)); + } Ok(zstd::stream::encode_all(data, level)?) } -pub fn zstd_decode(bytes: &[u8]) -> Result> { - Ok(zstd::stream::decode_all(bytes)?) +/// The most bytes a `wai.text.zstd` or `wai.text.xz` payload may decode to that a reader +/// must accept (SPEC §2, *Reader limits*): 2^27. The decoders here write no more than +/// this, whatever the payload asks for: one that decodes to more is refused +/// (`text_decoded_too_long`) when its output would pass the limit, so the output buffer +/// never holds more. The reference writers refuse to encode more. +pub const TEXT_DECODED_MAX_LEN: usize = 1 << 27; + +/// The base-2 logarithm of [`TEXT_WINDOW_MAX_LEN`]. +pub const TEXT_WINDOW_MAX_LOG: u32 = 27; + +/// The most history a `wai.text.zstd` or `wai.text.xz` stream may ask its decoder to +/// keep that a reader must accept (SPEC §2, *Reader limits*): 2^27 bytes, as the zstd +/// frame's window size or the LZMA2 dictionary size. A stream that declares more is +/// refused (`text_window_too_large`) from the declaration, before the decoder allocates +/// it. Together with [`TEXT_DECODED_MAX_LEN`] it bounds the memory a decode takes: the +/// length limit bounds the output, and this bounds what the decoder holds beside it. +pub const TEXT_WINDOW_MAX_LEN: usize = 1 << TEXT_WINDOW_MAX_LOG; + +/// What liblzma may allocate to decode one stream: the dictionary, at most +/// [`TEXT_WINDOW_MAX_LEN`], and the decoder's own fixed state beside it. +const XZ_DECODER_MEMLIMIT: u64 = TEXT_WINDOW_MAX_LEN as u64 + (1 << 20); + +fn too_long(max_out: usize) -> TextError { + TextError(format!( + "text_decoded_too_long: the payload decodes to more than {max_out} bytes, the most this reader accepts (SPEC §2)" + )) } -/// XZ/LZMA2 at the given preset (0..=9; 6 is the default, 9 is "extreme"). +fn window_too_large() -> TextError { + TextError(format!( + "text_window_too_large: the stream asks its decoder to keep more than {TEXT_WINDOW_MAX_LEN} bytes of history, the most a reader need give it (SPEC §2)" + )) +} + +/// Reads `r` to its end into a buffer that never holds, and is never sized for, more +/// than `max_out` bytes. `Ok(None)` when the reader has more than that to give. +fn read_within(mut r: R, max_out: usize) -> std::io::Result>> { + let mut out: Vec = Vec::new(); + let mut chunk = vec![0u8; 64 * 1024]; + loop { + let n = match r.read(&mut chunk) { + Ok(0) => return Ok(Some(out)), + Ok(n) => n, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + }; + let room = max_out - out.len(); + if n > room { + return Ok(None); + } + if out.capacity() - out.len() < n { + // Grow as a `Vec` would, by about its length, but never past the limit. + out.reserve_exact(out.len().max(chunk.len()).min(room).max(n)); + } + out.extend_from_slice(&chunk[..n]); + } +} + +/// Decodes a `wai.text.zstd` payload within the reader limits of SPEC §2: +/// [`TEXT_DECODED_MAX_LEN`] bytes of output and [`TEXT_WINDOW_MAX_LEN`] bytes of window. +pub fn zstd_decode(bytes: &[u8]) -> Result> { + zstd_decode_within(bytes, TEXT_DECODED_MAX_LEN) +} + +/// [`zstd_decode`] with the output limit the caller gives: for a caller that knows what +/// its payload may hold, and for tests. `max_out` is capped at +/// [`TEXT_DECODED_MAX_LEN`]. The window limit is not the caller's to change. +pub fn zstd_decode_within(bytes: &[u8], max_out: usize) -> Result> { + let max_out = max_out.min(TEXT_DECODED_MAX_LEN); + // A frame that declares its content size past the limit is refused from the + // declaration, before anything is decoded. A frame need not declare it, and a + // payload may hold more frames than the first, so the output is counted too. + if let Ok(Some(declared)) = zstd::zstd_safe::get_frame_content_size(bytes) { + if declared > max_out as u64 { + return Err(too_long(max_out)); + } + } + let mut dec = zstd::stream::read::Decoder::with_buffer(bytes)?; + dec.window_log_max(TEXT_WINDOW_MAX_LOG)?; + match read_within(dec, max_out) { + Ok(Some(out)) => Ok(out), + Ok(None) => Err(too_long(max_out)), + // libzstd's name for a frame whose window is past the decoder's limit. + Err(e) if e.to_string().contains("too much memory") => Err(window_too_large()), + Err(e) => Err(e.into()), + } +} + +/// XZ/LZMA2 at the given preset (0..=9; 6 is the default, 9 is "extreme"). Refuses more +/// than [`TEXT_DECODED_MAX_LEN`] bytes (`text_decoded_too_long`), as [`zstd_encode`] +/// does. No preset asks for a dictionary past [`TEXT_WINDOW_MAX_LEN`]. pub fn xz_encode(data: &[u8], level: u32) -> Result> { + if data.len() > TEXT_DECODED_MAX_LEN { + return Err(too_long(TEXT_DECODED_MAX_LEN)); + } let mut out = Vec::with_capacity(data.len() / 4); let mut enc = xz2::write::XzEncoder::new(&mut out, level); enc.write_all(data)?; @@ -30,10 +125,32 @@ pub fn xz_encode(data: &[u8], level: u32) -> Result> { Ok(out) } +/// Decodes a `wai.text.xz` payload within the reader limits of SPEC §2: +/// [`TEXT_DECODED_MAX_LEN`] bytes of output and [`TEXT_WINDOW_MAX_LEN`] bytes of +/// dictionary. pub fn xz_decode(bytes: &[u8]) -> Result> { - let mut out = Vec::new(); - xz2::read::XzDecoder::new(bytes).read_to_end(&mut out)?; - Ok(out) + xz_decode_within(bytes, TEXT_DECODED_MAX_LEN) +} + +/// [`xz_decode`] with the output limit the caller gives: for a caller that knows what +/// its payload may hold, and for tests. `max_out` is capped at +/// [`TEXT_DECODED_MAX_LEN`]. The dictionary limit is not the caller's to change. +pub fn xz_decode_within(bytes: &[u8], max_out: usize) -> Result> { + let max_out = max_out.min(TEXT_DECODED_MAX_LEN); + // The decoder is given a memory limit, so a block that declares a dictionary past + // it is refused from its header, before the dictionary is allocated. An xz stream + // states its decoded length only at its end, so the output is counted as it comes. + let stream = xz2::stream::Stream::new_stream_decoder(XZ_DECODER_MEMLIMIT, 0)?; + let dec = xz2::read::XzDecoder::new_stream(bytes, stream); + match read_within(dec, max_out) { + Ok(Some(out)) => Ok(out), + Ok(None) => Err(too_long(max_out)), + Err(e) if matches!( + e.get_ref().and_then(|i| i.downcast_ref::()), + Some(xz2::stream::Error::MemLimit) + ) => Err(window_too_large()), + Err(e) => Err(e.into()), + } } #[cfg(test)] diff --git a/wai/wai-rs/src/container.rs b/wai/wai-rs/src/container.rs index afc5363ae..b7b223f05 100644 --- a/wai/wai-rs/src/container.rs +++ b/wai/wai-rs/src/container.rs @@ -41,12 +41,22 @@ pub const MAGIC_V2: &[u8; 4] = b"WAI2"; // v1.1 multi-rendition /// `WIR1` layer may cover (staged-delivery Appendix A.3, `n_units ≤ 2^20`), about 90 /// bytes a run. The reference readers refuse a longer one before they read it, and the /// reference writers refuse to write one, so `man_len` never asks a reader to read -/// more. It bounds the bytes read, not the memory a parse takes: the reference readers -/// build a JSON tree, which can take some 40 times the manifest's length. The -/// manifest's nesting has its own limit: the JSON reader refuses 128 nested arrays and -/// objects and takes 127 (`check_unique_members`). +/// more. It bounds the bytes read; [`MANIFEST_MAX_VALUES`] bounds what a parse builds +/// from them. The manifest's nesting has its own limit: the JSON reader refuses 128 +/// nested arrays and objects and takes 127 ([`check_manifest_text`]). pub const MANIFEST_MAX_LEN: usize = 1 << 27; +/// The most JSON values a manifest may hold that a reader must accept (SPEC §2, *Reader +/// limits*): 2^22, counting the manifest object and every value nested in it, at any +/// depth; a member's name is not a value. It is sized to the same layer as +/// [`MANIFEST_MAX_LEN`]: a `prev` that names a different producer for each of 2^20 units +/// is three values a run (the run, its `producer` and its `count`), 3·2^20 and a few more +/// in all. The reference readers count a manifest's values before they parse it +/// ([`check_manifest_text`]) and refuse it as soon as the count passes the limit +/// (`manifest_too_many_values`), so the JSON tree a parse builds holds at most this many +/// values, whatever the manifest's length. +pub const MANIFEST_MAX_VALUES: usize = 1 << 22; + /// SPEC §2, *Reader limits*: `Err` with the code `manifest_too_large` when a manifest /// of `len` bytes is longer than [`MANIFEST_MAX_LEN`]. pub fn check_manifest_len(len: u64) -> Result<(), String> { @@ -75,7 +85,10 @@ impl WriteLimits { /// A manifest's JSON as the writers emit it (no insignificant whitespace, members in /// order), refused with `manifest_too_large` as soon as it would pass `cap` bytes -/// ([`MANIFEST_MAX_LEN`] in the writers), so a writer never builds more of it than that. +/// ([`MANIFEST_MAX_LEN`] in the writers), so a writer never builds more of it than that; +/// then read as a reader reads it ([`check_manifest_text`]), so a writer never emits a +/// manifest past [`MANIFEST_MAX_VALUES`] (`manifest_too_many_values`), nested past 127, or +/// naming a member twice. fn manifest_json(m: &T, cap: usize) -> std::io::Result> { struct Capped(Vec, usize); impl std::io::Write for Capped { @@ -93,6 +106,7 @@ fn manifest_json(m: &T, cap: usize) -> std::io::Result> { } let mut w = Capped(Vec::new(), cap); serde_json::to_writer(&mut w, m).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + check_manifest_text(&w.0).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; Ok(w.0) } @@ -1299,9 +1313,7 @@ impl Wai { check_manifest_len(ml as u64) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; need(8, ml, "manifest body")?; - check_unique_members(&bytes[8..8 + ml]) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - let m: Manifest = serde_json::from_slice(&bytes[8..8 + ml]) + let m = read_manifest(&bytes[8..8 + ml]) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; let p_off = 8 + ml; need(p_off, 4, "payload length")?; @@ -1531,18 +1543,72 @@ impl Wai { /// (SPEC §2: member names are unique within an object). `serde_json` keeps the last of /// two equal names in a `Value` and refuses them only in a struct's own fields, so a /// manifest is read once more by this check, which refuses any repeat. Names compare -/// after escapes are read, so `"a"` and `"\u0061"` are one name. The text's own JSON -/// errors are reported as they are. +/// after escapes are read, so `"a"` and `"a"` are one name. The text's own JSON +/// errors are reported as they are. A manifest is read by [`check_manifest_text`], +/// which makes this check and counts the manifest's values; this one counts nothing. pub fn check_unique_members(text: &[u8]) -> Result<(), serde_json::Error> { + scan_json(text, usize::MAX, None).map(|_| ()) +} + +/// The checks a reader makes on a manifest's text before it builds anything from it +/// (SPEC §2, *Reader limits*): it is JSON, it names no member twice in one object +/// ([`check_unique_members`]), it nests at most 127 deep, and it holds at most +/// [`MANIFEST_MAX_VALUES`] values, refused with `manifest_too_many_values` as soon as the +/// count passes the limit, so a manifest past it is read only that far. The pass builds +/// no value: it holds the member names of each object open on the path to the value it +/// reads, borrowed from `text` unless a name carries an escape. Returns the number of +/// values. +pub fn check_manifest_text(text: &[u8]) -> Result { + scan_json(text, MANIFEST_MAX_VALUES, None) +} + +/// One pass over the JSON text `text`: refuses a member named twice in one object, and +/// more than `max_values` values; returns how many values it holds. With `sizes`, pushes +/// each array's and object's number of elements or members, the containers in the order +/// they open. +fn scan_json(text: &[u8], max_values: usize, sizes: Option<&std::cell::RefCell>>) -> Result { use serde::de::{DeserializeSeed, Deserializer, Error as _, MapAccess, SeqAccess, Visitor}; - struct Unique; - impl<'de> DeserializeSeed<'de> for Unique { - type Value = (); - fn deserialize>(self, d: D) -> Result<(), D::Error> { - d.deserialize_any(Unique) + use std::borrow::Cow; + use std::cell::{Cell, RefCell}; + /// One value, counted before it is read. + #[derive(Clone, Copy)] + struct Scan<'c> { + values: &'c Cell, + max: usize, + sizes: Option<&'c RefCell>>, + } + impl Scan<'_> { + /// A slot for the size of the container that opens now. + fn open(self) -> Option { + self.sizes.map(|s| { + let mut s = s.borrow_mut(); + s.push(0); + s.len() - 1 + }) + } + /// The container in `slot` closed with `n` elements or members (at most + /// `max_values`, so within `u32` for a manifest). + fn close(self, slot: Option, n: usize) { + if let (Some(s), Some(i)) = (self.sizes, slot) { + s.borrow_mut()[i] = u32::try_from(n).unwrap_or(u32::MAX); + } } } - impl<'de> Visitor<'de> for Unique { + impl<'de> DeserializeSeed<'de> for Scan<'_> { + type Value = (); + fn deserialize>(self, d: D) -> Result<(), D::Error> { + let n = self.values.get() + 1; + if n > self.max { + return Err(D::Error::custom(format_args!( + "manifest_too_many_values: the manifest holds more than {} values, the most a reader need accept (SPEC §2)", + self.max + ))); + } + self.values.set(n); + d.deserialize_any(self) + } + } + impl<'de> Visitor<'de> for Scan<'_> { type Value = (); fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("a JSON value") @@ -1566,23 +1632,291 @@ pub fn check_unique_members(text: &[u8]) -> Result<(), serde_json::Error> { Ok(()) } fn visit_seq>(self, mut seq: A) -> Result<(), A::Error> { - while seq.next_element_seed(Unique)?.is_some() {} + let slot = self.open(); + let mut n = 0; + while seq.next_element_seed(self)?.is_some() { + n += 1; + } + self.close(slot, n); Ok(()) } fn visit_map>(self, mut map: A) -> Result<(), A::Error> { + let slot = self.open(); let mut seen = std::collections::HashSet::new(); - while let Some(k) = map.next_key::()? { + while let Some(k) = map.next_key_seed(Name)? { if !seen.insert(k) { return Err(A::Error::custom("an object names a member twice")); } - map.next_value_seed(Unique)?; + map.next_value_seed(self)?; } + self.close(slot, seen.len()); Ok(()) } } + /// A member's name, borrowed from the text when it carries no escape. + struct Name; + impl<'de> DeserializeSeed<'de> for Name { + type Value = Cow<'de, str>; + fn deserialize>(self, d: D) -> Result, D::Error> { + d.deserialize_str(Name) + } + } + impl<'de> Visitor<'de> for Name { + type Value = Cow<'de, str>; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("a member name") + } + fn visit_borrowed_str(self, s: &'de str) -> Result, E> { + Ok(Cow::Borrowed(s)) + } + fn visit_str(self, s: &str) -> Result, E> { + Ok(Cow::Owned(s.to_owned())) + } + } + let values = Cell::new(0); let mut de = serde_json::Deserializer::from_slice(text); - Unique.deserialize(&mut de)?; - de.end() + Scan { values: &values, max: max_values, sizes }.deserialize(&mut de)?; + de.end()?; + Ok(values.get()) +} + +/// A `WAI1` manifest read from its text as the reference readers read it (SPEC §2): the +/// checks of [`check_manifest_text`], then the [`Manifest`] that +/// `serde_json::from_slice::` builds from the same text, built with less +/// memory. The text is read once more into one JSON tree, each array and object +/// allocated at the size the first pass counted, and the manifest's members are moved +/// out of that tree into their fields; nothing is held twice, where serde's derived +/// reader holds every member a type does not define as an intermediate copy while it +/// builds the tree it keeps. So what a parse allocates is bounded by the value limit +/// and the text's length (SPEC §2, *Reader limits*; `tests/manifest_parse_memory.rs`). +pub fn read_manifest(text: &[u8]) -> Result { + manifest_from_tree(read_tree(text)?) +} + +/// A `WAI2` manifest (SPEC §7.1) read as [`read_manifest`] reads a `WAI1`'s. A +/// `renditions` longer than the 65 535 rows a `WAI2` table holds is refused before its +/// entries are built. +pub fn read_manifest_v2(text: &[u8]) -> Result { + manifest_v2_from_tree(read_tree(text)?) +} + +/// The JSON tree of the manifest text `text`, after [`check_manifest_text`]'s checks, +/// each array and object allocated at its size. +fn read_tree(text: &[u8]) -> Result { + use serde::de::{DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor}; + use std::cell::{Cell, RefCell}; + let sizes = RefCell::new(Vec::new()); + scan_json(text, MANIFEST_MAX_VALUES, Some(&sizes))?; + let mut sizes = sizes.into_inner(); + sizes.shrink_to_fit(); + /// One value of the tree; `at` is the next container's index in `sizes`. + #[derive(Clone, Copy)] + struct Tree<'s> { + sizes: &'s [u32], + at: &'s Cell, + } + impl Tree<'_> { + fn next_size(self) -> usize { + let i = self.at.get(); + self.at.set(i + 1); + self.sizes.get(i).map_or(0, |&n| n as usize) + } + } + impl<'de> DeserializeSeed<'de> for Tree<'_> { + type Value = Value; + fn deserialize>(self, d: D) -> Result { + d.deserialize_any(self) + } + } + // As `serde_json`'s own `Value` visitor builds each value. + impl<'de> Visitor<'de> for Tree<'_> { + type Value = Value; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("a JSON value") + } + fn visit_bool(self, b: bool) -> Result { + Ok(Value::Bool(b)) + } + fn visit_i64(self, n: i64) -> Result { + Ok(Value::Number(n.into())) + } + fn visit_u64(self, n: u64) -> Result { + Ok(Value::Number(n.into())) + } + fn visit_f64(self, x: f64) -> Result { + Ok(serde_json::Number::from_f64(x).map_or(Value::Null, Value::Number)) + } + fn visit_str(self, s: &str) -> Result { + Ok(Value::String(s.to_owned())) + } + fn visit_string(self, s: String) -> Result { + Ok(Value::String(s)) + } + fn visit_unit(self) -> Result { + Ok(Value::Null) + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut v = Vec::with_capacity(self.next_size()); + while let Some(x) = seq.next_element_seed(self)? { + v.push(x); + } + Ok(Value::Array(v)) + } + fn visit_map>(self, mut map: A) -> Result { + let mut m = Map::with_capacity(self.next_size()); + while let Some(k) = map.next_key::()? { + let v = map.next_value_seed(self)?; + m.insert(k, v); + } + Ok(Value::Object(m)) + } + } + let at = Cell::new(0); + let mut de = serde_json::Deserializer::from_slice(text); + let v = Tree { sizes: &sizes, at: &at }.deserialize(&mut de)?; + de.end()?; + Ok(v) +} + +// The manifest types built from a tree by moving its members, each member read as +// serde's derived reader reads it: a field `T` from its value as `T` reads it, a missing +// field refused unless the type gives it a default (an `Option` is `None`, `target` is +// `null`), an `Option` whose value is `null` is `None`, and the members a type does not +// define left in order in `extra`. `tests/manifest_reader_equivalence.rs` holds the two +// readers to the same result. +type Members = Map; + +fn object_of(v: Value, what: &str) -> Result { + match v { + Value::Object(m) => Ok(m), + _ => Err(serde::de::Error::custom(format_args!("invalid type: {what} is not an object"))), + } +} + +fn required(m: &mut Members, k: &'static str) -> Result { + match m.shift_remove(k) { + Some(v) => serde_json::from_value(v), + None => Err(serde::de::Error::missing_field(k)), + } +} + +fn optional(m: &mut Members, k: &str) -> Result, serde_json::Error> { + match m.shift_remove(k) { + Some(v) => serde_json::from_value(v), + None => Ok(None), + } +} + +/// An `Option` member, moved: `null` or absent is `None`. +fn optional_tree(m: &mut Members, k: &str) -> Option { + m.shift_remove(k).filter(|v| !v.is_null()) +} + +/// An `Option` member whose `T` is one of the manifest types. +fn optional_of(m: &mut Members, k: &str, build: fn(Members) -> Result) -> Result, serde_json::Error> { + match m.shift_remove(k) { + None | Some(Value::Null) => Ok(None), + Some(v) => build(object_of(v, k)?).map(Some), + } +} + +fn required_of(m: &mut Members, k: &'static str, build: fn(Members) -> Result) -> Result { + match m.shift_remove(k) { + None => Err(serde::de::Error::missing_field(k)), + Some(v) => build(object_of(v, k)?), + } +} + +fn model_requirement_from(mut m: Members) -> Result { + Ok(ModelRequirement { + capability: required(&mut m, "capability")?, + fallback: optional(&mut m, "fallback")?, + min_sim: optional(&mut m, "min_sim")?, + sim_metric: optional(&mut m, "sim_metric")?, + extra: m, + }) +} + +fn supersedes_from(mut m: Members) -> Result { + Ok(Supersedes { sha256: required(&mut m, "sha256")?, form: optional(&mut m, "form")?, extra: m }) +} + +fn prior_from(mut m: Members) -> Result { + Ok(Prior { + id: required(&mut m, "id")?, + sha256: required(&mut m, "sha256")?, + determinism: required(&mut m, "determinism")?, + form: optional(&mut m, "form")?, + tier: optional(&mut m, "tier")?, + supersedes: optional_of(&mut m, "supersedes", supersedes_from)?, + extra: m, + }) +} + +fn conditioning_from(mut m: Members) -> Result { + Ok(Conditioning { kind: required(&mut m, "kind")?, prior: optional_of(&mut m, "prior", prior_from)?, extra: m }) +} + +fn security_from(mut m: Members) -> Result { + Ok(Security { encryption: optional(&mut m, "encryption")?, kid: optional(&mut m, "kid")?, extra: m }) +} + +fn manifest_from_tree(v: Value) -> Result { + let mut m = object_of(v, "the manifest")?; + Ok(Manifest { + wai: required(&mut m, "wai")?, + media: required(&mut m, "media")?, + intent: required(&mut m, "intent")?, + model_requirement: required_of(&mut m, "model_requirement", model_requirement_from)?, + conditioning: required_of(&mut m, "conditioning", conditioning_from)?, + security: optional_of(&mut m, "security", security_from)?, + refines: optional_tree(&mut m, "refines"), + derived_from: optional_tree(&mut m, "derived_from"), + target: m.shift_remove("target").unwrap_or(Value::Null), + extra: m, + }) +} + +fn rendition_from(mut m: Members) -> Result { + Ok(RenditionMeta { + capability: required(&mut m, "capability")?, + kind: required(&mut m, "kind")?, + id: optional(&mut m, "id")?, + sha256: optional(&mut m, "sha256")?, + prior: optional_of(&mut m, "prior", prior_from)?, + role: optional(&mut m, "role")?, + companion_of: optional(&mut m, "companion_of")?, + companion_of_id: optional(&mut m, "companion_of_id")?, + refines: optional_tree(&mut m, "refines"), + envelope_manifest: optional_tree(&mut m, "envelope_manifest"), + derived_from: optional_tree(&mut m, "derived_from"), + extra: m, + }) +} + +fn manifest_v2_from_tree(v: Value) -> Result { + let mut m = object_of(v, "the manifest")?; + let wai = required(&mut m, "wai")?; + let media = required(&mut m, "media")?; + let intent = required(&mut m, "intent")?; + let renditions = match m.shift_remove("renditions") { + None => return Err(serde::de::Error::missing_field("renditions")), + Some(Value::Array(rs)) if rs.len() > u16::MAX as usize => { + return Err(serde::de::Error::custom(format_args!( + "the manifest lists {} renditions, more than the 65535 rows a WAI2 table holds (SPEC §7.1)", + rs.len() + ))); + } + Some(Value::Array(rs)) => { + let mut out = Vec::with_capacity(rs.len()); + for r in rs { + out.push(rendition_from(object_of(r, "a rendition")?)?); + } + out + } + Some(_) => return Err(serde::de::Error::custom("invalid type: renditions is not an array")), + }; + Ok(ManifestV2 { wai, media, intent, renditions, target: m.shift_remove("target").unwrap_or(Value::Null), extra: m }) } /// The manifest bytes of a `WAI1` or `WAI2` envelope, exactly as they were @@ -2611,9 +2945,7 @@ impl WaiMulti { check_manifest_len(ml as u64) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; need(8, ml, "manifest body")?; - check_unique_members(&bytes[8..8 + ml]) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; - let manifest: ManifestV2 = serde_json::from_slice(&bytes[8..8 + ml]) + let manifest = read_manifest_v2(&bytes[8..8 + ml]) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; let mut o = 8 + ml; @@ -2965,7 +3297,7 @@ pub fn detect_version(bytes: &[u8]) -> std::io::Result { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; /// A pin verifier that accepts every pin: for tests about selection order. @@ -4500,6 +4832,50 @@ mod tests { } } + /// A `WAI1` (or, with `v2`, a `WAI2`) envelope whose manifest holds exactly `values` + /// values: the small manifest of [`envelope_with_manifest_len`], whose `pad` is an + /// array of zeros. Its other members are 10 values in either form, `pad` among them. + pub(crate) fn envelope_with_manifest_values(v2: bool, values: usize) -> Vec { + let base = envelope_with_manifest_len(v2, 300); + let ml = u32::from_le_bytes(base[4..8].try_into().unwrap()) as usize; + let text = std::str::from_utf8(&base[8..8 + ml]).unwrap(); + let pad = text.find("\"pad\":").unwrap(); + // A reader that read one value more or less than this would fail the tests that + // use it at the limit and one past it. + let m = format!("{}\"pad\":[0{}]}}", &text[..pad], ",0".repeat(values - 11)); + [&base[..4], &(m.len() as u32).to_le_bytes(), m.as_bytes(), &base[8 + ml..]].concat() + } + + /// SPEC §2, *Reader limits*: a manifest of [`MANIFEST_MAX_VALUES`] values is read and + /// one value more is refused (`manifest_too_many_values`), by both readers, in `WAI1` + /// and `WAI2` manifests alike; the writers refuse to write one past the limit. The + /// count takes every value at any depth, the manifest object among them, and no + /// member name. + #[test] + fn a_manifest_past_the_value_limit_is_refused_and_never_written() { + assert_eq!(MANIFEST_MAX_VALUES, 4_194_304); + assert_eq!(check_manifest_text(br#"{"a":[1,{"b":null}],"c":"d"}"#).unwrap(), 6); + assert_eq!(check_manifest_text(br#"{}"#).unwrap(), 1); + let too_many = |e: String| assert!(e.starts_with("manifest_too_many_values:"), "{e}"); + for v2 in [false, true] { + let read = |b: &[u8]| if v2 { WaiMulti::from_bytes(b).map(|_| ()) } else { Wai::from_bytes(b).map(|_| ()) }; + let at = envelope_with_manifest_values(v2, MANIFEST_MAX_VALUES); + read(&at).unwrap_or_else(|e| panic!("v2={v2}: the limit is read: {e}")); + too_many(read(&envelope_with_manifest_values(v2, MANIFEST_MAX_VALUES + 1)).unwrap_err().to_string()); + } + // The writers: the envelope at the limit is written back; one zero more is not. + let at = envelope_with_manifest_values(false, MANIFEST_MAX_VALUES); + let mut w = Wai::from_bytes(&at).unwrap(); + assert_eq!(w.to_bytes().unwrap(), at); + w.manifest.extra["pad"].as_array_mut().unwrap().push(Value::from(0)); + too_many(w.to_bytes().unwrap_err().to_string()); + let at = envelope_with_manifest_values(true, MANIFEST_MAX_VALUES); + let mut w2 = WaiMulti::from_bytes(&at).unwrap(); + assert_eq!(w2.to_bytes().unwrap(), at); + w2.manifest.extra["pad"].as_array_mut().unwrap().push(Value::Null); + too_many(w2.to_bytes().unwrap_err().to_string()); + } + /// A length a container writes into a u32 field must fit it: the writers refuse a /// payload past 2^32 − 1 bytes instead of writing its length modulo 2^32. #[test] diff --git a/wai/wai-rs/src/ffi_codecs.rs b/wai/wai-rs/src/ffi_codecs.rs index 6b10c8ca8..44bd304e9 100644 --- a/wai/wai-rs/src/ffi_codecs.rs +++ b/wai/wai-rs/src/ffi_codecs.rs @@ -269,6 +269,11 @@ pub unsafe extern "C" fn wai_text_zstd_encode( }) } +/// Decode a `wai.text.zstd` payload within the reader limits of SPEC §2: at most +/// [`TEXT_DECODED_MAX_LEN`](crate::codecs::text::TEXT_DECODED_MAX_LEN) bytes of output +/// and [`TEXT_WINDOW_MAX_LEN`](crate::codecs::text::TEXT_WINDOW_MAX_LEN) bytes of +/// window. A payload past either is refused with `WAI_ERR_CODEC`; the message begins +/// `text_decoded_too_long` or `text_window_too_large`. #[unsafe(no_mangle)] pub unsafe extern "C" fn wai_text_zstd_decode( bytes: *const u8, len: usize, out: *mut WaiBuffer, @@ -293,6 +298,8 @@ pub unsafe extern "C" fn wai_text_xz_encode( }) } +/// Decode a `wai.text.xz` payload within the same limits as +/// [`wai_text_zstd_decode`], the window limit applying to the dictionary. #[unsafe(no_mangle)] pub unsafe extern "C" fn wai_text_xz_decode( bytes: *const u8, len: usize, out: *mut WaiBuffer, diff --git a/wai/wai-rs/src/http_binding/head.rs b/wai/wai-rs/src/http_binding/head.rs index 83fe94929..98d7c4e22 100644 --- a/wai/wai-rs/src/http_binding/head.rs +++ b/wai/wai-rs/src/http_binding/head.rs @@ -4,7 +4,7 @@ //! component's byte range and, through each entry's `sha256`, its digest. use super::{RangeRefusal, DOMAIN_LEN}; -use crate::container::{check_unique_members, Manifest, ManifestV2}; +use crate::container::{read_manifest, read_manifest_v2, Manifest, ManifestV2}; /// Which container an object is. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -136,9 +136,7 @@ impl Head { if self.version != Version::Wai1 { return Err(RangeRefusal::HeadMalformed("not a WAI1 head".into())); } - let b = self.manifest_bytes(); - check_unique_members(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))?; - serde_json::from_slice(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string())) + read_manifest(self.manifest_bytes()).map_err(|e| RangeRefusal::HeadMalformed(e.to_string())) } /// The `WAI2` manifest. @@ -146,9 +144,7 @@ impl Head { if self.version != Version::Wai2 { return Err(RangeRefusal::HeadMalformed("not a WAI2 head".into())); } - let b = self.manifest_bytes(); - check_unique_members(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))?; - serde_json::from_slice(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string())) + read_manifest_v2(self.manifest_bytes()).map_err(|e| RangeRefusal::HeadMalformed(e.to_string())) } /// The component at `index`. diff --git a/wai/wai-rs/src/http_binding/tests.rs b/wai/wai-rs/src/http_binding/tests.rs index 00f4f8fd6..c35c36d21 100644 --- a/wai/wai-rs/src/http_binding/tests.rs +++ b/wai/wai-rs/src/http_binding/tests.rs @@ -119,6 +119,22 @@ fn head_parse_refuses_a_manifest_past_the_reader_limit() { } } +/// SPEC §2, *Reader limits*: a head whose manifest holds more than +/// `MANIFEST_MAX_VALUES` values is refused (`head_malformed`, as +/// `manifest_too_many_values`); one at the limit is read. +#[test] +fn head_parse_refuses_a_manifest_past_the_value_limit() { + let max = crate::container::MANIFEST_MAX_VALUES; + for v2 in [false, true] { + let at = crate::container::tests::envelope_with_manifest_values(v2, max); + assert!(matches!(parse_head(&at, at.len() as u64), Ok(HeadParse::Head(_))), "v2={v2}"); + let over = crate::container::tests::envelope_with_manifest_values(v2, max + 1); + let e = parse_head(&over, over.len() as u64).unwrap_err(); + assert_eq!(e.code(), "head_malformed"); + assert!(matches!(&e, RangeRefusal::HeadMalformed(m) if m.starts_with("manifest_too_many_values:")), "{e:?}"); + } +} + #[test] fn object_length_rule() { let d = appendix_d(); diff --git a/wai/wai-rs/tests/manifest_parse_memory.rs b/wai/wai-rs/tests/manifest_parse_memory.rs new file mode 100644 index 000000000..36950ed2c --- /dev/null +++ b/wai/wai-rs/tests/manifest_parse_memory.rs @@ -0,0 +1,284 @@ +//! What reading a manifest allocates (SPEC §2, *Reader limits*; SECURITY.md, *Manifest +//! parsing*). A counting global allocator records every byte the reader asks for, and +//! each case requires the peak to stay within the bound the reference states: 128 bytes +//! for each value the manifest holds, one for each byte of its text, and, for a `WAI2`, +//! one [`RenditionMeta`] for each entry it lists, with 64 KiB for fixed costs. With the +//! value limit (2^22) and the length limit (2^27) that is at most 640 MiB for a `WAI1` +//! manifest, whatever its shape. +//! +//! The cases: a manifest at the length limit made of small values, refused for its +//! values before anything is built from it (the reader that parsed such a manifest into a +//! JSON tree took some 5 GB); one at the length limit within the value limit, which is +//! read; manifests of small values in each shape at the value limit, read, and one value +//! past it, refused; the largest manifest a conforming staged layer has, a `prev` with a +//! different producer for each of 2^20 units; and a `WAI2` listing as many renditions as +//! its table can hold, read, and one more, refused. +//! +//! The allocator counts what is asked of it, so the figures are the same on every 64-bit +//! target. A reallocation is counted as the new block allocated before the old one is +//! freed, which is the most it can hold at once. One test runs at a time (`ONE`), so the +//! count is this reader's alone. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; +use std::time::Instant; + +use wai::container::{ + check_manifest_text, RenditionMeta, Wai, WaiMulti, MANIFEST_MAX_LEN, MANIFEST_MAX_VALUES, +}; + +struct Counting; +static NOW: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +fn grow(n: usize) { + let now = NOW.fetch_add(n, Relaxed) + n; + PEAK.fetch_max(now, Relaxed); +} + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + let p = unsafe { System.alloc(l) }; + if !p.is_null() { + grow(l.size()); + } + p + } + unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 { + let p = unsafe { System.alloc_zeroed(l) }; + if !p.is_null() { + grow(l.size()); + } + p + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + unsafe { System.dealloc(p, l) }; + NOW.fetch_sub(l.size(), Relaxed); + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { + let q = unsafe { System.realloc(p, l, new) }; + if !q.is_null() { + grow(new); + NOW.fetch_sub(l.size(), Relaxed); + } + q + } +} + +#[global_allocator] +static COUNTING: Counting = Counting; + +static ONE: Mutex<()> = Mutex::new(()); + +/// The most the reference may allocate reading a manifest of `len` bytes holding +/// `values` values and listing `renditions` `WAI2` entries (SPEC §2; SECURITY.md). +fn bound(len: usize, values: usize, renditions: usize) -> usize { + 128 * values + len + renditions * std::mem::size_of::() + (1 << 16) +} + +/// What `f` allocates at its peak beyond what was held when it started, with its result +/// (which the peak includes) and the seconds it took. +fn peak_of(f: impl FnOnce() -> T) -> (T, usize, f64) { + let base = NOW.load(Relaxed); + PEAK.store(base, Relaxed); + let t = Instant::now(); + let r = f(); + (r, PEAK.load(Relaxed) - base, t.elapsed().as_secs_f64()) +} + +fn envelope(magic: &[u8; 4], m: &str, rows: usize) -> Vec { + let mut b = magic.to_vec(); + b.extend_from_slice(&(m.len() as u32).to_le_bytes()); + b.extend_from_slice(m.as_bytes()); + if magic == b"WAI2" { + b.extend_from_slice(&(rows as u16).to_le_bytes()); + for _ in 0..rows { + b.extend_from_slice(&[0; 8]); + } + } else { + b.extend_from_slice(&0u32.to_le_bytes()); + } + b +} + +/// A `WAI1` manifest with a member no revision defines, `pad` (a reader ignores it, SPEC +/// §8), whose value is `open`, `n` elements written by `elem`, and `close`: with `pad`, +/// 10 values and the elements'. +const HEAD: &str = r#"{"wai":"1.0","media":"text","intent":"replicate","model_requirement":{"capability":"wai.text.zstd"},"conditioning":{"kind":"zstd"},"target":null,"pad":"#; +const FIXED: usize = 10; + +fn padded(n: usize, open: &str, close: &str, mut elem: impl FnMut(usize, &mut String)) -> String { + let mut m = String::with_capacity(HEAD.len() + 2 + n * 8); + m.push_str(HEAD); + m.push_str(open); + for i in 0..n { + if i > 0 { + m.push(','); + } + elem(i, &mut m); + } + m.push_str(close); + m.push('}'); + m +} + +/// Read `m` as a `WAI1` envelope; the peak and time, and whether it was read. +fn read_wai1(m: &str) -> (Result<(), String>, usize, f64) { + let env = envelope(b"WAI1", m, 0); + peak_of(|| Wai::from_bytes(&env).map(|_| ()).map_err(|e| e.to_string())) +} + +fn report(name: &str, len: usize, values: usize, peak: usize, secs: f64, bound: usize) { + eprintln!( + "{name}: {len} bytes, {values} values: peak {peak} bytes ({:.2} per byte, {:.1} per value) in {secs:.2} s; bound {bound}", + peak as f64 / len as f64, + peak as f64 / values as f64 + ); + assert!(peak <= bound, "{name}: the read allocated {peak} bytes, past the bound {bound}"); +} + +/// A manifest at the length limit made of small values is refused from its value count, +/// having built nothing: the first pass holds no value, and stops at the value past the +/// limit, some 8 MB into the text. +#[test] +fn a_manifest_of_small_values_at_the_length_limit_is_refused_before_it_is_built() { + let _one = ONE.lock().unwrap_or_else(|e| e.into_inner()); + // Zeros, the last of them a number of ones long enough to bring the text to exactly + // 2^27 bytes. + let k = (MANIFEST_MAX_LEN - HEAD.len() - 3) / 2; + let mut m = padded(k, "[", "]", |_, m| m.push('0')); + let short = MANIFEST_MAX_LEN - m.len(); + let last = m.len() - 3; + m.replace_range(last..last + 1, &"1".repeat(short + 1)); + assert_eq!(m.len(), MANIFEST_MAX_LEN); + let (r, peak, secs) = read_wai1(&m); + let e = r.expect_err("refused"); + assert!(e.starts_with("manifest_too_many_values:"), "{e}"); + report("2^27 bytes of zeros (refused)", m.len(), MANIFEST_MAX_VALUES + 1, peak, secs, 1 << 16); +} + +/// A manifest at the length limit holding exactly the value limit is read within the +/// bound: 2^22 values, the most a reader must accept, each a string, to 2^27 bytes. +#[test] +fn a_manifest_at_both_limits_is_read_within_the_bound() { + let _one = ONE.lock().unwrap_or_else(|e| e.into_inner()); + let n = MANIFEST_MAX_VALUES - FIXED; + // Each string `"…"` and its comma: share out what is left of 2^27 bytes. + let room = MANIFEST_MAX_LEN - HEAD.len() - 3 - (n - 1) - 2 * n; + let (q, r) = (room / n, room % n); + let m = padded(n, "[", "]", |i, m| { + m.push('"'); + m.extend(std::iter::repeat_n('s', q + usize::from(i < r))); + m.push('"'); + }); + assert_eq!(m.len(), MANIFEST_MAX_LEN); + assert_eq!(check_manifest_text(m.as_bytes()).unwrap(), MANIFEST_MAX_VALUES); + let (r, peak, secs) = read_wai1(&m); + r.expect("read"); + report("2^27 bytes, 2^22 strings", m.len(), MANIFEST_MAX_VALUES, peak, secs, bound(m.len(), MANIFEST_MAX_VALUES, 0)); +} + +/// Manifests of small values, in every shape that costs a reader the most for each +/// value, at exactly the value limit, are read within the bound; one value more is +/// refused (`manifest_too_many_values`). +#[test] +fn small_values_at_the_value_limit_are_read_within_the_bound_and_one_more_is_refused() { + let _one = ONE.lock().unwrap_or_else(|e| e.into_inner()); + let n = MANIFEST_MAX_VALUES - FIXED; + type Shape = (&'static str, usize, &'static str, &'static str, fn(usize, &mut String)); + let shapes: [Shape; 6] = [ + ("an array of zeros", n, "[", "]", |_, m| m.push('0')), + ("an array of empty arrays", n, "[", "]", |_, m| m.push_str("[]")), + ("an array of empty objects", n, "[", "]", |_, m| m.push_str("{}")), + ("an array of one-element arrays", n / 2, "[", "]", |_, m| m.push_str("[0]")), + ("an array of one-member objects", n / 2, "[", "]", |_, m| m.push_str(r#"{"a":0}"#)), + ("one object of many members", n, "{", "}", |i, m| { + m.push('"'); + m.push_str(&format!("{i:x}")); + m.push_str("\":0"); + }), + ]; + for (name, k, open, close, elem) in shapes { + // A shape of two values an element takes one zero more when `n` is odd. + let mut m = padded(k, open, close, elem); + if k != n && n % 2 == 1 { + m.insert_str(m.len() - 2, ",0"); + } + let values = check_manifest_text(m.as_bytes()).unwrap(); + assert_eq!(values, MANIFEST_MAX_VALUES, "{name}"); + let (r, peak, secs) = read_wai1(&m); + r.expect(name); + report(name, m.len(), values, peak, secs, bound(m.len(), values, 0)); + + m.insert_str(m.len() - 2, if open == "{" { r#","x":0"# } else { ",0" }); + let (r, peak, _) = read_wai1(&m); + let e = r.expect_err("one value past the limit is refused"); + assert!(e.starts_with("manifest_too_many_values:"), "{name}: {e}"); + // The first pass holds the names of an object's members to find a repeat, so the + // refusal of one object of many members holds them; it stays within the bound. + assert!(peak <= bound(m.len(), MANIFEST_MAX_VALUES, 0), "{name}: the refusal allocated {peak} bytes"); + } +} + +/// The largest manifest a conforming staged layer has (SPEC §2): a `prev` that names a +/// different producer for each of the 2^20 units a `WIR1` layer may cover, about 94 MB +/// and 3·2^20 values, is read within the bound, and its `refines` reads well formed. +#[test] +fn the_largest_conforming_layer_manifest_is_read_within_the_bound() { + let _one = ONE.lock().unwrap_or_else(|e| e.into_inner()); + let (a, b) = ("ab".repeat(32), "cd".repeat(32)); + let units = 1usize << 20; + let mut m = String::with_capacity(95 << 20); + m.push_str(r#"{"wai":"1.0","media":"image","intent":"replicate","model_requirement":{"capability":"wai.image.int_refine","fallback":null},"conditioning":{"kind":"int_refine"},"refines":{"base":""#); + m.push_str(&a); + m.push_str(r#"","prev":["#); + for i in 0..units { + if i > 0 { + m.push(','); + } + m.push_str(r#"{"producer":""#); + m.push_str(if i % 2 == 0 { &a } else { &b }); + m.push_str(r#"","count":1}"#); + } + m.push_str(r#"],"layer":2},"target":null}"#); + let values = check_manifest_text(m.as_bytes()).unwrap(); + assert_eq!(values, 3 * units + 14); + let env = envelope(b"WAI1", &m, 0); + let (w, peak, secs) = peak_of(|| Wai::from_bytes(&env).expect("read")); + report("a layer naming 2^20 producers", m.len(), values, peak, secs, bound(m.len(), values, 0)); + let r = w.refines().expect("refines").expect("well formed"); + assert_eq!(r.units(units as u32), (0, units as u32)); + r.check_units(units as u32).expect("the runs cover the units"); +} + +/// A `WAI2` manifest listing the 65 535 renditions its table can hold is read within the +/// bound, which counts the entries it builds; a `renditions` of 65 536 is refused before +/// its entries are built. +#[test] +fn a_wai2_listing_its_tables_rows_is_read_within_the_bound() { + let _one = ONE.lock().unwrap_or_else(|e| e.into_inner()); + let head = r#"{"wai":"1.1","media":"text","intent":"replicate","renditions":["#; + let manifest = |rows: usize| { + let mut m = String::from(head); + for i in 0..rows { + if i > 0 { + m.push(','); + } + m.push_str(r#"{"capability":"wai.text.zstd","kind":"zstd"}"#); + } + m.push_str(r#"],"target":null}"#); + m + }; + let rows = u16::MAX as usize; + let m = manifest(rows); + let values = check_manifest_text(m.as_bytes()).unwrap(); + let env = envelope(b"WAI2", &m, rows); + let (w, peak, secs) = peak_of(|| WaiMulti::from_bytes(&env).map(|w| w.manifest.renditions.len())); + assert_eq!(w.expect("read"), rows); + report("a WAI2 of 65535 renditions", m.len(), values, peak, secs, bound(m.len(), values, rows)); + + let m = manifest(rows + 1); + let e = WaiMulti::from_bytes(&envelope(b"WAI2", &m, 0)).err().expect("refused").to_string(); + assert!(e.contains("more than the 65535 rows a WAI2 table holds"), "{e}"); +} diff --git a/wai/wai-rs/tests/manifest_reader_equivalence.rs b/wai/wai-rs/tests/manifest_reader_equivalence.rs new file mode 100644 index 000000000..921b31d7a --- /dev/null +++ b/wai/wai-rs/tests/manifest_reader_equivalence.rs @@ -0,0 +1,171 @@ +//! The reference readers build a manifest with `container::read_manifest` and +//! `read_manifest_v2`, which move its members out of one JSON tree (SPEC §2, *Reader +//! limits*: so a parse allocates within a bound), where they used serde's derived reader +//! (`serde_json::from_slice::`). This test holds the two to the same result: +//! for the manifest of every envelope committed under `wai/`, and for each of them with +//! every member, at the manifest's top level and one level down, removed or given a +//! value of each JSON type, both readers read it or both refuse it, and what they read +//! writes back to the same JSON. + +use std::path::{Path, PathBuf}; + +use serde_json::{json, Value}; +use wai::container::{check_unique_members, manifest_bytes, read_manifest, read_manifest_v2, Manifest, ManifestV2}; + +fn envelopes(dir: &Path, out: &mut Vec) { + let Ok(rd) = std::fs::read_dir(dir) else { return }; + let mut entries: Vec<_> = rd.flatten().map(|e| e.path()).collect(); + entries.sort(); + for p in entries { + let name = p.file_name().and_then(|n| n.to_str()).unwrap_or(""); + if p.is_dir() { + if name != "target" && name != "node_modules" && !name.starts_with('.') { + envelopes(&p, out); + } + } else if name.ends_with(".wai") { + out.push(p); + } + } +} + +/// Both readers on `text`: the same verdict, and the same JSON when both read it. +fn same(text: &[u8], v2: bool, what: &str) -> bool { + let (derived, moved) = if v2 { + let d = check_unique_members(text).and_then(|_| serde_json::from_slice::(text)).map(|m| serde_json::to_string(&m).unwrap()); + let m = read_manifest_v2(text).map(|m| serde_json::to_string(&m).unwrap()); + (d, m) + } else { + let d = check_unique_members(text).and_then(|_| serde_json::from_slice::(text)).map(|m| serde_json::to_string(&m).unwrap()); + let m = read_manifest(text).map(|m| serde_json::to_string(&m).unwrap()); + (d, m) + }; + match (&derived, &moved) { + (Ok(a), Ok(b)) => assert_eq!(a, b, "{what}: the readers read different manifests"), + (Err(_), Err(_)) => {} + _ => panic!("{what}: the derived reader gives {derived:?}, the reference reader {moved:?}"), + } + moved.is_ok() +} + +/// The values a member is given: one of each JSON type, and a few of each kind the +/// manifest types read (a string, an integer, a float, `null`). +fn replacements() -> Vec { + vec![json!(null), json!(true), json!(0), json!(-1), json!(1.5), json!(18446744073709551615u64), json!("s"), json!([]), json!({}), json!([{}]), json!({"x": 1})] +} + +fn mutations(text: &[u8], v2: bool, what: &str) -> usize { + let Ok(Value::Object(top)) = serde_json::from_slice::(text) else { return 0 }; + let mut n = 0; + let mut check = |m: &serde_json::Map, why: String| { + same(serde_json::to_string(m).unwrap().as_bytes(), v2, &format!("{what}: {why}")); + n += 1; + }; + for (k, v) in &top { + let mut m = top.clone(); + m.shift_remove(k); + check(&m, format!("without {k}")); + for r in replacements() { + let mut m = top.clone(); + m.insert(k.clone(), r.clone()); + check(&m, format!("{k} = {r}")); + } + // One level down: each member of an object member, and of the first object of + // an array member (a WAI2's first rendition entry). + let (inner, at): (Option<&serde_json::Map>, Option) = match v { + Value::Object(o) => (Some(o), None), + Value::Array(a) => match a.first() { + Some(Value::Object(o)) => (Some(o), Some(0)), + _ => (None, None), + }, + _ => (None, None), + }; + let Some(inner) = inner else { continue }; + let put = |o: serde_json::Map| { + let mut m = top.clone(); + let nv = match at { + None => Value::Object(o), + Some(i) => { + let mut a = v.as_array().unwrap().clone(); + a[i] = Value::Object(o); + Value::Array(a) + } + }; + m.insert(k.clone(), nv); + m + }; + for ik in inner.keys() { + let mut o = inner.clone(); + o.shift_remove(ik); + check(&put(o), format!("{k} without {ik}")); + for r in replacements() { + let mut o = inner.clone(); + o.insert(ik.clone(), r.clone()); + check(&put(o), format!("{k}.{ik} = {r}")); + } + } + } + n +} + +#[test] +fn the_reference_reader_reads_every_committed_manifest_as_the_derived_reader_does() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join(".."); + let mut files = Vec::new(); + envelopes(&root, &mut files); + assert!(files.len() >= 400, "found only {} envelopes under {}", files.len(), root.display()); + let (mut read, mut refused, mut mutated) = (0, 0, 0); + // Mutate the manifests of each distinct set of top-level members once. + let mut shapes = std::collections::BTreeSet::new(); + for f in &files { + let b = std::fs::read(f).unwrap(); + let Ok(text) = manifest_bytes(&b) else { continue }; + let v2 = &b[..4] == b"WAI2"; + let what = f.strip_prefix(&root).unwrap_or(f).display().to_string(); + if same(text, v2, &what) { + read += 1; + } else { + refused += 1; + } + if let Ok(Value::Object(o)) = serde_json::from_slice::(text) { + let keys: Vec = o.keys().cloned().collect(); + if shapes.insert((v2, keys)) { + mutated += mutations(text, v2, &what); + } + } + } + eprintln!("{} envelopes: {read} manifests read alike, {refused} refused alike; {mutated} mutations agree", files.len()); + assert!(read >= 300, "only {read} manifests read"); + assert!(mutated >= 1000, "only {mutated} mutations"); +} + +/// The cases the derived reader treats specially, each read alike: a member whose value +/// is `null` where an `Option` is (`None`, not written back), `target` absent (`null`), +/// a member no revision defines kept in order, a float `min_sim`, and an integer past +/// `u64` as `kid`. +#[test] +fn the_derived_readers_special_cases_are_read_alike() { + let base = r#"{"wai":"1.0","media":"image","intent":"replicate","model_requirement":{"capability":"wai.image.png","fallback":null,"min_sim":0.5,"z":1,"a":[1]},"conditioning":{"kind":"png","prior":null,"q":{}},"security":null,"refines":null,"derived_from":null,"later":{"b":2},"earlier":3}"#; + for t in [ + base.to_string(), + base.replace(r#""min_sim":0.5"#, r#""min_sim":1"#), + base.replace(r#""security":null"#, r#""security":{"encryption":"none","kid":18446744073709551616}"#), + base.replace(r#""security":null"#, r#""security":{"kid":-1}"#), + base.replace(r#""conditioning":{"kind":"png","prior":null,"q":{}}"#, r#""conditioning":{"kind":"png","prior":{"id":"x","sha256":"y","determinism":"z","supersedes":null,"k":[]}}"#), + base.replace(r#""conditioning":{"kind":"png","prior":null,"q":{}}"#, r#""conditioning":{"kind":"png","prior":{"id":"x","sha256":"y","determinism":"z","supersedes":{"sha256":"w","form":null}}}"#), + base.replace(r#""conditioning":{"kind":"png","prior":null,"q":{}}"#, r#""conditioning":{"kind":"png","prior":{"id":"x","sha256":"y"}}"#), + "[]".into(), + "null".into(), + "{}".into(), + ] { + same(t.as_bytes(), false, &t); + } + let v2 = r#"{"wai":"1.1","media":"text","intent":"replicate","renditions":[{"capability":"wai.text.zstd","kind":"zstd","refines":null,"envelope_manifest":{"a":1},"x":[]}],"security":{"encryption":"none"},"z":1}"#; + for t in [ + v2.to_string(), + v2.replace(r#""renditions":[{"#, r#""renditions":[1,{"#), + v2.replace(r#""renditions":["#, r#""renditions":{"a":"#).replace(r#""x":[]}],"#, r#""x":[]}},"#), + v2.replace(r#""kind":"zstd","#, ""), + ] { + same(t.as_bytes(), true, &t); + } +} diff --git a/wai/wai-rs/tests/staged_resources.rs b/wai/wai-rs/tests/staged_resources.rs index 3e5991480..8cd7240b6 100644 --- a/wai/wai-rs/tests/staged_resources.rs +++ b/wai/wai-rs/tests/staged_resources.rs @@ -351,9 +351,17 @@ fn staged_sink_resources() { let chain = sink.chain(&bh).unwrap(); assert!(chain.units().iter().all(|s| s.stage == 0)); assert_eq!(chain.state_bytes(), unit_bytes); - // The refusal held the envelope as parsed and the payload's parse, and no unit's output. - assert!((refused.peak as u64) < 2 * l1.len() as u64 + c.working_bytes - c.output_bytes / 4, - "a refused layer held {} bytes", refused.peak); + // The refusal read the envelope and its payload's header, and decoded no unit. What it + // allocated is set against what reading the same envelope allocates, measured here: + // the read is the larger part, and SPEC §2 bounds it by the manifest's values and + // length. Beyond the read, the refusal allocated less than the envelope's length, the + // payload's parse and half of one unit's output (the layer covers four units). Bytes + // allocated, not the peak: a unit decoded after the read would sit under the read's + // own peak and never raise it. + let (_, read) = measure(|| Wai::from_bytes(&l1).unwrap()); + let beyond = (refused.allocated as u64).saturating_sub(read.allocated as u64); + assert!(beyond < l1.len() as u64 + c.working_bytes + c.output_bytes / 8, + "a refused layer allocated {beyond} bytes beyond reading its envelope"); // At the budget: applied; the state is the units' new output and 32 bytes a stage. let mut sink = StageSink::new(cfg(16, unit_bytes + reserve)); sink.add_base(info.clone(), units.clone()); diff --git a/wai/wai-rs/tests/text_decode_bounds.rs b/wai/wai-rs/tests/text_decode_bounds.rs new file mode 100644 index 000000000..6094e2eec --- /dev/null +++ b/wai/wai-rs/tests/text_decode_bounds.rs @@ -0,0 +1,195 @@ +//! `wai.text.zstd` and `wai.text.xz` decode within the reader limits of SPEC §2: a +//! payload that decodes to more than the length limit, or whose stream asks its decoder +//! to keep more history than the window limit, is refused by name, and one at either +//! limit decodes. +//! +//! - The length limit, at a small limit the caller gives and at the limit the +//! specification sets, for a frame that declares its length and for one that does not. +//! - A zstd frame that declares a length past the limit is refused from the declaration: +//! the same frame cut off after its header is refused the same way. +//! - The window limit, on streams made here that declare one at the limit and one past +//! it. Each holds a byte or a line of content: what is refused is the declaration. +//! - The writers refuse what the readers may refuse. +//! +//! The memory the output buffer takes is measured in `text_decode_memory.rs`. +//! +//! Exercises the codec wrappers, so it requires the `codecs` feature. +#![cfg(feature = "codecs")] + +use std::io::Write; + +use wai::codecs::text::{ + xz_decode, xz_decode_within, xz_encode, zstd_decode, zstd_decode_within, zstd_encode, + TEXT_DECODED_MAX_LEN, TEXT_WINDOW_MAX_LEN, +}; + +fn refusal(r: Result) -> String { + r.expect_err("the payload must be refused").0 +} + +/// `n` bytes that compress well and are not all one value. +fn pattern(n: usize) -> Vec { + (0..n).map(|i| (i / 4096) as u8).collect() +} + +/// A zstd stream of `n` zero bytes whose frame declares no content size, written in +/// pieces so that making it takes no buffer of `n` bytes. +fn zstd_zeros(n: usize) -> Vec { + let piece = vec![0u8; 1 << 20]; + let mut enc = zstd::stream::write::Encoder::new(Vec::new(), 1).unwrap(); + let mut left = n; + while left > 0 { + let k = left.min(piece.len()); + enc.write_all(&piece[..k]).unwrap(); + left -= k; + } + enc.finish().unwrap() +} + +/// The same for xz. +fn xz_zeros(n: usize) -> Vec { + let piece = vec![0u8; 1 << 20]; + let mut enc = xz2::write::XzEncoder::new(Vec::new(), 0); + let mut left = n; + while left > 0 { + let k = left.min(piece.len()); + enc.write_all(&piece[..k]).unwrap(); + left -= k; + } + enc.finish().unwrap() +} + +#[test] +fn a_payload_at_the_callers_limit_decodes_and_one_byte_past_it_is_refused() { + const LIMIT: usize = 1 << 20; + let at = pattern(LIMIT); + let past = pattern(LIMIT + 1); + + // zstd, the frame declaring no content size (a streamed encode). + assert_eq!(zstd_decode_within(&zstd_encode(&at, 3).unwrap(), LIMIT).unwrap(), at); + let why = refusal(zstd_decode_within(&zstd_encode(&past, 3).unwrap(), LIMIT)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + + // zstd, the frame declaring its content size (a one-shot encode). + let declared_at = zstd::bulk::compress(&at, 3).unwrap(); + let declared_past = zstd::bulk::compress(&past, 3).unwrap(); + assert_eq!(zstd::zstd_safe::get_frame_content_size(&declared_past).ok().flatten(), Some(LIMIT as u64 + 1)); + assert_eq!(zstd_decode_within(&declared_at, LIMIT).unwrap(), at); + let why = refusal(zstd_decode_within(&declared_past, LIMIT)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + + // xz. + assert_eq!(xz_decode_within(&xz_encode(&at, 1).unwrap(), LIMIT).unwrap(), at); + let why = refusal(xz_decode_within(&xz_encode(&past, 1).unwrap(), LIMIT)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); +} + +#[test] +fn a_declared_length_past_the_limit_is_refused_from_the_declaration() { + const LIMIT: usize = 1 << 16; + let frame = zstd::bulk::compress(&pattern(LIMIT + 1), 3).unwrap(); + // The frame's first 18 bytes: its header, which is never longer, and at most a few + // bytes of its first block. Nothing can be decoded from them. + assert!(frame.len() > 18); + let header_only = &frame[..18]; + assert_eq!(zstd::zstd_safe::get_frame_content_size(header_only).ok().flatten(), Some(LIMIT as u64 + 1)); + let why = refusal(zstd_decode_within(header_only, LIMIT)); + assert!(why.starts_with("text_decoded_too_long:"), "refused for another reason: {why}"); + // The same cut, within the limit, is the truncated frame it is. + let why = refusal(zstd_decode_within(header_only, LIMIT + 1)); + assert!(!why.starts_with("text_decoded_too_long:"), "{why}"); +} + +#[test] +fn the_limit_without_an_argument_is_the_one_the_specification_sets() { + assert_eq!(TEXT_DECODED_MAX_LEN, 1 << 27); + + let out = zstd_decode(&zstd_zeros(TEXT_DECODED_MAX_LEN)).unwrap(); + assert_eq!(out.len(), TEXT_DECODED_MAX_LEN); + assert!(out.iter().all(|&b| b == 0)); + drop(out); + let past = zstd_zeros(TEXT_DECODED_MAX_LEN + 1); + let why = refusal(zstd_decode(&past)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + // A caller cannot raise it. + let why = refusal(zstd_decode_within(&past, usize::MAX)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + + let out = xz_decode(&xz_zeros(TEXT_DECODED_MAX_LEN)).unwrap(); + assert_eq!(out.len(), TEXT_DECODED_MAX_LEN); + drop(out); + let past = xz_zeros(TEXT_DECODED_MAX_LEN + 1); + let why = refusal(xz_decode(&past)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + let why = refusal(xz_decode_within(&past, usize::MAX)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); +} + +/// A zstd frame of one byte of content that declares a window of `2^window_log` bytes +/// (RFC 8878 §3.1.1): the magic number; a frame header descriptor of 0, which says a +/// window descriptor follows and nothing else does; the window descriptor, whose high +/// five bits are `window_log − 10`; and one last block, of type RLE and size 1. +fn zstd_frame_declaring_window(window_log: u32) -> Vec { + let mut f = vec![0x28, 0xB5, 0x2F, 0xFD, 0x00, ((window_log - 10) << 3) as u8]; + f.extend_from_slice(&[0x0B, 0x00, 0x00, b'A']); + f +} + +#[test] +fn a_zstd_window_at_the_limit_decodes_and_one_past_it_is_refused() { + assert_eq!(TEXT_WINDOW_MAX_LEN, 1 << 27); + assert_eq!(zstd_decode(&zstd_frame_declaring_window(27)).unwrap(), b"A"); + let why = refusal(zstd_decode(&zstd_frame_declaring_window(28))); + assert!(why.starts_with("text_window_too_large:"), "{why}"); +} + +/// CRC-32 as the xz format uses it (IEEE 802.3, reflected). +fn crc32(bytes: &[u8]) -> u32 { + let mut c = !0u32; + for &b in bytes { + c ^= b as u32; + for _ in 0..8 { + c = (c >> 1) ^ (0xEDB8_8320 & (!(c & 1)).wrapping_add(1)); + } + } + !c +} + +/// An xz stream of a line of text whose one block declares the LZMA2 dictionary size +/// coded as `code` (the xz file format, §5.3.1: 30 is 2^27 bytes, 31 is 3·2^26). Made by +/// encoding the line, then rewriting the dictionary size in the block header and the +/// header's CRC-32. The stream header is 12 bytes; the block header follows it: its +/// size, the block flags, then the filter (id 0x21, one byte of properties, which is the +/// dictionary size), padding, and the CRC-32 of all of it. +fn xz_stream_declaring_dictionary(code: u8) -> (Vec, Vec) { + let line = b"one line of text, long enough to be worth a block\n".to_vec(); + let mut s = xz_encode(&line, 0).unwrap(); + let start = 12; + let size = (s[start] as usize + 1) * 4; + let header = start..start + size; + let at = (header.start + 2..header.end - 5) + .find(|&i| s[i] == 0x21 && s[i + 1] == 0x01) + .expect("the block header names the LZMA2 filter with one byte of properties"); + s[at + 2] = code; + let crc = crc32(&s[header.start..header.end - 4]); + s[header.end - 4..header.end].copy_from_slice(&crc.to_le_bytes()); + (s, line) +} + +#[test] +fn an_xz_dictionary_at_the_limit_decodes_and_one_past_it_is_refused() { + let (at, line) = xz_stream_declaring_dictionary(30); + assert_eq!(xz_decode(&at).unwrap(), line); + let (past, _) = xz_stream_declaring_dictionary(31); + let why = refusal(xz_decode(&past)); + assert!(why.starts_with("text_window_too_large:"), "{why}"); +} + +#[test] +fn the_writers_refuse_what_a_reader_may_refuse() { + let over = vec![0u8; TEXT_DECODED_MAX_LEN + 1]; + let why = refusal(zstd_encode(&over, 1)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + let why = refusal(xz_encode(&over, 0)); + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); +} diff --git a/wai/wai-rs/tests/text_decode_memory.rs b/wai/wai-rs/tests/text_decode_memory.rs new file mode 100644 index 000000000..a33e3b319 --- /dev/null +++ b/wai/wai-rs/tests/text_decode_memory.rs @@ -0,0 +1,108 @@ +//! Decoding a `wai.text.zstd` or `wai.text.xz` payload that would expand past the +//! length limit takes no more output buffer than the limit (SPEC §2, *Reader limits*). +//! Measured with a global allocator that counts the bytes live and their peak. +//! +//! Each payload here is a few kilobytes and decodes to four times the limit the call is +//! given. The call must refuse it, and the bytes it allocated on the way must stay +//! within the limit and a small allowance for its read buffer. A decode that grew its +//! buffer past the limit before checking, or that decoded everything and then measured +//! it, would not. +//! +//! The counter sees what this crate allocates: the output buffer and the read buffer. +//! The decoding library's own working memory is allocated by the library and is bounded +//! by the window limit, which `text_decode_bounds.rs` tests. +//! +//! Its own test binary with a single test, so that no other test's allocations land in +//! the windows it measures. Requires the `codecs` feature. +#![cfg(feature = "codecs")] + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::io::Write; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; + +use wai::codecs::text::{xz_decode_within, zstd_decode_within}; + +/// The system allocator, counting the bytes live and their peak. +struct Count; +static CUR: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +fn grow(n: usize) { + PEAK.fetch_max(CUR.fetch_add(n, Relaxed) + n, Relaxed); +} + +unsafe impl GlobalAlloc for Count { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + grow(l.size()); + unsafe { System.alloc(l) } + } + unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 { + grow(l.size()); + unsafe { System.alloc_zeroed(l) } + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { + if new >= l.size() { + grow(new - l.size()); + } else { + CUR.fetch_sub(l.size() - new, Relaxed); + } + unsafe { System.realloc(p, l, new) } + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + CUR.fetch_sub(l.size(), Relaxed); + unsafe { System.dealloc(p, l) } + } +} + +#[global_allocator] +static ALLOC: Count = Count; + +/// The most bytes live at once while `f` runs, above what was live when it started. +fn peak_of(f: impl FnOnce() -> T) -> (T, usize) { + let base = CUR.load(Relaxed); + PEAK.store(base, Relaxed); + let r = f(); + (r, PEAK.load(Relaxed).saturating_sub(base)) +} + +const LIMIT: usize = 8 << 20; +/// The read buffer, and room for the error's text. +const ALLOWANCE: usize = 1 << 20; + +fn zeros_through(mut w: W, n: usize) -> W { + let piece = vec![0u8; 1 << 20]; + let mut left = n; + while left > 0 { + let k = left.min(piece.len()); + w.write_all(&piece[..k]).unwrap(); + left -= k; + } + w +} + +#[test] +fn a_payload_that_expands_past_the_limit_takes_no_more_buffer_than_the_limit() { + let zstd_payload = zeros_through(zstd::stream::write::Encoder::new(Vec::new(), 1).unwrap(), 4 * LIMIT) + .finish() + .unwrap(); + let xz_payload = zeros_through(xz2::write::XzEncoder::new(Vec::new(), 0), 4 * LIMIT).finish().unwrap(); + assert!(zstd_payload.len() < LIMIT / 64 && xz_payload.len() < LIMIT / 64, "the payloads are small"); + + let (r, peak) = peak_of(|| zstd_decode_within(&zstd_payload, LIMIT)); + let why = r.expect_err("zstd: the payload must be refused").0; + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + assert!(peak <= LIMIT + ALLOWANCE, "zstd: refusing took {peak} bytes of buffer, limit {LIMIT}"); + assert!(peak >= LIMIT / 2, "zstd: the refusal came before the output neared the limit ({peak} bytes): is the payload what it should be?"); + + let (r, peak) = peak_of(|| xz_decode_within(&xz_payload, LIMIT)); + let why = r.expect_err("xz: the payload must be refused").0; + assert!(why.starts_with("text_decoded_too_long:"), "{why}"); + assert!(peak <= LIMIT + ALLOWANCE, "xz: refusing took {peak} bytes of buffer, limit {LIMIT}"); + assert!(peak >= LIMIT / 2, "xz: the refusal came before the output neared the limit ({peak} bytes): is the payload what it should be?"); + + // At the limit the decode succeeds, in a buffer of the limit. + let zstd_at = zeros_through(zstd::stream::write::Encoder::new(Vec::new(), 1).unwrap(), LIMIT).finish().unwrap(); + let (r, peak) = peak_of(|| zstd_decode_within(&zstd_at, LIMIT)); + assert_eq!(r.unwrap().len(), LIMIT); + assert!(peak <= LIMIT + ALLOWANCE, "zstd: decoding to the limit took {peak} bytes of buffer, limit {LIMIT}"); +} diff --git a/wai/wai-web/src/lib.rs b/wai/wai-web/src/lib.rs index 453448cc2..1ceb23f63 100644 --- a/wai/wai-web/src/lib.rs +++ b/wai/wai-web/src/lib.rs @@ -501,10 +501,11 @@ fn pack_refusal(m: &wai::container::Manifest, payload: &[u8]) -> Result<(), Stri } /// A manifest a writer emits as given, read as every reader reads it (SPEC §2): a -/// member named twice in one object, at any depth, is refused, so a writer never emits -/// an envelope the reader refuses. +/// member named twice in one object, at any depth, and a manifest past the value limit +/// (`manifest_too_many_values`) are refused, so a writer never emits an envelope the +/// reader refuses. fn read_manifest(json: &str) -> Result { - wai::container::check_unique_members(json.as_bytes()).map_err(|e| format!("manifest parse: {e}"))?; + wai::container::check_manifest_text(json.as_bytes()).map_err(|e| format!("manifest parse: {e}"))?; serde_json::from_str(json).map_err(|e| format!("manifest parse: {e}")) }