wai: reader limits that bound memory (manifest values, text decode), and the codec wrappers in CI #231
24 changed files with 1705 additions and 98 deletions
47
.github/workflows/wai-conformance.yml
vendored
47
.github/workflows/wai-conformance.yml
vendored
|
|
@ -248,12 +248,18 @@ jobs:
|
|||
# instr-v0.json lists as accepted (one at each bound) made and wrapped, every object
|
||||
# it lists as refused refused for the reason it states, and each refusal writing
|
||||
# nothing; tests/generate.rs holds the same lists to a second statement of form v0.
|
||||
# tests/manifest_parse_memory.rs (its own binary, a counting allocator) checks that
|
||||
# reading a manifest allocates within the bound SPEC §2's reader limits give, at the
|
||||
# length limit and at the value limit, and tests/manifest_reader_equivalence.rs holds
|
||||
# the readers to the derived reader's result on every committed envelope. Both run in
|
||||
# the release step only: the first writes manifests of 2^27 bytes, which takes a debug
|
||||
# build most of a minute.
|
||||
- name: Fast integer kernels ≡ reference kernels (overflow checks on)
|
||||
working-directory: wai/wai-rs
|
||||
run: cargo test --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds
|
||||
- name: Fast integer kernels ≡ reference kernels (release, overflow checks off)
|
||||
working-directory: wai/wai-rs
|
||||
run: cargo test --release --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds
|
||||
run: cargo test --release --no-default-features --features neural_int,video_int,audio_int,world_int,splat_int,generate,score,staged,stage_measure --lib --test int_kernels_vectors --test int_kernels_geometry --test int_mlicpp --test int_mlicpp_preflight --test int_audio --test int_splat --test generate --test cli_generate_score --test int_payload --test int_payload_preflight --test int_payload_cost --test int_video_enc_memory --test cli_stage --test staged_resources --test stage_measure --test stage_measure_bounds --test manifest_parse_memory --test manifest_reader_equivalence
|
||||
# The integer decode path builds under each of its features alone, so a sink can take
|
||||
# one medium without the others, and for wasm32 (a browser sink); `int_payload` arms a
|
||||
# medium only under its feature, and none needs `neural`. So does the staged-measurement
|
||||
|
|
@ -882,6 +888,45 @@ jobs:
|
|||
working-directory: wai/byte-exact-conformance
|
||||
run: cargo test --release
|
||||
|
||||
# The codec wrappers are behind `codecs`, which binds C and C++ libraries that no
|
||||
# container image of this fleet carries, so no other job compiles them. A feature no
|
||||
# workflow enables has no CI coverage at all: without this job the wrappers' tests,
|
||||
# and the limits on what a text payload may decode to (SPEC §2), pass by not running.
|
||||
#
|
||||
# One platform, the `macos` runner, whose host has the libraries. What the job proves
|
||||
# is how the wrappers treat hostile bytes and where their limits fall, which is the
|
||||
# same code on every host. A missing library fails the build step by name.
|
||||
codec-wrappers:
|
||||
name: codec wrappers · fuzz smoke, text decode limits, C entry points
|
||||
runs-on: macos
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Install Rust 1.98.0
|
||||
shell: bash
|
||||
run: |
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
rustup toolchain install 1.98.0 --profile minimal
|
||||
rustup default 1.98.0
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
|
||||
# Every one of these test files is `#![cfg(feature = "codecs")]`: built without
|
||||
# the feature each reports "0 passed" and exits 0. So the step names the feature,
|
||||
# and then requires that each of the four binaries ran at least one test.
|
||||
- name: Fuzz smoke, the text decode limits and the C entry points
|
||||
working-directory: wai/wai-rs
|
||||
shell: bash
|
||||
run: |
|
||||
set -o pipefail
|
||||
cargo test --locked --features codecs \
|
||||
--test fuzz_smoke --test text_decode_bounds --test text_decode_memory --test ffi_robustness \
|
||||
2>&1 | tee "$RUNNER_TEMP/codec-wrappers.log"
|
||||
ran=$(grep -cE '^test result: ok\. [1-9][0-9]* passed' "$RUNNER_TEMP/codec-wrappers.log" || true)
|
||||
if [ "$ran" -ne 4 ]; then
|
||||
echo "::error::expected four test binaries that each ran tests; $ran did"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The C2PA emitter is gated behind an off-by-default feature, so the four-platform
|
||||
# job above — which builds a fixed feature list — never compiles it. A feature no
|
||||
# workflow enables has no CI coverage at all: the suite passes, and the tests in it
|
||||
|
|
|
|||
|
|
@ -6,6 +6,70 @@ wire format may change and such changes are recorded here.
|
|||
|
||||
## v0.4.0 — current
|
||||
|
||||
### Reader limits that bound memory: manifest values, and text decode
|
||||
|
||||
Two reader limits join the manifest length limit (SPEC §2, *Reader limits*), and with
|
||||
them a reader's memory is bounded by what the specification states, not by what an
|
||||
envelope asks for. `SECURITY.md` listed both as known gaps; it now states the bounds
|
||||
and the tests that measure them.
|
||||
|
||||
- **Manifest values.** A reader need accept no manifest of more than 2²² JSON values,
|
||||
the manifest object and every value in it at any depth. The reference readers count
|
||||
the values before they build anything, refuse one past the limit
|
||||
(`manifest_too_many_values`), and build what they read at the size counted: at most
|
||||
128 bytes a value and one a byte on a 64-bit target, where a manifest of small
|
||||
values at the length limit took several gigabytes to parse. The stdlib Python
|
||||
verifiers refuse the same manifests, and the HTTP corpus holds a case at the limit
|
||||
and one past it.
|
||||
- **Text decode.** A `wai.text.zstd` or `wai.text.xz` payload decodes to at most 2²⁷
|
||||
bytes and declares at most 2²⁷ bytes of window or dictionary; a reader may refuse
|
||||
more (`text_decoded_too_long`, `text_window_too_large`). The reference decoders
|
||||
write into a buffer never sized past the limit, refuse a Zstandard frame that
|
||||
declares a longer content size before decoding anything, and give the decoding
|
||||
library the window limit, so a declaration past it is refused from the stream's
|
||||
header. The reference encoders refuse input past the length limit. The C entry
|
||||
points `wai_text_zstd_decode` and `wai_text_xz_decode` are bounded the same way; the
|
||||
header gains `TEXT_DECODED_MAX_LEN` and `TEXT_WINDOW_MAX_LEN`.
|
||||
- **Planned break for writers.** A conforming encoder writes no manifest past the value
|
||||
limit and no text payload past either text limit. No committed envelope or vector is
|
||||
past any of them, and no digest changes.
|
||||
- **The codec wrappers in CI.** `wai-conformance.yml` gains `codec-wrappers`, on the
|
||||
macOS runner: the wrappers' fuzz smoke, the text decode limits and the C entry
|
||||
points, built with `codecs`. The step fails if any of its test binaries ran no test,
|
||||
which is what a build without the feature does.
|
||||
|
||||
### Integer operators: the reference crate, the bounds pass and its certificates
|
||||
|
||||
The crate, the tool and the corpus that
|
||||
[integer-operators](extensions/integer-operators.md) (Draft) names exist. No capability
|
||||
decodes through them yet: each capability's reference decoder is still the one its
|
||||
payload extension names.
|
||||
|
||||
- **`wai-ops`**, the reference crate, `no_std` with `alloc` and no dependencies: the
|
||||
512-bit saturating integer of §8.1, the canonical JSON writer of §8.5, the `WHG1`
|
||||
parser and writer, the load-time and run-time plans, every graph operator of §5, the
|
||||
stage operators E1–E13, and the lowerings of §13.
|
||||
- **Load step 8.** `wai_ops::bounds` runs the bounds pass of §8.1–§8.5 and writes the
|
||||
canonical certificate. The interpreter runs only a loaded graph, so nothing decodes a
|
||||
parameter set its certificate does not admit. `interp::run_observed` reports every
|
||||
value a run forms; `bounds::Within` holds a run to its certificate, and every golden
|
||||
decode runs that way.
|
||||
- **`wai-verify`**: `bounds`, `lower` and `vectors`, with `--check`, `--witness` and
|
||||
`--advise`.
|
||||
- **`wai/ops-conformance/`**: per-operator, stage, graph, load and lowering vectors, the
|
||||
lowered graph of every committed parameter set, and each set's certificate.
|
||||
- **Two certifiers.** `tools/wai_ops_check.py`, written from the extension text and
|
||||
sharing no code with the crate, recomputes every vector and every certificate byte
|
||||
for byte.
|
||||
- **`wai_ops::limb32`**, a reference model of the 32-bit arithmetic the `acc32` profile's
|
||||
two products need, checked against 64-bit arithmetic.
|
||||
- **Model checking.** The operator set's harnesses are in `wai-ops/src/proofs/`, hosted
|
||||
by `wai-verify` and run by `wai-formal-verification.yml`. A harness that does not yet
|
||||
verify within the job's budget is kept, behind `cfg(kani_open)`, and is not run or
|
||||
counted as proved.
|
||||
- **`wai-ops.yml`** runs the crate's tests and `vectors verify` natively, under
|
||||
wasm32-wasip2 and under wasm32 in Node, and runs the checker.
|
||||
|
||||
### wai-quantum 0.4.3: metered runs, placement by run, and the fabric wrapper
|
||||
|
||||
Additive. The default features, `full`, both implementations of the WebGPU member, and
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ What the readers bound:
|
|||
|---|---|---|
|
||||
| every length field (`man_len`, `payload_len`, each rendition's offset and length) | checked against the bytes in hand before anything is sliced; sums formed so they cannot wrap (`saturating_add`; the `WAI2` payload total in `u64`) | SPEC §2, §7.1 |
|
||||
| manifest length | at most 134 217 728 bytes (2²⁷), sized to the largest manifest a conforming staged layer has. A longer one is refused from `man_len` alone, before it is read or, over HTTP, requested (`manifest_too_large`; `head_malformed` from `parse_head`) | SPEC §2, *Reader limits* |
|
||||
| manifest values | at most 4 194 304 JSON values (2²²), the manifest object and every value in it at any depth; a member's name is not one. One more is refused while the values are counted, before anything is built from the manifest (`manifest_too_many_values`; `head_malformed` from `parse_head`) | SPEC §2, *Reader limits* |
|
||||
| manifest nesting | at most 127 nested arrays and objects, the manifest object counting as one; 128 are refused | SPEC §2, *Reader limits* |
|
||||
| member names | a member named twice in one object, at any depth, is refused | SPEC §2 |
|
||||
| `WAI2` rendition table | at most 65 535 rows (`u16`), as many as the manifest's `renditions`, packed contiguously from offset 0 | SPEC §7.1 |
|
||||
|
|
@ -52,16 +53,26 @@ it allocate more than it was given. A `WAI1` reader ignores bytes after the payl
|
|||
HTTP object that carries any is refused ([http-delivery](extensions/http-delivery.md)
|
||||
§5.1).
|
||||
|
||||
**Parsing a manifest takes more memory than its length.** The reference readers parse
|
||||
the manifest into a JSON tree (`serde_json::Value`, for `target`, `refines`,
|
||||
`derived_from` and every member this revision does not define), and a parsed value
|
||||
takes more memory than its text: about seven times the length of a 94 MB `prev` that
|
||||
names a producer for each of 2²⁰ units, the largest manifest a conforming layer has,
|
||||
and nearly 40 times the length of a manifest of many small values, such as one long
|
||||
array of zeros. A manifest at the length limit can take some 5 GB and several seconds
|
||||
to parse, and a reader may not refuse it for its length (SPEC §2). The length limit
|
||||
bounds what a reader must read, not what parsing it costs; the reference sets no limit
|
||||
of its own on the parse.
|
||||
**What parsing a manifest takes.** A parsed value takes more memory than its text, and
|
||||
many small values can take many times their length, so the length limit bounds what a
|
||||
reader must read and not what a parse costs. The value limit bounds the rest. The
|
||||
reference readers count a manifest's values before they build anything from it
|
||||
(`check_manifest_text`) and refuse it as soon as the count passes the limit; they then
|
||||
build each array and object at the size counted, and move the manifest's members out of
|
||||
that one tree (`read_manifest`, `read_manifest_v2`). On a 64-bit target a read allocates
|
||||
at most 128 bytes for each value and one for each byte of the manifest, and for a `WAI2`
|
||||
one rendition entry of 768 bytes for each it lists: at most 640 MiB for a `WAI1`
|
||||
manifest at both limits, whatever its shape, and 48 MiB more for a `WAI2`'s entries.
|
||||
|
||||
`tests/manifest_parse_memory.rs` measures this with an allocator that counts every byte
|
||||
asked for: a manifest of small values at the length limit is refused for its values
|
||||
before anything is built; one at the length limit within the value limit is read; a
|
||||
manifest of each shape at the value limit is read and one value past it refused; and so
|
||||
are the largest manifest a conforming staged layer has and a `WAI2` listing as many
|
||||
renditions as its table holds. Each stays within the bound above.
|
||||
`tests/manifest_reader_equivalence.rs` holds these readers to the result of the derived
|
||||
reader they replace, for every envelope committed under `wai/` and for each with every
|
||||
member removed or given a value of each JSON type.
|
||||
|
||||
The library tests in `container.rs` and `http_binding/tests.rs` check the manifest
|
||||
limits at their edges: from its first eight bytes, every reader refuses a manifest one
|
||||
|
|
@ -77,9 +88,10 @@ limit.
|
|||
|
||||
The stdlib Python verifiers read envelopes too. `staged_verify.py`,
|
||||
`claim_verify.py`, `pack_verify.py` and `http_verify.py` refuse a manifest past the
|
||||
length limit as the reference does. `http_verify.py` and `wai_http_vectors` agree on
|
||||
`parse_head` at the limit in the HTTP corpus; the other three each build an envelope in
|
||||
memory with a manifest one byte past it and require it refused. Those that read
|
||||
length limit, or past the value limit, as the reference does. `http_verify.py` and `wai_http_vectors` agree on
|
||||
`parse_head` at each limit in the HTTP corpus; the other three each build an envelope in
|
||||
memory with a manifest one byte past the length limit, and one value past the value
|
||||
limit, and require each refused. Those that read
|
||||
manifests through `staged_verify.strict_json` refuse nesting past 127; `pack_verify.py`
|
||||
reads up to the interpreter's recursion limit and refuses past it as unparseable.
|
||||
|
||||
|
|
@ -90,6 +102,21 @@ bytes and 2²⁰ frames, and a sink cost limit (`int_payload::check_cost`,
|
|||
`wai.neural.int_mlicpp` refuses a header whose shapes, or whose buffers, pass its cap
|
||||
before it allocates them (`tests/int_mlicpp_preflight.rs`).
|
||||
|
||||
`wai.text.zstd` and `wai.text.xz` carry streams whose formats bound neither what a
|
||||
payload decodes to nor the history its decoder must keep, so the specification sets
|
||||
both (SPEC §2): at most 2²⁷ decoded bytes, and at most 2²⁷ bytes of window or
|
||||
dictionary. The reference decoders (`codecs/text.rs`) write their output into a buffer
|
||||
that is never sized past the length limit and refuse the payload when its output would
|
||||
pass it (`text_decoded_too_long`); a Zstandard frame that declares a longer content
|
||||
size is refused before anything is decoded. They give the decoding library the window
|
||||
limit, so a stream that declares more is refused from its header, before the library
|
||||
allocates it (`text_window_too_large`). The encoders refuse input past the length limit.
|
||||
`tests/text_decode_bounds.rs` checks each limit at its edge, on streams made in the test
|
||||
that are at it and one step past it; `tests/text_decode_memory.rs` measures, with a
|
||||
counting allocator, that refusing a payload of a few kilobytes that would expand to
|
||||
four times a limit takes no more output buffer than the limit. The C entry points
|
||||
`wai_text_zstd_decode` and `wai_text_xz_decode` are these decoders.
|
||||
|
||||
## Native codec libraries and sandboxed decode
|
||||
|
||||
The default `codecs` feature binds C and C++ libraries for the floor and the
|
||||
|
|
@ -108,11 +135,12 @@ run it in a WebAssembly runtime.
|
|||
|
||||
What does not touch them:
|
||||
|
||||
- CI builds, except the neural workflows: no workflow enables `codecs` or `semantic`,
|
||||
so no CI build links the codec libraries. `wai-neural-gates.yml` and
|
||||
- Most CI builds. One job builds `codecs`: `codec-wrappers` in `wai-conformance.yml`,
|
||||
on the macOS runner, which runs the wrappers' fuzz smoke, the text decode limits and
|
||||
the C entry points through the libraries. `wai-neural-gates.yml` and
|
||||
`wai-neural-e2e.yml` build the `neural` feature, which links ONNX Runtime and
|
||||
libzstd, and run the float neural decoders through them; no other workflow's build
|
||||
links any of these libraries.
|
||||
libzstd, and run the float neural decoders through them. No workflow enables
|
||||
`semantic`, and no other workflow's build links any of these libraries.
|
||||
- WAI's own decoders are Rust: the integer codecs, worlds, scores, films and the
|
||||
generator. Outside the C ABI (`ffi.rs`, `ffi_codecs.rs`, `ffi_neural.rs`), the only
|
||||
`unsafe` in `wai-rs`'s library code is in the AArch64 SIMD kernels of
|
||||
|
|
@ -132,15 +160,12 @@ element beside each kernel and over every committed integer-decoder vector
|
|||
|
||||
Known gaps:
|
||||
|
||||
- **Decompression output is not bounded.** `wai.text.zstd`, in the floor, and
|
||||
`wai.text.xz` decode the whole stream into memory, so a small payload can expand to
|
||||
any size the format allows. The image, audio and video decoders allocate what the
|
||||
bitstream declares, within the libraries' own limits. The reference sets no output
|
||||
limit of its own.
|
||||
- **The image, audio and video decoders set no output limit of their own.** They
|
||||
allocate what the bitstream declares, within the libraries' own limits.
|
||||
- **Fuzzing is shallow.** `tests/fuzz_smoke.rs` feeds seeded random bytes to each
|
||||
codec wrapper and to `Wai::from_bytes` and requires that nothing panics. It needs
|
||||
the `codecs` feature, which no CI workflow builds, so it runs only by hand. There is
|
||||
no coverage-guided fuzzing of the manifest parser, the integer payload parsers or the
|
||||
codec wrapper and to `Wai::from_bytes` and requires that nothing panics. It runs in
|
||||
CI on one platform (`codec-wrappers` in `wai-conformance.yml`). There is no
|
||||
coverage-guided fuzzing of the manifest parser, the integer payload parsers or the
|
||||
codec wrappers.
|
||||
|
||||
## Module authorisation
|
||||
|
|
|
|||
57
wai/SPEC.md
57
wai/SPEC.md
|
|
@ -86,20 +86,51 @@ OPTIONAL; §5 says which are RECOMMENDED.
|
|||
|
||||
All WAI multi-byte integers are little-endian unless stated otherwise.
|
||||
|
||||
**Reader limits.** A reader MUST NOT refuse a manifest for its length or its nesting
|
||||
while it is at most 134 217 728 bytes (`man_len` ≤ 2²⁷) and its arrays and objects nest
|
||||
at most 127 deep, the manifest object itself counting as one. It MAY refuse a longer or
|
||||
deeper manifest. The length limit holds the largest manifest a conforming layer has: a
|
||||
`prev` ([staged-delivery](extensions/staged-delivery.md) §3) that names a different
|
||||
producer for each of the 2²⁰ units a `WIR1` layer may cover (staged-delivery
|
||||
Appendix A.3, `n_units` ≤ 2²⁰) is about 94 MB. A reader that refuses one past the
|
||||
length limit does so from `man_len` alone, before it reads or requests the manifest
|
||||
(`manifest_too_large`). The reference readers refuse both. The same limits apply to the
|
||||
`WAI2` manifest (§7.1). They bound what a reader must read, not the memory its parse
|
||||
takes, and move no byte of the layout above, which stays frozen (§8). Since a
|
||||
**Reader limits.** A reader MUST NOT refuse a manifest for its length, its nesting or
|
||||
the number of its values while it is at most 134 217 728 bytes (`man_len` ≤ 2²⁷), its
|
||||
arrays and objects nest at most 127 deep, the manifest object itself counting as one,
|
||||
and it holds at most 4 194 304 JSON values (2²²), counting the manifest object and every
|
||||
value nested in it at any depth; a member's name is not a value. It MAY refuse a longer
|
||||
or deeper manifest, or one that holds more values. The limits hold the largest manifest
|
||||
a conforming layer has: a `prev` ([staged-delivery](extensions/staged-delivery.md) §3)
|
||||
that names a different producer for each of the 2²⁰ units a `WIR1` layer may cover
|
||||
(staged-delivery Appendix A.3, `n_units` ≤ 2²⁰) is about 94 MB, and three values a run
|
||||
(the run, its `producer` and its `count`), 3·2²⁰ and a few more in all. A reader that
|
||||
refuses one past the length limit does so from `man_len` alone, before it reads or
|
||||
requests the manifest (`manifest_too_large`). A reader that refuses one past the value
|
||||
limit does so while it counts the values, before it builds anything from the manifest
|
||||
(`manifest_too_many_values`). The reference readers refuse all three. The same limits
|
||||
apply to the `WAI2` manifest (§7.1), whose `renditions` lists no more entries than its
|
||||
table holds. They move no byte of the layout above, which stays frozen (§8). Since a
|
||||
conforming encoder emits only containers every conforming sink can read (§7), a
|
||||
conforming encoder writes no manifest past either limit; the reference writers refuse
|
||||
one past the length limit.
|
||||
conforming encoder writes no manifest past any of these limits; the reference writers
|
||||
refuse one past any of them.
|
||||
|
||||
Together the length and value limits bound what parsing a manifest takes, which the
|
||||
length limit alone does not: a parsed value takes more memory than its text, and many
|
||||
small values can take many times their length. The reference readers count a manifest's
|
||||
values before they build anything from it, then build each array and object at the size
|
||||
counted, and on a 64-bit target allocate at most 128 bytes for each value and one for
|
||||
each byte of the manifest, with, for a `WAI2`, a rendition entry of 768 bytes for each it
|
||||
lists: at most 640 MiB for a `WAI1` manifest at both limits, whatever its shape, and
|
||||
48 MiB more for a `WAI2`'s entries (informative; `SECURITY.md`).
|
||||
|
||||
A `wai.text.zstd` or `wai.text.xz` payload (§5) has reader limits of its own, since
|
||||
neither stream format bounds what a payload may ask of its decoder. A reader MUST NOT
|
||||
refuse one for the length it decodes to, or for the history its decoder must keep, while
|
||||
it decodes to at most 134 217 728 bytes (2²⁷) and every frame or block of it declares at
|
||||
most 2²⁷ bytes of history: for Zstandard the frame's window size, which for a
|
||||
single-segment frame is its content size (RFC 8878 §3.1.1.1); for XZ the LZMA2
|
||||
dictionary size. It MAY refuse one that decodes to more (`text_decoded_too_long`) or
|
||||
declares more (`text_window_too_large`). A reader that refuses for length does so when
|
||||
its output would pass the limit, in a buffer it never sizes past the limit; where a
|
||||
Zstandard frame declares a content size past the limit, it does so from the declaration,
|
||||
before it decodes anything. A reader that refuses for the history does so from the
|
||||
declaration, before its decoder allocates it. The reference readers refuse both. A
|
||||
decode then takes its output, at most the length limit, the decoder's history, at most
|
||||
the other, and the decoder's fixed state. As for a manifest, a conforming encoder writes
|
||||
no payload past either limit, and the reference writers refuse one; bytes longer than
|
||||
the length limit are carried as more than one object.
|
||||
|
||||
An envelope delivered over HTTP is one object of the proposed media type
|
||||
`application/wai`, named by its receipt content hash, and fetched whole or, for a
|
||||
|
|
|
|||
|
|
@ -771,7 +771,7 @@ def wai2_read(data):
|
|||
if ml > SV.MANIFEST_MAX or 8 + ml > len(data):
|
||||
return None
|
||||
try:
|
||||
m = SV.strict_json(data[8:8 + ml])
|
||||
m = SV.strict_json(data[8:8 + ml], SV.MANIFEST_MAX_VALUES)
|
||||
SV.check_manifest_v2(m)
|
||||
except (SV.NotJSON, SV.Schema):
|
||||
return None
|
||||
|
|
@ -1965,7 +1965,7 @@ def main(argv):
|
|||
print(("PASS" if not bm else "FAIL") + " bracket arithmetic equals the literals pinned in wai-rs " + ",".join(bm))
|
||||
fail += bool(bm)
|
||||
rl = SV.reader_limit_failures(lambda b: wai2_read(b) is not None, True)
|
||||
print(("PASS" if not rl else "FAIL") + " wai2_read refuses a manifest past the reader limit (SPEC §2) " + "; ".join(rl))
|
||||
print(("PASS" if not rl else "FAIL") + " wai2_read refuses a manifest past the reader limits (SPEC §2) " + "; ".join(rl))
|
||||
fail += bool(rl)
|
||||
rd = OER.check_canonical_form() # the OER reference's own fixtures, its reading of -0 among them
|
||||
print(("PASS" if not rd else "FAIL") + " the OER reference's canonical form and number reading hold " + ",".join(rd))
|
||||
|
|
|
|||
|
|
@ -14,8 +14,12 @@
|
|||
{"name":"head_malformed_offset","op":"parse_head","input":{"object":"OD","patch_object":[[574,"01"]]},"expect":{"error":"head_malformed"}},
|
||||
{"name":"head_malformed_manifest_too_large_O5","op":"parse_head","input":{"object":"O5","prefix":8,"patch_object":[[4,"01"],[5,"00"],[6,"00"],[7,"08"]]},"expect":{"error":"head_malformed"}},
|
||||
{"name":"parse_head_need_manifest_at_limit_O5","op":"parse_head","input":{"object":"O5","prefix":8,"patch_object":[[4,"00"],[5,"00"],[6,"00"],[7,"08"]]},"expect":{"need":134217740}},
|
||||
{"name":"parse_head_manifest_values_at_limit_O5","op":"parse_head","input":{"object":"O5","pad_values":4194294},"expect":{"version":"WAI1","head_len":8388753,"envelope_len":8388847,"covered_len":8388847,"components":[[8388753,94]]}},
|
||||
{"name":"head_malformed_manifest_too_many_values_O5","op":"parse_head","input":{"object":"O5","pad_values":4194295},"expect":{"error":"head_malformed"}},
|
||||
{"name":"head_malformed_manifest_too_large_OD","op":"parse_head","input":{"object":"OD","prefix":8,"patch_object":[[4,"01"],[5,"00"],[6,"00"],[7,"08"]]},"expect":{"error":"head_malformed"}},
|
||||
{"name":"parse_head_need_manifest_at_limit_OD","op":"parse_head","input":{"object":"OD","prefix":8,"patch_object":[[4,"00"],[5,"00"],[6,"00"],[7,"08"]]},"expect":{"need":134217738}},
|
||||
{"name":"parse_head_manifest_values_at_limit_OD","op":"parse_head","input":{"object":"OD","pad_values":4194279},"expect":{"version":"WAI2","head_len":8389160,"envelope_len":8442360,"covered_len":8389617,"components":[[8389160,3000],[8392160,50000],[8442160,200]]}},
|
||||
{"name":"head_malformed_manifest_too_many_values_OD","op":"parse_head","input":{"object":"OD","pad_values":4194280},"expect":{"error":"head_malformed"}},
|
||||
{"name":"head_proof_ok_O1","op":"head_proof","input":{"object":"O1"},"expect":{"covered_len":859,"n_cv":0,"envelope_len":859}},
|
||||
{"name":"head_proof_ok_O10","op":"head_proof","input":{"object":"O10"},"expect":{"covered_len":1009,"n_cv":1,"envelope_len":1825}},
|
||||
{"name":"head_proof_ok_O10L1","op":"head_proof","input":{"object":"O10L1"},"expect":{"covered_len":1009,"n_cv":4,"envelope_len":12711}},
|
||||
|
|
|
|||
15
wai/http-conformance/falsify/manifest-values-uncounted.patch
Normal file
15
wai/http-conformance/falsify/manifest-values-uncounted.patch
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
# must fail: head_malformed_manifest_too_many_values_O5 head_malformed_manifest_too_many_values_OD
|
||||
# a manifest past the value limit is built, not refused (container::read_tree)
|
||||
diff --git a/wai/wai-rs/src/container.rs b/wai/wai-rs/src/container.rs
|
||||
index b7b223f0..a06a74fd 100644
|
||||
--- a/wai/wai-rs/src/container.rs
|
||||
+++ b/wai/wai-rs/src/container.rs
|
||||
@@ -1706,7 +1706,7 @@ fn read_tree(text: &[u8]) -> Result<Value, serde_json::Error> {
|
||||
use serde::de::{DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor};
|
||||
use std::cell::{Cell, RefCell};
|
||||
let sizes = RefCell::new(Vec::new());
|
||||
- scan_json(text, MANIFEST_MAX_VALUES, Some(&sizes))?;
|
||||
+ scan_json(text, usize::MAX, Some(&sizes))?;
|
||||
let mut sizes = sizes.into_inner();
|
||||
sizes.shrink_to_fit();
|
||||
/// One value of the tree; `at` is the next container's index in `sizes`.
|
||||
|
|
@ -30,6 +30,7 @@ key-unchecked live_edge_other_key live_edge_other_key_walked
|
|||
walk-unbounded live_edge_reanchored live_edge_reanchored_then_reloaded live_edge_reanchored_skipping
|
||||
reload-walked live_edge_unchanged_reload live_edge_reanchored_then_reloaded
|
||||
manifest-limit-dropped head_malformed_manifest_too_large_O5 head_malformed_manifest_too_large_OD
|
||||
manifest-values-uncounted head_malformed_manifest_too_many_values_O5 head_malformed_manifest_too_many_values_OD
|
||||
xml-char-unchecked mpd_rep_id_not_a_char
|
||||
xml-attribute-twice dash_adjacency_attribute_twice
|
||||
dash-start-ignored dash_adjacency_period_start
|
||||
|
|
|
|||
|
|
@ -116,7 +116,8 @@ def read_manifest(b, v2):
|
|||
"""SPEC §2, §3, §7.1: the manifest read strictly, with the members SPEC requires and
|
||||
their types (staged_verify's reader); `head_malformed` otherwise."""
|
||||
try:
|
||||
m = SV.strict_json(b)
|
||||
# SPEC §2, *Reader limits* (--mutate manifest-values-uncounted drops the value limit).
|
||||
m = SV.strict_json(b, None if MUTATE == "manifest-values-uncounted" else SV.MANIFEST_MAX_VALUES)
|
||||
(SV.check_manifest_v2 if v2 else SV.check_manifest_v1)(m)
|
||||
except (SV.NotJSON, SV.Schema):
|
||||
raise Refusal("head_malformed")
|
||||
|
|
@ -3284,6 +3285,15 @@ def receipt_evaluate(d, objs, op, inp):
|
|||
return {"fault": str(e)}
|
||||
|
||||
|
||||
def pad_values(b, n):
|
||||
"""SPEC §2, *Reader limits*: the envelope `b` with a member no revision defines added
|
||||
at the end of its manifest, "pad", an array of `n` zeros, and its man_len set to the
|
||||
manifest's new length; the manifest holds n + 1 values more."""
|
||||
ml = u32(b, 4)
|
||||
m = b[8:8 + ml - 1] + b',"pad":[0' + b",0" * (n - 1) + b"]}"
|
||||
return b[:4] + len(m).to_bytes(4, "little") + m + b[8 + ml:]
|
||||
|
||||
|
||||
def evaluate(d, objs, op, inp):
|
||||
def obj(i, patch=True):
|
||||
if "envelope_hex" in i:
|
||||
|
|
@ -3292,6 +3302,8 @@ def evaluate(d, objs, op, inp):
|
|||
b = objs[i["object"]]
|
||||
else:
|
||||
b = open(os.path.join(d, i["file"]), "rb").read()
|
||||
if "pad_values" in i:
|
||||
b = pad_values(b, i["pad_values"])
|
||||
return apply_patch(b, i.get("patch_object")) if patch else b
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -90,6 +90,9 @@ HV.load_registry()
|
|||
MAX_INTEGER = (1 << 53) - 1
|
||||
# The longest manifest a reader must accept (SPEC §2, *Reader limits*).
|
||||
MANIFEST_MAX = 1 << 27
|
||||
# The most JSON values a manifest may hold that a reader must accept (SPEC §2, *Reader
|
||||
# limits*): the manifest and every value in it, at any depth, and no member name.
|
||||
MANIFEST_MAX_VALUES = 1 << 22
|
||||
U32 = (1 << 32) - 1
|
||||
# The frames of each piece an encode vector codes: the first ENCODE_FRAMES.
|
||||
ENCODE_FRAMES = 12
|
||||
|
|
@ -1414,6 +1417,19 @@ def read_json(b, minus_zero=False):
|
|||
return None
|
||||
|
||||
|
||||
def json_values(v):
|
||||
"""The JSON values `v` holds: itself and every value in it, at any depth."""
|
||||
n, stack = 0, [v]
|
||||
while stack:
|
||||
x = stack.pop()
|
||||
n += 1
|
||||
if isinstance(x, dict):
|
||||
stack.extend(x.values())
|
||||
elif isinstance(x, list):
|
||||
stack.extend(x)
|
||||
return n
|
||||
|
||||
|
||||
def opt_str(o, k):
|
||||
"""An OPTIONAL string member: absent, null, or a string."""
|
||||
return k not in o or o[k] is None or isinstance(o[k], str)
|
||||
|
|
@ -1476,6 +1492,8 @@ def read_envelope(b):
|
|||
if ml > MANIFEST_MAX or 8 + ml > len(b):
|
||||
return None
|
||||
man = read_json(b[8:8 + ml])
|
||||
if man is not None and json_values(man) > MANIFEST_MAX_VALUES:
|
||||
return None
|
||||
# §6.3: "does not parse" is the typed parse with SPEC §3's members.
|
||||
if not manifest_typed(man, b[:4] == b"WAI2"):
|
||||
return None
|
||||
|
|
@ -1504,9 +1522,11 @@ def read_envelope(b):
|
|||
|
||||
def check_reader_limit(rep):
|
||||
"""SPEC §2, *Reader limits*, checked in memory, since no file in the corpus is past
|
||||
128 MiB: read_envelope reads a WAI1 and a WAI2 whose manifests are 300 bytes, and
|
||||
refuses each with its manifest padded to MANIFEST_MAX + 1 bytes by a member no
|
||||
revision defines, which a reader ignores (SPEC §8), so only the limit refuses it."""
|
||||
128 MiB or holds 4 Mi values: read_envelope reads a WAI1 and a WAI2 whose manifests
|
||||
are 300 bytes, and refuses each with its manifest padded to MANIFEST_MAX + 1 bytes by
|
||||
a member no revision defines, which a reader ignores (SPEC §8), so only the limit
|
||||
refuses it; it reads each with that member an array of zeros that brings the manifest
|
||||
to MANIFEST_MAX_VALUES values, and refuses each with one zero more."""
|
||||
for magic, head, rest in (
|
||||
(b"WAI1", b'{"wai":"1.0","media":"text","intent":"replicate","model_requirement":'
|
||||
b'{"capability":"wai.text.zstd"},"conditioning":{"kind":"zstd"},"pad":"',
|
||||
|
|
@ -1520,6 +1540,15 @@ def check_reader_limit(rep):
|
|||
rep.check(read_envelope(env(MANIFEST_MAX + 1)) is None,
|
||||
f"reader limit: a {magic.decode()} manifest of {MANIFEST_MAX + 1} bytes, past the limit, is refused")
|
||||
|
||||
def valued(n):
|
||||
fixed = json_values(read_json(head[:-1] + b"[]}"))
|
||||
m = head[:-1] + b"[0" + b",0" * (n - fixed - 1) + b"]}"
|
||||
return b"".join([magic, len(m).to_bytes(4, "little"), m, rest, b"abc"])
|
||||
rep.check(read_envelope(valued(MANIFEST_MAX_VALUES)) is not None,
|
||||
f"reader limit: a {magic.decode()} manifest of {MANIFEST_MAX_VALUES} values is read")
|
||||
rep.check(read_envelope(valued(MANIFEST_MAX_VALUES + 1)) is None,
|
||||
f"reader limit: a {magic.decode()} manifest of {MANIFEST_MAX_VALUES + 1} values, past the limit, is refused")
|
||||
|
||||
|
||||
WIR1_KIND = {"wai.image.int_refine": "rgb8", "wai.video.int_refine": "rgb8", "wai.audio.int_refine": "pcm",
|
||||
"wai.splat.int_refine": "tensor"}
|
||||
|
|
|
|||
|
|
@ -70,13 +70,19 @@ DEPTH_LIMIT = 127
|
|||
# reader refuses a longer one from its length field alone (`manifest_too_large`).
|
||||
MANIFEST_MAX = 1 << 27
|
||||
|
||||
# The most JSON values a manifest may hold that a reader must accept (SPEC §2, *Reader
|
||||
# limits*), counting the manifest and every value in it, at any depth, and no member
|
||||
# name; the reference reader refuses one more (`manifest_too_many_values`).
|
||||
MANIFEST_MAX_VALUES = 1 << 22
|
||||
|
||||
|
||||
class NotJSON(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def strict_json(b):
|
||||
"""The JSON value of the bytes `b`, read strictly, or NotJSON."""
|
||||
def strict_json(b, max_values=None):
|
||||
"""The JSON value of the bytes `b`, read strictly, or NotJSON. With `max_values`, a
|
||||
text holding more values than that is refused (`manifest_too_many_values`)."""
|
||||
if b[:3] == b"\xef\xbb\xbf":
|
||||
raise NotJSON("a byte-order mark")
|
||||
try:
|
||||
|
|
@ -121,8 +127,12 @@ def strict_json(b):
|
|||
except (ValueError, RecursionError) as e:
|
||||
raise NotJSON(str(e))
|
||||
stack = [(v, 1)]
|
||||
values = 0
|
||||
while stack:
|
||||
x, depth = stack.pop()
|
||||
values += 1
|
||||
if max_values is not None and values > max_values:
|
||||
raise NotJSON("manifest_too_many_values")
|
||||
if isinstance(x, (dict, list)):
|
||||
if depth > DEPTH_LIMIT:
|
||||
raise NotJSON("nested past the depth limit")
|
||||
|
|
@ -261,7 +271,7 @@ def read_wai1(b):
|
|||
raise NotWai1("truncated manifest")
|
||||
text = b[8:8 + ml]
|
||||
try:
|
||||
m = strict_json(text)
|
||||
m = strict_json(text, MANIFEST_MAX_VALUES)
|
||||
check_manifest_v1(m)
|
||||
except (NotJSON, Schema) as e:
|
||||
raise NotWai1(str(e))
|
||||
|
|
@ -1180,7 +1190,7 @@ def read_wai2(b):
|
|||
if 8 + ml > len(b):
|
||||
raise NotWai2("truncated manifest")
|
||||
try:
|
||||
m = strict_json(b[8:8 + ml])
|
||||
m = strict_json(b[8:8 + ml], MANIFEST_MAX_VALUES)
|
||||
check_manifest_v2(m)
|
||||
except (NotJSON, Schema) as e:
|
||||
raise NotWai2(str(e))
|
||||
|
|
@ -1226,17 +1236,34 @@ def envelope_with_manifest_len(v2, n):
|
|||
b'"}', rest, b"abc"])
|
||||
|
||||
|
||||
def envelope_with_manifest_values(v2, n):
|
||||
"""The envelope of envelope_with_manifest_len(v2, 300) with its manifest holding
|
||||
exactly `n` values: its `pad` an array of zeros, the manifest's other members 10
|
||||
values, `pad` among them."""
|
||||
b = envelope_with_manifest_len(v2, 300)
|
||||
ml = struct.unpack_from("<I", b, 4)[0]
|
||||
text = b[8:8 + ml]
|
||||
m = text[:text.index(b'"pad":')] + b'"pad":[0' + b",0" * (n - 11) + b"]}"
|
||||
return b[:4] + struct.pack("<I", len(m)) + m + b[8 + ml:]
|
||||
|
||||
|
||||
def reader_limit_failures(reads, v2):
|
||||
"""SPEC §2, *Reader limits*, checked in memory, since no file in a corpus is past
|
||||
128 MiB: `reads(bytes)` (True when the reader takes the envelope) takes an envelope
|
||||
whose manifest is 300 bytes and refuses the same envelope with a manifest of
|
||||
MANIFEST_MAX + 1 bytes. What failed, as a list of lines."""
|
||||
128 MiB, or holds 4 Mi values: `reads(bytes)` (True when the reader takes the
|
||||
envelope) takes an envelope whose manifest is 300 bytes and one whose manifest holds
|
||||
MANIFEST_MAX_VALUES values, and refuses the same envelope with a manifest of
|
||||
MANIFEST_MAX + 1 bytes or MANIFEST_MAX_VALUES + 1 values. What failed, as a list of
|
||||
lines."""
|
||||
form = "WAI2" if v2 else "WAI1"
|
||||
out = []
|
||||
if not reads(envelope_with_manifest_len(v2, 300)):
|
||||
out.append(f"{form}: an envelope with a 300-byte manifest is refused")
|
||||
if reads(envelope_with_manifest_len(v2, MANIFEST_MAX + 1)):
|
||||
out.append(f"{form}: a manifest of {MANIFEST_MAX + 1} bytes, past the reader limit, is read")
|
||||
if not reads(envelope_with_manifest_values(v2, MANIFEST_MAX_VALUES)):
|
||||
out.append(f"{form}: a manifest of {MANIFEST_MAX_VALUES} values, at the value limit, is refused")
|
||||
if reads(envelope_with_manifest_values(v2, MANIFEST_MAX_VALUES + 1)):
|
||||
out.append(f"{form}: a manifest of {MANIFEST_MAX_VALUES + 1} values, past the value limit, is read")
|
||||
return out
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,30 @@
|
|||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
/**
|
||||
* The most bytes a `wai.text.zstd` or `wai.text.xz` payload may decode to that a reader
|
||||
* must accept (SPEC §2, *Reader limits*): 2^27. The decoders here write no more than
|
||||
* this, whatever the payload asks for: one that decodes to more is refused
|
||||
* (`text_decoded_too_long`) when its output would pass the limit, so the output buffer
|
||||
* never holds more. The reference writers refuse to encode more.
|
||||
*/
|
||||
#define TEXT_DECODED_MAX_LEN (1 << 27)
|
||||
|
||||
/**
|
||||
* The base-2 logarithm of [`TEXT_WINDOW_MAX_LEN`].
|
||||
*/
|
||||
#define TEXT_WINDOW_MAX_LOG 27
|
||||
|
||||
/**
|
||||
* The most history a `wai.text.zstd` or `wai.text.xz` stream may ask its decoder to
|
||||
* keep that a reader must accept (SPEC §2, *Reader limits*): 2^27 bytes, as the zstd
|
||||
* frame's window size or the LZMA2 dictionary size. A stream that declares more is
|
||||
* refused (`text_window_too_large`) from the declaration, before the decoder allocates
|
||||
* it. Together with [`TEXT_DECODED_MAX_LEN`] it bounds the memory a decode takes: the
|
||||
* length limit bounds the output, and this bounds what the decoder holds beside it.
|
||||
*/
|
||||
#define TEXT_WINDOW_MAX_LEN (1 << TEXT_WINDOW_MAX_LOG)
|
||||
|
||||
/**
|
||||
* The longest manifest, in bytes, a reader must accept (SPEC §2, *Reader limits*):
|
||||
* 2^27 (128 MiB). It is sized to the largest manifest a conforming layer has: a `prev`
|
||||
|
|
@ -37,13 +61,25 @@
|
|||
* `WIR1` layer may cover (staged-delivery Appendix A.3, `n_units ≤ 2^20`), about 90
|
||||
* bytes a run. The reference readers refuse a longer one before they read it, and the
|
||||
* reference writers refuse to write one, so `man_len` never asks a reader to read
|
||||
* more. It bounds the bytes read, not the memory a parse takes: the reference readers
|
||||
* build a JSON tree, which can take some 40 times the manifest's length. The
|
||||
* manifest's nesting has its own limit: the JSON reader refuses 128 nested arrays and
|
||||
* objects and takes 127 (`check_unique_members`).
|
||||
* more. It bounds the bytes read; [`MANIFEST_MAX_VALUES`] bounds what a parse builds
|
||||
* from them. The manifest's nesting has its own limit: the JSON reader refuses 128
|
||||
* nested arrays and objects and takes 127 ([`check_manifest_text`]).
|
||||
*/
|
||||
#define MANIFEST_MAX_LEN (1 << 27)
|
||||
|
||||
/**
|
||||
* The most JSON values a manifest may hold that a reader must accept (SPEC §2, *Reader
|
||||
* limits*): 2^22, counting the manifest object and every value nested in it, at any
|
||||
* depth; a member's name is not a value. It is sized to the same layer as
|
||||
* [`MANIFEST_MAX_LEN`]: a `prev` that names a different producer for each of 2^20 units
|
||||
* is three values a run (the run, its `producer` and its `count`), 3·2^20 and a few more
|
||||
* in all. The reference readers count a manifest's values before they parse it
|
||||
* ([`check_manifest_text`]) and refuse it as soon as the count passes the limit
|
||||
* (`manifest_too_many_values`), so the JSON tree a parse builds holds at most this many
|
||||
* values, whatever the manifest's length.
|
||||
*/
|
||||
#define MANIFEST_MAX_VALUES (1 << 22)
|
||||
|
||||
#define WAI_OK 0
|
||||
|
||||
#define WAI_ERR_INVALID_ARG 1
|
||||
|
|
@ -668,6 +704,13 @@ int wai_text_zstd_encode(const uint8_t *data,
|
|||
int32_t level,
|
||||
struct wai_buffer_t *out);
|
||||
|
||||
/**
|
||||
* Decode a `wai.text.zstd` payload within the reader limits of SPEC §2: at most
|
||||
* [`TEXT_DECODED_MAX_LEN`](crate::codecs::text::TEXT_DECODED_MAX_LEN) bytes of output
|
||||
* and [`TEXT_WINDOW_MAX_LEN`](crate::codecs::text::TEXT_WINDOW_MAX_LEN) bytes of
|
||||
* window. A payload past either is refused with `WAI_ERR_CODEC`; the message begins
|
||||
* `text_decoded_too_long` or `text_window_too_large`.
|
||||
*/
|
||||
int wai_text_zstd_decode(const uint8_t *bytes, uintptr_t len, struct wai_buffer_t *out);
|
||||
|
||||
|
||||
|
|
@ -676,6 +719,10 @@ int wai_text_xz_encode(const uint8_t *data,
|
|||
uint32_t level,
|
||||
struct wai_buffer_t *out);
|
||||
|
||||
/**
|
||||
* Decode a `wai.text.xz` payload within the same limits as
|
||||
* [`wai_text_zstd_decode`], the window limit applying to the dictionary.
|
||||
*/
|
||||
int wai_text_xz_decode(const uint8_t *bytes, uintptr_t len, struct wai_buffer_t *out);
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -99,6 +99,23 @@ pub fn main(args: &[String]) -> Result<bool, String> {
|
|||
}
|
||||
}
|
||||
|
||||
/// `pad_values` (SPEC §2, *Reader limits*): the envelope `b` with a member no revision
|
||||
/// defines added at the end of its manifest, `"pad"`, an array of `n` zeros, and its
|
||||
/// `man_len` set to the manifest's new length. The manifest then holds `n + 1` values
|
||||
/// more.
|
||||
fn pad_values(b: &[u8], n: usize) -> Result<Vec<u8>, String> {
|
||||
let ml = wai::container::manifest_bytes(b).map_err(|e| e.to_string())?.len();
|
||||
let text = &b[8..8 + ml];
|
||||
if n == 0 || text.last() != Some(&b'}') {
|
||||
return Err("pad_values needs a manifest object and at least one zero".into());
|
||||
}
|
||||
let mut m = text[..ml - 1].to_vec();
|
||||
m.extend_from_slice(b",\"pad\":[0");
|
||||
m.extend_from_slice(&b",0".repeat(n - 1));
|
||||
m.extend_from_slice(b"]}");
|
||||
Ok([&b[..4], &(m.len() as u32).to_le_bytes(), &m, &b[8 + ml..]].concat())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- the objects
|
||||
|
||||
fn read(dir: &Path, rel: &str) -> Result<Vec<u8>, String> {
|
||||
|
|
@ -339,6 +356,9 @@ impl Corpus {
|
|||
}
|
||||
fn object(&self, input: &Value) -> Result<Vec<u8>, String> {
|
||||
let mut b = self.original(input)?;
|
||||
if let Some(n) = input["pad_values"].as_u64() {
|
||||
b = pad_values(&b, n as usize)?;
|
||||
}
|
||||
patch(&mut b, &input["patch_object"])?;
|
||||
Ok(b)
|
||||
}
|
||||
|
|
@ -872,6 +892,14 @@ fn cases(objs: &BTreeMap<String, Vec<u8>>, extra: &mut BTreeMap<String, Vec<u8>>
|
|||
json!({ "object": id, "prefix": 8, "patch_object": man_len((1 << 27) + 1) }), "head_malformed"));
|
||||
v.push(case(format!("parse_head_need_manifest_at_limit_{id}"), "parse_head",
|
||||
json!({ "object": id, "prefix": 8, "patch_object": man_len(1 << 27) })));
|
||||
// The value limit: the object's manifest with a `pad` of zeros bringing it to
|
||||
// exactly 2^22 values is read; one zero more is refused.
|
||||
let text = wai::container::manifest_bytes(&objs[id]).map_err(|e| e.to_string())?;
|
||||
let held = wai::container::check_manifest_text(text).map_err(|e| e.to_string())?;
|
||||
let at = wai::container::MANIFEST_MAX_VALUES - held - 1;
|
||||
v.push(case(format!("parse_head_manifest_values_at_limit_{id}"), "parse_head", json!({ "object": id, "pad_values": at })));
|
||||
v.push(refused(format!("head_malformed_manifest_too_many_values_{id}"), "parse_head",
|
||||
json!({ "object": id, "pad_values": at + 1 }), "head_malformed"));
|
||||
}
|
||||
|
||||
// head_proof: every object, and every way a proof can be wrong.
|
||||
|
|
|
|||
|
|
@ -12,17 +12,112 @@ impl<E: std::fmt::Display> From<E> for TextError {
|
|||
|
||||
pub type Result<T> = std::result::Result<T, TextError>;
|
||||
|
||||
/// zstd at the given compression level (1..=22; 22 is "ultra", slow).
|
||||
/// zstd at the given compression level (1..=22; 22 is "ultra", slow). Refuses more
|
||||
/// than [`TEXT_DECODED_MAX_LEN`] bytes (`text_decoded_too_long`): a conforming encoder
|
||||
/// writes no payload a conforming reader may refuse (SPEC §2, *Reader limits*). No
|
||||
/// level asks for a window past [`TEXT_WINDOW_MAX_LEN`].
|
||||
pub fn zstd_encode(data: &[u8], level: i32) -> Result<Vec<u8>> {
|
||||
if data.len() > TEXT_DECODED_MAX_LEN {
|
||||
return Err(too_long(TEXT_DECODED_MAX_LEN));
|
||||
}
|
||||
Ok(zstd::stream::encode_all(data, level)?)
|
||||
}
|
||||
|
||||
pub fn zstd_decode(bytes: &[u8]) -> Result<Vec<u8>> {
|
||||
Ok(zstd::stream::decode_all(bytes)?)
|
||||
/// The most bytes a `wai.text.zstd` or `wai.text.xz` payload may decode to that a reader
|
||||
/// must accept (SPEC §2, *Reader limits*): 2^27. The decoders here write no more than
|
||||
/// this, whatever the payload asks for: one that decodes to more is refused
|
||||
/// (`text_decoded_too_long`) when its output would pass the limit, so the output buffer
|
||||
/// never holds more. The reference writers refuse to encode more.
|
||||
pub const TEXT_DECODED_MAX_LEN: usize = 1 << 27;
|
||||
|
||||
/// The base-2 logarithm of [`TEXT_WINDOW_MAX_LEN`].
|
||||
pub const TEXT_WINDOW_MAX_LOG: u32 = 27;
|
||||
|
||||
/// The most history a `wai.text.zstd` or `wai.text.xz` stream may ask its decoder to
|
||||
/// keep that a reader must accept (SPEC §2, *Reader limits*): 2^27 bytes, as the zstd
|
||||
/// frame's window size or the LZMA2 dictionary size. A stream that declares more is
|
||||
/// refused (`text_window_too_large`) from the declaration, before the decoder allocates
|
||||
/// it. Together with [`TEXT_DECODED_MAX_LEN`] it bounds the memory a decode takes: the
|
||||
/// length limit bounds the output, and this bounds what the decoder holds beside it.
|
||||
pub const TEXT_WINDOW_MAX_LEN: usize = 1 << TEXT_WINDOW_MAX_LOG;
|
||||
|
||||
/// What liblzma may allocate to decode one stream: the dictionary, at most
|
||||
/// [`TEXT_WINDOW_MAX_LEN`], and the decoder's own fixed state beside it.
|
||||
const XZ_DECODER_MEMLIMIT: u64 = TEXT_WINDOW_MAX_LEN as u64 + (1 << 20);
|
||||
|
||||
fn too_long(max_out: usize) -> TextError {
|
||||
TextError(format!(
|
||||
"text_decoded_too_long: the payload decodes to more than {max_out} bytes, the most this reader accepts (SPEC §2)"
|
||||
))
|
||||
}
|
||||
|
||||
/// XZ/LZMA2 at the given preset (0..=9; 6 is the default, 9 is "extreme").
|
||||
fn window_too_large() -> TextError {
|
||||
TextError(format!(
|
||||
"text_window_too_large: the stream asks its decoder to keep more than {TEXT_WINDOW_MAX_LEN} bytes of history, the most a reader need give it (SPEC §2)"
|
||||
))
|
||||
}
|
||||
|
||||
/// Reads `r` to its end into a buffer that never holds, and is never sized for, more
|
||||
/// than `max_out` bytes. `Ok(None)` when the reader has more than that to give.
|
||||
fn read_within<R: Read>(mut r: R, max_out: usize) -> std::io::Result<Option<Vec<u8>>> {
|
||||
let mut out: Vec<u8> = Vec::new();
|
||||
let mut chunk = vec![0u8; 64 * 1024];
|
||||
loop {
|
||||
let n = match r.read(&mut chunk) {
|
||||
Ok(0) => return Ok(Some(out)),
|
||||
Ok(n) => n,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
let room = max_out - out.len();
|
||||
if n > room {
|
||||
return Ok(None);
|
||||
}
|
||||
if out.capacity() - out.len() < n {
|
||||
// Grow as a `Vec` would, by about its length, but never past the limit.
|
||||
out.reserve_exact(out.len().max(chunk.len()).min(room).max(n));
|
||||
}
|
||||
out.extend_from_slice(&chunk[..n]);
|
||||
}
|
||||
}
|
||||
|
||||
/// Decodes a `wai.text.zstd` payload within the reader limits of SPEC §2:
|
||||
/// [`TEXT_DECODED_MAX_LEN`] bytes of output and [`TEXT_WINDOW_MAX_LEN`] bytes of window.
|
||||
pub fn zstd_decode(bytes: &[u8]) -> Result<Vec<u8>> {
|
||||
zstd_decode_within(bytes, TEXT_DECODED_MAX_LEN)
|
||||
}
|
||||
|
||||
/// [`zstd_decode`] with the output limit the caller gives: for a caller that knows what
|
||||
/// its payload may hold, and for tests. `max_out` is capped at
|
||||
/// [`TEXT_DECODED_MAX_LEN`]. The window limit is not the caller's to change.
|
||||
pub fn zstd_decode_within(bytes: &[u8], max_out: usize) -> Result<Vec<u8>> {
|
||||
let max_out = max_out.min(TEXT_DECODED_MAX_LEN);
|
||||
// A frame that declares its content size past the limit is refused from the
|
||||
// declaration, before anything is decoded. A frame need not declare it, and a
|
||||
// payload may hold more frames than the first, so the output is counted too.
|
||||
if let Ok(Some(declared)) = zstd::zstd_safe::get_frame_content_size(bytes) {
|
||||
if declared > max_out as u64 {
|
||||
return Err(too_long(max_out));
|
||||
}
|
||||
}
|
||||
let mut dec = zstd::stream::read::Decoder::with_buffer(bytes)?;
|
||||
dec.window_log_max(TEXT_WINDOW_MAX_LOG)?;
|
||||
match read_within(dec, max_out) {
|
||||
Ok(Some(out)) => Ok(out),
|
||||
Ok(None) => Err(too_long(max_out)),
|
||||
// libzstd's name for a frame whose window is past the decoder's limit.
|
||||
Err(e) if e.to_string().contains("too much memory") => Err(window_too_large()),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// XZ/LZMA2 at the given preset (0..=9; 6 is the default, 9 is "extreme"). Refuses more
|
||||
/// than [`TEXT_DECODED_MAX_LEN`] bytes (`text_decoded_too_long`), as [`zstd_encode`]
|
||||
/// does. No preset asks for a dictionary past [`TEXT_WINDOW_MAX_LEN`].
|
||||
pub fn xz_encode(data: &[u8], level: u32) -> Result<Vec<u8>> {
|
||||
if data.len() > TEXT_DECODED_MAX_LEN {
|
||||
return Err(too_long(TEXT_DECODED_MAX_LEN));
|
||||
}
|
||||
let mut out = Vec::with_capacity(data.len() / 4);
|
||||
let mut enc = xz2::write::XzEncoder::new(&mut out, level);
|
||||
enc.write_all(data)?;
|
||||
|
|
@ -30,10 +125,32 @@ pub fn xz_encode(data: &[u8], level: u32) -> Result<Vec<u8>> {
|
|||
Ok(out)
|
||||
}
|
||||
|
||||
/// Decodes a `wai.text.xz` payload within the reader limits of SPEC §2:
|
||||
/// [`TEXT_DECODED_MAX_LEN`] bytes of output and [`TEXT_WINDOW_MAX_LEN`] bytes of
|
||||
/// dictionary.
|
||||
pub fn xz_decode(bytes: &[u8]) -> Result<Vec<u8>> {
|
||||
let mut out = Vec::new();
|
||||
xz2::read::XzDecoder::new(bytes).read_to_end(&mut out)?;
|
||||
Ok(out)
|
||||
xz_decode_within(bytes, TEXT_DECODED_MAX_LEN)
|
||||
}
|
||||
|
||||
/// [`xz_decode`] with the output limit the caller gives: for a caller that knows what
|
||||
/// its payload may hold, and for tests. `max_out` is capped at
|
||||
/// [`TEXT_DECODED_MAX_LEN`]. The dictionary limit is not the caller's to change.
|
||||
pub fn xz_decode_within(bytes: &[u8], max_out: usize) -> Result<Vec<u8>> {
|
||||
let max_out = max_out.min(TEXT_DECODED_MAX_LEN);
|
||||
// The decoder is given a memory limit, so a block that declares a dictionary past
|
||||
// it is refused from its header, before the dictionary is allocated. An xz stream
|
||||
// states its decoded length only at its end, so the output is counted as it comes.
|
||||
let stream = xz2::stream::Stream::new_stream_decoder(XZ_DECODER_MEMLIMIT, 0)?;
|
||||
let dec = xz2::read::XzDecoder::new_stream(bytes, stream);
|
||||
match read_within(dec, max_out) {
|
||||
Ok(Some(out)) => Ok(out),
|
||||
Ok(None) => Err(too_long(max_out)),
|
||||
Err(e) if matches!(
|
||||
e.get_ref().and_then(|i| i.downcast_ref::<xz2::stream::Error>()),
|
||||
Some(xz2::stream::Error::MemLimit)
|
||||
) => Err(window_too_large()),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -41,12 +41,22 @@ pub const MAGIC_V2: &[u8; 4] = b"WAI2"; // v1.1 multi-rendition
|
|||
/// `WIR1` layer may cover (staged-delivery Appendix A.3, `n_units ≤ 2^20`), about 90
|
||||
/// bytes a run. The reference readers refuse a longer one before they read it, and the
|
||||
/// reference writers refuse to write one, so `man_len` never asks a reader to read
|
||||
/// more. It bounds the bytes read, not the memory a parse takes: the reference readers
|
||||
/// build a JSON tree, which can take some 40 times the manifest's length. The
|
||||
/// manifest's nesting has its own limit: the JSON reader refuses 128 nested arrays and
|
||||
/// objects and takes 127 (`check_unique_members`).
|
||||
/// more. It bounds the bytes read; [`MANIFEST_MAX_VALUES`] bounds what a parse builds
|
||||
/// from them. The manifest's nesting has its own limit: the JSON reader refuses 128
|
||||
/// nested arrays and objects and takes 127 ([`check_manifest_text`]).
|
||||
pub const MANIFEST_MAX_LEN: usize = 1 << 27;
|
||||
|
||||
/// The most JSON values a manifest may hold that a reader must accept (SPEC §2, *Reader
|
||||
/// limits*): 2^22, counting the manifest object and every value nested in it, at any
|
||||
/// depth; a member's name is not a value. It is sized to the same layer as
|
||||
/// [`MANIFEST_MAX_LEN`]: a `prev` that names a different producer for each of 2^20 units
|
||||
/// is three values a run (the run, its `producer` and its `count`), 3·2^20 and a few more
|
||||
/// in all. The reference readers count a manifest's values before they parse it
|
||||
/// ([`check_manifest_text`]) and refuse it as soon as the count passes the limit
|
||||
/// (`manifest_too_many_values`), so the JSON tree a parse builds holds at most this many
|
||||
/// values, whatever the manifest's length.
|
||||
pub const MANIFEST_MAX_VALUES: usize = 1 << 22;
|
||||
|
||||
/// SPEC §2, *Reader limits*: `Err` with the code `manifest_too_large` when a manifest
|
||||
/// of `len` bytes is longer than [`MANIFEST_MAX_LEN`].
|
||||
pub fn check_manifest_len(len: u64) -> Result<(), String> {
|
||||
|
|
@ -75,7 +85,10 @@ impl WriteLimits {
|
|||
|
||||
/// A manifest's JSON as the writers emit it (no insignificant whitespace, members in
|
||||
/// order), refused with `manifest_too_large` as soon as it would pass `cap` bytes
|
||||
/// ([`MANIFEST_MAX_LEN`] in the writers), so a writer never builds more of it than that.
|
||||
/// ([`MANIFEST_MAX_LEN`] in the writers), so a writer never builds more of it than that;
|
||||
/// then read as a reader reads it ([`check_manifest_text`]), so a writer never emits a
|
||||
/// manifest past [`MANIFEST_MAX_VALUES`] (`manifest_too_many_values`), nested past 127, or
|
||||
/// naming a member twice.
|
||||
fn manifest_json<T: Serialize>(m: &T, cap: usize) -> std::io::Result<Vec<u8>> {
|
||||
struct Capped(Vec<u8>, usize);
|
||||
impl std::io::Write for Capped {
|
||||
|
|
@ -93,6 +106,7 @@ fn manifest_json<T: Serialize>(m: &T, cap: usize) -> std::io::Result<Vec<u8>> {
|
|||
}
|
||||
let mut w = Capped(Vec::new(), cap);
|
||||
serde_json::to_writer(&mut w, m).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
check_manifest_text(&w.0).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
Ok(w.0)
|
||||
}
|
||||
|
||||
|
|
@ -1299,9 +1313,7 @@ impl Wai {
|
|||
check_manifest_len(ml as u64)
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
need(8, ml, "manifest body")?;
|
||||
check_unique_members(&bytes[8..8 + ml])
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
let m: Manifest = serde_json::from_slice(&bytes[8..8 + ml])
|
||||
let m = read_manifest(&bytes[8..8 + ml])
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
let p_off = 8 + ml;
|
||||
need(p_off, 4, "payload length")?;
|
||||
|
|
@ -1531,18 +1543,72 @@ impl Wai {
|
|||
/// (SPEC §2: member names are unique within an object). `serde_json` keeps the last of
|
||||
/// two equal names in a `Value` and refuses them only in a struct's own fields, so a
|
||||
/// manifest is read once more by this check, which refuses any repeat. Names compare
|
||||
/// after escapes are read, so `"a"` and `"\u0061"` are one name. The text's own JSON
|
||||
/// errors are reported as they are.
|
||||
/// after escapes are read, so `"a"` and `"a"` are one name. The text's own JSON
|
||||
/// errors are reported as they are. A manifest is read by [`check_manifest_text`],
|
||||
/// which makes this check and counts the manifest's values; this one counts nothing.
|
||||
pub fn check_unique_members(text: &[u8]) -> Result<(), serde_json::Error> {
|
||||
scan_json(text, usize::MAX, None).map(|_| ())
|
||||
}
|
||||
|
||||
/// The checks a reader makes on a manifest's text before it builds anything from it
|
||||
/// (SPEC §2, *Reader limits*): it is JSON, it names no member twice in one object
|
||||
/// ([`check_unique_members`]), it nests at most 127 deep, and it holds at most
|
||||
/// [`MANIFEST_MAX_VALUES`] values, refused with `manifest_too_many_values` as soon as the
|
||||
/// count passes the limit, so a manifest past it is read only that far. The pass builds
|
||||
/// no value: it holds the member names of each object open on the path to the value it
|
||||
/// reads, borrowed from `text` unless a name carries an escape. Returns the number of
|
||||
/// values.
|
||||
pub fn check_manifest_text(text: &[u8]) -> Result<usize, serde_json::Error> {
|
||||
scan_json(text, MANIFEST_MAX_VALUES, None)
|
||||
}
|
||||
|
||||
/// One pass over the JSON text `text`: refuses a member named twice in one object, and
|
||||
/// more than `max_values` values; returns how many values it holds. With `sizes`, pushes
|
||||
/// each array's and object's number of elements or members, the containers in the order
|
||||
/// they open.
|
||||
fn scan_json(text: &[u8], max_values: usize, sizes: Option<&std::cell::RefCell<Vec<u32>>>) -> Result<usize, serde_json::Error> {
|
||||
use serde::de::{DeserializeSeed, Deserializer, Error as _, MapAccess, SeqAccess, Visitor};
|
||||
struct Unique;
|
||||
impl<'de> DeserializeSeed<'de> for Unique {
|
||||
type Value = ();
|
||||
fn deserialize<D: Deserializer<'de>>(self, d: D) -> Result<(), D::Error> {
|
||||
d.deserialize_any(Unique)
|
||||
use std::borrow::Cow;
|
||||
use std::cell::{Cell, RefCell};
|
||||
/// One value, counted before it is read.
|
||||
#[derive(Clone, Copy)]
|
||||
struct Scan<'c> {
|
||||
values: &'c Cell<usize>,
|
||||
max: usize,
|
||||
sizes: Option<&'c RefCell<Vec<u32>>>,
|
||||
}
|
||||
impl Scan<'_> {
|
||||
/// A slot for the size of the container that opens now.
|
||||
fn open(self) -> Option<usize> {
|
||||
self.sizes.map(|s| {
|
||||
let mut s = s.borrow_mut();
|
||||
s.push(0);
|
||||
s.len() - 1
|
||||
})
|
||||
}
|
||||
/// The container in `slot` closed with `n` elements or members (at most
|
||||
/// `max_values`, so within `u32` for a manifest).
|
||||
fn close(self, slot: Option<usize>, n: usize) {
|
||||
if let (Some(s), Some(i)) = (self.sizes, slot) {
|
||||
s.borrow_mut()[i] = u32::try_from(n).unwrap_or(u32::MAX);
|
||||
}
|
||||
}
|
||||
}
|
||||
impl<'de> Visitor<'de> for Unique {
|
||||
impl<'de> DeserializeSeed<'de> for Scan<'_> {
|
||||
type Value = ();
|
||||
fn deserialize<D: Deserializer<'de>>(self, d: D) -> Result<(), D::Error> {
|
||||
let n = self.values.get() + 1;
|
||||
if n > self.max {
|
||||
return Err(D::Error::custom(format_args!(
|
||||
"manifest_too_many_values: the manifest holds more than {} values, the most a reader need accept (SPEC §2)",
|
||||
self.max
|
||||
)));
|
||||
}
|
||||
self.values.set(n);
|
||||
d.deserialize_any(self)
|
||||
}
|
||||
}
|
||||
impl<'de> Visitor<'de> for Scan<'_> {
|
||||
type Value = ();
|
||||
fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("a JSON value")
|
||||
|
|
@ -1566,23 +1632,291 @@ pub fn check_unique_members(text: &[u8]) -> Result<(), serde_json::Error> {
|
|||
Ok(())
|
||||
}
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<(), A::Error> {
|
||||
while seq.next_element_seed(Unique)?.is_some() {}
|
||||
let slot = self.open();
|
||||
let mut n = 0;
|
||||
while seq.next_element_seed(self)?.is_some() {
|
||||
n += 1;
|
||||
}
|
||||
self.close(slot, n);
|
||||
Ok(())
|
||||
}
|
||||
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<(), A::Error> {
|
||||
let slot = self.open();
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
while let Some(k) = map.next_key::<String>()? {
|
||||
while let Some(k) = map.next_key_seed(Name)? {
|
||||
if !seen.insert(k) {
|
||||
return Err(A::Error::custom("an object names a member twice"));
|
||||
}
|
||||
map.next_value_seed(Unique)?;
|
||||
map.next_value_seed(self)?;
|
||||
}
|
||||
self.close(slot, seen.len());
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
/// A member's name, borrowed from the text when it carries no escape.
|
||||
struct Name;
|
||||
impl<'de> DeserializeSeed<'de> for Name {
|
||||
type Value = Cow<'de, str>;
|
||||
fn deserialize<D: Deserializer<'de>>(self, d: D) -> Result<Cow<'de, str>, D::Error> {
|
||||
d.deserialize_str(Name)
|
||||
}
|
||||
}
|
||||
impl<'de> Visitor<'de> for Name {
|
||||
type Value = Cow<'de, str>;
|
||||
fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("a member name")
|
||||
}
|
||||
fn visit_borrowed_str<E>(self, s: &'de str) -> Result<Cow<'de, str>, E> {
|
||||
Ok(Cow::Borrowed(s))
|
||||
}
|
||||
fn visit_str<E>(self, s: &str) -> Result<Cow<'de, str>, E> {
|
||||
Ok(Cow::Owned(s.to_owned()))
|
||||
}
|
||||
}
|
||||
let values = Cell::new(0);
|
||||
let mut de = serde_json::Deserializer::from_slice(text);
|
||||
Unique.deserialize(&mut de)?;
|
||||
de.end()
|
||||
Scan { values: &values, max: max_values, sizes }.deserialize(&mut de)?;
|
||||
de.end()?;
|
||||
Ok(values.get())
|
||||
}
|
||||
|
||||
/// A `WAI1` manifest read from its text as the reference readers read it (SPEC §2): the
|
||||
/// checks of [`check_manifest_text`], then the [`Manifest`] that
|
||||
/// `serde_json::from_slice::<Manifest>` builds from the same text, built with less
|
||||
/// memory. The text is read once more into one JSON tree, each array and object
|
||||
/// allocated at the size the first pass counted, and the manifest's members are moved
|
||||
/// out of that tree into their fields; nothing is held twice, where serde's derived
|
||||
/// reader holds every member a type does not define as an intermediate copy while it
|
||||
/// builds the tree it keeps. So what a parse allocates is bounded by the value limit
|
||||
/// and the text's length (SPEC §2, *Reader limits*; `tests/manifest_parse_memory.rs`).
|
||||
pub fn read_manifest(text: &[u8]) -> Result<Manifest, serde_json::Error> {
|
||||
manifest_from_tree(read_tree(text)?)
|
||||
}
|
||||
|
||||
/// A `WAI2` manifest (SPEC §7.1) read as [`read_manifest`] reads a `WAI1`'s. A
|
||||
/// `renditions` longer than the 65 535 rows a `WAI2` table holds is refused before its
|
||||
/// entries are built.
|
||||
pub fn read_manifest_v2(text: &[u8]) -> Result<ManifestV2, serde_json::Error> {
|
||||
manifest_v2_from_tree(read_tree(text)?)
|
||||
}
|
||||
|
||||
/// The JSON tree of the manifest text `text`, after [`check_manifest_text`]'s checks,
|
||||
/// each array and object allocated at its size.
|
||||
fn read_tree(text: &[u8]) -> Result<Value, serde_json::Error> {
|
||||
use serde::de::{DeserializeSeed, Deserializer, MapAccess, SeqAccess, Visitor};
|
||||
use std::cell::{Cell, RefCell};
|
||||
let sizes = RefCell::new(Vec::new());
|
||||
scan_json(text, MANIFEST_MAX_VALUES, Some(&sizes))?;
|
||||
let mut sizes = sizes.into_inner();
|
||||
sizes.shrink_to_fit();
|
||||
/// One value of the tree; `at` is the next container's index in `sizes`.
|
||||
#[derive(Clone, Copy)]
|
||||
struct Tree<'s> {
|
||||
sizes: &'s [u32],
|
||||
at: &'s Cell<usize>,
|
||||
}
|
||||
impl Tree<'_> {
|
||||
fn next_size(self) -> usize {
|
||||
let i = self.at.get();
|
||||
self.at.set(i + 1);
|
||||
self.sizes.get(i).map_or(0, |&n| n as usize)
|
||||
}
|
||||
}
|
||||
impl<'de> DeserializeSeed<'de> for Tree<'_> {
|
||||
type Value = Value;
|
||||
fn deserialize<D: Deserializer<'de>>(self, d: D) -> Result<Value, D::Error> {
|
||||
d.deserialize_any(self)
|
||||
}
|
||||
}
|
||||
// As `serde_json`'s own `Value` visitor builds each value.
|
||||
impl<'de> Visitor<'de> for Tree<'_> {
|
||||
type Value = Value;
|
||||
fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("a JSON value")
|
||||
}
|
||||
fn visit_bool<E>(self, b: bool) -> Result<Value, E> {
|
||||
Ok(Value::Bool(b))
|
||||
}
|
||||
fn visit_i64<E>(self, n: i64) -> Result<Value, E> {
|
||||
Ok(Value::Number(n.into()))
|
||||
}
|
||||
fn visit_u64<E>(self, n: u64) -> Result<Value, E> {
|
||||
Ok(Value::Number(n.into()))
|
||||
}
|
||||
fn visit_f64<E>(self, x: f64) -> Result<Value, E> {
|
||||
Ok(serde_json::Number::from_f64(x).map_or(Value::Null, Value::Number))
|
||||
}
|
||||
fn visit_str<E>(self, s: &str) -> Result<Value, E> {
|
||||
Ok(Value::String(s.to_owned()))
|
||||
}
|
||||
fn visit_string<E>(self, s: String) -> Result<Value, E> {
|
||||
Ok(Value::String(s))
|
||||
}
|
||||
fn visit_unit<E>(self) -> Result<Value, E> {
|
||||
Ok(Value::Null)
|
||||
}
|
||||
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Value, A::Error> {
|
||||
let mut v = Vec::with_capacity(self.next_size());
|
||||
while let Some(x) = seq.next_element_seed(self)? {
|
||||
v.push(x);
|
||||
}
|
||||
Ok(Value::Array(v))
|
||||
}
|
||||
fn visit_map<A: MapAccess<'de>>(self, mut map: A) -> Result<Value, A::Error> {
|
||||
let mut m = Map::with_capacity(self.next_size());
|
||||
while let Some(k) = map.next_key::<String>()? {
|
||||
let v = map.next_value_seed(self)?;
|
||||
m.insert(k, v);
|
||||
}
|
||||
Ok(Value::Object(m))
|
||||
}
|
||||
}
|
||||
let at = Cell::new(0);
|
||||
let mut de = serde_json::Deserializer::from_slice(text);
|
||||
let v = Tree { sizes: &sizes, at: &at }.deserialize(&mut de)?;
|
||||
de.end()?;
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
// The manifest types built from a tree by moving its members, each member read as
|
||||
// serde's derived reader reads it: a field `T` from its value as `T` reads it, a missing
|
||||
// field refused unless the type gives it a default (an `Option` is `None`, `target` is
|
||||
// `null`), an `Option` whose value is `null` is `None`, and the members a type does not
|
||||
// define left in order in `extra`. `tests/manifest_reader_equivalence.rs` holds the two
|
||||
// readers to the same result.
|
||||
type Members = Map<String, Value>;
|
||||
|
||||
fn object_of(v: Value, what: &str) -> Result<Members, serde_json::Error> {
|
||||
match v {
|
||||
Value::Object(m) => Ok(m),
|
||||
_ => Err(serde::de::Error::custom(format_args!("invalid type: {what} is not an object"))),
|
||||
}
|
||||
}
|
||||
|
||||
fn required<T: serde::de::DeserializeOwned>(m: &mut Members, k: &'static str) -> Result<T, serde_json::Error> {
|
||||
match m.shift_remove(k) {
|
||||
Some(v) => serde_json::from_value(v),
|
||||
None => Err(serde::de::Error::missing_field(k)),
|
||||
}
|
||||
}
|
||||
|
||||
fn optional<T: serde::de::DeserializeOwned>(m: &mut Members, k: &str) -> Result<Option<T>, serde_json::Error> {
|
||||
match m.shift_remove(k) {
|
||||
Some(v) => serde_json::from_value(v),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// An `Option<Value>` member, moved: `null` or absent is `None`.
|
||||
fn optional_tree(m: &mut Members, k: &str) -> Option<Value> {
|
||||
m.shift_remove(k).filter(|v| !v.is_null())
|
||||
}
|
||||
|
||||
/// An `Option<T>` member whose `T` is one of the manifest types.
|
||||
fn optional_of<T>(m: &mut Members, k: &str, build: fn(Members) -> Result<T, serde_json::Error>) -> Result<Option<T>, serde_json::Error> {
|
||||
match m.shift_remove(k) {
|
||||
None | Some(Value::Null) => Ok(None),
|
||||
Some(v) => build(object_of(v, k)?).map(Some),
|
||||
}
|
||||
}
|
||||
|
||||
fn required_of<T>(m: &mut Members, k: &'static str, build: fn(Members) -> Result<T, serde_json::Error>) -> Result<T, serde_json::Error> {
|
||||
match m.shift_remove(k) {
|
||||
None => Err(serde::de::Error::missing_field(k)),
|
||||
Some(v) => build(object_of(v, k)?),
|
||||
}
|
||||
}
|
||||
|
||||
fn model_requirement_from(mut m: Members) -> Result<ModelRequirement, serde_json::Error> {
|
||||
Ok(ModelRequirement {
|
||||
capability: required(&mut m, "capability")?,
|
||||
fallback: optional(&mut m, "fallback")?,
|
||||
min_sim: optional(&mut m, "min_sim")?,
|
||||
sim_metric: optional(&mut m, "sim_metric")?,
|
||||
extra: m,
|
||||
})
|
||||
}
|
||||
|
||||
fn supersedes_from(mut m: Members) -> Result<Supersedes, serde_json::Error> {
|
||||
Ok(Supersedes { sha256: required(&mut m, "sha256")?, form: optional(&mut m, "form")?, extra: m })
|
||||
}
|
||||
|
||||
fn prior_from(mut m: Members) -> Result<Prior, serde_json::Error> {
|
||||
Ok(Prior {
|
||||
id: required(&mut m, "id")?,
|
||||
sha256: required(&mut m, "sha256")?,
|
||||
determinism: required(&mut m, "determinism")?,
|
||||
form: optional(&mut m, "form")?,
|
||||
tier: optional(&mut m, "tier")?,
|
||||
supersedes: optional_of(&mut m, "supersedes", supersedes_from)?,
|
||||
extra: m,
|
||||
})
|
||||
}
|
||||
|
||||
fn conditioning_from(mut m: Members) -> Result<Conditioning, serde_json::Error> {
|
||||
Ok(Conditioning { kind: required(&mut m, "kind")?, prior: optional_of(&mut m, "prior", prior_from)?, extra: m })
|
||||
}
|
||||
|
||||
fn security_from(mut m: Members) -> Result<Security, serde_json::Error> {
|
||||
Ok(Security { encryption: optional(&mut m, "encryption")?, kid: optional(&mut m, "kid")?, extra: m })
|
||||
}
|
||||
|
||||
fn manifest_from_tree(v: Value) -> Result<Manifest, serde_json::Error> {
|
||||
let mut m = object_of(v, "the manifest")?;
|
||||
Ok(Manifest {
|
||||
wai: required(&mut m, "wai")?,
|
||||
media: required(&mut m, "media")?,
|
||||
intent: required(&mut m, "intent")?,
|
||||
model_requirement: required_of(&mut m, "model_requirement", model_requirement_from)?,
|
||||
conditioning: required_of(&mut m, "conditioning", conditioning_from)?,
|
||||
security: optional_of(&mut m, "security", security_from)?,
|
||||
refines: optional_tree(&mut m, "refines"),
|
||||
derived_from: optional_tree(&mut m, "derived_from"),
|
||||
target: m.shift_remove("target").unwrap_or(Value::Null),
|
||||
extra: m,
|
||||
})
|
||||
}
|
||||
|
||||
fn rendition_from(mut m: Members) -> Result<RenditionMeta, serde_json::Error> {
|
||||
Ok(RenditionMeta {
|
||||
capability: required(&mut m, "capability")?,
|
||||
kind: required(&mut m, "kind")?,
|
||||
id: optional(&mut m, "id")?,
|
||||
sha256: optional(&mut m, "sha256")?,
|
||||
prior: optional_of(&mut m, "prior", prior_from)?,
|
||||
role: optional(&mut m, "role")?,
|
||||
companion_of: optional(&mut m, "companion_of")?,
|
||||
companion_of_id: optional(&mut m, "companion_of_id")?,
|
||||
refines: optional_tree(&mut m, "refines"),
|
||||
envelope_manifest: optional_tree(&mut m, "envelope_manifest"),
|
||||
derived_from: optional_tree(&mut m, "derived_from"),
|
||||
extra: m,
|
||||
})
|
||||
}
|
||||
|
||||
fn manifest_v2_from_tree(v: Value) -> Result<ManifestV2, serde_json::Error> {
|
||||
let mut m = object_of(v, "the manifest")?;
|
||||
let wai = required(&mut m, "wai")?;
|
||||
let media = required(&mut m, "media")?;
|
||||
let intent = required(&mut m, "intent")?;
|
||||
let renditions = match m.shift_remove("renditions") {
|
||||
None => return Err(serde::de::Error::missing_field("renditions")),
|
||||
Some(Value::Array(rs)) if rs.len() > u16::MAX as usize => {
|
||||
return Err(serde::de::Error::custom(format_args!(
|
||||
"the manifest lists {} renditions, more than the 65535 rows a WAI2 table holds (SPEC §7.1)",
|
||||
rs.len()
|
||||
)));
|
||||
}
|
||||
Some(Value::Array(rs)) => {
|
||||
let mut out = Vec::with_capacity(rs.len());
|
||||
for r in rs {
|
||||
out.push(rendition_from(object_of(r, "a rendition")?)?);
|
||||
}
|
||||
out
|
||||
}
|
||||
Some(_) => return Err(serde::de::Error::custom("invalid type: renditions is not an array")),
|
||||
};
|
||||
Ok(ManifestV2 { wai, media, intent, renditions, target: m.shift_remove("target").unwrap_or(Value::Null), extra: m })
|
||||
}
|
||||
|
||||
/// The manifest bytes of a `WAI1` or `WAI2` envelope, exactly as they were
|
||||
|
|
@ -2611,9 +2945,7 @@ impl WaiMulti {
|
|||
check_manifest_len(ml as u64)
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
need(8, ml, "manifest body")?;
|
||||
check_unique_members(&bytes[8..8 + ml])
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
let manifest: ManifestV2 = serde_json::from_slice(&bytes[8..8 + ml])
|
||||
let manifest = read_manifest_v2(&bytes[8..8 + ml])
|
||||
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
|
||||
|
||||
let mut o = 8 + ml;
|
||||
|
|
@ -2965,7 +3297,7 @@ pub fn detect_version(bytes: &[u8]) -> std::io::Result<u8> {
|
|||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A pin verifier that accepts every pin: for tests about selection order.
|
||||
|
|
@ -4500,6 +4832,50 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
/// A `WAI1` (or, with `v2`, a `WAI2`) envelope whose manifest holds exactly `values`
|
||||
/// values: the small manifest of [`envelope_with_manifest_len`], whose `pad` is an
|
||||
/// array of zeros. Its other members are 10 values in either form, `pad` among them.
|
||||
pub(crate) fn envelope_with_manifest_values(v2: bool, values: usize) -> Vec<u8> {
|
||||
let base = envelope_with_manifest_len(v2, 300);
|
||||
let ml = u32::from_le_bytes(base[4..8].try_into().unwrap()) as usize;
|
||||
let text = std::str::from_utf8(&base[8..8 + ml]).unwrap();
|
||||
let pad = text.find("\"pad\":").unwrap();
|
||||
// A reader that read one value more or less than this would fail the tests that
|
||||
// use it at the limit and one past it.
|
||||
let m = format!("{}\"pad\":[0{}]}}", &text[..pad], ",0".repeat(values - 11));
|
||||
[&base[..4], &(m.len() as u32).to_le_bytes(), m.as_bytes(), &base[8 + ml..]].concat()
|
||||
}
|
||||
|
||||
/// SPEC §2, *Reader limits*: a manifest of [`MANIFEST_MAX_VALUES`] values is read and
|
||||
/// one value more is refused (`manifest_too_many_values`), by both readers, in `WAI1`
|
||||
/// and `WAI2` manifests alike; the writers refuse to write one past the limit. The
|
||||
/// count takes every value at any depth, the manifest object among them, and no
|
||||
/// member name.
|
||||
#[test]
|
||||
fn a_manifest_past_the_value_limit_is_refused_and_never_written() {
|
||||
assert_eq!(MANIFEST_MAX_VALUES, 4_194_304);
|
||||
assert_eq!(check_manifest_text(br#"{"a":[1,{"b":null}],"c":"d"}"#).unwrap(), 6);
|
||||
assert_eq!(check_manifest_text(br#"{}"#).unwrap(), 1);
|
||||
let too_many = |e: String| assert!(e.starts_with("manifest_too_many_values:"), "{e}");
|
||||
for v2 in [false, true] {
|
||||
let read = |b: &[u8]| if v2 { WaiMulti::from_bytes(b).map(|_| ()) } else { Wai::from_bytes(b).map(|_| ()) };
|
||||
let at = envelope_with_manifest_values(v2, MANIFEST_MAX_VALUES);
|
||||
read(&at).unwrap_or_else(|e| panic!("v2={v2}: the limit is read: {e}"));
|
||||
too_many(read(&envelope_with_manifest_values(v2, MANIFEST_MAX_VALUES + 1)).unwrap_err().to_string());
|
||||
}
|
||||
// The writers: the envelope at the limit is written back; one zero more is not.
|
||||
let at = envelope_with_manifest_values(false, MANIFEST_MAX_VALUES);
|
||||
let mut w = Wai::from_bytes(&at).unwrap();
|
||||
assert_eq!(w.to_bytes().unwrap(), at);
|
||||
w.manifest.extra["pad"].as_array_mut().unwrap().push(Value::from(0));
|
||||
too_many(w.to_bytes().unwrap_err().to_string());
|
||||
let at = envelope_with_manifest_values(true, MANIFEST_MAX_VALUES);
|
||||
let mut w2 = WaiMulti::from_bytes(&at).unwrap();
|
||||
assert_eq!(w2.to_bytes().unwrap(), at);
|
||||
w2.manifest.extra["pad"].as_array_mut().unwrap().push(Value::Null);
|
||||
too_many(w2.to_bytes().unwrap_err().to_string());
|
||||
}
|
||||
|
||||
/// A length a container writes into a u32 field must fit it: the writers refuse a
|
||||
/// payload past 2^32 − 1 bytes instead of writing its length modulo 2^32.
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -269,6 +269,11 @@ pub unsafe extern "C" fn wai_text_zstd_encode(
|
|||
})
|
||||
}
|
||||
|
||||
/// Decode a `wai.text.zstd` payload within the reader limits of SPEC §2: at most
|
||||
/// [`TEXT_DECODED_MAX_LEN`](crate::codecs::text::TEXT_DECODED_MAX_LEN) bytes of output
|
||||
/// and [`TEXT_WINDOW_MAX_LEN`](crate::codecs::text::TEXT_WINDOW_MAX_LEN) bytes of
|
||||
/// window. A payload past either is refused with `WAI_ERR_CODEC`; the message begins
|
||||
/// `text_decoded_too_long` or `text_window_too_large`.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn wai_text_zstd_decode(
|
||||
bytes: *const u8, len: usize, out: *mut WaiBuffer,
|
||||
|
|
@ -293,6 +298,8 @@ pub unsafe extern "C" fn wai_text_xz_encode(
|
|||
})
|
||||
}
|
||||
|
||||
/// Decode a `wai.text.xz` payload within the same limits as
|
||||
/// [`wai_text_zstd_decode`], the window limit applying to the dictionary.
|
||||
#[unsafe(no_mangle)]
|
||||
pub unsafe extern "C" fn wai_text_xz_decode(
|
||||
bytes: *const u8, len: usize, out: *mut WaiBuffer,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
//! component's byte range and, through each entry's `sha256`, its digest.
|
||||
|
||||
use super::{RangeRefusal, DOMAIN_LEN};
|
||||
use crate::container::{check_unique_members, Manifest, ManifestV2};
|
||||
use crate::container::{read_manifest, read_manifest_v2, Manifest, ManifestV2};
|
||||
|
||||
/// Which container an object is.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
|
|
@ -136,9 +136,7 @@ impl Head {
|
|||
if self.version != Version::Wai1 {
|
||||
return Err(RangeRefusal::HeadMalformed("not a WAI1 head".into()));
|
||||
}
|
||||
let b = self.manifest_bytes();
|
||||
check_unique_members(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))?;
|
||||
serde_json::from_slice(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))
|
||||
read_manifest(self.manifest_bytes()).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))
|
||||
}
|
||||
|
||||
/// The `WAI2` manifest.
|
||||
|
|
@ -146,9 +144,7 @@ impl Head {
|
|||
if self.version != Version::Wai2 {
|
||||
return Err(RangeRefusal::HeadMalformed("not a WAI2 head".into()));
|
||||
}
|
||||
let b = self.manifest_bytes();
|
||||
check_unique_members(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))?;
|
||||
serde_json::from_slice(b).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))
|
||||
read_manifest_v2(self.manifest_bytes()).map_err(|e| RangeRefusal::HeadMalformed(e.to_string()))
|
||||
}
|
||||
|
||||
/// The component at `index`.
|
||||
|
|
|
|||
|
|
@ -119,6 +119,22 @@ fn head_parse_refuses_a_manifest_past_the_reader_limit() {
|
|||
}
|
||||
}
|
||||
|
||||
/// SPEC §2, *Reader limits*: a head whose manifest holds more than
|
||||
/// `MANIFEST_MAX_VALUES` values is refused (`head_malformed`, as
|
||||
/// `manifest_too_many_values`); one at the limit is read.
|
||||
#[test]
|
||||
fn head_parse_refuses_a_manifest_past_the_value_limit() {
|
||||
let max = crate::container::MANIFEST_MAX_VALUES;
|
||||
for v2 in [false, true] {
|
||||
let at = crate::container::tests::envelope_with_manifest_values(v2, max);
|
||||
assert!(matches!(parse_head(&at, at.len() as u64), Ok(HeadParse::Head(_))), "v2={v2}");
|
||||
let over = crate::container::tests::envelope_with_manifest_values(v2, max + 1);
|
||||
let e = parse_head(&over, over.len() as u64).unwrap_err();
|
||||
assert_eq!(e.code(), "head_malformed");
|
||||
assert!(matches!(&e, RangeRefusal::HeadMalformed(m) if m.starts_with("manifest_too_many_values:")), "{e:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_length_rule() {
|
||||
let d = appendix_d();
|
||||
|
|
|
|||
284
wai/wai-rs/tests/manifest_parse_memory.rs
Normal file
284
wai/wai-rs/tests/manifest_parse_memory.rs
Normal file
|
|
@ -0,0 +1,284 @@
|
|||
//! What reading a manifest allocates (SPEC §2, *Reader limits*; SECURITY.md, *Manifest
|
||||
//! parsing*). A counting global allocator records every byte the reader asks for, and
|
||||
//! each case requires the peak to stay within the bound the reference states: 128 bytes
|
||||
//! for each value the manifest holds, one for each byte of its text, and, for a `WAI2`,
|
||||
//! one [`RenditionMeta`] for each entry it lists, with 64 KiB for fixed costs. With the
|
||||
//! value limit (2^22) and the length limit (2^27) that is at most 640 MiB for a `WAI1`
|
||||
//! manifest, whatever its shape.
|
||||
//!
|
||||
//! The cases: a manifest at the length limit made of small values, refused for its
|
||||
//! values before anything is built from it (the reader that parsed such a manifest into a
|
||||
//! JSON tree took some 5 GB); one at the length limit within the value limit, which is
|
||||
//! read; manifests of small values in each shape at the value limit, read, and one value
|
||||
//! past it, refused; the largest manifest a conforming staged layer has, a `prev` with a
|
||||
//! different producer for each of 2^20 units; and a `WAI2` listing as many renditions as
|
||||
//! its table can hold, read, and one more, refused.
|
||||
//!
|
||||
//! The allocator counts what is asked of it, so the figures are the same on every 64-bit
|
||||
//! target. A reallocation is counted as the new block allocated before the old one is
|
||||
//! freed, which is the most it can hold at once. One test runs at a time (`ONE`), so the
|
||||
//! count is this reader's alone.
|
||||
|
||||
use std::alloc::{GlobalAlloc, Layout, System};
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
|
||||
use std::time::Instant;
|
||||
|
||||
use wai::container::{
|
||||
check_manifest_text, RenditionMeta, Wai, WaiMulti, MANIFEST_MAX_LEN, MANIFEST_MAX_VALUES,
|
||||
};
|
||||
|
||||
struct Counting;
|
||||
static NOW: AtomicUsize = AtomicUsize::new(0);
|
||||
static PEAK: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
fn grow(n: usize) {
|
||||
let now = NOW.fetch_add(n, Relaxed) + n;
|
||||
PEAK.fetch_max(now, Relaxed);
|
||||
}
|
||||
|
||||
unsafe impl GlobalAlloc for Counting {
|
||||
unsafe fn alloc(&self, l: Layout) -> *mut u8 {
|
||||
let p = unsafe { System.alloc(l) };
|
||||
if !p.is_null() {
|
||||
grow(l.size());
|
||||
}
|
||||
p
|
||||
}
|
||||
unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 {
|
||||
let p = unsafe { System.alloc_zeroed(l) };
|
||||
if !p.is_null() {
|
||||
grow(l.size());
|
||||
}
|
||||
p
|
||||
}
|
||||
unsafe fn dealloc(&self, p: *mut u8, l: Layout) {
|
||||
unsafe { System.dealloc(p, l) };
|
||||
NOW.fetch_sub(l.size(), Relaxed);
|
||||
}
|
||||
unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 {
|
||||
let q = unsafe { System.realloc(p, l, new) };
|
||||
if !q.is_null() {
|
||||
grow(new);
|
||||
NOW.fetch_sub(l.size(), Relaxed);
|
||||
}
|
||||
q
|
||||
}
|
||||
}
|
||||
|
||||
#[global_allocator]
|
||||
static COUNTING: Counting = Counting;
|
||||
|
||||
static ONE: Mutex<()> = Mutex::new(());
|
||||
|
||||
/// The most the reference may allocate reading a manifest of `len` bytes holding
|
||||
/// `values` values and listing `renditions` `WAI2` entries (SPEC §2; SECURITY.md).
|
||||
fn bound(len: usize, values: usize, renditions: usize) -> usize {
|
||||
128 * values + len + renditions * std::mem::size_of::<RenditionMeta>() + (1 << 16)
|
||||
}
|
||||
|
||||
/// What `f` allocates at its peak beyond what was held when it started, with its result
|
||||
/// (which the peak includes) and the seconds it took.
|
||||
fn peak_of<T>(f: impl FnOnce() -> T) -> (T, usize, f64) {
|
||||
let base = NOW.load(Relaxed);
|
||||
PEAK.store(base, Relaxed);
|
||||
let t = Instant::now();
|
||||
let r = f();
|
||||
(r, PEAK.load(Relaxed) - base, t.elapsed().as_secs_f64())
|
||||
}
|
||||
|
||||
fn envelope(magic: &[u8; 4], m: &str, rows: usize) -> Vec<u8> {
|
||||
let mut b = magic.to_vec();
|
||||
b.extend_from_slice(&(m.len() as u32).to_le_bytes());
|
||||
b.extend_from_slice(m.as_bytes());
|
||||
if magic == b"WAI2" {
|
||||
b.extend_from_slice(&(rows as u16).to_le_bytes());
|
||||
for _ in 0..rows {
|
||||
b.extend_from_slice(&[0; 8]);
|
||||
}
|
||||
} else {
|
||||
b.extend_from_slice(&0u32.to_le_bytes());
|
||||
}
|
||||
b
|
||||
}
|
||||
|
||||
/// A `WAI1` manifest with a member no revision defines, `pad` (a reader ignores it, SPEC
|
||||
/// §8), whose value is `open`, `n` elements written by `elem`, and `close`: with `pad`,
|
||||
/// 10 values and the elements'.
|
||||
const HEAD: &str = r#"{"wai":"1.0","media":"text","intent":"replicate","model_requirement":{"capability":"wai.text.zstd"},"conditioning":{"kind":"zstd"},"target":null,"pad":"#;
|
||||
const FIXED: usize = 10;
|
||||
|
||||
fn padded(n: usize, open: &str, close: &str, mut elem: impl FnMut(usize, &mut String)) -> String {
|
||||
let mut m = String::with_capacity(HEAD.len() + 2 + n * 8);
|
||||
m.push_str(HEAD);
|
||||
m.push_str(open);
|
||||
for i in 0..n {
|
||||
if i > 0 {
|
||||
m.push(',');
|
||||
}
|
||||
elem(i, &mut m);
|
||||
}
|
||||
m.push_str(close);
|
||||
m.push('}');
|
||||
m
|
||||
}
|
||||
|
||||
/// Read `m` as a `WAI1` envelope; the peak and time, and whether it was read.
|
||||
fn read_wai1(m: &str) -> (Result<(), String>, usize, f64) {
|
||||
let env = envelope(b"WAI1", m, 0);
|
||||
peak_of(|| Wai::from_bytes(&env).map(|_| ()).map_err(|e| e.to_string()))
|
||||
}
|
||||
|
||||
fn report(name: &str, len: usize, values: usize, peak: usize, secs: f64, bound: usize) {
|
||||
eprintln!(
|
||||
"{name}: {len} bytes, {values} values: peak {peak} bytes ({:.2} per byte, {:.1} per value) in {secs:.2} s; bound {bound}",
|
||||
peak as f64 / len as f64,
|
||||
peak as f64 / values as f64
|
||||
);
|
||||
assert!(peak <= bound, "{name}: the read allocated {peak} bytes, past the bound {bound}");
|
||||
}
|
||||
|
||||
/// A manifest at the length limit made of small values is refused from its value count,
|
||||
/// having built nothing: the first pass holds no value, and stops at the value past the
|
||||
/// limit, some 8 MB into the text.
|
||||
#[test]
|
||||
fn a_manifest_of_small_values_at_the_length_limit_is_refused_before_it_is_built() {
|
||||
let _one = ONE.lock().unwrap_or_else(|e| e.into_inner());
|
||||
// Zeros, the last of them a number of ones long enough to bring the text to exactly
|
||||
// 2^27 bytes.
|
||||
let k = (MANIFEST_MAX_LEN - HEAD.len() - 3) / 2;
|
||||
let mut m = padded(k, "[", "]", |_, m| m.push('0'));
|
||||
let short = MANIFEST_MAX_LEN - m.len();
|
||||
let last = m.len() - 3;
|
||||
m.replace_range(last..last + 1, &"1".repeat(short + 1));
|
||||
assert_eq!(m.len(), MANIFEST_MAX_LEN);
|
||||
let (r, peak, secs) = read_wai1(&m);
|
||||
let e = r.expect_err("refused");
|
||||
assert!(e.starts_with("manifest_too_many_values:"), "{e}");
|
||||
report("2^27 bytes of zeros (refused)", m.len(), MANIFEST_MAX_VALUES + 1, peak, secs, 1 << 16);
|
||||
}
|
||||
|
||||
/// A manifest at the length limit holding exactly the value limit is read within the
|
||||
/// bound: 2^22 values, the most a reader must accept, each a string, to 2^27 bytes.
|
||||
#[test]
|
||||
fn a_manifest_at_both_limits_is_read_within_the_bound() {
|
||||
let _one = ONE.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let n = MANIFEST_MAX_VALUES - FIXED;
|
||||
// Each string `"…"` and its comma: share out what is left of 2^27 bytes.
|
||||
let room = MANIFEST_MAX_LEN - HEAD.len() - 3 - (n - 1) - 2 * n;
|
||||
let (q, r) = (room / n, room % n);
|
||||
let m = padded(n, "[", "]", |i, m| {
|
||||
m.push('"');
|
||||
m.extend(std::iter::repeat_n('s', q + usize::from(i < r)));
|
||||
m.push('"');
|
||||
});
|
||||
assert_eq!(m.len(), MANIFEST_MAX_LEN);
|
||||
assert_eq!(check_manifest_text(m.as_bytes()).unwrap(), MANIFEST_MAX_VALUES);
|
||||
let (r, peak, secs) = read_wai1(&m);
|
||||
r.expect("read");
|
||||
report("2^27 bytes, 2^22 strings", m.len(), MANIFEST_MAX_VALUES, peak, secs, bound(m.len(), MANIFEST_MAX_VALUES, 0));
|
||||
}
|
||||
|
||||
/// Manifests of small values, in every shape that costs a reader the most for each
|
||||
/// value, at exactly the value limit, are read within the bound; one value more is
|
||||
/// refused (`manifest_too_many_values`).
|
||||
#[test]
|
||||
fn small_values_at_the_value_limit_are_read_within_the_bound_and_one_more_is_refused() {
|
||||
let _one = ONE.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let n = MANIFEST_MAX_VALUES - FIXED;
|
||||
type Shape = (&'static str, usize, &'static str, &'static str, fn(usize, &mut String));
|
||||
let shapes: [Shape; 6] = [
|
||||
("an array of zeros", n, "[", "]", |_, m| m.push('0')),
|
||||
("an array of empty arrays", n, "[", "]", |_, m| m.push_str("[]")),
|
||||
("an array of empty objects", n, "[", "]", |_, m| m.push_str("{}")),
|
||||
("an array of one-element arrays", n / 2, "[", "]", |_, m| m.push_str("[0]")),
|
||||
("an array of one-member objects", n / 2, "[", "]", |_, m| m.push_str(r#"{"a":0}"#)),
|
||||
("one object of many members", n, "{", "}", |i, m| {
|
||||
m.push('"');
|
||||
m.push_str(&format!("{i:x}"));
|
||||
m.push_str("\":0");
|
||||
}),
|
||||
];
|
||||
for (name, k, open, close, elem) in shapes {
|
||||
// A shape of two values an element takes one zero more when `n` is odd.
|
||||
let mut m = padded(k, open, close, elem);
|
||||
if k != n && n % 2 == 1 {
|
||||
m.insert_str(m.len() - 2, ",0");
|
||||
}
|
||||
let values = check_manifest_text(m.as_bytes()).unwrap();
|
||||
assert_eq!(values, MANIFEST_MAX_VALUES, "{name}");
|
||||
let (r, peak, secs) = read_wai1(&m);
|
||||
r.expect(name);
|
||||
report(name, m.len(), values, peak, secs, bound(m.len(), values, 0));
|
||||
|
||||
m.insert_str(m.len() - 2, if open == "{" { r#","x":0"# } else { ",0" });
|
||||
let (r, peak, _) = read_wai1(&m);
|
||||
let e = r.expect_err("one value past the limit is refused");
|
||||
assert!(e.starts_with("manifest_too_many_values:"), "{name}: {e}");
|
||||
// The first pass holds the names of an object's members to find a repeat, so the
|
||||
// refusal of one object of many members holds them; it stays within the bound.
|
||||
assert!(peak <= bound(m.len(), MANIFEST_MAX_VALUES, 0), "{name}: the refusal allocated {peak} bytes");
|
||||
}
|
||||
}
|
||||
|
||||
/// The largest manifest a conforming staged layer has (SPEC §2): a `prev` that names a
|
||||
/// different producer for each of the 2^20 units a `WIR1` layer may cover, about 94 MB
|
||||
/// and 3·2^20 values, is read within the bound, and its `refines` reads well formed.
|
||||
#[test]
|
||||
fn the_largest_conforming_layer_manifest_is_read_within_the_bound() {
|
||||
let _one = ONE.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let (a, b) = ("ab".repeat(32), "cd".repeat(32));
|
||||
let units = 1usize << 20;
|
||||
let mut m = String::with_capacity(95 << 20);
|
||||
m.push_str(r#"{"wai":"1.0","media":"image","intent":"replicate","model_requirement":{"capability":"wai.image.int_refine","fallback":null},"conditioning":{"kind":"int_refine"},"refines":{"base":""#);
|
||||
m.push_str(&a);
|
||||
m.push_str(r#"","prev":["#);
|
||||
for i in 0..units {
|
||||
if i > 0 {
|
||||
m.push(',');
|
||||
}
|
||||
m.push_str(r#"{"producer":""#);
|
||||
m.push_str(if i % 2 == 0 { &a } else { &b });
|
||||
m.push_str(r#"","count":1}"#);
|
||||
}
|
||||
m.push_str(r#"],"layer":2},"target":null}"#);
|
||||
let values = check_manifest_text(m.as_bytes()).unwrap();
|
||||
assert_eq!(values, 3 * units + 14);
|
||||
let env = envelope(b"WAI1", &m, 0);
|
||||
let (w, peak, secs) = peak_of(|| Wai::from_bytes(&env).expect("read"));
|
||||
report("a layer naming 2^20 producers", m.len(), values, peak, secs, bound(m.len(), values, 0));
|
||||
let r = w.refines().expect("refines").expect("well formed");
|
||||
assert_eq!(r.units(units as u32), (0, units as u32));
|
||||
r.check_units(units as u32).expect("the runs cover the units");
|
||||
}
|
||||
|
||||
/// A `WAI2` manifest listing the 65 535 renditions its table can hold is read within the
|
||||
/// bound, which counts the entries it builds; a `renditions` of 65 536 is refused before
|
||||
/// its entries are built.
|
||||
#[test]
|
||||
fn a_wai2_listing_its_tables_rows_is_read_within_the_bound() {
|
||||
let _one = ONE.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let head = r#"{"wai":"1.1","media":"text","intent":"replicate","renditions":["#;
|
||||
let manifest = |rows: usize| {
|
||||
let mut m = String::from(head);
|
||||
for i in 0..rows {
|
||||
if i > 0 {
|
||||
m.push(',');
|
||||
}
|
||||
m.push_str(r#"{"capability":"wai.text.zstd","kind":"zstd"}"#);
|
||||
}
|
||||
m.push_str(r#"],"target":null}"#);
|
||||
m
|
||||
};
|
||||
let rows = u16::MAX as usize;
|
||||
let m = manifest(rows);
|
||||
let values = check_manifest_text(m.as_bytes()).unwrap();
|
||||
let env = envelope(b"WAI2", &m, rows);
|
||||
let (w, peak, secs) = peak_of(|| WaiMulti::from_bytes(&env).map(|w| w.manifest.renditions.len()));
|
||||
assert_eq!(w.expect("read"), rows);
|
||||
report("a WAI2 of 65535 renditions", m.len(), values, peak, secs, bound(m.len(), values, rows));
|
||||
|
||||
let m = manifest(rows + 1);
|
||||
let e = WaiMulti::from_bytes(&envelope(b"WAI2", &m, 0)).err().expect("refused").to_string();
|
||||
assert!(e.contains("more than the 65535 rows a WAI2 table holds"), "{e}");
|
||||
}
|
||||
171
wai/wai-rs/tests/manifest_reader_equivalence.rs
Normal file
171
wai/wai-rs/tests/manifest_reader_equivalence.rs
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
//! The reference readers build a manifest with `container::read_manifest` and
|
||||
//! `read_manifest_v2`, which move its members out of one JSON tree (SPEC §2, *Reader
|
||||
//! limits*: so a parse allocates within a bound), where they used serde's derived reader
|
||||
//! (`serde_json::from_slice::<Manifest>`). This test holds the two to the same result:
|
||||
//! for the manifest of every envelope committed under `wai/`, and for each of them with
|
||||
//! every member, at the manifest's top level and one level down, removed or given a
|
||||
//! value of each JSON type, both readers read it or both refuse it, and what they read
|
||||
//! writes back to the same JSON.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde_json::{json, Value};
|
||||
use wai::container::{check_unique_members, manifest_bytes, read_manifest, read_manifest_v2, Manifest, ManifestV2};
|
||||
|
||||
fn envelopes(dir: &Path, out: &mut Vec<PathBuf>) {
|
||||
let Ok(rd) = std::fs::read_dir(dir) else { return };
|
||||
let mut entries: Vec<_> = rd.flatten().map(|e| e.path()).collect();
|
||||
entries.sort();
|
||||
for p in entries {
|
||||
let name = p.file_name().and_then(|n| n.to_str()).unwrap_or("");
|
||||
if p.is_dir() {
|
||||
if name != "target" && name != "node_modules" && !name.starts_with('.') {
|
||||
envelopes(&p, out);
|
||||
}
|
||||
} else if name.ends_with(".wai") {
|
||||
out.push(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Both readers on `text`: the same verdict, and the same JSON when both read it.
|
||||
fn same(text: &[u8], v2: bool, what: &str) -> bool {
|
||||
let (derived, moved) = if v2 {
|
||||
let d = check_unique_members(text).and_then(|_| serde_json::from_slice::<ManifestV2>(text)).map(|m| serde_json::to_string(&m).unwrap());
|
||||
let m = read_manifest_v2(text).map(|m| serde_json::to_string(&m).unwrap());
|
||||
(d, m)
|
||||
} else {
|
||||
let d = check_unique_members(text).and_then(|_| serde_json::from_slice::<Manifest>(text)).map(|m| serde_json::to_string(&m).unwrap());
|
||||
let m = read_manifest(text).map(|m| serde_json::to_string(&m).unwrap());
|
||||
(d, m)
|
||||
};
|
||||
match (&derived, &moved) {
|
||||
(Ok(a), Ok(b)) => assert_eq!(a, b, "{what}: the readers read different manifests"),
|
||||
(Err(_), Err(_)) => {}
|
||||
_ => panic!("{what}: the derived reader gives {derived:?}, the reference reader {moved:?}"),
|
||||
}
|
||||
moved.is_ok()
|
||||
}
|
||||
|
||||
/// The values a member is given: one of each JSON type, and a few of each kind the
|
||||
/// manifest types read (a string, an integer, a float, `null`).
|
||||
fn replacements() -> Vec<Value> {
|
||||
vec![json!(null), json!(true), json!(0), json!(-1), json!(1.5), json!(18446744073709551615u64), json!("s"), json!([]), json!({}), json!([{}]), json!({"x": 1})]
|
||||
}
|
||||
|
||||
fn mutations(text: &[u8], v2: bool, what: &str) -> usize {
|
||||
let Ok(Value::Object(top)) = serde_json::from_slice::<Value>(text) else { return 0 };
|
||||
let mut n = 0;
|
||||
let mut check = |m: &serde_json::Map<String, Value>, why: String| {
|
||||
same(serde_json::to_string(m).unwrap().as_bytes(), v2, &format!("{what}: {why}"));
|
||||
n += 1;
|
||||
};
|
||||
for (k, v) in &top {
|
||||
let mut m = top.clone();
|
||||
m.shift_remove(k);
|
||||
check(&m, format!("without {k}"));
|
||||
for r in replacements() {
|
||||
let mut m = top.clone();
|
||||
m.insert(k.clone(), r.clone());
|
||||
check(&m, format!("{k} = {r}"));
|
||||
}
|
||||
// One level down: each member of an object member, and of the first object of
|
||||
// an array member (a WAI2's first rendition entry).
|
||||
let (inner, at): (Option<&serde_json::Map<String, Value>>, Option<usize>) = match v {
|
||||
Value::Object(o) => (Some(o), None),
|
||||
Value::Array(a) => match a.first() {
|
||||
Some(Value::Object(o)) => (Some(o), Some(0)),
|
||||
_ => (None, None),
|
||||
},
|
||||
_ => (None, None),
|
||||
};
|
||||
let Some(inner) = inner else { continue };
|
||||
let put = |o: serde_json::Map<String, Value>| {
|
||||
let mut m = top.clone();
|
||||
let nv = match at {
|
||||
None => Value::Object(o),
|
||||
Some(i) => {
|
||||
let mut a = v.as_array().unwrap().clone();
|
||||
a[i] = Value::Object(o);
|
||||
Value::Array(a)
|
||||
}
|
||||
};
|
||||
m.insert(k.clone(), nv);
|
||||
m
|
||||
};
|
||||
for ik in inner.keys() {
|
||||
let mut o = inner.clone();
|
||||
o.shift_remove(ik);
|
||||
check(&put(o), format!("{k} without {ik}"));
|
||||
for r in replacements() {
|
||||
let mut o = inner.clone();
|
||||
o.insert(ik.clone(), r.clone());
|
||||
check(&put(o), format!("{k}.{ik} = {r}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
n
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_reference_reader_reads_every_committed_manifest_as_the_derived_reader_does() {
|
||||
let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("..");
|
||||
let mut files = Vec::new();
|
||||
envelopes(&root, &mut files);
|
||||
assert!(files.len() >= 400, "found only {} envelopes under {}", files.len(), root.display());
|
||||
let (mut read, mut refused, mut mutated) = (0, 0, 0);
|
||||
// Mutate the manifests of each distinct set of top-level members once.
|
||||
let mut shapes = std::collections::BTreeSet::new();
|
||||
for f in &files {
|
||||
let b = std::fs::read(f).unwrap();
|
||||
let Ok(text) = manifest_bytes(&b) else { continue };
|
||||
let v2 = &b[..4] == b"WAI2";
|
||||
let what = f.strip_prefix(&root).unwrap_or(f).display().to_string();
|
||||
if same(text, v2, &what) {
|
||||
read += 1;
|
||||
} else {
|
||||
refused += 1;
|
||||
}
|
||||
if let Ok(Value::Object(o)) = serde_json::from_slice::<Value>(text) {
|
||||
let keys: Vec<String> = o.keys().cloned().collect();
|
||||
if shapes.insert((v2, keys)) {
|
||||
mutated += mutations(text, v2, &what);
|
||||
}
|
||||
}
|
||||
}
|
||||
eprintln!("{} envelopes: {read} manifests read alike, {refused} refused alike; {mutated} mutations agree", files.len());
|
||||
assert!(read >= 300, "only {read} manifests read");
|
||||
assert!(mutated >= 1000, "only {mutated} mutations");
|
||||
}
|
||||
|
||||
/// The cases the derived reader treats specially, each read alike: a member whose value
|
||||
/// is `null` where an `Option` is (`None`, not written back), `target` absent (`null`),
|
||||
/// a member no revision defines kept in order, a float `min_sim`, and an integer past
|
||||
/// `u64` as `kid`.
|
||||
#[test]
|
||||
fn the_derived_readers_special_cases_are_read_alike() {
|
||||
let base = r#"{"wai":"1.0","media":"image","intent":"replicate","model_requirement":{"capability":"wai.image.png","fallback":null,"min_sim":0.5,"z":1,"a":[1]},"conditioning":{"kind":"png","prior":null,"q":{}},"security":null,"refines":null,"derived_from":null,"later":{"b":2},"earlier":3}"#;
|
||||
for t in [
|
||||
base.to_string(),
|
||||
base.replace(r#""min_sim":0.5"#, r#""min_sim":1"#),
|
||||
base.replace(r#""security":null"#, r#""security":{"encryption":"none","kid":18446744073709551616}"#),
|
||||
base.replace(r#""security":null"#, r#""security":{"kid":-1}"#),
|
||||
base.replace(r#""conditioning":{"kind":"png","prior":null,"q":{}}"#, r#""conditioning":{"kind":"png","prior":{"id":"x","sha256":"y","determinism":"z","supersedes":null,"k":[]}}"#),
|
||||
base.replace(r#""conditioning":{"kind":"png","prior":null,"q":{}}"#, r#""conditioning":{"kind":"png","prior":{"id":"x","sha256":"y","determinism":"z","supersedes":{"sha256":"w","form":null}}}"#),
|
||||
base.replace(r#""conditioning":{"kind":"png","prior":null,"q":{}}"#, r#""conditioning":{"kind":"png","prior":{"id":"x","sha256":"y"}}"#),
|
||||
"[]".into(),
|
||||
"null".into(),
|
||||
"{}".into(),
|
||||
] {
|
||||
same(t.as_bytes(), false, &t);
|
||||
}
|
||||
let v2 = r#"{"wai":"1.1","media":"text","intent":"replicate","renditions":[{"capability":"wai.text.zstd","kind":"zstd","refines":null,"envelope_manifest":{"a":1},"x":[]}],"security":{"encryption":"none"},"z":1}"#;
|
||||
for t in [
|
||||
v2.to_string(),
|
||||
v2.replace(r#""renditions":[{"#, r#""renditions":[1,{"#),
|
||||
v2.replace(r#""renditions":["#, r#""renditions":{"a":"#).replace(r#""x":[]}],"#, r#""x":[]}},"#),
|
||||
v2.replace(r#""kind":"zstd","#, ""),
|
||||
] {
|
||||
same(t.as_bytes(), true, &t);
|
||||
}
|
||||
}
|
||||
|
|
@ -351,9 +351,17 @@ fn staged_sink_resources() {
|
|||
let chain = sink.chain(&bh).unwrap();
|
||||
assert!(chain.units().iter().all(|s| s.stage == 0));
|
||||
assert_eq!(chain.state_bytes(), unit_bytes);
|
||||
// The refusal held the envelope as parsed and the payload's parse, and no unit's output.
|
||||
assert!((refused.peak as u64) < 2 * l1.len() as u64 + c.working_bytes - c.output_bytes / 4,
|
||||
"a refused layer held {} bytes", refused.peak);
|
||||
// The refusal read the envelope and its payload's header, and decoded no unit. What it
|
||||
// allocated is set against what reading the same envelope allocates, measured here:
|
||||
// the read is the larger part, and SPEC §2 bounds it by the manifest's values and
|
||||
// length. Beyond the read, the refusal allocated less than the envelope's length, the
|
||||
// payload's parse and half of one unit's output (the layer covers four units). Bytes
|
||||
// allocated, not the peak: a unit decoded after the read would sit under the read's
|
||||
// own peak and never raise it.
|
||||
let (_, read) = measure(|| Wai::from_bytes(&l1).unwrap());
|
||||
let beyond = (refused.allocated as u64).saturating_sub(read.allocated as u64);
|
||||
assert!(beyond < l1.len() as u64 + c.working_bytes + c.output_bytes / 8,
|
||||
"a refused layer allocated {beyond} bytes beyond reading its envelope");
|
||||
// At the budget: applied; the state is the units' new output and 32 bytes a stage.
|
||||
let mut sink = StageSink::new(cfg(16, unit_bytes + reserve));
|
||||
sink.add_base(info.clone(), units.clone());
|
||||
|
|
|
|||
195
wai/wai-rs/tests/text_decode_bounds.rs
Normal file
195
wai/wai-rs/tests/text_decode_bounds.rs
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
//! `wai.text.zstd` and `wai.text.xz` decode within the reader limits of SPEC §2: a
|
||||
//! payload that decodes to more than the length limit, or whose stream asks its decoder
|
||||
//! to keep more history than the window limit, is refused by name, and one at either
|
||||
//! limit decodes.
|
||||
//!
|
||||
//! - The length limit, at a small limit the caller gives and at the limit the
|
||||
//! specification sets, for a frame that declares its length and for one that does not.
|
||||
//! - A zstd frame that declares a length past the limit is refused from the declaration:
|
||||
//! the same frame cut off after its header is refused the same way.
|
||||
//! - The window limit, on streams made here that declare one at the limit and one past
|
||||
//! it. Each holds a byte or a line of content: what is refused is the declaration.
|
||||
//! - The writers refuse what the readers may refuse.
|
||||
//!
|
||||
//! The memory the output buffer takes is measured in `text_decode_memory.rs`.
|
||||
//!
|
||||
//! Exercises the codec wrappers, so it requires the `codecs` feature.
|
||||
#![cfg(feature = "codecs")]
|
||||
|
||||
use std::io::Write;
|
||||
|
||||
use wai::codecs::text::{
|
||||
xz_decode, xz_decode_within, xz_encode, zstd_decode, zstd_decode_within, zstd_encode,
|
||||
TEXT_DECODED_MAX_LEN, TEXT_WINDOW_MAX_LEN,
|
||||
};
|
||||
|
||||
fn refusal<T: std::fmt::Debug>(r: Result<T, wai::codecs::text::TextError>) -> String {
|
||||
r.expect_err("the payload must be refused").0
|
||||
}
|
||||
|
||||
/// `n` bytes that compress well and are not all one value.
|
||||
fn pattern(n: usize) -> Vec<u8> {
|
||||
(0..n).map(|i| (i / 4096) as u8).collect()
|
||||
}
|
||||
|
||||
/// A zstd stream of `n` zero bytes whose frame declares no content size, written in
|
||||
/// pieces so that making it takes no buffer of `n` bytes.
|
||||
fn zstd_zeros(n: usize) -> Vec<u8> {
|
||||
let piece = vec![0u8; 1 << 20];
|
||||
let mut enc = zstd::stream::write::Encoder::new(Vec::new(), 1).unwrap();
|
||||
let mut left = n;
|
||||
while left > 0 {
|
||||
let k = left.min(piece.len());
|
||||
enc.write_all(&piece[..k]).unwrap();
|
||||
left -= k;
|
||||
}
|
||||
enc.finish().unwrap()
|
||||
}
|
||||
|
||||
/// The same for xz.
|
||||
fn xz_zeros(n: usize) -> Vec<u8> {
|
||||
let piece = vec![0u8; 1 << 20];
|
||||
let mut enc = xz2::write::XzEncoder::new(Vec::new(), 0);
|
||||
let mut left = n;
|
||||
while left > 0 {
|
||||
let k = left.min(piece.len());
|
||||
enc.write_all(&piece[..k]).unwrap();
|
||||
left -= k;
|
||||
}
|
||||
enc.finish().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_payload_at_the_callers_limit_decodes_and_one_byte_past_it_is_refused() {
|
||||
const LIMIT: usize = 1 << 20;
|
||||
let at = pattern(LIMIT);
|
||||
let past = pattern(LIMIT + 1);
|
||||
|
||||
// zstd, the frame declaring no content size (a streamed encode).
|
||||
assert_eq!(zstd_decode_within(&zstd_encode(&at, 3).unwrap(), LIMIT).unwrap(), at);
|
||||
let why = refusal(zstd_decode_within(&zstd_encode(&past, 3).unwrap(), LIMIT));
|
||||
assert!(why.starts_with("text_decoded_too_long:"), "{why}");
|
||||
|
||||
// zstd, the frame declaring its content size (a one-shot encode).
|
||||
let declared_at = zstd::bulk::compress(&at, 3).unwrap();
|
||||
let declared_past = zstd::bulk::compress(&past, 3).unwrap();
|
||||
assert_eq!(zstd::zstd_safe::get_frame_content_size(&declared_past).ok().flatten(), Some(LIMIT as u64 + 1));
|
||||
assert_eq!(zstd_decode_within(&declared_at, LIMIT).unwrap(), at);
|
||||
let why = refusal(zstd_decode_within(&declared_past, LIMIT));
|
||||
assert!(why.starts_with("text_decoded_too_long:"), "{why}");
|
||||
|
||||
// xz.
|
||||
assert_eq!(xz_decode_within(&xz_encode(&at, 1).unwrap(), LIMIT).unwrap(), at);
|
||||
let why = refusal(xz_decode_within(&xz_encode(&past, 1).unwrap(), LIMIT));
|
||||
assert!(why.starts_with("text_decoded_too_long:"), "{why}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_declared_length_past_the_limit_is_refused_from_the_declaration() {
|
||||
const LIMIT: usize = 1 << 16;
|
||||
let frame = zstd::bulk::compress(&pattern(LIMIT + 1), 3).unwrap();
|
||||
// The frame's first 18 bytes: its header, which is never longer, and at most a few
|
||||
// bytes of its first block. Nothing can be decoded from them.
|
||||
assert!(frame.len() > 18);
|
||||
let header_only = &frame[..18];
|
||||
assert_eq!(zstd::zstd_safe::get_frame_content_size(header_only).ok().flatten(), Some(LIMIT as u64 + 1));
|
||||
let why = refusal(zstd_decode_within(header_only, LIMIT));
|
||||
assert!(why.starts_with("text_decoded_too_long:"), "refused for another reason: {why}");
|
||||
// The same cut, within the limit, is the truncated frame it is.
|
||||
let why = refusal(zstd_decode_within(header_only, LIMIT + 1));
|
||||
assert!(!why.starts_with("text_decoded_too_long:"), "{why}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_limit_without_an_argument_is_the_one_the_specification_sets() {
|
||||
assert_eq!(TEXT_DECODED_MAX_LEN, 1 << 27);
|
||||
|
||||
let out = zstd_decode(&zstd_zeros(TEXT_DECODED_MAX_LEN)).unwrap();
|
||||
assert_eq!(out.len(), TEXT_DECODED_MAX_LEN);
|
||||
assert!(out.iter().all(|&b| b == 0));
|
||||
drop(out);
|
||||
let past = zstd_zeros(TEXT_DECODED_MAX_LEN + 1);
|
||||
let why = refusal(zstd_decode(&past));
|
||||
assert!(why.starts_with("text_decoded_too_long:"), "{why}");
|
||||
// A caller cannot raise it.
|
||||
let why = refusal(zstd_decode_within(&past, usize::MAX));
|
||||
assert!(why.starts_with("text_decoded_too_long:"), "{why}");
|
||||
|
||||
let out = xz_decode(&xz_zeros(TEXT_DECODED_MAX_LEN)).unwrap();
|
||||
assert_eq!(out.len(), TEXT_DECODED_MAX_LEN);
|
||||
drop(out);
|
||||
let past = xz_zeros(TEXT_DECODED_MAX_LEN + 1);
|
||||
let why = refusal(xz_decode(&past));
|
||||
assert!(why.starts_with("text_decoded_too_long:"), "{why}");
|
||||
let why = refusal(xz_decode_within(&past, usize::MAX));
|
||||
assert!(why.starts_with("text_decoded_too_long:"), "{why}");
|
||||
}
|
||||
|
||||
/// A zstd frame of one byte of content that declares a window of `2^window_log` bytes
|
||||
/// (RFC 8878 §3.1.1): the magic number; a frame header descriptor of 0, which says a
|
||||
/// window descriptor follows and nothing else does; the window descriptor, whose high
|
||||
/// five bits are `window_log − 10`; and one last block, of type RLE and size 1.
|
||||
fn zstd_frame_declaring_window(window_log: u32) -> Vec<u8> {
|
||||
let mut f = vec![0x28, 0xB5, 0x2F, 0xFD, 0x00, ((window_log - 10) << 3) as u8];
|
||||
f.extend_from_slice(&[0x0B, 0x00, 0x00, b'A']);
|
||||
f
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zstd_window_at_the_limit_decodes_and_one_past_it_is_refused() {
|
||||
assert_eq!(TEXT_WINDOW_MAX_LEN, 1 << 27);
|
||||
assert_eq!(zstd_decode(&zstd_frame_declaring_window(27)).unwrap(), b"A");
|
||||
let why = refusal(zstd_decode(&zstd_frame_declaring_window(28)));
|
||||
assert!(why.starts_with("text_window_too_large:"), "{why}");
|
||||
}
|
||||
|
||||
/// CRC-32 as the xz format uses it (IEEE 802.3, reflected).
|
||||
fn crc32(bytes: &[u8]) -> u32 {
|
||||
let mut c = !0u32;
|
||||
for &b in bytes {
|
||||
c ^= b as u32;
|
||||
for _ in 0..8 {
|
||||
c = (c >> 1) ^ (0xEDB8_8320 & (!(c & 1)).wrapping_add(1));
|
||||
}
|
||||
}
|
||||
!c
|
||||
}
|
||||
|
||||
/// An xz stream of a line of text whose one block declares the LZMA2 dictionary size
|
||||
/// coded as `code` (the xz file format, §5.3.1: 30 is 2^27 bytes, 31 is 3·2^26). Made by
|
||||
/// encoding the line, then rewriting the dictionary size in the block header and the
|
||||
/// header's CRC-32. The stream header is 12 bytes; the block header follows it: its
|
||||
/// size, the block flags, then the filter (id 0x21, one byte of properties, which is the
|
||||
/// dictionary size), padding, and the CRC-32 of all of it.
|
||||
fn xz_stream_declaring_dictionary(code: u8) -> (Vec<u8>, Vec<u8>) {
|
||||
let line = b"one line of text, long enough to be worth a block\n".to_vec();
|
||||
let mut s = xz_encode(&line, 0).unwrap();
|
||||
let start = 12;
|
||||
let size = (s[start] as usize + 1) * 4;
|
||||
let header = start..start + size;
|
||||
let at = (header.start + 2..header.end - 5)
|
||||
.find(|&i| s[i] == 0x21 && s[i + 1] == 0x01)
|
||||
.expect("the block header names the LZMA2 filter with one byte of properties");
|
||||
s[at + 2] = code;
|
||||
let crc = crc32(&s[header.start..header.end - 4]);
|
||||
s[header.end - 4..header.end].copy_from_slice(&crc.to_le_bytes());
|
||||
(s, line)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_xz_dictionary_at_the_limit_decodes_and_one_past_it_is_refused() {
|
||||
let (at, line) = xz_stream_declaring_dictionary(30);
|
||||
assert_eq!(xz_decode(&at).unwrap(), line);
|
||||
let (past, _) = xz_stream_declaring_dictionary(31);
|
||||
let why = refusal(xz_decode(&past));
|
||||
assert!(why.starts_with("text_window_too_large:"), "{why}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_writers_refuse_what_a_reader_may_refuse() {
|
||||
let over = vec![0u8; TEXT_DECODED_MAX_LEN + 1];
|
||||
let why = refusal(zstd_encode(&over, 1));
|
||||
assert!(why.starts_with("text_decoded_too_long:"), "{why}");
|
||||
let why = refusal(xz_encode(&over, 0));
|
||||
assert!(why.starts_with("text_decoded_too_long:"), "{why}");
|
||||
}
|
||||
108
wai/wai-rs/tests/text_decode_memory.rs
Normal file
108
wai/wai-rs/tests/text_decode_memory.rs
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
//! Decoding a `wai.text.zstd` or `wai.text.xz` payload that would expand past the
|
||||
//! length limit takes no more output buffer than the limit (SPEC §2, *Reader limits*).
|
||||
//! Measured with a global allocator that counts the bytes live and their peak.
|
||||
//!
|
||||
//! Each payload here is a few kilobytes and decodes to four times the limit the call is
|
||||
//! given. The call must refuse it, and the bytes it allocated on the way must stay
|
||||
//! within the limit and a small allowance for its read buffer. A decode that grew its
|
||||
//! buffer past the limit before checking, or that decoded everything and then measured
|
||||
//! it, would not.
|
||||
//!
|
||||
//! The counter sees what this crate allocates: the output buffer and the read buffer.
|
||||
//! The decoding library's own working memory is allocated by the library and is bounded
|
||||
//! by the window limit, which `text_decode_bounds.rs` tests.
|
||||
//!
|
||||
//! Its own test binary with a single test, so that no other test's allocations land in
|
||||
//! the windows it measures. Requires the `codecs` feature.
|
||||
#![cfg(feature = "codecs")]
|
||||
|
||||
use std::alloc::{GlobalAlloc, Layout, System};
|
||||
use std::io::Write;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
|
||||
|
||||
use wai::codecs::text::{xz_decode_within, zstd_decode_within};
|
||||
|
||||
/// The system allocator, counting the bytes live and their peak.
|
||||
struct Count;
|
||||
static CUR: AtomicUsize = AtomicUsize::new(0);
|
||||
static PEAK: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
fn grow(n: usize) {
|
||||
PEAK.fetch_max(CUR.fetch_add(n, Relaxed) + n, Relaxed);
|
||||
}
|
||||
|
||||
unsafe impl GlobalAlloc for Count {
|
||||
unsafe fn alloc(&self, l: Layout) -> *mut u8 {
|
||||
grow(l.size());
|
||||
unsafe { System.alloc(l) }
|
||||
}
|
||||
unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 {
|
||||
grow(l.size());
|
||||
unsafe { System.alloc_zeroed(l) }
|
||||
}
|
||||
unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 {
|
||||
if new >= l.size() {
|
||||
grow(new - l.size());
|
||||
} else {
|
||||
CUR.fetch_sub(l.size() - new, Relaxed);
|
||||
}
|
||||
unsafe { System.realloc(p, l, new) }
|
||||
}
|
||||
unsafe fn dealloc(&self, p: *mut u8, l: Layout) {
|
||||
CUR.fetch_sub(l.size(), Relaxed);
|
||||
unsafe { System.dealloc(p, l) }
|
||||
}
|
||||
}
|
||||
|
||||
#[global_allocator]
|
||||
static ALLOC: Count = Count;
|
||||
|
||||
/// The most bytes live at once while `f` runs, above what was live when it started.
|
||||
fn peak_of<T>(f: impl FnOnce() -> T) -> (T, usize) {
|
||||
let base = CUR.load(Relaxed);
|
||||
PEAK.store(base, Relaxed);
|
||||
let r = f();
|
||||
(r, PEAK.load(Relaxed).saturating_sub(base))
|
||||
}
|
||||
|
||||
const LIMIT: usize = 8 << 20;
|
||||
/// The read buffer, and room for the error's text.
|
||||
const ALLOWANCE: usize = 1 << 20;
|
||||
|
||||
fn zeros_through<W: Write>(mut w: W, n: usize) -> W {
|
||||
let piece = vec![0u8; 1 << 20];
|
||||
let mut left = n;
|
||||
while left > 0 {
|
||||
let k = left.min(piece.len());
|
||||
w.write_all(&piece[..k]).unwrap();
|
||||
left -= k;
|
||||
}
|
||||
w
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_payload_that_expands_past_the_limit_takes_no_more_buffer_than_the_limit() {
|
||||
let zstd_payload = zeros_through(zstd::stream::write::Encoder::new(Vec::new(), 1).unwrap(), 4 * LIMIT)
|
||||
.finish()
|
||||
.unwrap();
|
||||
let xz_payload = zeros_through(xz2::write::XzEncoder::new(Vec::new(), 0), 4 * LIMIT).finish().unwrap();
|
||||
assert!(zstd_payload.len() < LIMIT / 64 && xz_payload.len() < LIMIT / 64, "the payloads are small");
|
||||
|
||||
let (r, peak) = peak_of(|| zstd_decode_within(&zstd_payload, LIMIT));
|
||||
let why = r.expect_err("zstd: the payload must be refused").0;
|
||||
assert!(why.starts_with("text_decoded_too_long:"), "{why}");
|
||||
assert!(peak <= LIMIT + ALLOWANCE, "zstd: refusing took {peak} bytes of buffer, limit {LIMIT}");
|
||||
assert!(peak >= LIMIT / 2, "zstd: the refusal came before the output neared the limit ({peak} bytes): is the payload what it should be?");
|
||||
|
||||
let (r, peak) = peak_of(|| xz_decode_within(&xz_payload, LIMIT));
|
||||
let why = r.expect_err("xz: the payload must be refused").0;
|
||||
assert!(why.starts_with("text_decoded_too_long:"), "{why}");
|
||||
assert!(peak <= LIMIT + ALLOWANCE, "xz: refusing took {peak} bytes of buffer, limit {LIMIT}");
|
||||
assert!(peak >= LIMIT / 2, "xz: the refusal came before the output neared the limit ({peak} bytes): is the payload what it should be?");
|
||||
|
||||
// At the limit the decode succeeds, in a buffer of the limit.
|
||||
let zstd_at = zeros_through(zstd::stream::write::Encoder::new(Vec::new(), 1).unwrap(), LIMIT).finish().unwrap();
|
||||
let (r, peak) = peak_of(|| zstd_decode_within(&zstd_at, LIMIT));
|
||||
assert_eq!(r.unwrap().len(), LIMIT);
|
||||
assert!(peak <= LIMIT + ALLOWANCE, "zstd: decoding to the limit took {peak} bytes of buffer, limit {LIMIT}");
|
||||
}
|
||||
|
|
@ -501,10 +501,11 @@ fn pack_refusal(m: &wai::container::Manifest, payload: &[u8]) -> Result<(), Stri
|
|||
}
|
||||
|
||||
/// A manifest a writer emits as given, read as every reader reads it (SPEC §2): a
|
||||
/// member named twice in one object, at any depth, is refused, so a writer never emits
|
||||
/// an envelope the reader refuses.
|
||||
/// member named twice in one object, at any depth, and a manifest past the value limit
|
||||
/// (`manifest_too_many_values`) are refused, so a writer never emits an envelope the
|
||||
/// reader refuses.
|
||||
fn read_manifest<T: serde::de::DeserializeOwned>(json: &str) -> Result<T, String> {
|
||||
wai::container::check_unique_members(json.as_bytes()).map_err(|e| format!("manifest parse: {e}"))?;
|
||||
wai::container::check_manifest_text(json.as_bytes()).map_err(|e| format!("manifest parse: {e}"))?;
|
||||
serde_json::from_str(json).map_err(|e| format!("manifest parse: {e}"))
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue