diff --git a/.github/workflows/wai-breaking.yml b/.github/workflows/wai-breaking.yml new file mode 100644 index 000000000..8d4bc9951 --- /dev/null +++ b/.github/workflows/wai-breaking.yml @@ -0,0 +1,158 @@ +name: WAI breaking changes and the conformance runner + +# The gate WAI's libraries are published behind (toolchain v1, decision 7): every pull request +# that touches WAI runs `wai breaking --against origin/main`, and fails on a breaking change that +# wai/CHANGELOG.md does not name. Two jobs: +# +# - breaking: the check itself, against the merge base of the pull request and main, in all +# three categories: wire (the registry export, the media and delivery corpora), record (the +# signing domains, magics and versions, the record corpora, the published JSON Schemas) and +# API (the Rust public API of the WAI library crates through rustdoc's JSON, the C header, +# the room service's OpenAPI document). +# - runner: the tests of `wai breaking` (each kind of break reported and failing until a +# changelog entry names it) and of `wai conform` (the reference implementations conform, a +# changed verdict or output byte fails, a changed record is refused by `wai verify`); then +# `wai conform` runs the corpora's independent Python verifiers and the reference packager, +# and `wai verify` checks every record those runs wrote. +# +# Push filters match the push's delta, and the `relevant` job gates the work from the pull +# request's own diff (scripts/pr-touches-paths.py, reading the list below), as wai-pack.yml +# explains. + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + paths: + - 'wai/**' + # The room service's API is part of the API category. + - 'joule-code/joule-code-rs/crates/joule-moq-room/openapi.json' + # The toolchain pin decides which compiler documents both sides of the Rust API. + - 'rust-toolchain.toml' + - 'scripts/pr-touches-paths.py' + - '.github/workflows/wai-breaking.yml' + +jobs: + relevant: + name: WAI breaking · does this change touch WAI + runs-on: linux-arm64 + outputs: + run: ${{ steps.gate.outputs.run }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - id: gate + run: | + set -euo pipefail + r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-breaking.yml \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}") + echo "this change touches WAI: $r" + echo "run=$r" >> "$GITHUB_OUTPUT" + + breaking: + name: WAI breaking · wire, record and API against main · linux + needs: relevant + if: needs.relevant.outputs.run == 'true' + runs-on: linux + timeout-minutes: 60 + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Install Rust 1.98.0 + shell: bash + run: | + export PATH="$HOME/.cargo/bin:$PATH" + command -v rustup >/dev/null 2>&1 || \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.98.0 + rustup toolchain install 1.98.0 --profile minimal + rustup default 1.98.0 + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo" + command -v python3 >/dev/null || { $SUDO apt-get update -qq; $SUDO apt-get install -y -qq python3; } + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: Build wai + working-directory: wai/wai-rs + shell: bash + run: | + set -euo pipefail + cargo build --locked --no-default-features --bin wai + out=$(cargo metadata --format-version 1 --no-deps --locked | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])') + mkdir -p "$RUNNER_TEMP/bin" && cp "$out/debug/wai" "$RUNNER_TEMP/bin/" + # The base is main's tip as fetched now; `wai breaking` compares against its merge base + # with this pull request, so what main gained since the branch left it is not read as + # this change removing it. Every finding is printed: each break with its category, what + # changed and why it breaks, and the line that acknowledges it if it is intended. + - name: wai breaking --against origin/main + shell: bash + run: | + set -euo pipefail + # The job's container user may not own the checkout; git then refuses to read it. + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch -q --no-tags origin +refs/heads/main:refs/remotes/origin/main + "$RUNNER_TEMP/bin/wai" breaking --against origin/main --text --all + + runner: + name: WAI breaking and conform · their tests, and the independent verifiers through wai conform · linux + needs: relevant + if: needs.relevant.outputs.run == 'true' + runs-on: linux + timeout-minutes: 90 + steps: + - uses: actions/checkout@v5 + - name: Install Rust 1.98.0 + shell: bash + run: | + export PATH="$HOME/.cargo/bin:$PATH" + command -v rustup >/dev/null 2>&1 || \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.98.0 + rustup toolchain install 1.98.0 --profile minimal + rustup default 1.98.0 + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + # claim_verify.py, jwp_verify.py and signer_verify.py check Ed25519's equation with it. + - name: Install the Ed25519 library + shell: bash + run: python -m pip install cryptography + # The tests build fixture repositories with git and document a fixture crate with rustdoc. + - name: The tests of wai breaking and wai conform + working-directory: wai/wai-rs + shell: bash + run: | + set -euo pipefail + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git config --global user.email ci@example.invalid + git config --global user.name ci + cargo test --locked --no-default-features --features quickstart,score,key_discovery \ + --bin wai --test cli_breaking --test cli_conform + - name: Build wai and the reference packager + shell: bash + run: | + set -euo pipefail + out() { (cd "$1" && cargo metadata --format-version 1 --no-deps --locked | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])'); } + mkdir -p "$RUNNER_TEMP/bin" + (cd wai/wai-rs && cargo build --locked --no-default-features --features quickstart --bin wai) && cp "$(out wai/wai-rs)/debug/wai" "$RUNNER_TEMP/bin/" + (cd wai/wai-pack && cargo build --locked --bin wai-pack) && cp "$(out wai/wai-pack)/debug/wai-pack" "$RUNNER_TEMP/bin/" + # The second implementations of the claim, JWP and key-discovery corpora, written from the + # extension texts and sharing no code with wai-rs, run through the conformance runner as + # any other implementation would be; then the reference packager's policy reader. Each + # run must be conformant, and every record it signs must verify. + - name: wai conform with the independent verifiers and the reference packager + shell: bash + run: | + set -euo pipefail + w="$RUNNER_TEMP/bin/wai"; d="$RUNNER_TEMP/conform"; mkdir -p "$d" + for s in claim jwp signer; do + "$w" conform verifier --suite "$s" --text --root wai \ + --impl "python3 -I wai/$s-conformance/${s}_verify.py verdicts {corpus}" --out "$d/$s.wai-conform.json" + done + "$w" conform verifier --suite conform-result --text --root wai \ + --impl "python3 -I wai/conform-result-conformance/conform_result_verify.py verdicts {corpus}" --out "$d/conform-result.wai-conform.json" + "$w" conform packager --suite policy --text --root wai \ + --impl "$RUNNER_TEMP/bin/wai-pack policy-verdict {in}" --out "$d/policy.wai-conform.json" + "$w" verify "$d" diff --git a/.github/workflows/wai-conformance.yml b/.github/workflows/wai-conformance.yml index 82bdc3bc3..317fc77a8 100644 --- a/.github/workflows/wai-conformance.yml +++ b/.github/workflows/wai-conformance.yml @@ -131,6 +131,10 @@ on: # reference's codes, key_release_verify.py reads those tables and §3.3, and the registry's # wire identifiers test reads its sections. - 'wai/extensions/key-release.md' + # tests/conform_result.rs requires this extension's §5 to list the codes the reference + # reports, conform_result_verify.py reads that table, and the registry's wire identifiers + # test reads its sections. + - 'wai/extensions/conformance-results.md' # The ad-presentation step runs the browser client's reference producer against the # corpus, and the stitched-presentation step runs its stitched producer. - 'joule-code/joule-code-rs/crates/joule-moq-web/web/ads/**' @@ -1145,7 +1149,7 @@ jobs: name: independent verifiers · claims, JWP, key discovery and integer payloads (Python) runs-on: linux-arm64 env: - WAI_FEATURES: key_discovery,http,video_manifest,stage_measure,live_events,live_room,capture_take,live_compose,live_record,timed_text,access_services,live_metering,energy_steering,live_moderation,dred_exact,live_ads,live_provenance,synthesis,synthesis_mark,neural_int,exact_upscale,robustness,ai_usage,contribution,settlement,browser_sink,receipt_claims,takedown,influence,direction,co_performance,age_assurance,bearer_receipts,exploration,co_watch,telemetry_aggregate,exact_tonemap,splat_exact,live_watermark,derive_edge,sink_energy,key_release,stitched_presentation,prior_wire,scene_map,avatar_face,receipt_ledger,stream_audit,fleet,native_sink,platform_encode,remote_desktop,stream_termination,live_world,format_probe,live_messaging + WAI_FEATURES: key_discovery,http,video_manifest,stage_measure,live_events,live_room,capture_take,live_compose,live_record,timed_text,access_services,live_metering,energy_steering,live_moderation,dred_exact,live_ads,live_provenance,synthesis,synthesis_mark,neural_int,exact_upscale,robustness,ai_usage,contribution,settlement,browser_sink,receipt_claims,takedown,influence,direction,co_performance,age_assurance,bearer_receipts,exploration,co_watch,telemetry_aggregate,exact_tonemap,splat_exact,live_watermark,derive_edge,sink_energy,key_release,stitched_presentation,prior_wire,scene_map,avatar_face,receipt_ledger,stream_audit,fleet,native_sink,platform_encode,remote_desktop,stream_termination,live_world,format_probe,live_messaging,conform_result WAI_PQ_FEATURES: synthesis,pq_signatures WAI_RELEASE_FEATURES: dred_exact,synthesis_mark,robustness,avatar_face steps: @@ -1215,11 +1219,11 @@ jobs: --bin wai_http_vectors --bin wai_stage_measure --bin wai_browser_sink_vectors \ --bin wai_claim_vectors --bin wai_jwp_vectors --bin wai_signer_vectors \ --bin wai_sink_energy_vectors --bin wai_stitched_presentation \ - --bin wai_stream_termination_vectors --bin wai_format_probe + --bin wai_stream_termination_vectors --bin wai_format_probe --bin wai_conform_result_vectors for b in wai_http_vectors wai_stage_measure wai_browser_sink_vectors \ wai_claim_vectors wai_jwp_vectors wai_signer_vectors \ wai_sink_energy_vectors wai_stitched_presentation \ - wai_stream_termination_vectors wai_format_probe; do + wai_stream_termination_vectors wai_format_probe wai_conform_result_vectors; do cp "${CARGO_TARGET_DIR:-target}/debug/$b" "$RUNNER_TEMP/wai-bins/union/" done # Copied before this build replaces target/debug's copies of the three it shares. @@ -1239,7 +1243,8 @@ jobs: --test derive_edge --test det_cost_bin --test sink_energy --test key_release \ --test stitched_presentation --test prior_wire --test scene_map --test receipt_ledger \ --test stream_audit --test fleet --test native_sink --test platform_encode --test remote_desktop \ - --test stream_termination --test live_world --test format_probe --test live_messaging + --test stream_termination --test live_world --test format_probe --test live_messaging \ + --test conform_result cargo test --release --no-run --no-default-features --features "$WAI_RELEASE_FEATURES" \ --test dred_exact --test dred_exact_measure --test synthesis_mark --test robustness \ --test avatar_face @@ -1467,6 +1472,29 @@ jobs: (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib access_services::) (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test access_services) python3 -I live-conformance/access_services_verify.py live-conformance/access + # conform-result/1 (extensions/conformance-results.md): the reference module's unit tests + # (a signed record accepted and every member covered, the strict rule's small-order forgery + # refused, the date form, a vector for every reason) and tests/conform_result.rs (the + # committed corpus regenerates byte for byte, wai-rs gives every vector its stated outcome, + # every reason of §5 refuses a vector, §5 lists exactly the reference's codes, and no leaf + # of a record survives a change after signing); then wai_conform_result_vectors verify, and + # the corpus's second implementation, conform_result_verify.py, written from the + # extension's text and sharing no code with wai-rs (WAI's strict Ed25519 rule from + # claim-conformance/_ed25519.py), which reads the extension's reason table; and with each + # check of §4 left out in turn (--skip), the corpus must fail it. Standard library only, + # run isolated (-I). + - name: conformance results · wai-rs and conform_result_verify.py + working-directory: wai + shell: bash + run: | + set -euo pipefail + (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --lib conform_result::) + (cd wai-rs && cargo test --no-default-features --features "$WAI_FEATURES" --test conform_result) + "$RUNNER_TEMP/wai-bins/union/wai_conform_result_vectors" verify conform-result-conformance + python3 -I conform-result-conformance/conform_result_verify.py conform-result-conformance + for c in duplicates format members signature summary verdict; do + python3 -I conform-result-conformance/conform_result_verify.py --skip "$c" conform-result-conformance | tail -1 + done # wai.live.metering (extensions/live-metering.md): the reference module's unit tests # (records and ledgers round-trip, the energy rules, the size bound at its limit, a # total of two classes told from a wrong one, funds rounding, the CMCD event-timeline @@ -2629,7 +2657,8 @@ jobs: telemetry_aggregate exact_tonemap splat_exact live_watermark derive_edge pq_signatures \ synthesis,pq_signatures sink_energy key_release stitched_presentation prior_wire \ prior_wire,neural_int scene_map avatar_face receipt_ledger stream_audit fleet native_sink \ - platform_encode remote_desktop stream_termination live_world format_probe live_messaging; do + platform_encode remote_desktop stream_termination live_world format_probe live_messaging \ + conform_result; do echo "::group::$f" cargo check --no-default-features --features "$f" --tests echo "::endgroup::" diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index 8e67127c4..01b409787 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -6,6 +6,55 @@ wire format may change and such changes are recorded here. ## v0.4.0 — current +### `wai breaking` and `wai conform`: the compatibility gate, and conformance anyone can run + +- **`wai breaking --against `** (operation `compat.check`) reports what the working + tree breaks against a ref, in three cumulative categories, strictest by default: **wire** (the + registry export: capabilities and their classes, deterministic identifiers, wire identifiers; + the media and delivery corpora), **record** (the domain-separation prefixes, magics and + versions records are built from; the corpora of signed records and their verdicts; the + published JSON Schemas) and **API** (the Rust public API of `wai` and `wai-quantum`, read from + the compiler's rustdoc JSON; the C header `wai-rs/include/wai.h`; the room service's OpenAPI + document). It compares against the merge base of the ref and `HEAD`. Each finding names its + item, its category, what changed and why that breaks; additions are listed as compatible. +- **An intended break passes only when this file names it**: a line the change adds here, in + an entry that says **Breaking**, with the finding's id (as `wai breaking` prints it, such as + api:c:wai_envelope_pack) in backticks. Otherwise the run exits 1. +- **`.github/workflows/wai-breaking.yml`** runs it on every pull request that touches WAI, + against main. WAI's libraries go to public registries behind this gate. +- **`wai conform --suite --impl `** (operation + `conformance.run`) runs one of WAI's corpora against another implementation: verifiers + (`claim`, `jwp`, `signer`), a decoder (`score`), a packager (`policy`) and an origin + (`http`). A command-line adapter gives each case as a file (`{case}`, `{in}`), the corpus + whole (`{corpus}`), or standard input, and reads the answer from standard output or `{out}`; + an HTTP adapter posts each case to a URL, or, for an origin, fetches from it. Every + expectation is the corpus's own. +- **The result is a signed record**, `conform-result/1` (`*.wai-conform.json`): the corpus + digest, the implementation (its command and the SHA-256 of each file it names, or its URL), + each case's outcome, the summary and the standard's version, signed with Ed25519 by a key + made for the run or one given with `--key`. **`wai verify`** checks its signature and that its + summary and verdict are what its cases say. CI runs the corpora's independent Python + verifiers and the reference packager through it. +- **[conformance-results](extensions/conformance-results.md)** specifies the record: its + members (§2) and corpus digest (§2.1), its signing input (the domain `wai:conform-result` and + `0x01`, then the record without `sig` in RFC 8785 form) and Ed25519 signature under SPEC §7's + strict rule (§3), what a verifier checks and in what order (§4) and the codes it refuses with + (§5). `conform-result/1` is registered as a document kind. The reference is the `wai-rs` + module `conform_result` (feature `conform_result`, which `quickstart` includes); the corpus, + [`conform-result-conformance/`](conform-result-conformance/), holds accepted records and + refused ones (a case flipped, the corpus digest changed, the summary alone changed, a bad + signature, `S` past L, a key of small order, a summary or verdict that disagrees with its + cases though signed, an unknown version, members missing, added or named twice, a count with + a fraction, a digest out of form, bytes that are not JSON), generated by + `wai_conform_result_vectors`; its second implementation is `conform_result_verify.py`. +- **Measurements are not compared**: `wai breaking` leaves out the corpus files it lists as + records of a measured run (the external decodes of the format probes, the relay's recorded + metering run), each with its reason; the vectors built from them are still compared. +- Both follow the toolchain's agent rules: the JSON envelope `envelope/1` with `--json` or off a + terminal, one exit status for each outcome (0 held, 1 a check failed, 2 usage, 4 a record + would be replaced without `--confirm`, 5 not found, 6 the implementation does not answer, 7 + internal), and `wai conform --dry-run`. + ### Fleet: an organisation's nodes enrolled by their keys, their records verified, and a report it signs for a period - **The text** ([`extensions/fleet.md`](extensions/fleet.md), `wai.delivery.fleet`): the diff --git a/wai/README.md b/wai/README.md index 2dbe8f5b3..49d4e7bee 100644 --- a/wai/README.md +++ b/wai/README.md @@ -86,6 +86,20 @@ cargo run --no-default-features --bin wai -- capabilities # the registry, one r cargo build --release --lib # with the codecs feature: libwai.dylib/.so/.dll + libwai.a ``` +Two commands hold what is published to what was published before, and other implementations +to WAI's corpora: + +```bash +wai breaking --against origin/main # what this tree breaks: wire, record and API (CI runs it on every pull request) +wai conform verifier --suite claim --impl "my-verifier verdicts {corpus}" # a corpus run against your implementation +wai verify claim.wai-conform.json # the signed result, checked +``` + +A break passes only when [`CHANGELOG.md`](CHANGELOG.md) names it: an entry this change adds that +says **Breaking** and gives the finding's id in backticks. `wai conform --list` names every kind +(verifier, decoder, packager, origin) and suite, with its adapters: a command line, or an +`http://` URL. + With the `codecs` feature, `wai-rs/build.rs` finds libFLAC through pkg-config and, on macOS without pkg-config, in the Homebrew "flac" formula prefix (`/opt/homebrew/opt/flac/lib` or `/usr/local/opt/flac/lib`). Where neither finds diff --git a/wai/REGISTRY.md b/wai/REGISTRY.md index c145e716b..b56e889e5 100644 --- a/wai/REGISTRY.md +++ b/wai/REGISTRY.md @@ -452,6 +452,7 @@ and a role are fixed once registered. | `X-WAI-EVENT` | playlist-attribute | defined | live-events §8.3 | | `X-WAI-FLOOR` | playlist-attribute | defined | http-delivery §11.1 | | `application/wai` | media-type | proposed | http-delivery §3 | +| `conform-result/1` | document-kind | defined | conformance-results §2 | | `contentBinding` | frame-metadata | proposed | browser-sink §7.1 | | `id-MLDSA65-Ed25519-SHA512` | scheme | defined | pq-signatures §2 | | `mr/` | path | defined | http-delivery §8.6 | diff --git a/wai/SPEC.md b/wai/SPEC.md index 25c74cc8e..e9d290e44 100644 --- a/wai/SPEC.md +++ b/wai/SPEC.md @@ -2516,6 +2516,12 @@ it does for plaintext; in Lane A there is no output for it to cover. Rust reference impl in `wai-rs/` ships a test suite that carries every registered capability through the full WAI envelope (`cargo test --lib`, `tests/carriage.rs`). +- A **conformance result**, the outcome of running one of WAI's conformance corpora against an + implementation, is the signed record `conform-result/1` of + [conformance-results](extensions/conformance-results.md) §2. A verifier checks its signature + and that its summary and its verdict are what its cases say + ([conformance-results](extensions/conformance-results.md) §4). `wai conform` writes such + records and `wai verify` checks them. ### Replicate-soundness — the integer-entropy contract (normative) diff --git a/wai/conform-result-conformance/README.md b/wai/conform-result-conformance/README.md new file mode 100644 index 000000000..879494f8c --- /dev/null +++ b/wai/conform-result-conformance/README.md @@ -0,0 +1,73 @@ +# conform-result-conformance + +The conformance corpus of [conformance-results](../extensions/conformance-results.md) (§7): +records of `conform-result/1`, the signed outcome of a conformance run, each with the outcome +§4 gives it. + +**Provenance.** Generated here by `wai_conform_result_vectors gen`, from records written in +code and signed with published one-byte seeds: 32 bytes of `0x43` (the key of every record but +one) and of `0x44` (a key a runner holds, in `valid_http_supplied_key`). Both are test keys and +attest nothing. Ed25519 signing is deterministic (RFC 8032), so the file is the same bytes on +every machine. Every run, corpus file, program, path and URL a record names is an example; no +record states a run that took place. + +Each vector declares **one outcome**, set from the extension's text in the generator and never +copied from a verifier: `accept conformant=true`, `accept conformant=false`, or +`reject step= reason=` at a step of §4 with a code of §5. A verifier that reaches +another outcome does not conform. + +## Files + +| file | what it is | +|---|---| +| `vectors.json` | The corpus: each vector's `name`, what it is (`about`), the record's bytes (`record`, a string, since some vectors are bytes a JSON reader must refuse) and its outcome (`expected`); and the corpus keys' public keys. | +| `conform_result_verify.py` | The second implementation, written from the extension's text and sharing no code with `wai-rs`. It takes WAI's strict Ed25519 rule from [`../claim-conformance/_ed25519.py`](../claim-conformance/_ed25519.py), which is written from RFC 8032; it reads the JSON and writes the RFC 8785 form itself. Standard library only. | + +## Vectors + +- **Accepted:** a conformant run (`valid`); a run with a failed case (`valid_not_conformant`), + a genuine record of a run that did not conform; a run with no case (`valid_no_cases`); and a + decoder's run from the HTTP adapter, signed with a runner's key, with a name and version + (`valid_http_supplied_key`). +- **Refused at step 4, `signature_invalid`:** a failed case turned into a pass after signing, + its summary and verdict made to agree (`flipped_case`); the corpus digest changed + (`corpus_digest_changed`); the summary alone changed (`summary_changed`); one bit of the + signature changed (`bad_signature`); the signature's `S` replaced by `S + L` + (`signature_s_not_reduced`); the identity as the key with `(R, S) = (B, 1)`, which the + cofactorless equation accepts over any message (`key_small_order`). +- **Signed as they are, and refused:** a summary that does not count its cases (step 5, + `summary_mismatch`): two counts wrong, and each of the three counts wrong alone with the + others right; conformant though a case failed, and conformant with no case (step 6, + `conformant_mismatch`); another version of the format (step 2, `unknown_format`); a member + missing and a member added (step 3, `malformed`). +- **Bytes refused before a signature is read:** the verdict named twice (step 1); a count + written with a fraction and a digest in upper-case hexadecimal (step 3); bytes that are not a + JSON text (step 1). + +## Checks + +From `wai/wai-rs`: + +``` +cargo run --no-default-features --features conform_result --bin wai_conform_result_vectors -- verify ../conform-result-conformance +cargo test --no-default-features --features conform_result --test conform_result +``` + +`verify` requires `vectors.json` to regenerate byte for byte, no other file but this README and +the second implementation, and every vector to reach its outcome. The test adds that every +reason of §5 refuses a vector, that §5 lists exactly the reference's codes, and that a record +changed anywhere after it is signed is refused. + +From `wai/`: + +``` +python3 -I conform-result-conformance/conform_result_verify.py conform-result-conformance +python3 -I conform-result-conformance/conform_result_verify.py --skip conform-result-conformance +``` + +The first must report every vector's outcome and that the corpus refuses with every reason of §5. +The second leaves out one check of §4 (`duplicates`, `format`, `members`, `signature`, +`summary`, `verdict`) and succeeds only when the corpus then fails: each check is guarded by a +vector. `wai_conform_result_vectors verdicts` and `conform_result_verify.py verdicts` print +`nameoutcome` for any corpus-format file, and `wai conform verifier --suite conform-result` +runs a verifier against this corpus. diff --git a/wai/conform-result-conformance/conform_result_verify.py b/wai/conform-result-conformance/conform_result_verify.py new file mode 100644 index 000000000..13598b4d1 --- /dev/null +++ b/wai/conform-result-conformance/conform_result_verify.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +"""Second verifier for conform-result-conformance/: conformance results, the record +conform-result/1 (extensions/conformance-results.md §2-§5). + +Written from the extension's text. It shares no code with wai-rs (`wai::conform_result`), +which generates the corpus. It takes only WAI's strict Ed25519 rule (SPEC §7 *Signatures*) +from ../claim-conformance/_ed25519.py, itself written from RFC 8032 and independent of +wai-rs; the JSON reading and RFC 8785 serialisation are written here. Standard library only. + + python3 -I conform_result_verify.py [corpus-dir] every vector reaches its stated outcome + python3 -I conform_result_verify.py verdicts `nameoutcome` for every vector + python3 -I conform_result_verify.py --skip [dir] the same with one check of §4 left + out; succeeds only if the corpus then + fails, showing it guards that check + +The checks --skip names: duplicates (step 1's member named twice), format (step 2), members +(step 3), signature (step 4), summary (step 5), verdict (step 6). + +Exit status 0 only if every vector reaches its stated outcome (or, with --skip, only if one +does not). +""" +import hashlib +import json +import os +import re +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..", "claim-conformance")) +import _ed25519 as ED # noqa: E402 + +FORMAT = "conform-result/1" +DOMAIN = b"wai:conform-result" + b"\x01" +MAX_INT = 2 ** 53 - 1 +MAX_DEPTH = 64 +CHECKS = ("duplicates", "format", "members", "signature", "summary", "verdict") +SKIP = set() + + +class Refuse(Exception): + def __init__(self, step, reason): + super().__init__(f"step {step} {reason}") + self.step, self.reason = step, reason + + +class Num: + """A JSON number as written: the reader keeps its spelling.""" + + def __init__(self, text): + self.text = text + + def __eq__(self, other): + return isinstance(other, Num) and other.text == self.text + + def uint(self): + """The value of a plain decimal integer (no sign, fraction or exponent) up to 2^53 - 1.""" + t = self.text + if re.fullmatch(r"0|[1-9][0-9]*", t) and int(t) <= MAX_INT: + return int(t) + return None + + +def _pairs(ps): + d = {} + for k, v in ps: + if k in d and "duplicates" not in SKIP: + raise ValueError("a member named twice: " + k) + d[k] = v + return d + + +def _no_constant(c): + raise ValueError("not JSON: " + c) + + +def _depth_and_strings(v, depth): + """§4 step 1: no value nested more than 64 levels deep, and no string that is not + Unicode text (an escape that leaves a surrogate unpaired).""" + if depth >= MAX_DEPTH: + raise ValueError("nested too deep") + if isinstance(v, str): + v.encode("utf-8") + elif isinstance(v, dict): + for k, x in v.items(): + k.encode("utf-8") + _depth_and_strings(x, depth + 1) + elif isinstance(v, list): + for x in v: + _depth_and_strings(x, depth + 1) + + +def read(raw): + """§4 step 1: one JSON text in UTF-8 whose value is an object, no member named twice.""" + try: + text = raw.decode("utf-8") + v = json.loads(text, object_pairs_hook=_pairs, parse_constant=_no_constant, parse_int=Num, parse_float=Num, strict=True) + _depth_and_strings(v, 0) + except (ValueError, UnicodeError, RecursionError): + raise Refuse(1, "malformed") + if not isinstance(v, dict): + raise Refuse(1, "malformed") + return v + + +def is_hex(s, n): + return isinstance(s, str) and len(s) == n and re.fullmatch(r"[0-9a-f]*", s) is not None + + +def is_str(v): + return isinstance(v, str) + + +def exactly(v, names): + return isinstance(v, dict) and set(v) == set(names) and len(v) == len(names) + + +def is_uint(v): + return isinstance(v, Num) and v.uint() is not None + + +def utc_second(s): + m = isinstance(s, str) and re.fullmatch(r"([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2}):([0-9]{2})Z", s) + if not m: + return False + mo, d, h, mi, se = (int(x) for x in m.groups()[1:]) + return 1 <= mo <= 12 and 1 <= d <= 31 and h <= 23 and mi <= 59 and se <= 60 + + +TOP = ["wai", "kind", "suite", "standard", "toolchain", "corpus", "implementation", "started_at", + "cases", "summary", "conformant", "key", "signer_pubkey", "sig"] + + +def members(r): + """§4 step 3: exactly the members of §2, each of its type and form.""" + ok = exactly(r, TOP) + ok = ok and all(is_str(r[k]) for k in ("kind", "suite", "standard")) + t = r.get("toolchain") + ok = ok and exactly(t, ["name", "version"]) and is_str(t["name"]) and is_str(t["version"]) + c = r.get("corpus") + ok = ok and exactly(c, ["dir", "files", "digest"]) and is_str(c["dir"]) and is_uint(c["files"]) + ok = ok and is_str(c["digest"]) and c["digest"].startswith("sha256:") and is_hex(c["digest"][7:], 64) + i = r.get("implementation") + ok = ok and exactly(i, ["adapter", "impl", "name", "version", "files"]) and i["adapter"] in ("command", "http") + ok = ok and is_str(i["impl"]) and all(x is None or is_str(x) for x in (i["name"], i["version"])) + ok = ok and isinstance(i["files"], list) and all( + exactly(f, ["path", "sha256"]) and is_str(f["path"]) and is_hex(f["sha256"], 64) for f in i["files"]) + ok = ok and utc_second(r["started_at"]) + ok = ok and isinstance(r["cases"], list) and all( + exactly(x, ["name", "pass", "expected", "got"]) and is_str(x["name"]) and isinstance(x["pass"], bool) + and is_str(x["expected"]) and is_str(x["got"]) for x in r["cases"]) + s = r.get("summary") + ok = ok and exactly(s, ["cases", "passed", "failed"]) and all(is_uint(s[k]) for k in ("cases", "passed", "failed")) + ok = ok and isinstance(r["conformant"], bool) and r["key"] in ("ephemeral", "supplied") + ok = ok and is_hex(r["signer_pubkey"], 64) and is_hex(r["sig"], 128) + return bool(ok) + + +def jcs(v): + """RFC 8785 over the values a record holds: members sorted by their UTF-16 code units, + strings escaped as ECMAScript's JSON.stringify escapes them, integers in decimal.""" + if v is None: + return "null" + if v is True: + return "true" + if v is False: + return "false" + if isinstance(v, Num): + return str(v.uint()) + if isinstance(v, str): + return json.dumps(v, ensure_ascii=False) + if isinstance(v, list): + return "[" + ",".join(jcs(x) for x in v) + "]" + keys = sorted(v, key=lambda k: k.encode("utf-16-be")) + return "{" + ",".join(json.dumps(k, ensure_ascii=False) + ":" + jcs(v[k]) for k in keys) + "}" + + +def check(raw): + """§4, in order. Returns the verdict line.""" + try: + r = read(raw) + if "format" not in SKIP and r.get("wai") != FORMAT: + raise Refuse(2, "unknown_format") + if "members" not in SKIP and not members(r): + raise Refuse(3, "malformed") + if "signature" not in SKIP: + body = {k: v for k, v in r.items() if k != "sig"} + msg = DOMAIN + jcs(body).encode("utf-8") + if ED.verify(bytes.fromhex(r["signer_pubkey"]), msg, bytes.fromhex(r["sig"])) is not None: + raise Refuse(4, "signature_invalid") + cases = r["cases"] + passed = sum(1 for c in cases if c["pass"] is True) + s = r["summary"] + if "summary" not in SKIP and (s["cases"].uint(), s["passed"].uint(), s["failed"].uint()) != (len(cases), passed, len(cases) - passed): + raise Refuse(5, "summary_mismatch") + conformant = len(cases) > 0 and passed == len(cases) + if "verdict" not in SKIP and r["conformant"] is not conformant: + raise Refuse(6, "conformant_mismatch") + return "accept conformant=" + ("true" if r["conformant"] else "false") + except Refuse as e: + return f"reject step={e.step} reason={e.reason}" + except (KeyError, TypeError, AttributeError, ValueError): + # Reached only when a check is skipped and a later one reads what it would refuse. + return "error" + + +def reasons_table(): + """§5's reasons, read from the extension.""" + with open(os.path.join(HERE, "..", "extensions", "conformance-results.md"), encoding="utf-8") as f: + text = f.read() + section = text.split("\n## 5. Reasons", 1)[1].split("\n## ", 1)[0] + return {m for m in re.findall(r"^\| `([a-z_]+)` \|", section, re.M)} + + +def vectors(path): + with open(path, encoding="utf-8") as f: + return json.load(f)["vectors"] + + +def run(path): + fail = 0 + b = ED._self_test() if hasattr(ED, "_self_test") else None + if b: + print("FAIL Ed25519 self-test: " + str(b)) + fail += 1 + used = set() + vs = vectors(path) + for v in vs: + got = check(v["record"].encode("utf-8")) + ok = got == v["expected"] + fail += not ok + m = re.search(r"reason=([a-z_]+)$", v["expected"]) + if m: + used.add(m.group(1)) + print(f"{'PASS' if ok else 'FAIL'} {v['name']} expected={v['expected']} got={got}") + table = reasons_table() + same = table == used + print(("PASS" if same else "FAIL") + f" the corpus refuses with every reason of §5 and no other ({sorted(table ^ used)})") + fail += not same + print(f"{len(vs)} vectors, {fail} failure(s)") + return fail + + +def main(argv): + global SKIP + if argv[1:2] == ["verdicts"] and len(argv) == 3: + for v in vectors(argv[2]): + print(f"{v['name']}\t{check(v['record'].encode('utf-8'))}") + return 0 + args = argv[1:] + if args[:1] == ["--skip"]: + if len(args) < 2 or args[1] not in CHECKS: + print("--skip takes one of " + ", ".join(CHECKS)) + return 2 + SKIP = {args[1]} + d = args[2] if len(args) > 2 else HERE + fail = run(os.path.join(d, "vectors.json")) + print(f"with the {args[1]} check left out: {'caught' if fail else 'NOT caught'}") + return 0 if fail else 1 + d = args[0] if args else HERE + return 1 if run(os.path.join(d, "vectors.json")) else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/wai/conform-result-conformance/vectors.json b/wai/conform-result-conformance/vectors.json new file mode 100644 index 000000000..b96a9731c --- /dev/null +++ b/wai/conform-result-conformance/vectors.json @@ -0,0 +1,147 @@ +{ + "about": "conformance-results §7: records of conform-result/1, each with the outcome §4 gives it. Signed with the published test keys of seeds 32 bytes of 0x43 and of 0x44, which attest nothing; every run, program and path in them is an example.", + "public_keys": [ + "22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6", + "d759793bbc13a2819a827c76adb6fba8a49aee007f49f2d0992d99b825ad2c48" + ], + "vectors": [ + { + "name": "valid", + "about": "a conformant run, every case passed, from the command adapter with a key made for the run", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"e9dd68e9c855e98fe4c7a1221429fb48712d5cd14d96a9bd05c74e92577df3d8f531c9c274cb6c024103dd8569d5f0ed8fc4199929e78dc05dcfbdc17d07a804\"}", + "expected": "accept conformant=true" + }, + { + "name": "valid_not_conformant", + "about": "a run with one failed case: a genuine record of a run that did not conform", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":false,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"accept\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":2,\"failed\":1},\"conformant\":false,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"988b7c0a1141b8793f049c84a64033f6138be7c0898a5ac9f13ad0f83055a8533dc1778586a4b31edadef53a20947cd0787b0fc194b01ab5cb156484cb400a0c\"}", + "expected": "accept conformant=false" + }, + { + "name": "valid_no_cases", + "about": "a run with no case: not conformant", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[],\"summary\":{\"cases\":0,\"passed\":0,\"failed\":0},\"conformant\":false,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"efcb2c1b3554fa1115990d121acb0545011df768cdc517c48c9d4e0b100da19316b3082b32d9aa497478e5a25eb86c0e156943f156152cb5ac4053752066990f\"}", + "expected": "accept conformant=false" + }, + { + "name": "valid_http_supplied_key", + "about": "a conformant run of a decoder from the HTTP adapter, signed with a key the runner holds, its name and version given", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"decoder\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"http\",\"impl\":\"http://127.0.0.1:8380/verify\",\"name\":\"an example verifier\",\"version\":\"1.0\",\"files\":[]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":true,\"key\":\"supplied\",\"signer_pubkey\":\"d759793bbc13a2819a827c76adb6fba8a49aee007f49f2d0992d99b825ad2c48\",\"sig\":\"74a2f68e0bd8a05a9ad772a3ffd4f3a90e39d8aacc177d545e9d35a95c44119d8ac0db1f29c65f9d06c2c3fc512e1f013ec50173d39970a710932eed03c1110c\"}", + "expected": "accept conformant=true" + }, + { + "name": "flipped_case", + "about": "the failed case turned into a pass after signing, its summary and verdict made to agree", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"accept\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"988b7c0a1141b8793f049c84a64033f6138be7c0898a5ac9f13ad0f83055a8533dc1778586a4b31edadef53a20947cd0787b0fc194b01ab5cb156484cb400a0c\"}", + "expected": "reject step=4 reason=signature_invalid" + }, + { + "name": "corpus_digest_changed", + "about": "the corpus digest's last digit changed after signing", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad0\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"e9dd68e9c855e98fe4c7a1221429fb48712d5cd14d96a9bd05c74e92577df3d8f531c9c274cb6c024103dd8569d5f0ed8fc4199929e78dc05dcfbdc17d07a804\"}", + "expected": "reject step=4 reason=signature_invalid" + }, + { + "name": "summary_changed", + "about": "the summary alone changed after signing", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":false,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"accept\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":2,\"failed\":0},\"conformant\":false,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"988b7c0a1141b8793f049c84a64033f6138be7c0898a5ac9f13ad0f83055a8533dc1778586a4b31edadef53a20947cd0787b0fc194b01ab5cb156484cb400a0c\"}", + "expected": "reject step=4 reason=signature_invalid" + }, + { + "name": "bad_signature", + "about": "one bit of the signature changed", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"e8dd68e9c855e98fe4c7a1221429fb48712d5cd14d96a9bd05c74e92577df3d8f531c9c274cb6c024103dd8569d5f0ed8fc4199929e78dc05dcfbdc17d07a804\"}", + "expected": "reject step=4 reason=signature_invalid" + }, + { + "name": "signature_s_not_reduced", + "about": "the signature's S replaced by S + L, which verifies under the cofactorless equation reduced mod L but not under the strict rule", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"e9dd68e9c855e98fe4c7a1221429fb48712d5cd14d96a9bd05c74e92577df3d8e205bf1f8f2e7f5a17a0d42848cfcf0290c4199929e78dc05dcfbdc17d07a814\"}", + "expected": "reject step=4 reason=signature_invalid" + }, + { + "name": "key_small_order", + "about": "the identity point as the key with (R, S) = (B, 1), which the cofactorless equation accepts over any message", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"0100000000000000000000000000000000000000000000000000000000000000\",\"sig\":\"58666666666666666666666666666666666666666666666666666666666666660100000000000000000000000000000000000000000000000000000000000000\"}", + "expected": "reject step=4 reason=signature_invalid" + }, + { + "name": "summary_mismatch_signed", + "about": "signed as it is: a summary that does not count its cases", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":false,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"accept\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":false,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"a817be73b9f129adf5e67e48ea19ea5ea122df1bcce00caac8fd06a4154f319786cbe6e26382402bc3a3f30b6f584ffea6decff2f755f468aba68ec76ac2f200\"}", + "expected": "reject step=5 reason=summary_mismatch" + }, + { + "name": "summary_cases_mismatch_signed", + "about": "signed as it is: a case count one more than its cases, its passed and failed right", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":false,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"accept\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":4,\"passed\":2,\"failed\":1},\"conformant\":false,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"5d30d9eb3b01a65112f048bbe9dcdf85abef30cbea3e5d8832c09237a51a0189fd262e1ee9aa5f10488be40259b75cddb06550b2b641cc5fee1d5872ab80d907\"}", + "expected": "reject step=5 reason=summary_mismatch" + }, + { + "name": "summary_passed_mismatch_signed", + "about": "signed as it is: a passed count one more than its cases pass, its other counts right", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":false,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"accept\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":1},\"conformant\":false,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"e77942ca2e7278a07eb78fe94e9f808c211e726d4b01d8d2d8b10ff8c5711dcae25038296d6a11a1c3a14cd31e71e85409e16775d5c71e5b9ef9c3392b473207\"}", + "expected": "reject step=5 reason=summary_mismatch" + }, + { + "name": "summary_failed_mismatch_signed", + "about": "signed as it is: a failed count one less than its cases fail, its other counts right", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":false,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"accept\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":2,\"failed\":0},\"conformant\":false,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"2f3270617bb5b746f4357175afcd7b0096aeb28a21cb8553cfcc5cb20dea07f38e4deda292447e8cf33ec5276d6f8c361c1aece305afbbbd07a999f9fe71630c\"}", + "expected": "reject step=5 reason=summary_mismatch" + }, + { + "name": "conformant_mismatch_signed", + "about": "signed as it is: conformant though a case failed", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":false,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"accept\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":2,\"failed\":1},\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"9a822a7581ce6828e53a8010b3e74af8042b9fa3e75aa093760f5f483fba3fd27ea7fde36c4f5f2615ab10a486ef62bd22150642733aefda27182f1170745700\"}", + "expected": "reject step=6 reason=conformant_mismatch" + }, + { + "name": "conformant_no_cases_signed", + "about": "signed as it is: conformant with no case", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[],\"summary\":{\"cases\":0,\"passed\":0,\"failed\":0},\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"ff8802bb7268c58e149e6e3a83ed90cda92b5d6b89d59658e80a274ff250624998136d3924139e30f6880a5e5f6df9996522204a3df73b0623368298da5c7500\"}", + "expected": "reject step=6 reason=conformant_mismatch" + }, + { + "name": "unknown_format", + "about": "signed as it is: a version this revision does not define", + "record": "{\"wai\":\"conform-result/2\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"9b004322af6b5d17316e311765435a037c531ad4617df6504fe3ce5c2cd6a8c01bfee4c56676115ae2e39bb18e456583ec600bd78c1ba009cba78af74663710c\"}", + "expected": "reject step=2 reason=unknown_format" + }, + { + "name": "missing_member", + "about": "signed as it is: no corpus member", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"key\":\"ephemeral\",\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":true,\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"aaaeec52c150ea472a45ef47b4a54f152fd2c2d6a79e9abb2397f409b6ff4eccb4c360c4c9413685140b7426a4b9df6f444d7a2fd67a45d445ba2de6cbe9dd06\"}", + "expected": "reject step=3 reason=malformed" + }, + { + "name": "extra_member", + "about": "signed as it is: a member §2 does not list", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":true,\"key\":\"ephemeral\",\"note\":\"a member §2 does not list\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"6ca3491e6a018d64a9fdb5fa0eaa096a6e6b744d5c6646e4b4d2d690a6e3c943b137023b39fbd7ba3bf0bd51157e99f729411228851443d69700f0c8303ed600\"}", + "expected": "reject step=3 reason=malformed" + }, + { + "name": "duplicate_member", + "about": "the valid record with its verdict named twice, the second time true", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":false,\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"e9dd68e9c855e98fe4c7a1221429fb48712d5cd14d96a9bd05c74e92577df3d8f531c9c274cb6c024103dd8569d5f0ed8fc4199929e78dc05dcfbdc17d07a804\"}", + "expected": "reject step=1 reason=malformed" + }, + { + "name": "fraction_count", + "about": "the valid record with a count written with a fraction (3.0)", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3.0,\"passed\":3,\"failed\":0},\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"e9dd68e9c855e98fe4c7a1221429fb48712d5cd14d96a9bd05c74e92577df3d8f531c9c274cb6c024103dd8569d5f0ed8fc4199929e78dc05dcfbdc17d07a804\"}", + "expected": "reject step=3 reason=malformed" + }, + { + "name": "digest_upper_case", + "about": "the valid record with its corpus digest in upper-case hexadecimal", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360CE9739BEFC69B6773ED9FC087B2A9611EB75910E62796EA1B886770DAD9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"e9dd68e9c855e98fe4c7a1221429fb48712d5cd14d96a9bd05c74e92577df3d8f531c9c274cb6c024103dd8569d5f0ed8fc4199929e78dc05dcfbdc17d07a804\"}", + "expected": "reject step=3 reason=malformed" + }, + { + "name": "not_json", + "about": "bytes that are not a JSON text", + "record": "{\"wai\":\"conform-result/1\",\"kind\":\"verifier\",\"suite\":\"claim\",\"standard\":\"v0.4.0\",\"toolchain\":{\"name\":\"wai\",\"version\":\"0.3.0\"},\"corpus\":{\"dir\":\"claim-conformance\",\"files\":1,\"digest\":\"sha256:50360ce9739befc69b6773ed9fc087b2a9611eb75910e62796ea1b886770dad9\"},\"implementation\":{\"adapter\":\"command\",\"impl\":\"example-verifier verdicts {corpus}\",\"name\":null,\"version\":null,\"files\":[{\"path\":\"/opt/example/bin/example-verifier\",\"sha256\":\"6b0cf49408a2a43a27d45717c016625528f9e09892b99827c8a3aaefe1aed964\"}]},\"started_at\":\"2026-10-27T12:00:00Z\",\"cases\":[{\"name\":\"signature_invalid\",\"pass\":true,\"expected\":\"reject step=3 reason=signature_invalid\",\"got\":\"reject step=3 reason=signature_invalid\"},{\"name\":\"bad_signer_key\",\"pass\":true,\"expected\":\"reject step=3 reason=bad_signer_key\",\"got\":\"reject step=3 reason=bad_signer_key\"},{\"name\":\"field_type\",\"pass\":true,\"expected\":\"reject step=3 reason=field_type\",\"got\":\"reject step=3 reason=field_type\"}],\"summary\":{\"cases\":3,\"passed\":3,\"failed\":0},\"conformant\":true,\"key\":\"ephemeral\",\"signer_pubkey\":\"22fc297792f0b6ffc0bfcfdb7edb0c0aa14e025a365ec0e342e86e3829cb74b6\",\"sig\":\"e9dd68e9c855e98fe4c7a1221429fb48712d5cd14d96a9bd05c74e92577df3d8f531c9c274cb6c024103dd8569d5f0ed8fc4199929e78dc05dcfbdc17d07a804\"", + "expected": "reject step=1 reason=malformed" + } + ] +} diff --git a/wai/extensions/conformance-results.md b/wai/extensions/conformance-results.md new file mode 100644 index 000000000..e2ea5200e --- /dev/null +++ b/wai/extensions/conformance-results.md @@ -0,0 +1,151 @@ +# WAI Extension: Conformance Results + +> Status: Draft. Defines the record `conform-result/1`: the signed outcome of running one of +> WAI's conformance corpora against an implementation, whoever wrote it. The record names the +> corpus by digest, the implementation by what ran, and each case by its expected and its +> actual answer; its summary and its verdict follow from its cases; and it is signed with +> Ed25519 over its RFC 8785 form. A verifier checks the record as written, offline: that it is +> well formed, that the signature holds, and that the summary and the verdict are what the +> cases say. No new transport. The reference implementation is the `conform_result` module of +> `wai-rs` (feature `conform_result`), which `wai conform` writes records with and `wai verify` +> checks them with; the corpus is +> [`conform-result-conformance/`](../conform-result-conformance/), and its second +> implementation is `conform-result-conformance/conform_result_verify.py`. +> Keywords **MUST**, **MUST NOT**, **SHOULD**, **MAY** are RFC 2119/8174. +> +> A section number in this text is this extension's unless it names another document. + +## 1. Scope + +In scope: the record (§2), its signing input and signature (§3), what a verifier checks and in +what order (§4), and the codes it refuses with (§5). + +Out of scope: how a run reaches an implementation (a command line, a URL), which corpora and +suites exist, and what makes an implementation conformant beyond the run a record states. A +record states what one run observed; it is not a certificate. + +## 2. The record + +A conformance result is one JSON text (RFC 8259) in UTF-8: an object with exactly the members +below, each of the type given, and no member named twice at any level. Every number in it is an +integer from 0 to 2^53 − 1 written in decimal digits, without a sign, a fraction or an exponent. +Every other number, and every member not listed, makes the record malformed. + +| member | type | value | +|---|---|---| +| `wai` | string | `conform-result/1`, the record's kind and version | +| `kind` | string | the kind of implementation run: `verifier`, `decoder`, `packager`, `origin`, or another kind a later revision names | +| `suite` | string | the suite that ran | +| `standard` | string | the version of WAI whose corpus ran (`v0.4.0`) | +| `toolchain` | object | the program that ran the corpus: `name` and `version`, each a string | +| `corpus` | object | `dir`, the corpus's directory under `wai/` (a string); `files`, the number of corpus files the run read (an integer); `digest`, the corpus digest of §2.1 | +| `implementation` | object | `adapter` (`command` or `http`); `impl`, the command line or the URL (a string); `name` and `version`, what the implementer calls it (each a string or `null`); `files`, an array of objects with exactly `path` (a string) and `sha256` (64 lowercase hexadecimal digits), the files the command names (§2.2) | +| `started_at` | string | when the run started, as `YYYY-MM-DDTHH:MM:SSZ`: RFC 3339 §5.6 in UTC, to the second, with month 01–12, day 01–31, hour 00–23, minute 00–59 and second 00–60 | +| `cases` | array | one object a case, with exactly `name` (a string), `pass` (a boolean), `expected` and `got` (each a string: what the corpus states, and what the implementation answered) | +| `summary` | object | exactly `cases`, `passed` and `failed`, each an integer | +| `conformant` | boolean | whether the run passed | +| `key` | string | `ephemeral`, for a key made for the run, or `supplied`, for a key the runner holds (§6) | +| `signer_pubkey` | string | the Ed25519 public key that signed the record, 64 lowercase hexadecimal digits | +| `sig` | string | its signature, 128 lowercase hexadecimal digits (§3) | + +A writer MUST set `summary.cases` to the number of cases, `summary.passed` to the number whose +`pass` is `true` and `summary.failed` to the number whose `pass` is `false`; and MUST set +`conformant` to `true` exactly when there is at least one case and every case passed. + +### 2.1 The corpus digest + +The corpus digest is `sha256:` followed by 64 lowercase hexadecimal digits: the SHA-256 of the +concatenation of one line for every corpus file the run read, each line the file's SHA-256 in +lowercase hexadecimal, two spaces, the file's path relative to `wai/` with `/` between its +segments, and a line feed; the lines sorted in ascending byte order, and a line that occurs twice +taken once. A party that holds the corpus can compute the digest and compare it; a record made +against another corpus has another digest. + +### 2.2 The implementation + +For the `command` adapter, `files` holds the program the command line runs, as found on the +runner's `PATH`, and every argument of the command line that is a file, each with the SHA-256 of +its bytes when the run started. For the `http` adapter it is empty, and `impl` is the URL. These +name what ran on the runner's machine; they are statements of the runner, covered by the +signature like every other member. + +## 3. The signature + +The signing input is the 18 bytes `wai:conform-result` followed by the byte `0x01`, followed by +the record without its `sig` member, serialised as RFC 8785 specifies (its members sorted, no +insignificant whitespace). `signer_pubkey` is part of the signed record. + +`sig` is a pure Ed25519 signature (RFC 8032 §5.1) over the signing input by the key +`signer_pubkey`. It is verified by WAI's strict rule ([SPEC](../SPEC.md) §7 *Signatures*): +the key canonical and not of small order, `S` < L, `R` not of small order, and the cofactorless +equation compared byte for byte. + +## 4. Verification + +A verifier checks a record in this order and refuses it at the first step that fails, with the +code that step names (§5): + +1. **It reads.** The bytes are one JSON text in UTF-8 whose value is an object, and no object in + it names a member twice. Otherwise `malformed`. +2. **It is this format.** The member `wai` is the string `conform-result/1`. Otherwise + `unknown_format`. +3. **Its members.** It has exactly the members of §2, each of the type and form §2 gives. + Otherwise `malformed`. +4. **Its signature.** `sig` verifies over the signing input of §3 under `signer_pubkey`, by the + strict rule. Otherwise `signature_invalid`. +5. **Its summary.** `summary.cases`, `summary.passed` and `summary.failed` are what its cases + count (§2). Otherwise `summary_mismatch`. +6. **Its verdict.** `conformant` is `true` exactly when there is at least one case and every case + passed. Otherwise `conformant_mismatch`. + +A record that passes every step is accepted, and its verdict is its `conformant`. A verifier +writes its outcome as `accept conformant=true`, `accept conformant=false`, or +`reject step= reason=`. A record that says its run was not conformant is a genuine +record of that run, and is accepted as one. + +## 5. Reasons + +The codes this extension defines, each with the step that raises it: + +| reason | raised at | +|---|---| +| `malformed` | §4 steps 1 and 3 | +| `unknown_format` | §4 step 2 | +| `signature_invalid` | §4 step 4 | +| `summary_mismatch` | §4 step 5 | +| `conformant_mismatch` | §4 step 6 | + +## 6. What a signature shows + +A record signed with a key made for its run (`"key": "ephemeral"`) shows that the record is the +one the run wrote: no case, count, digest or name was changed after it was signed. It does not +say who ran it. A record signed with a key the runner holds (`"key": "supplied"`) says, in +addition, that the holder of that key ran it; which party that is, a relying party learns as it +learns any key, for instance from a key log (jwp-receipts §7). Neither makes the implementation's +other behaviour conform, and neither makes the run's statements true: a runner can run a corpus +other than the one it names, and the digest of §2.1 is how a party holding the corpus checks +which one ran. + +## 7. Conformance + +[`conform-result-conformance/`](../conform-result-conformance/) holds records signed with a +published test key: accepted ones (conformant, not conformant, with no cases, from either +adapter and either kind of key) and refused ones (a case's outcome changed after signing, the +corpus digest changed, the summary alone changed, a signature changed in one bit, an `S` past +L, a key of small order, a summary that disagrees with the cases though signed (each of its +three counts wrong alone, and two at once), a verdict that disagrees with them, an unknown +version, a member missing, a member added, a member named twice, a count with a fraction, a +digest out of form, bytes that are not JSON), each with the outcome §4 gives it. +`wai_conform_result_vectors verify` (in `wai-rs`, feature `conform_result`) and the independent +`conform_result_verify.py`, which shares no code with `wai-rs`, reach every vector's outcome. + +## 8. Security and privacy considerations + +- **What a record reveals.** The command line and the paths of the files it names are the + runner's, and can name a user or a machine; the URL names where the implementation ran. A + runner SHOULD run from paths and addresses it is willing to publish with the record. +- **Accepted is not trusted.** Acceptance shows the record is intact and consistent. Whether + the corpus it names is WAI's, and whether its signer is a party to rely on, a relying party + decides from the digest (§2.1) and from what it knows of the key (§6). +- **Bounded reading.** A verifier reads one JSON object and its members once. It MAY refuse a + record larger than it is prepared to read, as `malformed`. diff --git a/wai/registry/capabilities.json b/wai/registry/capabilities.json index 3cd6b599a..7598e0868 100644 --- a/wai/registry/capabilities.json +++ b/wai/registry/capabilities.json @@ -3255,6 +3255,13 @@ "section": "§3", "status": "proposed" }, + { + "name": "conform-result/1", + "kind": "document-kind", + "document": "extensions/conformance-results.md", + "section": "§2", + "status": "defined" + }, { "name": "contentBinding", "kind": "frame-metadata", diff --git a/wai/wai-rs/Cargo.toml b/wai/wai-rs/Cargo.toml index e84042286..fd68611b5 100644 --- a/wai/wai-rs/Cargo.toml +++ b/wai/wai-rs/Cargo.toml @@ -159,7 +159,9 @@ floor = ["dep:image", "dep:claxon", "dep:zstd"] # extensions/fleet.md §7 verifies it), and `wai mcp`, the server # `wai setup-agent` registers. Pure Rust and the floor's zstd: no system library. `wai create`, # `wai dev` and `wai setup-agent` need no feature. -quickstart = ["floor", "http", "receipt_claims", "mcp", "fleet", "dep:getrandom"] +quickstart = ["floor", "http", "receipt_claims", "conform_result", "mcp", "fleet", "dep:getrandom"] +# extensions/conformance-results.md: the record `conform-result/1`, its signature and verification. +conform_result = ["ed25519"] # Pulls in the ONNX runtime + zstd unpack so the native sink can # decode `wai.neural.<...>` envelopes. Off by default — the bare # container + zeroth codecs build with no ML dep chain. @@ -1427,6 +1429,12 @@ name = "wai_signer_vectors" path = "src/bin/wai_signer_vectors.rs" required-features = ["key_discovery"] +# The corpus of extensions/conformance-results.md §7 (`conform-result-conformance/`). +[[bin]] +name = "wai_conform_result_vectors" +path = "src/bin/wai_conform_result_vectors.rs" +required-features = ["conform_result"] + [[bin]] name = "wai_pq_vectors" path = "src/bin/wai_pq_vectors.rs" @@ -1499,6 +1507,22 @@ name = "cli_dx" path = "tests/cli_dx.rs" required-features = ["quickstart"] +# `wai breaking` needs no feature; `wai conform` runs the reference verifiers, the score +# renderer and the origin of this crate as the implementations under test. +[[test]] +name = "cli_breaking" +path = "tests/cli_breaking.rs" + +[[test]] +name = "conform_result" +path = "tests/conform_result.rs" +required-features = ["conform_result"] + +[[test]] +name = "cli_conform" +path = "tests/cli_conform.rs" +required-features = ["quickstart", "score", "key_discovery"] + [[test]] name = "cli_fleet" path = "tests/cli_fleet.rs" diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/c_header.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/c_header.rs new file mode 100644 index 000000000..0669138f1 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/c_header.rs @@ -0,0 +1,368 @@ +//! API: the C header (`wai/wai-rs/include/wai.h`, generated by cbindgen from the FFI and +//! committed). A program compiled and linked against the old header must still build and +//! run against the new library. So a function that is gone, or whose return type or +//! parameter types changed (their names are not compared), breaks; so does a struct whose +//! fields changed in any way (its size or layout differs), an opaque type or typedef that +//! is gone or changed, an enumerator or macro constant that is gone or holds another +//! value, and a declaration moved under a preprocessor condition. New declarations are +//! compatible. + +use super::{Category, Finding}; +use std::collections::BTreeMap; + +pub const PATH: &str = "wai/wai-rs/include/wai.h"; + +/// One declaration of the header. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Decl { + /// A function: its return type and its parameters' types, in order. + Function { ret: String, params: Vec }, + /// A struct or union with its fields (`type name`, in order), or none for an opaque one. + Struct { fields: Option> }, + /// An enumeration: each enumerator and its value. + Enum { values: Vec<(String, String)> }, + Typedef { ty: String }, + /// A macro constant, with every value it is defined to (cbindgen repeats some). + Macro { values: Vec }, + Variable { ty: String }, +} + +/// A declaration and the preprocessor conditions it sits under (empty for none). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Item { + pub decl: Decl, + pub cond: String, +} + +fn strip_comments(src: &str) -> String { + let mut out = String::with_capacity(src.len()); + let b = src.as_bytes(); + let mut i = 0; + while i < b.len() { + if b[i] == b'/' && b.get(i + 1) == Some(&b'*') { + i += 2; + while i + 1 < b.len() && !(b[i] == b'*' && b[i + 1] == b'/') { + if b[i] == b'\n' { + out.push('\n'); + } + i += 1; + } + i += 2; + } else if b[i] == b'/' && b.get(i + 1) == Some(&b'/') { + while i < b.len() && b[i] != b'\n' { + i += 1; + } + } else { + let ch = src[i..].chars().next().unwrap_or(' '); + out.push(ch); + i += ch.len_utf8(); + } + } + out +} + +/// Whitespace collapsed, with `*`, `(`, `)`, `[`, `]` and `,` set apart so spellings +/// that differ only in spacing compare equal. +fn norm(s: &str) -> String { + let spaced: String = s.chars().flat_map(|c| if "*()[],".contains(c) { vec![' ', c, ' '] } else { vec![c] }).collect(); + spaced.split_whitespace().collect::>().join(" ") +} + +fn ident(s: &str) -> bool { + !s.is_empty() && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') && !s.starts_with(|c: char| c.is_ascii_digit()) +} + +/// A parameter's type: its text without the trailing name. +fn param_type(p: &str) -> String { + let n = norm(p); + let toks: Vec<&str> = n.split(' ').collect(); + let last = toks.last().copied().unwrap_or(""); + let basic = matches!(last, "void" | "int" | "char" | "long" | "short" | "unsigned" | "signed" | "float" | "double" | "bool"); + if toks.len() > 1 && ident(last) && !basic { toks[..toks.len() - 1].join(" ") } else { n.clone() } +} + +fn split_top(s: &str, sep: char) -> Vec { + let mut out = Vec::new(); + let (mut depth, mut cur) = (0i32, String::new()); + for c in s.chars() { + match c { + '(' | '{' | '[' => depth += 1, + ')' | '}' | ']' => depth -= 1, + _ => {} + } + if c == sep && depth == 0 { + out.push(std::mem::take(&mut cur)); + } else { + cur.push(c); + } + } + if !cur.trim().is_empty() { + out.push(cur); + } + out +} + +fn declaration(text: &str, cond: &str, out: &mut BTreeMap) { + let t = norm(text); + if t.is_empty() { + return; + } + let item = |decl| Item { decl, cond: cond.to_string() }; + let body = |t: &str| -> Option<(String, String, String)> { + let open = t.find('{')?; + let close = t.rfind('}')?; + Some((t[..open].trim().to_string(), t[open + 1..close].trim().to_string(), t[close + 1..].trim().to_string())) + }; + if let Some((head, inner, tail)) = body(&t) { + let words: Vec<&str> = head.split(' ').collect(); + let is_enum = words.contains(&"enum"); + let tag = words.last().filter(|w| ident(w) && !matches!(**w, "struct" | "union" | "enum" | "typedef")).map(|w| w.to_string()); + let name = if tail.is_empty() { tag } else { Some(tail.trim_end_matches(';').trim().to_string()) }; + let Some(name) = name else { return }; + if is_enum { + let mut values = Vec::new(); + let mut next: i128 = 0; + for e in split_top(&inner, ',') { + let e = e.trim(); + if e.is_empty() { + continue; + } + let (n, v) = match e.split_once('=') { + Some((n, v)) => (n.trim().to_string(), norm(v)), + None => (e.to_string(), next.to_string()), + }; + next = v.parse::().map(|x| x + 1).unwrap_or(next + 1); + values.push((n, v)); + } + out.insert(name, item(Decl::Enum { values })); + } else { + let fields = split_top(&inner, ';').into_iter().map(|f| norm(&f)).filter(|f| !f.is_empty()).collect(); + out.insert(name, item(Decl::Struct { fields: Some(fields) })); + } + return; + } + if let Some(rest) = t.strip_prefix("typedef ") { + let words: Vec<&str> = rest.split(' ').collect(); + if matches!(words.first(), Some(&"struct") | Some(&"union")) && words.len() == 3 && !rest.contains('(') { + out.entry(words[2].to_string()).or_insert_with(|| item(Decl::Struct { fields: None })); + return; + } + // `typedef `, or a function pointer `typedef R ( * name ) ( … )`. + let name = if let Some(i) = rest.find("( * ") { + rest[i + 4..].split(' ').next().unwrap_or("").to_string() + } else { + words.last().unwrap_or(&"").to_string() + }; + if ident(&name) { + out.insert(name, item(Decl::Typedef { ty: rest.to_string() })); + } + return; + } + if let Some(open) = t.find(" ( ") { + let head = &t[..open]; + let name = head.split(' ').next_back().unwrap_or("").to_string(); + if !ident(&name) { + return; + } + let ret = head[..head.len() - name.len()].trim().trim_start_matches("extern ").to_string(); + let close = t.rfind(')').unwrap_or(t.len()); + let inner = &t[open + 3..close]; + let params: Vec = split_top(inner, ',').iter().map(|p| param_type(p)).filter(|p| !p.is_empty()).collect(); + let params = if params == ["void"] { Vec::new() } else { params }; + out.insert(name, item(Decl::Function { ret, params })); + return; + } + let words: Vec<&str> = t.split(' ').collect(); + if words.len() >= 2 && ident(words[words.len() - 1]) { + let name = words[words.len() - 1].to_string(); + let ty = words[..words.len() - 1].join(" ").trim_start_matches("extern ").to_string(); + out.insert(name, item(Decl::Variable { ty })); + } +} + +/// Every declaration of a header, by name. +pub fn parse(src: &str) -> BTreeMap { + let text = strip_comments(src); + let mut out: BTreeMap = BTreeMap::new(); + let mut conds: Vec = Vec::new(); + let mut guard: Option = None; + let mut pending = String::new(); + let mut lines = text.lines().peekable(); + while let Some(line) = lines.next() { + let mut l = line.trim().to_string(); + while l.ends_with('\\') { + l.pop(); + l.push(' '); + l.push_str(lines.next().unwrap_or("").trim()); + } + if let Some(d) = l.strip_prefix('#') { + let d = d.trim(); + let (word, rest) = d.split_once(char::is_whitespace).map(|(a, b)| (a, b.trim())).unwrap_or((d, "")); + match word { + "ifndef" if guard.is_none() && conds.is_empty() => { + guard = Some(rest.to_string()); + conds.push(String::new()); + } + "if" | "ifdef" | "ifndef" => conds.push(format!("{word} {rest}")), + "else" => { + if let Some(c) = conds.last_mut() { + *c = format!("not ({c})"); + } + } + "elif" => { + if let Some(c) = conds.last_mut() { + *c = format!("elif {rest}"); + } + } + "endif" => { + conds.pop(); + } + "define" => { + let (name, value) = rest.split_once(char::is_whitespace).map(|(a, b)| (a, norm(b))).unwrap_or((rest, String::new())); + if Some(name) != guard.as_deref() && ident(name) { + let cond = conds.iter().filter(|c| !c.is_empty()).cloned().collect::>().join(" && "); + match out.get_mut(name) { + Some(Item { decl: Decl::Macro { values }, .. }) => { + if !values.contains(&value) { + values.push(value); + values.sort(); + } + } + _ => { + out.insert(name.to_string(), Item { decl: Decl::Macro { values: vec![value] }, cond }); + } + } + } + } + _ => {} + } + continue; + } + // The `extern "C" {` wrapper sits under `__cplusplus`; nothing there is a declaration. + if conds.iter().any(|c| c.contains("__cplusplus")) { + continue; + } + let cond = conds.iter().filter(|c| !c.is_empty()).cloned().collect::>().join(" && "); + for ch in l.chars().chain(std::iter::once(' ')) { + pending.push(ch); + if ch == ';' { + let depth: i32 = pending.chars().map(|c| match c { + '{' => 1, + '}' => -1, + _ => 0, + }).sum(); + if depth == 0 { + let decl = std::mem::take(&mut pending); + declaration(decl.trim().trim_end_matches(';'), &cond, &mut out); + } + } + } + } + out +} + +fn show(d: &Decl, name: &str) -> String { + match d { + Decl::Function { ret, params } => format!("{ret} {name}({})", params.join(", ")), + Decl::Struct { fields: Some(f) } => format!("struct {name} {{ {} }}", f.join("; ")), + Decl::Struct { fields: None } => format!("struct {name} (opaque)"), + Decl::Enum { values } => format!("enum {name} {{ {} }}", values.iter().map(|(n, v)| format!("{n} = {v}")).collect::>().join(", ")), + Decl::Typedef { ty } => format!("typedef {ty}"), + Decl::Macro { values } => format!("#define {name} {}", values.join(" | ")), + Decl::Variable { ty } => format!("{ty} {name}"), + } +} + +fn kind(d: &Decl) -> &'static str { + match d { + Decl::Function { .. } => "function", + Decl::Struct { .. } => "struct", + Decl::Enum { .. } => "enum", + Decl::Typedef { .. } => "typedef", + Decl::Macro { .. } => "constant", + Decl::Variable { .. } => "variable", + } +} + +/// The header's findings, from the two headers (`None` where a tree has none). +pub fn compare(old: Option<&str>, new: Option<&str>) -> Vec { + let a = old.map(parse).unwrap_or_default(); + let b = new.map(parse).unwrap_or_default(); + let mut out = Vec::new(); + for (name, x) in &a { + let k = kind(&x.decl); + let Some(y) = b.get(name) else { + out.push(Finding::broke( + Category::Api, + "c", + name.clone(), + "removed".into(), + format!("the {k} {} is gone", show(&x.decl, name)), + format!("a program that uses {name} no longer compiles against the header, or no longer links against the library"), + )); + continue; + }; + let why = match (&x.decl, &y.decl) { + _ if x.cond != y.cond => Some(format!("now declared under `{}` (was `{}`)", y.cond, x.cond)), + (Decl::Struct { fields: Some(_) }, Decl::Struct { fields: None }) => Some("its fields are no longer declared".into()), + (Decl::Struct { fields: None }, Decl::Struct { fields: Some(_) }) => None, + (Decl::Enum { values: va }, Decl::Enum { values: vb }) => { + let gone: Vec = va.iter().filter(|e| !vb.contains(e)).map(|(n, v)| format!("{n} = {v}")).collect(); + (!gone.is_empty()).then(|| format!("{} gone or renumbered", gone.join(", "))) + } + (Decl::Macro { values: va }, Decl::Macro { values: vb }) => { + (va.iter().any(|v| !vb.contains(v))).then(|| format!("value {} → {}", va.join(" | "), vb.join(" | "))) + } + (p, q) if p != q => Some(format!("{} → {}", show(p, name), show(q, name))), + _ => None, + }; + if let Some(what) = why { + let reason = match &x.decl { + Decl::Function { .. } => "a caller compiled against the old prototype passes or reads the wrong types: undefined behaviour, or a build that fails", + Decl::Struct { .. } => "the struct's size or layout differs, so a caller compiled against the old header reads and writes the wrong bytes", + Decl::Macro { .. } | Decl::Enum { .. } => "a program compiled with the old value passes or compares a value the library now reads differently", + _ => "a program compiled against the old declaration no longer matches the library", + }; + out.push(Finding::broke(Category::Api, "c", name.clone(), "changed".into(), format!("the {k} {name}: {what}"), reason.into())); + } else if x.decl != y.decl { + out.push(Finding::compatible(Category::Api, "c", name.clone(), format!("{} → {}", show(&x.decl, name), show(&y.decl, name)))); + } + } + for (name, y) in &b { + if !a.contains_key(name) { + out.push(Finding::compatible(Category::Api, "c", name.clone(), format!("the {} {} is new", kind(&y.decl), show(&y.decl, name)))); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + const H: &str = "#ifndef WAI_H\n#define WAI_H\n#include \n/** doc */\n#define WAI_OK 0\n#define CREATE 1\n#define CREATE 1\n\ + typedef struct wai_sink_t wai_sink_t;\ntypedef struct wai_buffer_t {\n uint8_t *data; // the bytes\n uintptr_t len;\n} wai_buffer_t;\n\ + #ifdef __cplusplus\nextern \"C\" {\n#endif\n int wai_pack(const char *manifest,\n const uint8_t *payload, uintptr_t len,\n struct wai_buffer_t *out);\n\ + uint32_t wai_abi(void);\n#ifdef __cplusplus\n}\n#endif\n#endif\n"; + + #[test] + fn parses_what_cbindgen_writes() { + let m = parse(H); + assert_eq!(m["wai_pack"].decl, Decl::Function { ret: "int".into(), params: vec!["const char *".into(), "const uint8_t *".into(), "uintptr_t".into(), "struct wai_buffer_t *".into()] }); + assert_eq!(m["wai_abi"].decl, Decl::Function { ret: "uint32_t".into(), params: vec![] }); + assert_eq!(m["wai_buffer_t"].decl, Decl::Struct { fields: Some(vec!["uint8_t * data".into(), "uintptr_t len".into()]) }); + assert_eq!(m["wai_sink_t"].decl, Decl::Struct { fields: None }); + assert_eq!(m["CREATE"].decl, Decl::Macro { values: vec!["1".into()] }); + assert!(!m.contains_key("WAI_H")); + assert_eq!(m["wai_pack"].cond, ""); + } + + #[test] + fn a_changed_parameter_type_breaks_and_a_renamed_one_does_not() { + let renamed = H.replace("uintptr_t len,\n", "uintptr_t payload_len,\n"); + assert!(compare(Some(H), Some(&renamed)).iter().all(|f| !f.breaking)); + let changed = H.replace("uintptr_t len,\n", "uint32_t len,\n"); + let f = compare(Some(H), Some(&changed)); + assert!(f.iter().any(|f| f.breaking && f.id == "api:c:wai_pack"), "{f:?}"); + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/corpora.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/corpora.rs new file mode 100644 index 000000000..1bb9cbbd3 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/corpora.rs @@ -0,0 +1,160 @@ +//! Wire and record: the conformance corpora (`wai/*-conformance/`) and the pinned vectors +//! of the reference (`wai/wai-rs/tests/vectors/`). Each vector states bytes a conforming +//! reader must accept or refuse, and what it must make of them: a decoded output's digest, +//! a verdict, a reason. The conformance workflow holds every corpus to the code that +//! generates it, byte for byte, so a change in what the code writes or signs (a field, a +//! signing preimage, a magic) reaches these files, or that workflow fails. +//! +//! So a vector file that is gone, or whose bytes changed, breaks; a JSON file breaks where +//! an entry or member is gone or a value changed, and is compatible where it only gained +//! entries; a JSON Lines file breaks where a line is gone. A new file is compatible. +//! Verifier code, documentation and scripts beside a corpus are not vectors and are not +//! compared, and neither are the files [`MEASUREMENTS`] lists: records of what one run +//! measured, kept beside a corpus, which a second measurement changes without changing any +//! rule an implementation is held to. The record corpora (signed records and their verdicts) +//! report under `record`; the others, under `wire`. + +use super::git::{Repo, Status}; +use super::jsondiff; +use super::super::envelope::OpError; +use super::{Category, Finding}; + +pub const PATHSPECS: &[&str] = &[":(glob)wai/*-conformance/**", ":(glob)wai/wai-rs/tests/vectors/**"]; + +/// The corpora of signed records and their verdicts. +const RECORD_CORPORA: &[&str] = &[ + "claim-conformance", + "signer-conformance", + "jwp-conformance", + "pq-conformance", + "oer-conformance", + "pin-conformance", + "edge-conformance", + "live-conformance", + "conform-result-conformance", +]; + +/// Corpus files that hold measurements, not vectors an implementation must match: each with +/// why. The list is explicit, one entry a file; a file is never left out by its name or its +/// contents. What a corpus derives from a measurement (vectors built around a recorded run) +/// stays compared. +pub const MEASUREMENTS: &[(&str, &str)] = &[ + ( + "wai/format-probe-conformance/provenance.json", + "the decodes of the external vectors as measured on one build host by tools/make_external.sh; no verifier reads it as an expectation", + ), + ( + "wai/live-conformance/metering/relay-record.wmh", + "a relay's metering record from one live run on loopback: its counts, span and subscriber time are what that run measured, and a second run gives other bytes; the vectors built around it (vectors.json) stay compared", + ), + ("wai/live-conformance/metering/relay-ledger.wml", "the ledger of that run, recorded with it"), +]; + +/// Why `path` is a measurement, when [`MEASUREMENTS`] lists it. +pub fn measurement(path: &str) -> Option<&'static str> { + MEASUREMENTS.iter().find(|m| m.0 == path).map(|m| m.1) +} + +/// Files beside a corpus that are not vectors: verifiers, generators, scripts, notes. +const NOT_VECTORS: &[&str] = &[ + ".py", ".pyc", ".mjs", ".js", ".ts", ".md", ".sh", ".rs", ".toml", ".lock", ".patch", ".txt", ".gitattributes", ".gitignore", +]; + +fn is_vector(path: &str) -> bool { + !path.split('/').any(|c| c == "__pycache__" || (c == "src" && path.starts_with("wai/byte-exact-conformance/"))) + && !NOT_VECTORS.iter().any(|x| path.ends_with(x)) +} + +fn category(path: &str) -> Category { + let dir = path.split('/').nth(1).unwrap_or(""); + if RECORD_CORPORA.contains(&dir) { Category::Record } else { Category::Wire } +} + +fn list(xs: &[String]) -> String { + let n = xs.len(); + let shown: Vec<&str> = xs.iter().take(4).map(String::as_str).collect(); + if n > 4 { format!("{} (and {} more)", shown.join("; "), n - 4) } else { shown.join("; ") } +} + +/// The corpora's findings between `base` and the working tree; a changed measurement is a +/// note, not a finding. +pub fn compare(repo: &Repo, base: &str, notes: &mut Vec) -> Result, OpError> { + let mut changed: Vec<(Status, String)> = repo.changed(base, PATHSPECS)?.into_iter().filter(|(_, p)| is_vector(p)).collect(); + changed.retain(|(_, p)| match measurement(p) { + Some(why) => { + notes.push(format!("corpora: {p} changed, a measurement and not compared ({why})")); + false + } + None => true, + }); + let modified: Vec = changed.iter().filter(|(s, p)| *s == Status::Modified && (p.ends_with(".json") || p.ends_with(".jsonl"))).map(|x| x.1.clone()).collect(); + let olds = repo.base_files(base, &modified)?; + let mut out = Vec::new(); + for (status, path) in changed { + let cat = category(&path); + let why_bytes = if cat == Category::Record { + "a verifier conforms by reaching each vector's stated verdict on these bytes; records signed or read as before no longer match it" + } else { + "a reader conforms by decoding these bytes to the stated output; readers built against the old vector disagree with the new one" + }; + match status { + Status::Added => out.push(Finding::compatible(cat, "corpus", path, "a new vector file".into())), + Status::Removed => out.push(Finding::broke(cat, "corpus", path, "removed".into(), "the vector file is gone".into(), why_bytes.into())), + Status::Modified => { + let json = path.ends_with(".json"); + let lines = path.ends_with(".jsonl"); + let parsed = if json || lines { + let old = olds.get(&path).cloned().unwrap_or_default(); + let new = repo.head_file(&path).unwrap_or_default(); + if lines { + Some(jsondiff::diff_lines(&String::from_utf8_lossy(&old), &String::from_utf8_lossy(&new))) + } else { + match (serde_json::from_slice(&old), serde_json::from_slice(&new)) { + (Ok(a), Ok(b)) => Some(jsondiff::diff(&a, &b)), + _ => None, + } + } + } else { + None + }; + match parsed { + None => out.push(Finding::broke(cat, "corpus", path, "changed".into(), "the vector's bytes changed".into(), why_bytes.into())), + Some(d) if d.is_empty() => {} + Some(d) if !d.breaks() => out.push(Finding::compatible(cat, "corpus", path, format!("added {}", list(&d.added)))), + Some(d) => { + let mut what = Vec::new(); + if !d.removed.is_empty() { + what.push(format!("removed {}", list(&d.removed))); + } + if !d.changed.is_empty() { + what.push(format!("changed {}", list(&d.changed))); + } + if !d.added.is_empty() { + what.push(format!("added {}", d.added.len())); + } + out.push(Finding::broke(cat, "corpus", path, "changed".into(), what.join("; "), why_bytes.into())); + } + } + } + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::MEASUREMENTS; + + /// Every listed measurement is a file of this tree, named once: a list that names what is + /// gone excuses nothing and hides its own rot. + #[test] + fn every_listed_measurement_exists() { + let top = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let mut seen = std::collections::BTreeSet::new(); + for (p, why) in MEASUREMENTS { + assert!(top.join(p).is_file(), "{p} is listed as a measurement and is not in the tree"); + assert!(seen.insert(*p), "{p} is listed twice"); + assert!(!why.is_empty(), "{p} has no reason"); + } + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/domains.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/domains.rs new file mode 100644 index 000000000..679dcde48 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/domains.rs @@ -0,0 +1,165 @@ +//! Record: the constants a signed or framed record is built from, in the WAI crates' library +//! sources (`wai/*/src/`, binaries and tests left out). A constant whose name has the word +//! `DOMAIN`, `MAGIC` or `VERSION` and whose value is a literal is one of: +//! +//! - a domain-separation prefix of a signing preimage or a hash (`SIGN_DOMAIN`, +//! `LINK_DOMAIN`): another value changes every signature and link over the record, and a +//! verifier of the old records refuses the new ones; +//! - a record's magic (`MAGIC`, `GRANT_MAGIC`): a reader that matches the old magic refuses +//! the new records; +//! - a record's version (`RECORD_VERSION`): a reader that knows the old version refuses +//! the new one. +//! +//! So such a constant breaks when its value changes or it is gone (unless the same name +//! and value now sit in another file: a move). A new one is compatible. A record's other +//! fields and its preimage's shape reach the corpora (see `corpora`), which hold the bytes +//! the code writes and signs. + +use super::git::Repo; +use super::super::envelope::OpError; +use super::{Category, Finding}; +use std::collections::BTreeMap; + +pub const PATHSPECS: &[&str] = &[":(glob)wai/*/src/**/*.rs", ":(exclude,glob)wai/*/src/bin/**"]; + +/// The prefilter `git grep` runs (POSIX extended syntax). +const GREP: &str = "const[[:space:]]+[A-Z0-9_]*(DOMAIN|MAGIC|VERSION)"; + +/// One constant: its name, its type and its value as written (spaces dropped). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Const { + pub name: String, + pub ty: String, + pub value: String, +} + +fn named(name: &str) -> bool { + let words: Vec<&str> = name.split('_').collect(); + words.iter().any(|w| matches!(*w, "DOMAIN" | "MAGIC" | "VERSION")) + && !words.iter().any(|w| matches!(*w, "MAX" | "MIN" | "LEN" | "UNKNOWN" | "API" | "HEADER" | "VERSIONS")) +} + +fn literal(v: &str) -> bool { + v.starts_with("b\"") || v.starts_with("*b\"") || v.starts_with('"') || v.starts_with("b'") || v.starts_with('[') || v.starts_with("&[") || v.starts_with(|c: char| c.is_ascii_digit()) +} + +/// The constants of one source file, keyed by name (`NAME`, then `NAME#2`… for a name +/// declared again in the same file). Scanning stops at the file's test module. +pub fn scan(src: &str) -> BTreeMap { + let mut out = BTreeMap::new(); + let lines: Vec<&str> = src.lines().collect(); + let mut i = 0; + while i < lines.len() { + let t = lines[i].trim(); + if t == "#[cfg(test)]" && lines.get(i + 1).is_some_and(|n| n.trim_start().starts_with("mod ")) { + break; + } + let rest = t.strip_prefix("pub ").or_else(|| t.strip_prefix("pub(crate) ")).or_else(|| t.strip_prefix("pub(super) ")).unwrap_or(t); + if let Some(decl) = rest.strip_prefix("const ") { + // The declaration, joined across lines until its `;`. + let mut text = decl.to_string(); + let mut j = i; + while !text.contains(';') && j + 1 < lines.len() && j < i + 6 { + j += 1; + text.push(' '); + text.push_str(lines[j].trim()); + } + if let Some((name, after)) = text.split_once(':') { + let name = name.trim(); + if name.chars().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_') + && named(name) + && let Some((ty, value)) = after.split_once('=') + { + let value: String = value.split(';').next().unwrap_or("").split_whitespace().collect::>().join(""); + if literal(&value) { + let ty: String = ty.split_whitespace().collect(); + let mut key = name.to_string(); + let mut n = 1; + while out.contains_key(&key) { + n += 1; + key = format!("{name}#{n}"); + } + out.insert(key, Const { name: name.to_string(), ty, value }); + } + } + } + i = j; + } + i += 1; + } + out +} + +fn role(name: &str) -> (&'static str, &'static str) { + if name.split('_').any(|w| w == "DOMAIN") { + ("signing or hashing domain", "every signature and hash made under it changes; a verifier of the records made before refuses those made after, and the other way round") + } else if name.split('_').any(|w| w == "MAGIC") { + ("magic", "a reader that matches the old magic refuses the records written with the new one") + } else { + ("version", "a reader that knows the old version refuses records of the new one, and a writer of the old version is refused by the new reader") + } +} + +fn all(repo: &Repo, base: Option<&str>) -> Result, OpError> { + let files = repo.grep_files(base, GREP, PATHSPECS)?; + let texts: BTreeMap> = match base { + Some(b) => repo.base_files(b, &files)?, + None => files.iter().filter_map(|f| Some((f.clone(), repo.head_file(f)?))).collect(), + }; + let mut out = BTreeMap::new(); + for (f, t) in texts { + for (k, c) in scan(&String::from_utf8_lossy(&t)) { + out.insert((f.clone(), k), c); + } + } + Ok(out) +} + +/// The constants' findings between `base` and the working tree. +pub fn compare(repo: &Repo, base: &str) -> Result, OpError> { + let old = all(repo, Some(base))?; + let new = all(repo, None)?; + let mut out = Vec::new(); + let moved = |c: &Const| new.values().any(|n| n.name == c.name && n.value == c.value); + for ((file, key), c) in &old { + let item = format!("{file}::{key}"); + let (what_it_is, why) = role(&c.name); + match new.get(&(file.clone(), key.clone())) { + None if moved(c) => {} + None => out.push(Finding::broke(Category::Record, "domain", item, "removed".into(), format!("the {what_it_is} {} = {} is gone", c.name, c.value), why.into())), + Some(n) if n.value != c.value || n.ty != c.ty => out.push(Finding::broke( + Category::Record, + "domain", + item, + "changed".into(), + format!("the {what_it_is} {} changed: {} → {}", c.name, c.value, n.value), + why.into(), + )), + Some(_) => {} + } + } + for ((file, key), c) in &new { + if !old.contains_key(&(file.clone(), key.clone())) && !old.values().any(|o| o.name == c.name && o.value == c.value) { + out.push(Finding::compatible(Category::Record, "domain", format!("{file}::{key}"), format!("{} = {} is new", c.name, c.value))); + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::scan; + + #[test] + fn scans_domains_magics_and_versions() { + let src = "pub const SIGN_DOMAIN: &[u8] = b\"wai:x\\x01\";\nconst MAGIC: [u8; 4] = *b\"WEV1\";\npub const RECORD_VERSION: u8 = 1;\n\ + pub(crate) const DOMAIN_LEN: u64 = X.len() as u64;\nconst MAX_VERSION: u8 = 3;\nconst OTHER: u8 = 1;\n\ + pub const LONG_DOMAIN: &[u8] =\n b\"wai:long\\x01\";\n#[cfg(test)]\nmod tests {\n const TEST_DOMAIN: &[u8] = b\"t\";\n}\n"; + let m = scan(src); + let names: Vec<&str> = m.keys().map(String::as_str).collect(); + assert_eq!(names, vec!["LONG_DOMAIN", "MAGIC", "RECORD_VERSION", "SIGN_DOMAIN"]); + assert_eq!(m["SIGN_DOMAIN"].value, "b\"wai:x\\x01\""); + assert_eq!(m["LONG_DOMAIN"].value, "b\"wai:long\\x01\""); + assert_eq!(m["MAGIC"].ty, "[u8;4]"); + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/git.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/git.rs new file mode 100644 index 000000000..e1e404d5a --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/git.rs @@ -0,0 +1,332 @@ +//! The two trees `wai breaking` compares, read through `git`: the base (a commit) and the +//! working tree (what is on disk, committed or not, untracked files included). + +use super::super::envelope::{EXIT_INTERNAL, EXIT_NOT_FOUND, OpError}; +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +/// A checkout: its top-level directory. +pub struct Repo { + pub root: PathBuf, +} + +/// How a path differs between the base and the working tree. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Status { + Added, + Modified, + Removed, +} + +fn internal(e: impl std::fmt::Display) -> OpError { + OpError::new(EXIT_INTERNAL, "git-failed", e.to_string(), "run it in a git checkout with git on PATH") +} + +impl Repo { + /// The checkout holding `dir`. + pub fn open(dir: &Path) -> Result { + let out = Command::new("git").arg("-C").arg(dir).args(["rev-parse", "--show-toplevel"]).output().map_err(|e| { + OpError::new(EXIT_NOT_FOUND, "git-not-found", format!("git: {e}"), "install git; `wai breaking` reads both trees through it") + })?; + if !out.status.success() { + return Err(OpError::new( + EXIT_NOT_FOUND, + "not-a-checkout", + format!("{}: not in a git checkout", dir.display()), + "run it inside the repository, or pass --root ", + )); + } + let root = String::from_utf8_lossy(&out.stdout).trim().to_string(); + Ok(Repo { root: PathBuf::from(root) }) + } + + fn git(&self, args: &[&str]) -> Result, OpError> { + let out = Command::new("git").arg("-C").arg(&self.root).args(args).output().map_err(internal)?; + if !out.status.success() { + return Err(internal(format!("git {}: {}", args.join(" "), String::from_utf8_lossy(&out.stderr).trim()))); + } + Ok(out.stdout) + } + + /// The commit `rev` names, or `None` when it names none. + pub fn commit(&self, rev: &str) -> Option { + let out = Command::new("git").arg("-C").arg(&self.root).args(["rev-parse", "--verify", "--quiet", &format!("{rev}^{{commit}}")]).output().ok()?; + out.status.success().then(|| String::from_utf8_lossy(&out.stdout).trim().to_string()) + } + + /// The best common ancestor of two commits. + pub fn merge_base(&self, a: &str, b: &str) -> Option { + let out = Command::new("git").arg("-C").arg(&self.root).args(["merge-base", a, b]).output().ok()?; + out.status.success().then(|| String::from_utf8_lossy(&out.stdout).trim().to_string()) + } + + /// Every path under `pathspecs` that differs between `base` and the working tree. + pub fn changed(&self, base: &str, pathspecs: &[&str]) -> Result, OpError> { + let mut args = vec!["diff", "--name-status", "--no-renames", "-z", base, "--"]; + args.extend_from_slice(pathspecs); + let out = self.git(&args)?; + let mut v = Vec::new(); + let mut it = out.split(|b| *b == 0).filter(|s| !s.is_empty()); + while let (Some(st), Some(p)) = (it.next(), it.next()) { + let s = match st.first() { + Some(b'A') => Status::Added, + Some(b'D') => Status::Removed, + _ => Status::Modified, + }; + v.push((s, String::from_utf8_lossy(p).into_owned())); + } + let mut args = vec!["ls-files", "--others", "--exclude-standard", "-z", "--"]; + args.extend_from_slice(pathspecs); + for p in self.git(&args)?.split(|b| *b == 0).filter(|s| !s.is_empty()) { + v.push((Status::Added, String::from_utf8_lossy(p).into_owned())); + } + v.sort_by(|a, b| a.1.cmp(&b.1)); + v.dedup_by(|a, b| a.1 == b.1); + Ok(v) + } + + /// The paths under `pathspecs` in `base`. (`git ls-tree` takes no pathspec magic, so + /// the listing is filtered here, by [`matches`].) + pub fn ls_base(&self, base: &str, pathspecs: &[&str]) -> Result, OpError> { + let all = split0(&self.git(&["ls-tree", "-r", "--name-only", "-z", base])?); + Ok(all.into_iter().filter(|p| matches(pathspecs, p)).collect()) + } + + /// The paths under `pathspecs` in the working tree: tracked files still on disk, and + /// untracked files that are not ignored. + pub fn ls_head(&self, pathspecs: &[&str]) -> Result, OpError> { + let mut args = vec!["ls-files", "--cached", "--others", "--exclude-standard", "-z", "--"]; + args.extend_from_slice(pathspecs); + let mut v: Vec = split0(&self.git(&args)?).into_iter().filter(|p| self.root.join(p).is_file()).collect(); + v.sort(); + v.dedup(); + Ok(v) + } + + /// The files under `pathspecs` with a line matching the extended regular expression + /// `re`: in `base` when it is given, else in the working tree (untracked files too). + pub fn grep_files(&self, base: Option<&str>, re: &str, pathspecs: &[&str]) -> Result, OpError> { + let mut args = vec!["grep", "-l", "-I", "-z", "-E", "-e", re]; + match base { + Some(b) => args.push(b), + None => args.push("--untracked"), + } + args.push("--"); + args.extend_from_slice(pathspecs); + let out = Command::new("git").arg("-C").arg(&self.root).args(&args).output().map_err(internal)?; + // git grep exits 1 when nothing matched. + if !out.status.success() && out.status.code() != Some(1) { + return Err(internal(format!("git grep: {}", String::from_utf8_lossy(&out.stderr).trim()))); + } + let prefix = base.map(|b| format!("{b}:")).unwrap_or_default(); + Ok(split0(&out.stdout).into_iter().map(|p| p.strip_prefix(&prefix).map(String::from).unwrap_or(p)).collect()) + } + + /// A file of the working tree, or `None` when it is not there. + pub fn head_file(&self, path: &str) -> Option> { + std::fs::read(self.root.join(path)).ok() + } + + /// Files of `base`, read in one `git cat-file --batch`: each path's bytes, or no entry + /// when `base` does not hold it. + pub fn base_files(&self, base: &str, paths: &[String]) -> Result>, OpError> { + let mut out = BTreeMap::new(); + if paths.is_empty() { + return Ok(out); + } + let mut child = Command::new("git") + .arg("-C") + .arg(&self.root) + .args(["cat-file", "--batch"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(internal)?; + let mut stdin = child.stdin.take().ok_or_else(|| internal("git cat-file: no stdin"))?; + let req: String = paths.iter().map(|p| format!("{base}:{p}\n")).collect(); + let writer = std::thread::spawn(move || { + let _ = stdin.write_all(req.as_bytes()); + }); + let mut all = Vec::new(); + child.stdout.take().ok_or_else(|| internal("git cat-file: no stdout"))?.read_to_end(&mut all).map_err(internal)?; + let _ = writer.join(); + let _ = child.wait(); + let mut rest: &[u8] = &all; + for p in paths { + let nl = rest.iter().position(|b| *b == b'\n').ok_or_else(|| internal("git cat-file: short answer"))?; + let header = String::from_utf8_lossy(&rest[..nl]).into_owned(); + rest = &rest[nl + 1..]; + if header.ends_with(" missing") || header.ends_with(" ambiguous") { + continue; + } + let size: usize = header.rsplit(' ').next().and_then(|s| s.parse().ok()).ok_or_else(|| internal(format!("git cat-file: {header}")))?; + if rest.len() < size + 1 { + return Err(internal("git cat-file: short object")); + } + if header.split(' ').nth(1) == Some("blob") { + out.insert(p.clone(), rest[..size].to_vec()); + } + rest = &rest[size + 1..]; + } + Ok(out) + } + + /// One file of `base`. + pub fn base_file(&self, base: &str, path: &str) -> Result>, OpError> { + Ok(self.base_files(base, &[path.to_string()])?.remove(path)) + } + + /// `base`'s tree under `path`, written below `dest` at the same relative paths. + pub fn extract(&self, base: &str, path: &str, dest: &Path) -> Result<(), OpError> { + std::fs::create_dir_all(dest).map_err(internal)?; + let mut archive = Command::new("git") + .arg("-C") + .arg(&self.root) + .args(["archive", "--format=tar", base, "--", path]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(internal)?; + let tar_in = archive.stdout.take().ok_or_else(|| internal("git archive: no stdout"))?; + let tar = Command::new("tar").arg("-x").arg("-C").arg(dest).stdin(tar_in).output().map_err(internal)?; + let a = archive.wait_with_output().map_err(internal)?; + if !a.status.success() || !tar.status.success() { + return Err(internal(format!( + "extracting {path} of {base}: {}{}", + String::from_utf8_lossy(&a.stderr).trim(), + String::from_utf8_lossy(&tar.stderr).trim() + ))); + } + Ok(()) + } + + /// The lines `base`'s CHANGELOG at `path` does not have and the working tree's has. + pub fn added_lines(&self, base: &str, path: &str) -> Result, OpError> { + let old = self.base_file(base, path)?.unwrap_or_default(); + let new = self.head_file(path).unwrap_or_default(); + Ok(added_lines(&String::from_utf8_lossy(&old), &String::from_utf8_lossy(&new))) + } +} + +/// Whether `path` is under `pathspecs`, as git reads the forms used here: a plain path (the +/// file, or everything below the directory), `:(glob)` and `:(exclude,glob)`, +/// where `*` matches within one path segment and `**` matches any number of segments. +pub fn matches(pathspecs: &[&str], path: &str) -> bool { + let mut included = false; + for spec in pathspecs { + let (exclude, pattern, glob) = if let Some(p) = spec.strip_prefix(":(exclude,glob)") { + (true, p, true) + } else if let Some(p) = spec.strip_prefix(":(glob)") { + (false, p, true) + } else { + (false, *spec, false) + }; + let hit = if glob { glob_match(&pattern.split('/').collect::>(), &path.split('/').collect::>()) } else { path == pattern || path.starts_with(&format!("{}/", pattern.trim_end_matches('/'))) }; + if hit && exclude { + return false; + } + included |= hit; + } + included +} + +fn glob_match(pat: &[&str], path: &[&str]) -> bool { + match (pat.first(), path.first()) { + (None, None) => true, + (Some(&"**"), _) => glob_match(&pat[1..], path) || (!path.is_empty() && glob_match(pat, &path[1..])), + (Some(p), Some(s)) => segment_match(p.as_bytes(), s.as_bytes()) && glob_match(&pat[1..], &path[1..]), + _ => false, + } +} + +fn segment_match(p: &[u8], s: &[u8]) -> bool { + match (p.first(), s.first()) { + (None, None) => true, + (Some(b'*'), _) => segment_match(&p[1..], s) || (!s.is_empty() && segment_match(p, &s[1..])), + (Some(b'?'), Some(_)) => segment_match(&p[1..], &s[1..]), + (Some(a), Some(b)) if a == b => segment_match(&p[1..], &s[1..]), + _ => false, + } +} + +fn split0(b: &[u8]) -> Vec { + b.split(|x| *x == 0).filter(|s| !s.is_empty()).map(|s| String::from_utf8_lossy(s).into_owned()).collect() +} + +/// The lines of `new` that a longest common subsequence with `old` leaves over: what an +/// edit added, in order. A line moved within the file counts as added where it now is. +pub fn added_lines(old: &str, new: &str) -> Vec { + let a: Vec<&str> = old.lines().collect(); + let b: Vec<&str> = new.lines().collect(); + // Trim the common head and tail first: a changelog edit is a few lines near the top of a + // long file, and what is left between is small. + let mut lo = 0; + while lo < a.len() && lo < b.len() && a[lo] == b[lo] { + lo += 1; + } + let (mut ea, mut eb) = (a.len(), b.len()); + while ea > lo && eb > lo && a[ea - 1] == b[eb - 1] { + ea -= 1; + eb -= 1; + } + let (a, b) = (&a[lo..ea], &b[lo..eb]); + if a.is_empty() { + return b.iter().map(|s| s.to_string()).collect(); + } + if (a.len() as u64) * (b.len() as u64) > 40_000_000 { + // Too far apart for the table: every line not in the old file counts as added. + let old: std::collections::HashSet<&str> = a.iter().copied().collect(); + return b.iter().filter(|l| !old.contains(*l)).map(|s| s.to_string()).collect(); + } + let (n, m) = (a.len(), b.len()); + let mut t = vec![0u32; (n + 1) * (m + 1)]; + for i in (0..n).rev() { + for j in (0..m).rev() { + t[i * (m + 1) + j] = if a[i] == b[j] { t[(i + 1) * (m + 1) + j + 1] + 1 } else { t[(i + 1) * (m + 1) + j].max(t[i * (m + 1) + j + 1]) }; + } + } + let (mut i, mut j) = (0, 0); + let mut out = Vec::new(); + while j < m { + if i < n && a[i] == b[j] { + i += 1; + j += 1; + } else if i < n && t[(i + 1) * (m + 1) + j] >= t[i * (m + 1) + j + 1] { + i += 1; + } else { + out.push(b[j].to_string()); + j += 1; + } + } + out +} + +#[cfg(test)] +mod tests { + use super::added_lines; + + #[test] + fn pathspecs_match_as_git_reads_them() { + use super::matches; + let specs = [":(glob)wai/*/src/**/*.rs", ":(exclude,glob)wai/*/src/bin/**"]; + assert!(matches(&specs, "wai/wai-rs/src/live_events.rs")); + assert!(matches(&specs, "wai/wai-rs/src/a/b/c.rs")); + assert!(!matches(&specs, "wai/wai-rs/src/bin/wai.rs")); + assert!(!matches(&specs, "wai/wai-rs/tests/x.rs")); + assert!(matches(&[":(glob)wai/**/openapi*.json"], "wai/openapi.json")); + assert!(matches(&["joule-code/x/openapi.json"], "joule-code/x/openapi.json")); + assert!(matches(&[":(glob)wai/*-conformance/**"], "wai/claim-conformance/vectors.json")); + assert!(!matches(&[":(glob)wai/*-conformance/**"], "wai/wai-rs/x")); + } + + #[test] + fn added_lines_are_the_new_ones() { + assert_eq!(added_lines("a\nb\nc\n", "a\nx\nb\nc\ny\n"), vec!["x", "y"]); + assert_eq!(added_lines("", "a\n"), vec!["a"]); + assert!(added_lines("a\nb\n", "a\n").is_empty()); + assert_eq!(added_lines("a\nb\n", "b\na\n"), vec!["a"]); + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/jsondiff.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/jsondiff.rs new file mode 100644 index 000000000..521652ea5 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/jsondiff.rs @@ -0,0 +1,158 @@ +//! What changed between two JSON documents, as a reader of the old one sees it: members +//! and entries that went away, values that changed, and members and entries that are new. +//! +//! An array whose entries are objects named by one member (`name`, `file`, `id`, …) is +//! compared entry by entry under that name, so a corpus that gains a vector shows one added +//! entry, not a shifted list. Any other array is compared position by position, and one +//! that only grew at its end shows the new positions as added. Prose members (`about`, +//! `description`, `note`, `notes`, `comment`) are not compared: rewording them changes no +//! byte a reader acts on. + +use serde_json::Value; + +/// Members never compared: prose about the document, not data a reader acts on. +pub const PROSE: &[&str] = &["about", "description", "note", "notes", "comment"]; + +/// The members an array's entries may be named by, in the order they are tried. +const ENTRY_KEYS: &[&str] = &["name", "file", "id", "capability", "identifier", "label", "case"]; + +/// Paths (`$.vectors[name=x].expected`) of what was removed, changed and added. +#[derive(Default, Debug)] +pub struct Diff { + pub removed: Vec, + pub changed: Vec, + pub added: Vec, +} + +impl Diff { + pub fn breaks(&self) -> bool { + !self.removed.is_empty() || !self.changed.is_empty() + } + pub fn is_empty(&self) -> bool { + !self.breaks() && self.added.is_empty() + } +} + +/// The difference from `old` to `new`. +pub fn diff(old: &Value, new: &Value) -> Diff { + let mut d = Diff::default(); + walk(old, new, "$", &mut d); + d +} + +/// A value shortened for a message. +pub fn short(v: &Value) -> String { + let s = v.to_string(); + if s.chars().count() > 72 { format!("{}…", s.chars().take(72).collect::()) } else { s } +} + +fn entry_key(a: &[Value], b: &[Value]) -> Option<&'static str> { + ENTRY_KEYS.iter().copied().find(|k| { + let named = |xs: &[Value]| { + let mut seen = std::collections::HashSet::new(); + xs.iter().all(|x| x.get(k).and_then(Value::as_str).is_some_and(|n| seen.insert(n.to_string()))) + }; + !(a.is_empty() && b.is_empty()) && named(a) && named(b) + }) +} + +fn walk(old: &Value, new: &Value, path: &str, d: &mut Diff) { + match (old, new) { + (Value::Object(a), Value::Object(b)) => { + for (k, av) in a { + if PROSE.contains(&k.as_str()) { + continue; + } + let p = format!("{path}.{k}"); + match b.get(k) { + None => d.removed.push(p), + Some(bv) => walk(av, bv, &p, d), + } + } + for k in b.keys() { + if !a.contains_key(k) && !PROSE.contains(&k.as_str()) { + d.added.push(format!("{path}.{k}")); + } + } + } + (Value::Array(a), Value::Array(b)) => { + if let Some(key) = entry_key(a, b) { + let name = |x: &Value| x.get(key).and_then(Value::as_str).unwrap_or_default().to_string(); + let bm: std::collections::HashMap = b.iter().map(|x| (name(x), x)).collect(); + let am: std::collections::HashSet = a.iter().map(name).collect(); + for x in a { + let n = name(x); + let p = format!("{path}[{key}={n}]"); + match bm.get(&n) { + None => d.removed.push(p), + Some(y) => walk(x, y, &p, d), + } + } + for y in b { + let n = name(y); + if !am.contains(&n) { + d.added.push(format!("{path}[{key}={n}]")); + } + } + } else { + for (i, (x, y)) in a.iter().zip(b.iter()).enumerate() { + walk(x, y, &format!("{path}[{i}]"), d); + } + for i in b.len()..a.len() { + d.removed.push(format!("{path}[{i}]")); + } + for i in a.len()..b.len() { + d.added.push(format!("{path}[{i}]")); + } + } + } + (x, y) if x == y => {} + (x, y) => d.changed.push(format!("{path}: {} → {}", short(x), short(y))), + } +} + +/// The difference between two JSON Lines documents: lines that are gone, and lines that +/// are new. Lines are compared as text. +pub fn diff_lines(old: &str, new: &str) -> Diff { + let a: std::collections::HashSet<&str> = old.lines().filter(|l| !l.trim().is_empty()).collect(); + let b: std::collections::HashSet<&str> = new.lines().filter(|l| !l.trim().is_empty()).collect(); + let mut d = Diff::default(); + for (i, l) in old.lines().enumerate() { + if !l.trim().is_empty() && !b.contains(l) { + d.removed.push(format!("line {}: {}", i + 1, short(&Value::String(l.to_string())))); + } + } + for (i, l) in new.lines().enumerate() { + if !l.trim().is_empty() && !a.contains(l) { + d.added.push(format!("line {}", i + 1)); + } + } + d +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn named_entries_are_matched_by_name() { + let a = json!({"about": "x", "vectors": [{"name": "a", "expected": "accept"}, {"name": "b", "expected": "reject"}]}); + let b = json!({"about": "y", "vectors": [{"name": "c", "expected": "accept"}, {"name": "a", "expected": "accept"}, {"name": "b", "expected": "reject"}]}); + let d = diff(&a, &b); + assert!(!d.breaks(), "{d:?}"); + assert_eq!(d.added, vec!["$.vectors[name=c]"]); + let c = json!({"about": "x", "vectors": [{"name": "a", "expected": "reject"}]}); + let d = diff(&a, &c); + assert_eq!(d.removed, vec!["$.vectors[name=b]"]); + assert_eq!(d.changed, vec![r#"$.vectors[name=a].expected: "accept" → "reject""#]); + } + + #[test] + fn a_list_that_grew_at_its_end_only_added() { + let d = diff(&json!([1, 2]), &json!([1, 2, 3])); + assert!(!d.breaks()); + let d = diff(&json!([1, 2]), &json!([2, 1])); + assert!(d.breaks()); + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/mod.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/mod.rs new file mode 100644 index 000000000..baf5ac820 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/mod.rs @@ -0,0 +1,352 @@ +//! `wai breaking --against ` (operation `compat.check`): what the working tree +//! breaks, by category, for a reader, a verifier or a program written against the ref. +//! +//! - **wire**: the bytes a conforming reader must accept. The registry export (capability +//! names and their classes, deterministic identifiers, wire identifiers: magics, +//! document kinds, headers, schemes, tags) and the conformance corpora of media and +//! delivery (`registry`, `corpora`). +//! - **record**: signed record formats. The domain-separation prefixes, magics and +//! versions records are built from, the corpora of signed records and their verdicts, +//! and the published JSON Schemas (`domains`, `corpora`, `schemas`). +//! - **api**: the Rust public API of the WAI library crates, the C header and the HTTP +//! services' OpenAPI documents (`rust`, `c`, `openapi`). +//! +//! The categories are cumulative, strictest by default: `--category api` (the default) +//! reports all three, `record` reports wire and record, `wire` reports wire alone. The +//! base is the merge base of the ref and `HEAD`, so a ref that moved on since the branch +//! left it does not read as the branch removing what the ref gained. +//! +//! Each finding names its item, its category, what changed and why that breaks; additions +//! are listed as compatible. A break is intended only when `wai/CHANGELOG.md` says so: a +//! line this change adds to it, in an entry that has the word "breaking", naming the +//! finding's id in backticks. Then it is reported as acknowledged and passes; every other +//! break fails the run (exit 1). + +pub mod c_header; +pub mod corpora; +pub mod domains; +pub mod git; +pub mod jsondiff; +pub mod openapi; +pub mod registry; +pub mod rust_api; +pub mod schema; + +use super::envelope::{self, EXIT_FAILED, EXIT_NOT_FOUND, OpError}; +use serde_json::{Map, Value, json}; +use std::collections::BTreeSet; +use std::path::PathBuf; + +pub const OP: &str = "compat.check"; + +pub const USAGE: &str = "wai breaking --against [--category wire|record|api] [--only ,...] [--skip ,...]\n\ + [--root ] [--target-dir ] [--json|--text] what the working tree breaks against :\n\ + wire, record and API; a break passes only where wai/CHANGELOG.md names it (operation compat.check)"; + +/// The changelog whose added lines acknowledge a break. +pub const CHANGELOG: &str = "wai/CHANGELOG.md"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Category { + Wire, + Record, + Api, +} + +impl Category { + pub fn name(self) -> &'static str { + match self { + Category::Wire => "wire", + Category::Record => "record", + Category::Api => "api", + } + } + fn parse(s: &str) -> Option { + match s { + "wire" => Some(Category::Wire), + "record" => Some(Category::Record), + "api" => Some(Category::Api), + _ => None, + } + } +} + +/// One change between the base and the working tree. +#[derive(Clone, Debug)] +pub struct Finding { + /// `::`, the name a changelog entry acknowledges it by. + pub id: String, + pub category: Category, + pub surface: &'static str, + pub item: String, + /// `removed`, `changed` or `added`. + pub change: String, + pub what: String, + /// Why it breaks; empty for a compatible change. + pub why: String, + pub breaking: bool, + pub acknowledged: bool, +} + +impl Finding { + pub fn broke(category: Category, surface: &'static str, item: String, change: String, what: String, why: String) -> Finding { + Finding { id: format!("{}:{surface}:{item}", category.name()), category, surface, item, change, what, why, breaking: true, acknowledged: false } + } + pub fn compatible(category: Category, surface: &'static str, item: String, what: String) -> Finding { + Finding { id: format!("{}:{surface}:{item}", category.name()), category, surface, item, change: "added".into(), what, why: String::new(), breaking: false, acknowledged: false } + } + fn to_json(&self) -> Value { + json!({ + "id": self.id, "category": self.category.name(), "surface": self.surface, "item": self.item, + "change": self.change, "what": self.what, "why": self.why, "breaking": self.breaking, "acknowledged": self.acknowledged, + }) + } +} + +/// The surfaces, each with the most permissive category it reports under. +const SURFACES: &[(&str, Category)] = &[ + ("registry", Category::Wire), + ("corpora", Category::Wire), + ("domains", Category::Record), + ("schemas", Category::Record), + ("c", Category::Api), + ("openapi", Category::Api), + ("rust", Category::Api), +]; + +/// What to check. +pub struct Options { + pub against: String, + pub category: Category, + pub surfaces: BTreeSet<&'static str>, + pub root: PathBuf, + pub target_dir: Option, +} + +/// The report of one run. +pub struct Report { + pub against: String, + pub base: String, + pub category: Category, + pub surfaces: Vec<&'static str>, + pub findings: Vec, + pub notes: Vec, +} + +impl Report { + pub fn unacknowledged(&self) -> usize { + self.findings.iter().filter(|f| f.breaking && !f.acknowledged).count() + } + pub fn to_json(&self) -> Map { + let mut m = Map::new(); + m.insert("against".into(), json!(self.against)); + m.insert("base".into(), json!(self.base)); + m.insert("category".into(), json!(self.category.name())); + m.insert("surfaces".into(), json!(self.surfaces)); + m.insert("breaking".into(), json!(self.unacknowledged())); + m.insert("acknowledged".into(), json!(self.findings.iter().filter(|f| f.breaking && f.acknowledged).count())); + m.insert("compatible".into(), json!(self.findings.iter().filter(|f| !f.breaking).count())); + m.insert("findings".into(), Value::Array(self.findings.iter().map(Finding::to_json).collect())); + m.insert("notes".into(), json!(self.notes)); + m + } +} + +/// The ids a changelog's added lines acknowledge: every backticked span of an entry (a +/// bullet and its continuation lines, or a paragraph) that has the word "breaking". +pub fn acknowledgements(added: &[String]) -> BTreeSet { + let mut entries: Vec = Vec::new(); + for l in added { + let t = l.trim_start(); + let starts = t.starts_with("- ") || t.starts_with("* ") || t.starts_with('#') || (!l.starts_with(' ') && !t.is_empty()); + if starts || entries.is_empty() { + entries.push(String::new()); + } + if t.is_empty() { + entries.push(String::new()); + continue; + } + if let Some(e) = entries.last_mut() { + e.push_str(t); + e.push(' '); + } + } + let mut ids = BTreeSet::new(); + for e in entries { + if !e.to_lowercase().contains("breaking") { + continue; + } + let mut parts = e.split('`'); + parts.next(); + while let Some(span) = parts.next() { + ids.insert(span.trim().to_string()); + parts.next(); + } + } + ids +} + +/// Run the check. +pub fn run(o: &Options) -> Result { + let repo = git::Repo::open(&o.root)?; + let target = repo.commit(&o.against).ok_or_else(|| { + OpError::new(EXIT_NOT_FOUND, "ref-not-found", format!("{}: no such commit", o.against), "fetch it first (git fetch origin main), or name a commit, tag or branch this checkout holds") + })?; + let base = repo.commit("HEAD").and_then(|h| repo.merge_base(&target, &h)).unwrap_or(target); + let mut findings = Vec::new(); + let mut notes = Vec::new(); + let checked: Vec<&'static str> = SURFACES.iter().filter(|(s, c)| *c <= o.category && o.surfaces.contains(s)).map(|x| x.0).collect(); + for s in &checked { + eprintln!("wai breaking: {s}"); + match *s { + "registry" => { + let old = repo.base_file(&base, registry::PATH)?.and_then(|b| serde_json::from_slice::(&b).ok()); + let new = repo.head_file(registry::PATH).and_then(|b| serde_json::from_slice::(&b).ok()); + findings.extend(registry::compare(old.as_ref(), new.as_ref())); + } + "corpora" => findings.extend(corpora::compare(&repo, &base, &mut notes)?), + "domains" => findings.extend(domains::compare(&repo, &base)?), + "schemas" => findings.extend(schema::compare(&repo, &base)?), + "c" => { + let old = repo.base_file(&base, c_header::PATH)?.map(|b| String::from_utf8_lossy(&b).into_owned()); + let new = repo.head_file(c_header::PATH).map(|b| String::from_utf8_lossy(&b).into_owned()); + findings.extend(c_header::compare(old.as_deref(), new.as_deref())); + } + "openapi" => findings.extend(openapi::compare(&repo, &base)?), + "rust" => { + let scratch = std::env::temp_dir().join(format!("wai-breaking-{}-{}", std::process::id(), std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0))); + let toolchain = std::fs::read_to_string(repo.root.join("rust-toolchain.toml")).ok().and_then(|t| { + t.lines().find_map(|l| l.trim().strip_prefix("channel").and_then(|r| r.trim_start().strip_prefix('=')).map(|v| v.trim().trim_matches('"').to_string())) + }); + let builder = rust_api::Builder { target_dir: o.target_dir.clone().unwrap_or_else(|| scratch.join("target")), toolchain }; + let r = rust_api::compare(&repo, &base, &builder, &scratch, &mut notes); + let _ = std::fs::remove_dir_all(&scratch); + findings.extend(r?); + } + _ => {} + } + } + findings.retain(|f| f.category <= o.category); + let acks = acknowledgements(&repo.added_lines(&base, CHANGELOG)?); + for f in findings.iter_mut().filter(|f| f.breaking) { + f.acknowledged = acks.contains(&f.id); + } + findings.sort_by(|a, b| (!a.breaking, a.acknowledged, a.category, &a.id).cmp(&(!b.breaking, b.acknowledged, b.category, &b.id))); + Ok(Report { against: o.against.clone(), base, category: o.category, surfaces: checked, findings, notes }) +} + +fn options(args: &[String]) -> Result { + let flags = ["--against", "--category", "--only", "--skip", "--root", "--target-dir", "--json", "--text", "--all"]; + let with_value = ["--against", "--category", "--only", "--skip", "--root", "--target-dir"]; + envelope::check_args(args, &flags, &with_value, 0, USAGE)?; + let against = envelope::value(args, "--against", USAGE)?.ok_or_else(|| OpError::usage("--against is required", USAGE))?.to_string(); + let category = match envelope::value(args, "--category", USAGE)? { + None => Category::Api, + Some(c) => Category::parse(c).ok_or_else(|| OpError::usage(format!("--category {c}: wire, record or api"), USAGE))?, + }; + let names = |v: Option<&str>| -> Result>, OpError> { + let Some(v) = v else { return Ok(None) }; + v.split(',') + .map(|n| SURFACES.iter().find(|s| s.0 == n.trim()).map(|s| s.0).ok_or_else(|| OpError::usage(format!("{n}: not a surface ({})", SURFACES.iter().map(|s| s.0).collect::>().join(", ")), USAGE))) + .collect::, _>>() + .map(Some) + }; + let mut surfaces: BTreeSet<&'static str> = names(envelope::value(args, "--only", USAGE)?)?.unwrap_or_else(|| SURFACES.iter().map(|s| s.0).collect()); + if let Some(skip) = names(envelope::value(args, "--skip", USAGE)?)? { + surfaces.retain(|s| !skip.contains(s)); + } + Ok(Options { + against, + category, + surfaces, + root: envelope::value(args, "--root", USAGE)?.map(PathBuf::from).unwrap_or_else(|| PathBuf::from(".")), + target_dir: envelope::value(args, "--target-dir", USAGE)?.map(PathBuf::from), + }) +} + +fn text(r: &Report, all: bool) { + let short = r.base.get(..12).unwrap_or(&r.base); + println!("wai breaking: against {} (merge base {short}); {} and every category below it; checked {}", r.against, r.category.name(), r.surfaces.join(", ")); + let breaks: Vec<&Finding> = r.findings.iter().filter(|f| f.breaking).collect(); + let compat: Vec<&Finding> = r.findings.iter().filter(|f| !f.breaking).collect(); + if !breaks.is_empty() { + println!(); + println!("breaking changes ({}, {} acknowledged in {CHANGELOG}):", breaks.len(), breaks.iter().filter(|f| f.acknowledged).count()); + for f in &breaks { + println!(" {} {}", if f.acknowledged { "ACK " } else { "BREAK" }, f.id); + println!(" {} ({} · {}): {}", f.change, f.category.name(), f.surface, f.what); + println!(" why it breaks: {}", f.why); + } + } + if !compat.is_empty() { + println!(); + println!("compatible changes ({}):", compat.len()); + let shown = if all { compat.len() } else { compat.len().min(40) }; + for f in &compat[..shown] { + println!(" + {} {}", f.id, f.what); + } + if shown < compat.len() { + println!(" … and {} more (--all lists them)", compat.len() - shown); + } + } + for n in &r.notes { + println!("note: {n}"); + } + println!(); + let n = r.unacknowledged(); + if n == 0 { + println!("wai breaking: no unacknowledged breaking change"); + } else { + println!("wai breaking: {n} breaking change{} not acknowledged. If {} intended, add to {CHANGELOG}, under the current version:", if n == 1 { "" } else { "s" }, if n == 1 { "it is" } else { "they are" }); + for f in breaks.iter().filter(|f| !f.acknowledged).take(8) { + println!(" - **Breaking:** `{}`: ", f.id); + } + } +} + +pub fn main(args: &[String]) -> Result { + let json_mode = envelope::json_wanted(args); + let o = match options(args) { + Ok(o) => o, + Err(e) => envelope::fail(OP, json_mode, &e), + }; + let r = match run(&o) { + Ok(r) => r, + Err(e) => envelope::fail(OP, json_mode, &e), + }; + let ok = r.unacknowledged() == 0; + if json_mode { + println!("{}", envelope::envelope(OP, ok, r.to_json())); + } else { + text(&r, args.iter().any(|a| a == "--all")); + } + if !ok { + envelope::exit(EXIT_FAILED); + } + Ok(true) +} + +#[cfg(test)] +mod tests { + use super::acknowledgements; + + #[test] + fn an_entry_acknowledges_only_when_it_says_breaking() { + let added: Vec = [ + "- **Breaking:** `api:c:wai_pack` takes a", + " `uint64_t` now, and api:rust:x is not in backticks.", + "- `wire:capability:wai.x` is new (no such word here).", + "", + "Breaking, as a paragraph: `record:domain:a.rs::SIGN_DOMAIN`.", + ] + .iter() + .map(|s| s.to_string()) + .collect(); + let a = acknowledgements(&added); + assert!(a.contains("api:c:wai_pack")); + assert!(a.contains("uint64_t")); + assert!(a.contains("record:domain:a.rs::SIGN_DOMAIN")); + assert!(!a.contains("wire:capability:wai.x")); + assert!(!a.contains("api:rust:x")); + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/openapi.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/openapi.rs new file mode 100644 index 000000000..4edc469ac --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/openapi.rs @@ -0,0 +1,175 @@ +//! API: the HTTP services' OpenAPI documents (the room service's +//! `joule-moq-room/openapi.json`, and any `openapi*.json` under `wai/`). A client written +//! against the old document must still work. So a route (a path and method) that is gone +//! breaks; so does an operation whose `operationId` changed (the generated clients and +//! tools are named by it), a parameter that is gone, moved or newly required, a request +//! body that becomes required, a success response that is gone, and a schema a request +//! carries that gains a required property, or one a response carries that loses a +//! property or changes a property's type. New routes, optional parameters and response +//! properties are compatible. + +use super::git::Repo; +use super::super::envelope::OpError; +use super::{Category, Finding}; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; + +/// The documents read, by their path in the repository. +pub const PATHSPECS: &[&str] = &["joule-code/joule-code-rs/crates/joule-moq-room/openapi.json", ":(glob)wai/**/openapi*.json"]; + +const METHODS: &[&str] = &["get", "put", "post", "delete", "options", "head", "patch", "trace"]; + +/// A short name for a document: its crate's directory, else its path. +fn service(path: &str) -> String { + let parts: Vec<&str> = path.split('/').collect(); + if parts.len() >= 2 { parts[parts.len() - 2].to_string() } else { path.to_string() } +} + +fn resolve<'a>(doc: &'a Value, v: &'a Value) -> &'a Value { + let mut v = v; + for _ in 0..16 { + let Some(r) = v.get("$ref").and_then(Value::as_str) else { break }; + let Some(p) = r.strip_prefix("#/") else { break }; + let mut t = doc; + for seg in p.split('/') { + match t.get(seg.replace("~1", "/").replace("~0", "~")) { + Some(x) => t = x, + None => return v, + } + } + v = t; + } + v +} + +fn params(doc: &Value, path_item: &Value, op: &Value) -> BTreeMap<(String, String), bool> { + let mut m = BTreeMap::new(); + for list in [path_item.get("parameters"), op.get("parameters")].into_iter().flatten() { + for p in list.as_array().into_iter().flatten() { + let p = resolve(doc, p); + if let (Some(n), Some(i)) = (p.get("name").and_then(Value::as_str), p.get("in").and_then(Value::as_str)) { + m.insert((n.to_string(), i.to_string()), p.get("required").and_then(Value::as_bool).unwrap_or(i == "path")); + } + } + } + m +} + +/// The properties of a schema, resolved: each name with its type (or `$ref` name) and +/// whether it is required. +fn props(doc: &Value, schema: &Value) -> BTreeMap { + let s = resolve(doc, schema); + let required: BTreeSet = s.get("required").and_then(Value::as_array).into_iter().flatten().filter_map(|x| x.as_str().map(String::from)).collect(); + let mut m = BTreeMap::new(); + if let Some(p) = s.get("properties").and_then(Value::as_object) { + for (k, v) in p { + let ty = v.get("$ref").and_then(Value::as_str).map(String::from).or_else(|| v.get("type").map(|t| t.to_string())).unwrap_or_default(); + m.insert(k.clone(), (ty, required.contains(k))); + } + } + m +} + +fn json_schema<'a>(doc: &'a Value, body: &'a Value) -> Option<&'a Value> { + let body = resolve(doc, body); + let content = body.get("content")?.as_object()?; + content.get("application/json").or_else(|| content.values().next())?.get("schema") +} + +fn ops(doc: &Value) -> BTreeMap { + let mut m = BTreeMap::new(); + for (path, item) in doc.get("paths").and_then(Value::as_object).into_iter().flatten() { + for meth in METHODS { + if let Some(op) = item.get(*meth) { + m.insert(format!("{} {path}", meth.to_uppercase()), (item, op)); + } + } + } + m +} + +/// One document's findings (`None` where a tree has none). +pub fn compare_doc(path: &str, old: Option<&Value>, new: Option<&Value>) -> Vec { + let svc = service(path); + let mut out = Vec::new(); + let empty = Value::Null; + let (a, b) = (old.unwrap_or(&empty), new.unwrap_or(&empty)); + let (oa, ob) = (ops(a), ops(b)); + for (route, (ia, xa)) in &oa { + let item = format!("{svc}:{route}"); + let Some((ib, xb)) = ob.get(route) else { + out.push(Finding::broke(Category::Api, "openapi", item, "removed".into(), format!("the route {route} is gone from {path}"), "a client that calls it gets no such route".into())); + continue; + }; + let mut what = Vec::new(); + if xa.get("operationId") != xb.get("operationId") && xa.get("operationId").is_some() { + what.push(format!("operationId {} → {}", xa.get("operationId").unwrap_or(&empty), xb.get("operationId").unwrap_or(&empty))); + } + let (pa, pb) = (params(a, ia, xa), params(b, ib, xb)); + for ((n, i), req) in &pa { + match pb.get(&(n.clone(), i.clone())) { + None => what.push(format!("the {i} parameter {n} is gone")), + Some(r) if *r && !req => what.push(format!("the {i} parameter {n} is now required")), + _ => {} + } + } + for ((n, i), req) in &pb { + if *req && !pa.contains_key(&(n.clone(), i.clone())) { + what.push(format!("a required {i} parameter {n} is new")); + } + } + let body_required = |x: &Value, d: &Value| x.get("requestBody").map(|r| resolve(d, r).get("required").and_then(Value::as_bool).unwrap_or(false)).unwrap_or(false); + if body_required(xb, b) && !body_required(xa, a) { + what.push("the request body is now required".into()); + } + if let (Some(sa), Some(sb)) = (xa.get("requestBody").and_then(|r| json_schema(a, r)), xb.get("requestBody").and_then(|r| json_schema(b, r))) { + let (qa, qb) = (props(a, sa), props(b, sb)); + for (k, (_, req)) in &qb { + if *req && !qa.get(k).is_some_and(|x| x.1) { + what.push(format!("the request property {k} is now required")); + } + } + } + let codes = |x: &Value| -> Vec { x.get("responses").and_then(Value::as_object).map(|r| r.keys().cloned().collect()).unwrap_or_default() }; + let (ca, cb) = (codes(xa), codes(xb)); + for c in ca.iter().filter(|c| c.starts_with('2') && !cb.contains(c)) { + what.push(format!("the {c} response is gone")); + } + for c in ca.iter().filter(|c| c.starts_with('2') && cb.contains(c)) { + let schema = |x: &Value, d: &Value| x.get("responses").and_then(|r| r.get(c.as_str())).and_then(|r| json_schema(d, r)).map(|s| props(d, s)); + if let (Some(ra), Some(rb)) = (schema(xa, a), schema(xb, b)) { + for (k, (ty, _)) in &ra { + match rb.get(k) { + None => what.push(format!("the {c} response's property {k} is gone")), + Some((t, _)) if t != ty => what.push(format!("the {c} response's property {k}: {ty} → {t}")), + _ => {} + } + } + } + } + if !what.is_empty() { + out.push(Finding::broke(Category::Api, "openapi", item, "changed".into(), format!("{route}: {}", what.join("; ")), "a client written against the old document sends what is now refused, or reads what is no longer sent".into())); + } + } + for route in ob.keys() { + if !oa.contains_key(route) { + out.push(Finding::compatible(Category::Api, "openapi", format!("{svc}:{route}"), format!("the route {route} is new in {path}"))); + } + } + out +} + +/// Every document's findings between `base` and the working tree. +pub fn compare(repo: &Repo, base: &str) -> Result, OpError> { + let mut paths: BTreeSet = repo.ls_base(base, PATHSPECS)?.into_iter().collect(); + paths.extend(repo.ls_head(PATHSPECS)?); + let list: Vec = paths.into_iter().collect(); + let olds = repo.base_files(base, &list)?; + let mut out = Vec::new(); + for p in &list { + let old: Option = olds.get(p).and_then(|b| serde_json::from_slice(b).ok()); + let new: Option = repo.head_file(p).and_then(|b| serde_json::from_slice(&b).ok()); + out.extend(compare_doc(p, old.as_ref(), new.as_ref())); + } + Ok(out) +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/registry.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/registry.rs new file mode 100644 index 000000000..665fee30a --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/registry.rs @@ -0,0 +1,169 @@ +//! Wire: the registry export (`wai/registry/capabilities.json`, written by `wai registry +//! export` from REGISTRY.md and held to it by `registry export --check`). It names every +//! capability a reader dispatches on, the deterministic identifiers, and the wire +//! identifiers: magics, document kinds, headers, paths, schemes and tags. +//! +//! A capability, identifier or wire identifier that is gone breaks: a reader that +//! dispatches on it, or a writer that emits it, no longer conforms. So does a capability +//! whose kind, media, role, tier, licence, replicate class, edition or operator set +//! changed (one the old export did not state at all is new, and compatible), one that left a set or joined the mandatory floor (every old reader lacks +//! it), a registered capability demoted to candidate, a wire identifier (named by its kind +//! and name together) whose status fell back from `defined`, and an identifier whose +//! meaning changed. A section moving between documents is editorial and is not compared. + +use super::jsondiff; +use super::{Category, Finding}; +use serde_json::{Map, Value}; + +pub const PATH: &str = "wai/registry/capabilities.json"; + +const COMPARED: &[&str] = &["kind", "media", "role", "tier", "licence", "replicate", "edition", "ops"]; + +fn by<'a>(doc: &'a Value, list: &str, key: &str) -> Vec<(String, &'a Map)> { + doc.get(list) + .and_then(Value::as_array) + .map(|a| a.iter().filter_map(|x| Some((x.get(key)?.as_str()?.to_string(), x.as_object()?))).collect()) + .unwrap_or_default() +} + +/// The wire identifiers, each named by its kind and its name together (`fourcc/wai1`): one +/// name can be, say, a `codecs` prefix and a sample-entry 4CC at once. +fn wire(doc: &Value) -> Vec<(String, &Map)> { + by(doc, "wire_identifiers", "name").into_iter().map(|(n, m)| (format!("{}/{n}", m.get("kind").and_then(Value::as_str).unwrap_or("identifier")), m)).collect() +} + +fn s(m: &Map, k: &str) -> String { + match m.get(k) { + None => "absent".into(), + Some(Value::String(x)) => x.clone(), + Some(v) => jsondiff::short(v), + } +} + +/// The registry's findings, from the two exports (`None` where a tree has none). +pub fn compare(old: Option<&Value>, new: Option<&Value>) -> Vec { + let mut out = Vec::new(); + let (old, new) = match (old, new) { + (None, None) => return out, + (None, Some(_)) => { + out.push(Finding::compatible(Category::Wire, "registry", PATH.into(), "the registry export is new".into())); + return out; + } + (Some(_), None) => { + out.push(Finding::broke( + Category::Wire, + "registry", + PATH.into(), + "removed".into(), + "the registry export is gone".into(), + "every capability and wire identifier a reader dispatches on loses its registration".into(), + )); + return out; + } + (Some(a), Some(b)) => (a, b), + }; + + let caps_new: std::collections::HashMap> = by(new, "capabilities", "capability").into_iter().collect(); + let caps_old: std::collections::HashSet = by(old, "capabilities", "capability").into_iter().map(|x| x.0).collect(); + for (name, a) in by(old, "capabilities", "capability") { + let id = name.clone(); + let Some(b) = caps_new.get(&name) else { + out.push(Finding::broke(Category::Wire, "capability", id, "removed".into(), format!("capability {name} is no longer registered"), "a reader that dispatches on it, or a writer that names it, no longer conforms".into())); + continue; + }; + let mut what = Vec::new(); + let mut gained = Vec::new(); + for k in COMPARED { + match (a.get(*k), b.get(*k)) { + (x, y) if x == y => {} + // A member the old export did not have is new information, not a change. + (None, Some(_)) => gained.push(format!("{k} {}", s(b, k))), + _ => what.push(format!("{k} {} → {}", s(a, k), s(b, k))), + } + } + if a.get("status").and_then(Value::as_str) == Some("registered") && b.get("status").and_then(Value::as_str) != Some("registered") { + what.push(format!("status registered → {}", s(b, "status"))); + } + let sets = |m: &Map| -> Vec { m.get("sets").and_then(Value::as_array).map(|a| a.iter().filter_map(|x| x.as_str().map(String::from)).collect()).unwrap_or_default() }; + let (sa, sb) = (sets(a), sets(b)); + for x in sa.iter().filter(|x| !sb.contains(x)) { + what.push(format!("left the set {x}")); + } + if sb.iter().any(|x| x == "floor") && !sa.iter().any(|x| x == "floor") { + what.push("joined the mandatory floor, which every older reader lacks".into()); + } + if !what.is_empty() { + out.push(Finding::broke(Category::Wire, "capability", id, "changed".into(), what.join("; "), "readers and writers dispatch, select and verify by these fields".into())); + } else if a.get("status") != b.get("status") || sb.iter().any(|x| !sa.contains(x)) || !gained.is_empty() { + gained.insert(0, format!("status {} → {}, sets {:?} → {:?}", s(a, "status"), s(b, "status"), sa, sb)); + out.push(Finding::compatible(Category::Wire, "capability", name.clone(), gained.join("; "))); + } + } + for (name, _) in by(new, "capabilities", "capability") { + if !caps_old.contains(&name) { + out.push(Finding::compatible(Category::Wire, "capability", name.clone(), format!("capability {name} is registered"))); + } + } + + let ids_new: std::collections::HashMap> = by(new, "identifiers", "identifier").into_iter().collect(); + let ids_old: std::collections::HashSet = by(old, "identifiers", "identifier").into_iter().map(|x| x.0).collect(); + for (name, a) in by(old, "identifiers", "identifier") { + let id = name.clone(); + match ids_new.get(&name) { + None => out.push(Finding::broke(Category::Wire, "identifier", id, "removed".into(), format!("identifier {name} is gone"), "a manifest that names it is no longer read as it was".into())), + Some(b) if a.get("meaning") != b.get("meaning") => out.push(Finding::broke( + Category::Wire, + "identifier", + id, + "changed".into(), + format!("its meaning changed: {} → {}", s(a, "meaning"), s(b, "meaning")), + "a deterministic identifier names one arithmetic; another meaning gives other bytes under the same name".into(), + )), + Some(_) => {} + } + } + for (name, _) in by(new, "identifiers", "identifier") { + if !ids_old.contains(&name) { + out.push(Finding::compatible(Category::Wire, "identifier", name.clone(), format!("identifier {name} is new"))); + } + } + + let rank = |m: &Map| match m.get("status").and_then(Value::as_str) { + Some("defined") => 2, + Some("provisional") => 1, + _ => 0, + }; + let wire_new: std::collections::HashMap> = wire(new).into_iter().collect(); + let wire_old: std::collections::HashSet = wire(old).into_iter().map(|x| x.0).collect(); + for (id, a) in wire(old) { + let kind = a.get("kind").and_then(Value::as_str).unwrap_or("identifier"); + let name = a.get("name").and_then(Value::as_str).unwrap_or("?"); + match wire_new.get(&id) { + None => out.push(Finding::broke( + Category::Wire, + "wire-identifier", + id.clone(), + "removed".into(), + format!("the {kind} {name} ({}) is gone", s(a, "document")), + format!("a reader that matches this {kind} refuses what is written without it, and what was written with it is no longer defined"), + )), + Some(b) if rank(b) < rank(a) => out.push(Finding::broke( + Category::Wire, + "wire-identifier", + id.clone(), + "changed".into(), + format!("the {kind} {name}: status {} → {}", s(a, "status"), s(b, "status")), + format!("readers and writers rely on this {kind} as it was defined"), + )), + Some(_) => {} + } + } + for (id, b) in wire(new) { + if !wire_old.contains(&id) { + let kind = b.get("kind").and_then(Value::as_str).unwrap_or("identifier"); + let name = b.get("name").and_then(Value::as_str).unwrap_or("?"); + out.push(Finding::compatible(Category::Wire, "wire-identifier", id.clone(), format!("the {kind} {name} is new"))); + } + } + out +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/rust_api.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/rust_api.rs new file mode 100644 index 000000000..c8c575b00 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/rust_api.rs @@ -0,0 +1,683 @@ +//! API: the Rust public API of the WAI library crates, read from the compiler's own +//! description of it. Each crate is documented twice by `cargo rustdoc` in rustdoc's JSON +//! output, once from the base (its tree extracted to a scratch directory) and once from the +//! working tree, with the same toolchain and the same features: every feature but those +//! that need system libraries or downloads (`DENY`, and any feature that enables one). +//! +//! The public surface is walked from the crate root through public modules and re-exports: +//! every public path to a module, function, struct and its public fields, enum and its +//! variants, union, trait and its items, inherent method and associated item, constant, +//! static, type alias and macro, and every trait a public type implements (auto traits +//! included, blanket implementations left out). A path that is gone breaks. So does a +//! signature that changed (parameter and return types, generics and bounds, `unsafe`, +//! `const`, `async`, ABI; parameter names are not part of it), a field's or alias's type +//! that changed, a struct that can no longer be built with a literal, an enum or struct +//! that became `#[non_exhaustive]`, a new public field of a struct a literal could build, +//! a new variant of an exhaustive enum, and a new required method of a trait. Other new +//! paths are compatible. +//! +//! Rustdoc's JSON is an unstable output of a stable toolchain, read here with +//! `RUSTC_BOOTSTRAP=1`; both sides are written by one toolchain, so they share one format +//! version, and a run that finds two refuses to compare them. + +use super::git::Repo; +use super::super::envelope::{EXIT_INTERNAL, EXIT_NOT_FOUND, OpError}; +use super::{Category, Finding}; +use serde_json::{Map, Value}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// A library crate whose public API is held. +pub struct Crate { + pub name: &'static str, + pub dir: &'static str, + /// Features left out of the documented set: they need system libraries, a download, or + /// a GPU toolchain, which a check of the API does not. + pub deny: &'static [&'static str], +} + +pub const CRATES: &[Crate] = &[ + Crate { name: "wai", dir: "wai/wai-rs", deny: &["default", "codecs", "semantic", "neural", "c2pa_emit", "ops_gpu"] }, + Crate { name: "wai-quantum", dir: "wai/wai-quantum", deny: &["default"] }, +]; + +/// One public item: what it is, its comparable form, how it reads, and the facts the +/// semver rules above need. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ApiItem { + pub kind: &'static str, + pub sig: String, + pub show: String, + /// A struct every field of which is public and which is not `#[non_exhaustive]`, or an + /// enum with every variant public and not `#[non_exhaustive]`. + pub exhaustive: bool, + /// A trait method, type or constant with no default. + pub required: bool, +} + +/// The public surface of one crate: each public path, and its item. +pub type Surface = BTreeMap; + +struct Doc<'a> { + index: &'a Map, + paths: &'a Map, + crate_name: String, +} + +impl Doc<'_> { + fn item(&self, id: &Value) -> Option<&Value> { + self.index.get(&id_key(id)) + } + fn path_of(&self, id: &Value) -> Option { + let p = self.paths.get(&id_key(id))?.get("path")?.as_array()?; + Some(p.iter().filter_map(Value::as_str).collect::>().join("::")) + } +} + +fn id_key(id: &Value) -> String { + match id { + Value::String(s) => s.clone(), + other => other.to_string(), + } +} + +fn inner<'a>(item: &'a Value, kind: &str) -> Option<&'a Value> { + item.get("inner")?.get(kind) +} + +fn attrs_have(item: &Value, word: &str) -> bool { + item.get("attrs").and_then(Value::as_array).is_some_and(|a| a.iter().any(|x| x.as_str() == Some(word) || x.to_string().contains(word))) +} + +/// `v` with every `id` dropped and every resolved path written as its full path, keys +/// sorted: equal for two builds of one API. +fn canon(doc: &Doc, v: &Value) -> Value { + match v { + Value::Object(m) => { + let mut keys: Vec<&String> = m.keys().collect(); + keys.sort(); + let mut out = Map::new(); + for k in keys { + if k == "id" { + continue; + } + let mut x = canon(doc, &m[k]); + if k == "path" && x.is_string() && let Some(full) = m.get("id").and_then(|id| doc.path_of(id)) { + x = Value::String(full); + } + out.insert(k.clone(), x); + } + Value::Object(out) + } + Value::Array(a) => Value::Array(a.iter().map(|x| canon(doc, x)).collect()), + other => other.clone(), + } +} + +/// A type as Rust spells it, as far as these forms go; anything else as JSON. +fn ty(doc: &Doc, t: &Value) -> String { + let Some((k, v)) = t.as_object().and_then(|m| m.iter().next()) else { + return t.as_str().map(String::from).unwrap_or_else(|| t.to_string()); + }; + match k.as_str() { + "resolved_path" => path_ty(doc, v), + "generic" | "primitive" => v.as_str().unwrap_or("?").to_string(), + "borrowed_ref" => format!( + "&{}{}{}", + v.get("lifetime").and_then(Value::as_str).map(|l| format!("{l} ")).unwrap_or_default(), + if v.get("is_mutable").and_then(Value::as_bool) == Some(true) { "mut " } else { "" }, + ty(doc, &v["type"]) + ), + "raw_pointer" => format!("*{} {}", if v.get("is_mutable").and_then(Value::as_bool) == Some(true) { "mut" } else { "const" }, ty(doc, &v["type"])), + "slice" => format!("[{}]", ty(doc, v)), + "array" => format!("[{}; {}]", ty(doc, &v["type"]), v.get("len").and_then(Value::as_str).unwrap_or("?")), + "tuple" => format!("({})", v.as_array().map(|a| a.iter().map(|x| ty(doc, x)).collect::>().join(", ")).unwrap_or_default()), + "impl_trait" => format!("impl {}", bounds(doc, v)), + "dyn_trait" => format!( + "dyn {}", + v.get("traits").and_then(Value::as_array).map(|a| a.iter().map(|x| path_ty(doc, &x["trait"])).collect::>().join(" + ")).unwrap_or_default() + ), + "qualified_path" => format!( + "<{} as {}>::{}", + ty(doc, &v["self_type"]), + v.get("trait").filter(|t| !t.is_null()).map(|t| path_ty(doc, t)).unwrap_or_default(), + v.get("name").and_then(Value::as_str).unwrap_or("?") + ), + "function_pointer" => { + let sig = &v["sig"]; + format!("fn({}){}", inputs(doc, sig), output(doc, sig)) + } + "infer" => "_".into(), + _ => t.to_string(), + } +} + +fn path_ty(doc: &Doc, v: &Value) -> String { + let name = v.get("id").and_then(|id| doc.path_of(id)).or_else(|| v.get("path").and_then(Value::as_str).map(String::from)).unwrap_or_default(); + format!("{name}{}", args(doc, v.get("args"))) +} + +fn args(doc: &Doc, a: Option<&Value>) -> String { + let Some(a) = a.filter(|a| !a.is_null()) else { return String::new() }; + if let Some(ab) = a.get("angle_bracketed") { + let mut parts: Vec = ab + .get("args") + .and_then(Value::as_array) + .into_iter() + .flatten() + .map(|x| match x { + Value::Object(m) if m.contains_key("type") => ty(doc, &m["type"]), + Value::Object(m) if m.contains_key("lifetime") => m["lifetime"].as_str().unwrap_or("'_").to_string(), + other => other.to_string(), + }) + .collect(); + for c in ab.get("constraints").and_then(Value::as_array).into_iter().flatten() { + let n = c.get("name").and_then(Value::as_str).unwrap_or("?"); + match c.get("binding").and_then(|b| b.get("equality")) { + Some(e) => parts.push(format!("{n} = {}", e.get("type").map(|t| ty(doc, t)).unwrap_or_else(|| e.to_string()))), + None => parts.push(format!("{n}: …")), + } + } + return if parts.is_empty() { String::new() } else { format!("<{}>", parts.join(", ")) }; + } + if let Some(p) = a.get("parenthesized") { + let ins = p.get("inputs").and_then(Value::as_array).map(|i| i.iter().map(|x| ty(doc, x)).collect::>().join(", ")).unwrap_or_default(); + let out = p.get("output").filter(|o| !o.is_null()).map(|o| format!(" -> {}", ty(doc, o))).unwrap_or_default(); + return format!("({ins}){out}"); + } + a.to_string() +} + +fn bounds(doc: &Doc, b: &Value) -> String { + b.as_array() + .map(|a| { + a.iter() + .map(|x| match x.get("trait_bound") { + Some(tb) => path_ty(doc, &tb["trait"]), + None => x.get("outlives").and_then(Value::as_str).map(String::from).unwrap_or_else(|| x.to_string()), + }) + .collect::>() + .join(" + ") + }) + .unwrap_or_default() +} + +fn inputs(doc: &Doc, sig: &Value) -> String { + sig.get("inputs").and_then(Value::as_array).map(|a| a.iter().map(|p| ty(doc, &p[1])).collect::>().join(", ")).unwrap_or_default() +} + +fn output(doc: &Doc, sig: &Value) -> String { + sig.get("output").filter(|o| !o.is_null()).map(|o| format!(" -> {}", ty(doc, o))).unwrap_or_default() +} + +fn generics_show(doc: &Doc, g: &Value) -> String { + let params: Vec = g + .get("params") + .and_then(Value::as_array) + .into_iter() + .flatten() + .map(|p| { + let n = p.get("name").and_then(Value::as_str).unwrap_or("?").to_string(); + match p.get("kind").and_then(|k| k.get("type")).and_then(|t| t.get("bounds")) { + Some(b) if b.as_array().is_some_and(|a| !a.is_empty()) => format!("{n}: {}", bounds(doc, b)), + _ => n, + } + }) + .filter(|n| !n.starts_with("impl ")) + .collect(); + if params.is_empty() { String::new() } else { format!("<{}>", params.join(", ")) } +} + +/// A function's comparable form (types only, parameter names dropped) and how it reads. +fn function(doc: &Doc, name: &str, f: &Value) -> (String, String) { + let sig = &f["sig"]; + let types: Vec = sig.get("inputs").and_then(Value::as_array).into_iter().flatten().map(|p| canon(doc, &p[1])).collect(); + let cmp = serde_json::json!({"inputs": types, "output": canon(doc, &sig["output"]), "c_variadic": sig.get("is_c_variadic"), "generics": canon(doc, &f["generics"]), "header": canon(doc, &f["header"])}); + let h = &f["header"]; + let mut q = String::new(); + for (k, w) in [("is_const", "const "), ("is_async", "async "), ("is_unsafe", "unsafe ")] { + if h.get(k).and_then(Value::as_bool) == Some(true) { + q.push_str(w); + } + } + let show = format!("{q}fn {name}{}({}){}", generics_show(doc, &f["generics"]), inputs(doc, sig), output(doc, sig)); + (cmp.to_string(), show) +} + +struct Walk<'a, 'b> { + doc: &'b Doc<'a>, + out: Surface, + stack: Vec, +} + +impl Walk<'_, '_> { + fn put(&mut self, path: String, kind: &'static str, sig: String, show: String, exhaustive: bool, required: bool) { + self.out.insert(path, ApiItem { kind, sig, show, exhaustive, required }); + } + + fn module(&mut self, id: &Value, path: &str) { + let key = id_key(id); + if self.stack.contains(&key) || self.stack.len() > 32 { + return; + } + let Some(m) = self.doc.item(id).and_then(|i| inner(i, "module")) else { return }; + self.stack.push(key); + for child in m.get("items").and_then(Value::as_array).into_iter().flatten() { + self.child(child, path); + } + self.stack.pop(); + } + + fn child(&mut self, id: &Value, parent: &str) { + let doc = self.doc; + let Some(item) = doc.item(id) else { return }; + if item.get("visibility").and_then(Value::as_str) != Some("public") { + return; + } + if let Some(u) = inner(item, "use") { + let name = u.get("name").and_then(Value::as_str).unwrap_or("?"); + let source = u.get("source").and_then(Value::as_str).unwrap_or("?").to_string(); + let target = u.get("id").filter(|t| !t.is_null()).and_then(|t| doc.item(t).map(|i| (t, i))); + let glob = u.get("is_glob").and_then(Value::as_bool) == Some(true); + match (glob, target) { + (true, Some((t, i))) if inner(i, "module").is_some() => self.module(t, parent), + (false, Some((t, _))) => self.named(t, &format!("{parent}::{name}")), + (true, _) => self.put(format!("{parent}::*{source}"), "re-export", source.clone(), format!("pub use {source}::*"), false, false), + (false, None) => self.put(format!("{parent}::{name}"), "re-export", source.clone(), format!("pub use {source}"), false, false), + } + return; + } + let Some(name) = item.get("name").and_then(Value::as_str) else { return }; + self.named(id, &format!("{parent}::{name}")); + } + + /// The item `id` at the public path `path`, and its members below it. + fn named(&mut self, id: &Value, path: &str) { + let doc = self.doc; + let Some(item) = doc.item(id) else { return }; + let Some((kind, v)) = item.get("inner").and_then(Value::as_object).and_then(|m| m.iter().next()) else { return }; + let name = path.rsplit("::").next().unwrap_or(path).to_string(); + let non_exhaustive = attrs_have(item, "non_exhaustive"); + match kind.as_str() { + "module" => { + self.put(path.into(), "module", String::new(), format!("mod {name}"), false, false); + self.module(id, path); + } + "function" => { + let (sig, show) = function(doc, &name, v); + self.put(path.into(), "function", sig, show, false, false); + } + "struct" | "union" => { + let k = &v["kind"]; + let mut exhaustive = !non_exhaustive; + let fields: Vec<&Value> = if let Some(p) = k.get("plain") { + exhaustive &= p.get("has_stripped_fields").and_then(Value::as_bool) != Some(true); + p.get("fields").and_then(Value::as_array).map(|a| a.iter().collect()).unwrap_or_default() + } else if let Some(t) = k.get("tuple").and_then(Value::as_array) { + exhaustive &= t.iter().all(|f| !f.is_null()); + t.iter().filter(|f| !f.is_null()).collect() + } else if let Some(f) = v.get("fields").and_then(Value::as_array) { + exhaustive &= v.get("has_stripped_fields").and_then(Value::as_bool) != Some(true); + f.iter().collect() + } else { + Vec::new() + }; + let shape = if k.get("tuple").is_some() { "tuple" } else if k.as_str() == Some("unit") { "unit" } else { "plain" }; + let kind_name: &'static str = if kind == "union" { "union" } else { "struct" }; + self.put(path.into(), kind_name, format!("{shape} {}", canon(doc, &v["generics"])), format!("{kind_name} {name}{}", generics_show(doc, &v["generics"])), exhaustive, false); + for f in fields { + if let Some(fi) = doc.item(f) + && fi.get("visibility").and_then(Value::as_str) == Some("public") + { + let fname = fi.get("name").and_then(Value::as_str).unwrap_or("?"); + let t = inner(fi, "struct_field").cloned().unwrap_or(Value::Null); + self.put(format!("{path}::{fname}"), "field", canon(doc, &t).to_string(), format!("{fname}: {}", ty(doc, &t)), false, false); + } + } + self.impls(v, path); + } + "enum" => { + let exhaustive = !non_exhaustive && v.get("has_stripped_variants").and_then(Value::as_bool) != Some(true); + self.put(path.into(), "enum", canon(doc, &v["generics"]).to_string(), format!("enum {name}{}", generics_show(doc, &v["generics"])), exhaustive, false); + for var in v.get("variants").and_then(Value::as_array).into_iter().flatten() { + if let Some(vi) = doc.item(var) { + let vname = vi.get("name").and_then(Value::as_str).unwrap_or("?"); + let vk = inner(vi, "variant").cloned().unwrap_or(Value::Null); + let mut fields = Vec::new(); + let field_ids: Vec<&Value> = match vk.get("kind") { + Some(k) if k.get("tuple").is_some() => k["tuple"].as_array().map(|a| a.iter().collect()).unwrap_or_default(), + Some(k) if k.get("struct").is_some() => k["struct"].get("fields").and_then(Value::as_array).map(|a| a.iter().collect()).unwrap_or_default(), + _ => Vec::new(), + }; + for fid in field_ids { + let t = doc.item(fid).and_then(|fi| inner(fi, "struct_field").map(|t| (fi.get("name").and_then(Value::as_str).unwrap_or("?").to_string(), t.clone()))); + if let Some((fname, t)) = t { + fields.push(serde_json::json!([fname, canon(doc, &t)])); + } + } + let sig = serde_json::json!({"shape": vk.get("kind").map(|k| if k.is_string() { "plain" } else if k.get("tuple").is_some() { "tuple" } else { "struct" }), "fields": fields, "discriminant": vk.get("discriminant")}); + self.put(format!("{path}::{vname}"), "variant", sig.to_string(), format!("{name}::{vname}"), false, false); + } + } + self.impls(v, path); + } + "trait" => { + let sig = serde_json::json!({"generics": canon(doc, &v["generics"]), "bounds": canon(doc, &v["bounds"]), "unsafe": v.get("is_unsafe"), "auto": v.get("is_auto")}); + self.put(path.into(), "trait", sig.to_string(), format!("trait {name}{}", generics_show(doc, &v["generics"])), false, false); + for t in v.get("items").and_then(Value::as_array).into_iter().flatten() { + let Some(ti) = doc.item(t) else { continue }; + let tname = ti.get("name").and_then(Value::as_str).unwrap_or("?"); + let Some((tk, tv)) = ti.get("inner").and_then(Value::as_object).and_then(|m| m.iter().next()) else { continue }; + let p = format!("{path}::{tname}"); + match tk.as_str() { + "function" => { + let (sig, show) = function(doc, tname, tv); + let required = tv.get("has_body").and_then(Value::as_bool) != Some(true); + self.put(p, "trait method", sig, show, false, required); + } + "assoc_type" => { + let required = tv.get("type").is_none_or(Value::is_null); + self.put(p, "associated type", canon(doc, &tv["bounds"]).to_string(), format!("type {tname}"), false, required); + } + "assoc_const" => { + let required = tv.get("value").is_none_or(Value::is_null); + self.put(p, "associated constant", canon(doc, &tv["type"]).to_string(), format!("const {tname}: {}", ty(doc, &tv["type"])), false, required); + } + _ => {} + } + } + } + "constant" => { + let t = v.get("type").cloned().unwrap_or(Value::Null); + self.put(path.into(), "constant", canon(doc, &t).to_string(), format!("const {name}: {}", ty(doc, &t)), false, false); + } + "static" => { + let t = v.get("type").cloned().unwrap_or(Value::Null); + let sig = serde_json::json!({"type": canon(doc, &t), "mutable": v.get("is_mutable")}); + self.put(path.into(), "static", sig.to_string(), format!("static {name}: {}", ty(doc, &t)), false, false); + } + "type_alias" => { + let t = v.get("type").cloned().unwrap_or(Value::Null); + let sig = serde_json::json!({"type": canon(doc, &t), "generics": canon(doc, &v["generics"])}); + self.put(path.into(), "type alias", sig.to_string(), format!("type {name} = {}", ty(doc, &t)), false, false); + } + "macro" | "proc_macro" => self.put(path.into(), "macro", String::new(), format!("macro {name}!"), false, false), + _ => {} + } + } + + /// The inherent methods and associated items, and the traits implemented, of the type + /// at `path`. + fn impls(&mut self, v: &Value, path: &str) { + let doc = self.doc; + for imp in v.get("impls").and_then(Value::as_array).into_iter().flatten() { + let Some(ii) = doc.item(imp).and_then(|i| inner(i, "impl")) else { continue }; + if ii.get("blanket_impl").is_some_and(|b| !b.is_null()) || ii.get("is_negative").and_then(Value::as_bool) == Some(true) { + continue; + } + match ii.get("trait").filter(|t| !t.is_null()) { + Some(t) => { + let tr = path_ty(doc, t); + let sig = serde_json::json!({"generics": canon(doc, &ii["generics"]), "unsafe": ii.get("is_unsafe")}); + self.put(format!("impl {tr} for {path}"), "trait implementation", sig.to_string(), format!("impl {tr} for {path}"), false, false); + } + None => { + for m in ii.get("items").and_then(Value::as_array).into_iter().flatten() { + let Some(mi) = doc.item(m) else { continue }; + if mi.get("visibility").and_then(Value::as_str) != Some("public") { + continue; + } + let mname = mi.get("name").and_then(Value::as_str).unwrap_or("?"); + let Some((mk, mv)) = mi.get("inner").and_then(Value::as_object).and_then(|m| m.iter().next()) else { continue }; + let p = format!("{path}::{mname}"); + match mk.as_str() { + "function" => { + let (sig, show) = function(doc, mname, mv); + self.put(p, "method", sig, show, false, false); + } + "assoc_const" => self.put(p, "associated constant", canon(doc, &mv["type"]).to_string(), format!("const {mname}: {}", ty(doc, &mv["type"])), false, false), + "assoc_type" => self.put(p, "associated type", canon(doc, &mv["type"]).to_string(), format!("type {mname}"), false, false), + _ => {} + } + } + } + } + } + } +} + +/// The public surface a rustdoc JSON document describes, and its format version. +pub fn surface(json: &Value) -> Result<(Surface, u64), String> { + let index = json.get("index").and_then(Value::as_object).ok_or("rustdoc JSON without an index")?; + let paths = json.get("paths").and_then(Value::as_object).ok_or("rustdoc JSON without paths")?; + let root = json.get("root").ok_or("rustdoc JSON without a root")?; + let crate_name = index.get(&id_key(root)).and_then(|r| r.get("name")).and_then(Value::as_str).unwrap_or("crate").to_string(); + let doc = Doc { index, paths, crate_name }; + let mut w = Walk { doc: &doc, out: Surface::new(), stack: Vec::new() }; + let name = doc.crate_name.clone(); + w.module(root, &name); + Ok((w.out, json.get("format_version").and_then(Value::as_u64).unwrap_or(0))) +} + +fn parent(path: &str) -> &str { + path.rsplit_once("::").map(|x| x.0).unwrap_or("") +} + +/// The findings between two surfaces of the crate `name`. +pub fn compare_surfaces(old: &Surface, new: &Surface) -> Vec { + let mut out = Vec::new(); + for (path, a) in old { + match new.get(path) { + None => out.push(Finding::broke( + Category::Api, + "rust", + path.clone(), + "removed".into(), + format!("the {} {} is gone", a.kind, a.show), + format!("code that names {path} no longer compiles"), + )), + Some(b) if a.kind != b.kind => out.push(Finding::broke(Category::Api, "rust", path.clone(), "changed".into(), format!("a {} is now a {}", a.kind, b.kind), "code that uses it as it was no longer compiles".into())), + Some(b) if a.sig != b.sig => { + let what = if a.show != b.show { format!("{} → {}", a.show, b.show) } else { format!("the {} {}: its generics, bounds or qualifiers changed", a.kind, a.show) }; + out.push(Finding::broke(Category::Api, "rust", path.clone(), "changed".into(), what, "code written against the old signature no longer compiles, or means something else".into())); + } + Some(b) if a.exhaustive && !b.exhaustive => out.push(Finding::broke( + Category::Api, + "rust", + path.clone(), + "changed".into(), + format!("the {} {} is no longer exhaustive (a private field or variant, or #[non_exhaustive])", a.kind, a.show), + if a.kind == "enum" { "a match over every variant no longer compiles outside the crate".into() } else { "a struct literal or pattern that lists every field no longer compiles outside the crate".into() }, + )), + Some(b) if !a.required && b.required => out.push(Finding::broke(Category::Api, "rust", path.clone(), "changed".into(), format!("{} lost its default", b.show), "every implementation outside the crate must now provide it".into())), + Some(_) => {} + } + } + for (path, b) in new { + if old.contains_key(path) { + continue; + } + let up = old.get(parent(path)); + let (breaks, why) = match (b.kind, up) { + ("field", Some(s)) if s.exhaustive && new.get(parent(path)).is_some_and(|n| n.kind == s.kind) => (true, "a struct literal or pattern that lists every field no longer compiles outside the crate"), + ("variant", Some(e)) if e.exhaustive && e.kind == "enum" => (true, "a match over every variant no longer compiles outside the crate"), + ("trait method" | "associated type" | "associated constant", Some(t)) if b.required && t.kind == "trait" => (true, "every implementation outside the crate must now provide it"), + _ => (false, ""), + }; + if breaks { + out.push(Finding::broke(Category::Api, "rust", path.clone(), "added".into(), format!("the {} {} is new", b.kind, b.show), why.into())); + } else { + out.push(Finding::compatible(Category::Api, "rust", path.clone(), format!("the {} {} is new", b.kind, b.show))); + } + } + out +} + +/// The scratch space and toolchain the two documentation builds share. +pub struct Builder { + pub target_dir: PathBuf, + pub toolchain: Option, +} + +fn internal(e: impl std::fmt::Display) -> OpError { + OpError::new(EXIT_INTERNAL, "rustdoc-failed", e.to_string(), "the API check builds each crate's documentation with cargo; run `cargo rustdoc` in the crate to see why it fails") +} + +impl Builder { + fn cargo(&self, dir: &Path) -> Command { + let mut c = Command::new("cargo"); + c.current_dir(dir).env("RUSTC_BOOTSTRAP", "1").env_remove("RUSTDOCFLAGS"); + if let Some(t) = &self.toolchain { + c.env("RUSTUP_TOOLCHAIN", t); + } + c + } + + /// The features documented for `krate` in its directory `dir`: all, less the denied + /// ones and every feature that enables a denied one. + fn features(&self, krate: &Crate, dir: &Path) -> Result, OpError> { + let out = self.cargo(dir).args(["metadata", "--no-deps", "--format-version", "1", "--offline"]).output().map_err(|e| { + OpError::new(EXIT_NOT_FOUND, "cargo-not-found", format!("cargo: {e}"), "the API check needs a Rust toolchain on PATH, or `--skip rust`") + })?; + if !out.status.success() { + return Err(internal(format!("cargo metadata in {}: {}", dir.display(), String::from_utf8_lossy(&out.stderr).trim()))); + } + let meta: Value = serde_json::from_slice(&out.stdout).map_err(internal)?; + let pkg = meta["packages"].as_array().and_then(|a| a.iter().find(|p| p["name"] == krate.name)).ok_or_else(|| internal(format!("no package {} in {}", krate.name, dir.display())))?; + let feats: BTreeMap> = pkg["features"] + .as_object() + .map(|m| m.iter().map(|(k, v)| (k.clone(), v.as_array().map(|a| a.iter().filter_map(|x| x.as_str().map(String::from)).collect()).unwrap_or_default())).collect()) + .unwrap_or_default(); + let mut denied: BTreeSet = krate.deny.iter().map(|s| s.to_string()).collect(); + loop { + let more: Vec = feats.iter().filter(|(k, v)| !denied.contains(*k) && v.iter().any(|x| denied.contains(x))).map(|x| x.0.clone()).collect(); + if more.is_empty() { + break; + } + denied.extend(more); + } + Ok(feats.keys().filter(|k| !denied.contains(*k)).cloned().collect()) + } + + /// `krate`'s public surface in the tree whose top is `top`. + pub fn surface(&self, krate: &Crate, top: &Path) -> Result<(Surface, u64), OpError> { + let dir = top.join(krate.dir); + let feats = self.features(krate, &dir)?; + let mut c = self.cargo(&dir); + c.args(["rustdoc", "--lib", "--no-default-features", "--quiet"]); + if !feats.is_empty() { + c.arg("--features").arg(feats.join(",")); + } + if dir.join("Cargo.lock").exists() { + c.arg("--locked"); + } + c.arg("--target-dir").arg(&self.target_dir); + c.args(["--", "-Z", "unstable-options", "--output-format", "json", "--cap-lints", "allow"]); + eprintln!("wai breaking: documenting {} in {}", krate.name, dir.display()); + let out = c.output().map_err(internal)?; + if !out.status.success() { + let err = String::from_utf8_lossy(&out.stderr); + let tail: Vec<&str> = err.lines().rev().take(30).collect::>().into_iter().rev().collect(); + return Err(internal(format!("cargo rustdoc for {} in {}:\n{}", krate.name, dir.display(), tail.join("\n")))); + } + let file = self.target_dir.join("doc").join(format!("{}.json", krate.name.replace('-', "_"))); + let bytes = std::fs::read(&file).map_err(|e| internal(format!("{}: {e}", file.display())))?; + let json: Value = serde_json::from_slice(&bytes).map_err(internal)?; + let _ = std::fs::remove_file(&file); + surface(&json).map_err(internal) + } +} + +/// The directories `dir`'s crate depends on by path, `dir` included, found through each +/// manifest's `path = "…"` dependencies: what can change its API. +pub fn path_closure(read: &dyn Fn(&str) -> Option, dir: &str) -> BTreeSet { + let mut seen = BTreeSet::new(); + let mut todo = vec![dir.to_string()]; + while let Some(d) = todo.pop() { + if !seen.insert(d.clone()) { + continue; + } + let Some(toml) = read(&format!("{d}/Cargo.toml")) else { continue }; + let mut section = String::new(); + for line in toml.lines() { + let l = line.trim(); + if l.starts_with('[') { + section = l.to_string(); + continue; + } + if !section.contains("dependencies") { + continue; + } + if let Some(i) = l.find("path") { + let rest = l[i + 4..].trim_start(); + if let Some(rest) = rest.strip_prefix('=') + && let Some(p) = rest.trim_start().strip_prefix('"').and_then(|r| r.split('"').next()) + { + let mut parts: Vec<&str> = d.split('/').collect(); + for seg in p.split('/') { + match seg { + ".." => { + parts.pop(); + } + "." | "" => {} + s => parts.push(s), + } + } + todo.push(parts.join("/")); + } + } + } + } + seen +} + +/// The Rust API's findings between `base` and the working tree, for every crate in +/// [`CRATES`] whose sources (or those it depends on by path) differ. +pub fn compare(repo: &Repo, base: &str, builder: &Builder, scratch: &Path, notes: &mut Vec) -> Result, OpError> { + let mut out = Vec::new(); + let top = scratch.join("base"); + let mut extracted = BTreeSet::new(); + for krate in CRATES { + let read_head = |p: &str| std::fs::read_to_string(repo.root.join(p)).ok(); + let read_base = |p: &str| repo.base_file(base, p).ok().flatten().map(|b| String::from_utf8_lossy(&b).into_owned()); + let base_dirs = path_closure(&read_base, krate.dir); + let mut dirs = path_closure(&read_head, krate.dir); + dirs.extend(base_dirs.iter().cloned()); + let specs: Vec = dirs.iter().cloned().collect(); + let spec_refs: Vec<&str> = specs.iter().map(String::as_str).collect(); + let in_head = repo.root.join(krate.dir).join("Cargo.toml").exists(); + let in_base = read_base(&format!("{}/Cargo.toml", krate.dir)).is_some(); + if !in_head && !in_base { + continue; + } + if repo.changed(base, &spec_refs)?.is_empty() { + notes.push(format!("rust: {} unchanged (nothing under {} differs)", krate.name, specs.join(", "))); + continue; + } + let new = if in_head { Some(builder.surface(krate, &repo.root)?) } else { None }; + let old = if in_base { + // Each standard the base's crate reaches by path, whole: a path dependency in + // another workspace inherits fields from that workspace's root manifest. + for d in &base_dirs { + let standard = d.split('/').next().unwrap_or(d); + if read_base(&format!("{d}/Cargo.toml")).is_some() && extracted.insert(standard.to_string()) { + repo.extract(base, standard, &top)?; + } + } + Some(builder.surface(krate, &top)?) + } else { + None + }; + if let (Some((_, fa)), Some((_, fb))) = (&old, &new) + && fa != fb + { + return Err(internal(format!("rustdoc wrote format {fa} for the base and {fb} for the working tree; they are not compared"))); + } + let empty = Surface::new(); + let (a, b) = (old.as_ref().map(|x| &x.0).unwrap_or(&empty), new.as_ref().map(|x| &x.0).unwrap_or(&empty)); + notes.push(format!("rust: {} documented, {} public paths in the base and {} in the working tree", krate.name, a.len(), b.len())); + out.extend(compare_surfaces(a, b)); + } + Ok(out) +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/schema.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/schema.rs new file mode 100644 index 000000000..b63c8f7f6 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/schema.rs @@ -0,0 +1,112 @@ +//! Record: the JSON Schemas WAI publishes for its documents (`wai/**/*.schema.json`). A +//! document that validated before must validate after. So a schema breaks where a property +//! is gone, a property becomes required, a type or enumeration narrows, a pattern or +//! constant changes, or additional properties are no longer allowed; it is compatible +//! where it only gains optional properties, types or enumerated values. + +use super::git::{Repo, Status}; +use super::super::envelope::OpError; +use super::{Category, Finding}; +use serde_json::Value; + +pub const PATHSPECS: &[&str] = &[":(glob)wai/**/*.schema.json"]; + +fn strings(v: Option<&Value>) -> Vec { + match v { + Some(Value::Array(a)) => a.iter().map(|x| x.as_str().map(String::from).unwrap_or_else(|| x.to_string())).collect(), + Some(Value::String(s)) => vec![s.clone()], + _ => Vec::new(), + } +} + +/// What a document valid under `old` may fail under `new` (`breaks`), and what `new` admits +/// that `old` did not (`adds`), each a line naming the schema path. +pub fn walk(old: &Value, new: &Value, path: &str, breaks: &mut Vec, adds: &mut Vec) { + let (Some(a), Some(b)) = (old.as_object(), new.as_object()) else { + if old != new { + breaks.push(format!("{path}: {old} → {new}")); + } + return; + }; + let (ta, tb) = (strings(a.get("type")), strings(b.get("type"))); + if (!tb.is_empty() && ta.iter().any(|t| !tb.contains(t))) || (ta.is_empty() && !tb.is_empty()) { + breaks.push(format!("{path}: type {ta:?} → {tb:?}")); + } + if let (Some(Value::Array(ea)), Some(Value::Array(eb))) = (a.get("enum"), b.get("enum")) { + for x in ea.iter().filter(|x| !eb.contains(x)) { + breaks.push(format!("{path}: the value {x} is no longer allowed")); + } + for x in eb.iter().filter(|x| !ea.contains(x)) { + adds.push(format!("{path}: the value {x} is allowed")); + } + } else if a.get("enum").is_none() && b.get("enum").is_some() { + breaks.push(format!("{path}: values are now enumerated")); + } + for k in ["pattern", "const", "format", "maxLength", "maximum", "maxItems", "minLength", "minimum", "minItems"] { + if a.get(k) != b.get(k) && b.get(k).is_some() { + breaks.push(format!("{path}: {k} {} → {}", a.get(k).map(Value::to_string).unwrap_or_else(|| "absent".into()), b.get(k).map(Value::to_string).unwrap_or_default())); + } + } + let (ra, rb) = (strings(a.get("required")), strings(b.get("required"))); + for r in rb.iter().filter(|r| !ra.contains(r)) { + breaks.push(format!("{path}: {r} is now required")); + } + if b.get("additionalProperties") == Some(&Value::Bool(false)) && a.get("additionalProperties") != Some(&Value::Bool(false)) { + breaks.push(format!("{path}: other properties are no longer allowed")); + } + for key in ["properties", "$defs", "definitions"] { + let (pa, pb) = (a.get(key).and_then(Value::as_object), b.get(key).and_then(Value::as_object)); + if let Some(pa) = pa { + for (name, sa) in pa { + match pb.and_then(|pb| pb.get(name)) { + None => breaks.push(format!("{path}/{key}/{name}: removed")), + Some(sb) => walk(sa, sb, &format!("{path}/{key}/{name}"), breaks, adds), + } + } + } + if let Some(pb) = pb { + for name in pb.keys().filter(|n| pa.is_none_or(|pa| !pa.contains_key(*n))) { + adds.push(format!("{path}/{key}/{name}: new")); + } + } + } + for key in ["items", "additionalProperties", "not"] { + if let (Some(x @ Value::Object(_)), Some(y @ Value::Object(_))) = (a.get(key), b.get(key)) { + walk(x, y, &format!("{path}/{key}"), breaks, adds); + } + } + for key in ["anyOf", "oneOf", "allOf"] { + if a.get(key) != b.get(key) { + breaks.push(format!("{path}: {key} changed")); + } + } +} + +/// The schemas' findings between `base` and the working tree. +pub fn compare(repo: &Repo, base: &str) -> Result, OpError> { + let changed = repo.changed(base, PATHSPECS)?; + let olds = repo.base_files(base, &changed.iter().map(|x| x.1.clone()).collect::>())?; + let mut out = Vec::new(); + for (status, path) in changed { + match status { + Status::Added => out.push(Finding::compatible(Category::Record, "schema", path, "a new schema".into())), + Status::Removed => out.push(Finding::broke(Category::Record, "schema", path, "removed".into(), "the schema is gone".into(), "documents written against it lose their published shape".into())), + Status::Modified => { + let old: Option = olds.get(&path).and_then(|b| serde_json::from_slice(b).ok()); + let new: Option = repo.head_file(&path).and_then(|b| serde_json::from_slice(&b).ok()); + let (Some(old), Some(new)) = (old, new) else { + out.push(Finding::broke(Category::Record, "schema", path, "changed".into(), "the schema is no longer JSON".into(), "documents cannot be checked against it".into())); + continue; + }; + let (mut breaks, mut adds) = (Vec::new(), Vec::new()); + walk(&old, &new, "#", &mut breaks, &mut adds); + if !breaks.is_empty() { + out.push(Finding::broke(Category::Record, "schema", path, "changed".into(), breaks.join("; "), "a document valid under the old schema is refused under the new one".into())); + } else if !adds.is_empty() { + out.push(Finding::compatible(Category::Record, "schema", path, adds.join("; "))); + } + } + } + } + Ok(out) +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/conform/adapter.rs b/wai/wai-rs/src/bin/wai_cli/dx/conform/adapter.rs new file mode 100644 index 000000000..9bb61a70b --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/conform/adapter.rs @@ -0,0 +1,206 @@ +//! The two ways `wai conform` reaches an implementation. +//! +//! **Command.** `--impl` is a command line, run by the system shell once a case (or once for +//! the whole corpus, where the suite allows it), with these placeholders replaced by quoted +//! paths: `{case}` a file holding the case (a verifier's vector, as the corpus writes it), +//! `{corpus}` the corpus file whole, `{in}` the input file, `{out}` a file to write the +//! output to. Without `{case}` or `{in}`, the case or input is given on standard input; +//! without `{out}`, the output is read from standard output. What the command prints first +//! on standard output is its verdict; its exit status says whether it refused (non-zero) +//! where the suite asks. +//! +//! **HTTP.** `--impl` is an `http://` URL: each case is a `POST` of the case or input to +//! it (an origin is instead fetched from, below it), and the response's status and body are +//! the answer. + +use sha2::{Digest, Sha256}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +/// What a command did. +pub struct Ran { + pub code: Option, + pub stdout: Vec, + pub stderr: Vec, + pub timed_out: bool, +} + +/// A path quoted for the shell that runs the command. +pub fn quote(p: &Path) -> String { + let s = p.to_string_lossy(); + if cfg!(windows) { format!("\"{s}\"") } else { format!("'{}'", s.replace('\'', "'\\''")) } +} + +/// `cmd` with each `{name}` replaced by its quoted path. +pub fn fill(cmd: &str, with: &[(&str, &Path)]) -> String { + let mut s = cmd.to_string(); + for (k, p) in with { + s = s.replace(&format!("{{{k}}}"), "e(p)); + } + s +} + +/// Run `cmd` through the shell, `stdin` given to it, killed after `timeout`. +/// +/// Its output is read until the command has exited and then for at most [`DRAIN`] more: a +/// program the command started in the background, or one still running after the shell was +/// killed, can hold the output open, and the run does not wait for it. The command stays in +/// the caller's process group, so an interrupt of `wai conform` reaches it too. +pub fn run(cmd: &str, stdin: Option<&[u8]>, timeout: Duration) -> Result { + let mut c = if cfg!(windows) { + let mut c = Command::new("cmd"); + c.arg("/C").arg(cmd); + c + } else { + let mut c = Command::new("sh"); + c.arg("-c").arg(cmd); + c + }; + let mut child = c + .stdin(if stdin.is_some() { Stdio::piped() } else { Stdio::null() }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("{cmd}: {e}"))?; + if let (Some(b), Some(mut w)) = (stdin, child.stdin.take()) { + let b = b.to_vec(); + // Detached: a reader that never drains its input must not hold the run. + std::thread::spawn(move || { + let _ = w.write_all(&b); + }); + } + let read = |r: Option>| { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut v = Vec::new(); + if let Some(mut r) = r { + let _ = r.read_to_end(&mut v); + } + let _ = tx.send(v); + }); + rx + }; + let out = read(child.stdout.take().map(|x| Box::new(x) as Box)); + let err = read(child.stderr.take().map(|x| Box::new(x) as Box)); + let t0 = Instant::now(); + let mut timed_out = false; + let status = loop { + match child.try_wait().map_err(|e| e.to_string())? { + Some(s) => break Some(s), + None if t0.elapsed() >= timeout => { + timed_out = true; + let _ = child.kill(); + break child.wait().ok(); + } + None => std::thread::sleep(Duration::from_millis(5)), + } + }; + let until = Instant::now() + DRAIN; + let stdout = out.recv_timeout(until.saturating_duration_since(Instant::now())).unwrap_or_default(); + let stderr = err.recv_timeout(until.saturating_duration_since(Instant::now())).unwrap_or_default(); + Ok(Ran { code: status.and_then(|s| s.code()), stdout, stderr, timed_out }) +} + +/// How long the output of a command that has exited is still read. +pub const DRAIN: Duration = Duration::from_secs(2); + +/// The first line of `b`, trimmed. +pub fn first_line(b: &[u8]) -> String { + String::from_utf8_lossy(b).lines().map(str::trim).find(|l| !l.is_empty()).unwrap_or("").to_string() +} + +pub fn sha256_hex(b: &[u8]) -> String { + Sha256::digest(b).iter().map(|x| format!("{x:02x}")).collect() +} + +/// The words of a command line, quotes removed (the shell's rules, roughly). +fn words(cmd: &str) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + let mut quote: Option = None; + let mut any = false; + for c in cmd.chars() { + match (quote, c) { + (Some(q), c) if c == q => quote = None, + (Some(_), c) => cur.push(c), + (None, '\'' | '"') => { + quote = Some(c); + any = true; + } + (None, c) if c.is_whitespace() => { + if any || !cur.is_empty() { + out.push(std::mem::take(&mut cur)); + any = false; + } + } + (None, c) => cur.push(c), + } + } + if any || !cur.is_empty() { + out.push(cur); + } + out +} + +fn on_path(name: &str) -> Option { + if name.contains('/') || name.contains('\\') { + return Path::new(name).is_file().then(|| PathBuf::from(name)); + } + let path = std::env::var_os("PATH")?; + std::env::split_paths(&path).map(|d| d.join(name)).find(|p| p.is_file()) +} + +/// The files a command line names (its program found on `PATH`, and every argument that +/// is a file), each with its SHA-256: what identifies the implementation that ran. +pub fn identity(cmd: &str) -> Vec<(String, String)> { + let mut out = Vec::new(); + for (i, w) in words(cmd).iter().enumerate() { + if w.starts_with('{') && w.ends_with('}') { + continue; + } + let p = if i == 0 { on_path(w) } else { Path::new(w).is_file().then(|| PathBuf::from(w)) }; + if let Some(p) = p + && let Ok(b) = std::fs::read(&p) + { + out.push((p.to_string_lossy().into_owned(), sha256_hex(&b))); + } + } + out +} + +/// Whether the program a command line runs can be found. +pub fn program_found(cmd: &str) -> bool { + match words(cmd).first() { + None => false, + Some(p) => on_path(p).is_some() || matches!(p.as_str(), "cd" | "exec" | "env" | "echo" | "printf" | "test" | "[" | "true" | "false"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(unix)] + #[test] + fn placeholders_are_quoted_paths() { + let s = fill("dec {in} {out}", &[("in", Path::new("/t/a b.wscr")), ("out", Path::new("/t/it's.pcm"))]); + assert_eq!(s, "dec '/t/a b.wscr' '/t/it'\\''s.pcm'"); + assert_eq!(words("python3 -I 'a b.py' verdicts {corpus}"), vec!["python3", "-I", "a b.py", "verdicts", "{corpus}"]); + } + + #[cfg(unix)] + #[test] + fn a_command_runs_and_one_that_hangs_is_stopped() { + let r = run("cat", Some(b"x\ny"), Duration::from_secs(20)).unwrap(); + assert_eq!(r.stdout, b"x\ny"); + assert_eq!(first_line(&r.stdout), "x"); + // The shell is killed at the timeout; the `sleep` it started keeps the output open, + // and is not waited for past the drain. + let t0 = Instant::now(); + let r = run("sleep 30; echo done", None, Duration::from_millis(200)).unwrap(); + assert!(r.timed_out); + assert!(t0.elapsed() < Duration::from_secs(10), "{:?}", t0.elapsed()); + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/conform/http.rs b/wai/wai-rs/src/bin/wai_cli/dx/conform/http.rs new file mode 100644 index 000000000..3c4cf8a41 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/conform/http.rs @@ -0,0 +1,169 @@ +//! A small HTTP/1.1 client over plain TCP, enough to drive an implementation behind a URL: +//! one request a connection (`Connection: close`), a body by `Content-Length` or chunked, +//! and a response read whole up to a bound. + +use std::io::{Read, Write}; +use std::net::{TcpStream, ToSocketAddrs}; +use std::time::{Duration, Instant}; + +/// The most a response may hold. +const MAX_RESPONSE: usize = 512 << 20; + +/// An `http://` URL: where to connect, the `Host` header, and the path it starts with. +#[derive(Clone, Debug)] +pub struct Url { + pub authority: String, + pub prefix: String, +} + +impl Url { + pub fn parse(u: &str) -> Result { + let rest = u.strip_prefix("http://").ok_or_else(|| { + if u.starts_with("https://") { + format!("{u}: this build speaks plain HTTP to an implementation; serve it on an http:// address (a loopback port, or a tunnel to one)") + } else { + format!("{u}: not an http:// URL") + } + })?; + let (authority, path) = match rest.find('/') { + Some(i) => (&rest[..i], &rest[i..]), + None => (rest, "/"), + }; + if authority.is_empty() { + return Err(format!("{u}: no host")); + } + Ok(Url { authority: authority.to_string(), prefix: path.trim_end_matches('/').to_string() }) + } + + /// The path `p` (which starts with `/`, or is empty for the URL itself) below the URL. + pub fn join(&self, p: &str) -> String { + let s = format!("{}{p}", self.prefix); + if s.is_empty() { "/".into() } else { s } + } +} + +#[derive(Debug)] +pub struct Response { + pub status: u16, + pub headers: Vec<(String, String)>, + pub body: Vec, +} + +impl Response { + /// Every value of the header `name` (case-insensitive), joined with `, `. + pub fn header(&self, name: &str) -> Option { + let v: Vec<&str> = self.headers.iter().filter(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str()).collect(); + if v.is_empty() { None } else { Some(v.join(", ")) } + } +} + +/// Why a request got no response. +#[derive(Debug)] +pub enum Failure { + /// No connection: the implementation is not there (or not yet). + Unreachable(String), + /// A connection, and no readable response. + Broken(String), +} + +impl std::fmt::Display for Failure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Failure::Unreachable(e) => write!(f, "unreachable: {e}"), + Failure::Broken(e) => write!(f, "{e}"), + } + } +} + +fn dechunk(mut b: &[u8]) -> Option> { + let mut out = Vec::new(); + loop { + let nl = b.windows(2).position(|w| w == b"\r\n")?; + let size_text = std::str::from_utf8(&b[..nl]).ok()?; + let size = usize::from_str_radix(size_text.split(';').next()?.trim(), 16).ok()?; + b = &b[nl + 2..]; + if size == 0 { + return Some(out); + } + if b.len() < size + 2 { + return None; + } + out.extend_from_slice(&b[..size]); + b = &b[size + 2..]; + } +} + +/// One request, answered whole within `timeout`. +pub fn request(url: &Url, method: &str, path: &str, headers: &[(&str, &str)], body: Option<&[u8]>, timeout: Duration) -> Result { + let deadline = Instant::now() + timeout; + let addrs: Vec<_> = url.authority.to_socket_addrs().map_err(|e| Failure::Unreachable(format!("{}: {e}", url.authority)))?.collect(); + let mut last = String::from("no address"); + let mut stream = None; + for a in &addrs { + match TcpStream::connect_timeout(a, timeout.min(Duration::from_secs(10))) { + Ok(s) => { + stream = Some(s); + break; + } + Err(e) => last = format!("{a}: {e}"), + } + } + let mut s = stream.ok_or(Failure::Unreachable(last))?; + let left = || deadline.saturating_duration_since(Instant::now()).max(Duration::from_millis(1)); + let _ = s.set_read_timeout(Some(left())); + let _ = s.set_write_timeout(Some(left())); + let mut head = format!("{method} {} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\nUser-Agent: wai-conform\r\n", url.join(path), url.authority); + for (k, v) in headers { + head.push_str(&format!("{k}: {v}\r\n")); + } + if let Some(b) = body { + head.push_str(&format!("Content-Length: {}\r\n", b.len())); + } + head.push_str("\r\n"); + let broken = |e: std::io::Error| Failure::Broken(format!("{method} {path}: {e}")); + s.write_all(head.as_bytes()).map_err(broken)?; + if let Some(b) = body { + s.write_all(b).map_err(broken)?; + } + let mut buf = Vec::new(); + let mut chunk = [0u8; 64 * 1024]; + loop { + let _ = s.set_read_timeout(Some(left())); + match s.read(&mut chunk) { + Ok(0) => break, + Ok(n) => { + buf.extend_from_slice(&chunk[..n]); + if buf.len() > MAX_RESPONSE { + return Err(Failure::Broken(format!("{method} {path}: the response is larger than {MAX_RESPONSE} bytes"))); + } + } + Err(e) if Instant::now() >= deadline => return Err(Failure::Broken(format!("{method} {path}: no complete response in {timeout:?} ({e})"))), + Err(e) if matches!(e.kind(), std::io::ErrorKind::Interrupted) => {} + Err(e) => { + if buf.is_empty() { + return Err(broken(e)); + } + break; + } + } + } + let end = buf.windows(4).position(|w| w == b"\r\n\r\n").ok_or_else(|| Failure::Broken(format!("{method} {path}: no response head")))?; + let head = String::from_utf8_lossy(&buf[..end]).into_owned(); + let mut lines = head.split("\r\n"); + let status_line = lines.next().unwrap_or(""); + let status: u16 = status_line.split(' ').nth(1).and_then(|s| s.parse().ok()).ok_or_else(|| Failure::Broken(format!("{method} {path}: status line {status_line:?}")))?; + let headers: Vec<(String, String)> = lines.filter_map(|l| l.split_once(':').map(|(k, v)| (k.trim().to_string(), v.trim().to_string()))).collect(); + let mut body = buf[end + 4..].to_vec(); + let r = Response { status, headers, body: Vec::new() }; + if method == "HEAD" || status == 204 || status == 304 { + body.clear(); + } else if r.header("Transfer-Encoding").is_some_and(|t| t.to_ascii_lowercase().contains("chunked")) { + body = dechunk(&body).ok_or_else(|| Failure::Broken(format!("{method} {path}: a chunked body that does not read")))?; + } else if let Some(n) = r.header("Content-Length").and_then(|v| v.trim().parse::().ok()) { + if body.len() < n { + return Err(Failure::Broken(format!("{method} {path}: {} of {n} body bytes", body.len()))); + } + body.truncate(n); + } + Ok(Response { body, ..r }) +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/conform/mod.rs b/wai/wai-rs/src/bin/wai_cli/dx/conform/mod.rs new file mode 100644 index 000000000..f9b22c511 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/conform/mod.rs @@ -0,0 +1,576 @@ +//! `wai conform --suite --impl ` (operation +//! `conformance.run`): one of WAI's conformance corpora run against another implementation, +//! and the outcome written as a signed result record (`record`), which `wai verify` checks. +//! +//! The kinds and their suites are in `suites`; the two adapters (a command line, or an +//! `http://` URL) in `adapter`. Every case's expectation is the corpus's own. A run reads +//! the corpus from a checkout of the standard (`--root `, or found from the +//! current directory), and records the digest of every corpus file it read, the +//! implementation it ran (the command and the SHA-256 of every file it names, or the URL), +//! each case's outcome, and the standard's version. +//! +//! Exit status: 0 when every case passed, 1 when one did not, 2 for a usage error, 4 when +//! the record would replace a file and `--confirm` was not given, 5 when the suite, corpus +//! or program is not found, 6 when the implementation's URL does not answer, 7 otherwise. + +mod adapter; +mod http; +pub mod record; +pub mod suites; + +use super::envelope::{self, EXIT_CONFIRM, EXIT_FAILED, EXIT_INTERNAL, EXIT_NOT_FOUND, EXIT_UNAVAILABLE, OpError}; +use serde_json::{Map, Value, json}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use suites::{Case, Expect, Input, SUITES, Suite}; + +pub const OP: &str = "conformance.run"; + +const USAGE: &str = "wai conform --suite --impl [--key ] [--out ] [--confirm]\n\ + [--root ] [--timeout ] [--name ] [--version ] [--dry-run] [--all] [--json|--text]\n\ + wai conform --list"; + +/// How the implementation is reached. +enum Impl { + Command(String), + Http(http::Url), +} + +/// One case's outcome. +struct Outcome { + name: String, + pass: bool, + expected: String, + got: String, +} + +fn cut(s: &str) -> String { + if s.chars().count() > 300 { format!("{}…", s.chars().take(300).collect::()) } else { s.to_string() } +} + +/// The `wai/` directory of a checkout: `--root`, else the first directory up from here +/// that is one, or holds one. +fn wai_root(given: Option<&str>) -> Result { + let is_root = |d: &Path| d.join("SPEC.md").is_file() && d.join("registry").join("capabilities.json").is_file(); + if let Some(g) = given { + let d = PathBuf::from(g); + return if is_root(&d) { + Ok(d) + } else { + Err(OpError::new(EXIT_NOT_FOUND, "no-corpus", format!("{g}: not the wai/ directory of a checkout of the standard"), "--root names the directory that holds SPEC.md and the *-conformance corpora")) + }; + } + let mut d = std::env::current_dir().map_err(|e| OpError::new(EXIT_INTERNAL, "internal", e.to_string(), ""))?; + loop { + if is_root(&d) { + return Ok(d); + } + if is_root(&d.join("wai")) { + return Ok(d.join("wai")); + } + if !d.pop() { + return Err(OpError::new( + EXIT_NOT_FOUND, + "no-corpus", + "no checkout of the WAI standard here, so no corpus to run", + "run it inside a checkout (git clone https://git.transaction.science/Transaction-Science/open-standards), or pass --root /wai", + )); + } + } +} + +/// The standard's version: the first `## v…` heading of `CHANGELOG.md`. +fn standard_version(root: &Path) -> String { + std::fs::read_to_string(root.join("CHANGELOG.md")) + .ok() + .and_then(|t| t.lines().find_map(|l| l.strip_prefix("## v").map(|r| format!("v{}", r.split_whitespace().next().unwrap_or(""))))) + .unwrap_or_else(|| "unknown".into()) +} + +/// Now, in RFC 3339 UTC to the second. +fn now_rfc3339() -> String { + let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0); + let (days, rem) = (secs.div_euclid(86_400), secs.rem_euclid(86_400)); + // Civil date from days since 1970-01-01 (Howard Hinnant's algorithm). + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z", rem / 3600, rem % 3600 / 60, rem % 60) +} + +/// The corpus digest (conformance-results §2.1) of the files a run read. +fn corpus_digest(root: &Path, files: &[PathBuf]) -> Result { + let mut read: Vec<(String, Vec)> = Vec::new(); + for f in files { + let b = std::fs::read(f).map_err(|e| OpError::new(EXIT_NOT_FOUND, "corpus-file-missing", format!("{}: {e}", f.display()), "the corpus is incomplete; check out the standard again"))?; + read.push((f.strip_prefix(root).unwrap_or(f).to_string_lossy().into_owned(), b)); + } + Ok(wai::conform_result::corpus_digest(read.iter().map(|(p, b)| (p.as_str(), b.as_slice())))) +} + +/// A packager's verdict line read: `ok `, or a refusal's reason, a leading copy of +/// the input's path ignored. +fn packager_tokens(line: &str, input: &Path) -> Vec { + let mut t: Vec = line.split_whitespace().map(String::from).collect(); + if t.first().is_some_and(|w| Path::new(w) == input || input.file_name().is_some_and(|n| w.ends_with(&*n.to_string_lossy()))) { + t.remove(0); + } + t +} + +fn expected_text(e: &Expect) -> String { + match e { + Expect::Verdict { line, prefix: true } => format!("{line}…"), + Expect::Verdict { line, .. } => line.clone(), + Expect::Output { sha256 } => format!("output sha256 {sha256}"), + Expect::Admit { digest } => format!("ok {digest}"), + Expect::Refuse { reason } => reason.clone(), + Expect::Object { cid, .. } => format!("{cid}.wai served as http-delivery §5.3 and §6.2 require"), + } +} + +fn judge(e: &Expect, got: &str, input: Option<&Path>) -> bool { + match e { + Expect::Verdict { line, prefix } => { + if *prefix { + got.starts_with(line.as_str()) + } else { + got == line + } + } + Expect::Output { sha256 } => got == format!("output sha256 {sha256}"), + Expect::Admit { digest } => { + let t = packager_tokens(got, input.unwrap_or(Path::new(""))); + t.len() >= 2 && t[0] == "ok" && t[1] == *digest + } + Expect::Refuse { reason } => packager_tokens(got, input.unwrap_or(Path::new(""))).first() == Some(reason), + Expect::Object { .. } => got.is_empty(), + } +} + +/// What an origin serving the object answers, against http-delivery §5.3 and §6.2: the +/// problems found, none when it conforms. +fn check_object(url: &http::Url, cid: &str, bytes: &[u8], head_proof: Option<&[u8]>, timeout: Duration) -> Result, http::Failure> { + let mut p = Vec::new(); + let path = format!("/{cid}.wai"); + let r = http::request(url, "GET", &path, &[], None, timeout)?; + if r.status != 200 { + p.push(format!("GET {path}: status {} (200)", r.status)); + } else if r.body != bytes { + p.push(format!("GET {path}: the body is not the object ({} bytes, sha256 {})", r.body.len(), &adapter::sha256_hex(&r.body)[..16])); + } + let want_etag = format!("\"{cid}\""); + let h = |name: &str| r.header(name).unwrap_or_default(); + if h("ETag") != want_etag { + p.push(format!("ETag {:?} ({want_etag})", h("ETag"))); + } + if !h("Content-Type").starts_with("application/wai") { + p.push(format!("Content-Type {:?} (application/wai)", h("Content-Type"))); + } + let cc: Vec = h("Cache-Control").split(',').map(|x| x.trim().to_ascii_lowercase()).collect(); + for want in ["public", "max-age=31536000", "immutable", "no-transform"] { + if !cc.iter().any(|x| x == want) { + p.push(format!("Cache-Control {:?} lacks {want}", h("Cache-Control"))); + } + } + if !h("Accept-Ranges").eq_ignore_ascii_case("bytes") { + p.push(format!("Accept-Ranges {:?} (bytes)", h("Accept-Ranges"))); + } + if h("Access-Control-Allow-Origin") != "*" { + p.push(format!("Access-Control-Allow-Origin {:?} (*)", h("Access-Control-Allow-Origin"))); + } + let expose = h("Access-Control-Expose-Headers").to_ascii_lowercase(); + for want in ["content-range", "content-length", "etag"] { + if !expose.split(',').any(|x| x.trim() == want) { + p.push(format!("Access-Control-Expose-Headers {:?} lacks {want}", h("Access-Control-Expose-Headers"))); + } + } + if h("Timing-Allow-Origin") != "*" { + p.push(format!("Timing-Allow-Origin {:?} (*)", h("Timing-Allow-Origin"))); + } + if r.header("Vary").is_some() { + p.push(format!("Vary {:?} (an object is never selected by a request header)", h("Vary"))); + } + if r.header("Content-Encoding").is_some_and(|e| !e.eq_ignore_ascii_case("identity")) { + p.push(format!("Content-Encoding {:?} (none)", h("Content-Encoding"))); + } + // A single byte range: 206 with the bytes and the whole length, or 200 with the object. + let last = bytes.len().min(32).saturating_sub(1); + let range = format!("bytes=0-{last}"); + let r = http::request(url, "GET", &path, &[("Range", &range)], None, timeout)?; + match r.status { + 206 => { + let want = format!("bytes 0-{last}/{}", bytes.len()); + if r.header("Content-Range").as_deref() != Some(want.as_str()) { + p.push(format!("Range {range}: Content-Range {:?} ({want})", r.header("Content-Range"))); + } + if r.body != bytes[..=last] { + p.push(format!("Range {range}: the body is not those bytes")); + } + } + 200 if r.body == bytes => {} + s => p.push(format!("Range {range}: status {s} (206 with the bytes, or 200 with the object)")), + } + let r = http::request(url, "HEAD", &path, &[], None, timeout)?; + if r.status != 200 || r.header("ETag").as_deref() != Some(want_etag.as_str()) { + p.push(format!("HEAD {path}: status {}, ETag {:?}", r.status, r.header("ETag"))); + } + let r = http::request(url, "OPTIONS", &path, &[("Origin", "https://conformance.example"), ("Access-Control-Request-Method", "GET"), ("Access-Control-Request-Headers", "range, priority")], None, timeout)?; + let allow = r.header("Access-Control-Allow-Headers").unwrap_or_default().to_ascii_lowercase(); + for want in ["range", "priority"] { + if !allow.split(',').any(|x| x.trim() == want) { + p.push(format!("preflight: Access-Control-Allow-Headers {allow:?} lacks {want}")); + } + } + if r.header("Access-Control-Max-Age").is_none() { + p.push("preflight: no Access-Control-Max-Age".into()); + } + if let Some(hp) = head_proof { + let hpath = format!("/{cid}.wai.hp"); + let r = http::request(url, "GET", &hpath, &[], None, timeout)?; + if r.status != 200 || r.body != hp { + p.push(format!("GET {hpath}: status {}, {} bytes (the head proof, {} bytes)", r.status, r.body.len(), hp.len())); + } else if r.header("Content-Type").as_deref() != Some("application/octet-stream") { + p.push(format!("GET {hpath}: Content-Type {:?} (application/octet-stream)", r.header("Content-Type"))); + } + } + Ok(p) +} + +struct Run<'a> { + imp: &'a Impl, + timeout: Duration, + scratch: PathBuf, +} + +impl Run<'_> { + fn http_failure(&self, f: http::Failure) -> Result { + match f { + http::Failure::Unreachable(e) => Err(OpError::new(EXIT_UNAVAILABLE, "implementation-unreachable", format!("the implementation does not answer: {e}"), "start it, then run again")), + http::Failure::Broken(e) => Ok(format!("no answer: {e}")), + } + } + + /// One case's answer, as the text `judge` reads. + fn answer(&self, case: &Case) -> Result { + let internal = |e: String| OpError::new(EXIT_INTERNAL, "internal", e, ""); + match (self.imp, &case.input, &case.expect) { + (Impl::Http(url), _, Expect::Object { cid, bytes_file, head_proof }) => { + let bytes = std::fs::read(bytes_file).map_err(|e| internal(e.to_string()))?; + let hp = head_proof.as_ref().and_then(|p| std::fs::read(p).ok()); + match check_object(url, cid, &bytes, hp.as_deref(), self.timeout) { + Ok(problems) => Ok(problems.join("; ")), + Err(f) => self.http_failure(f), + } + } + (Impl::Http(url), input, expect) => { + let (body, ctype) = match input { + Input::Json(v) => (v.to_string().into_bytes(), "application/json"), + Input::File(p) => (std::fs::read(p).map_err(|e| internal(format!("{}: {e}", p.display())))?, "application/octet-stream"), + Input::Served => (Vec::new(), "application/octet-stream"), + }; + match http::request(url, "POST", "", &[("Content-Type", ctype)], Some(&body), self.timeout) { + Err(f) => self.http_failure(f), + Ok(r) => Ok(match expect { + Expect::Output { .. } if r.status == 200 => format!("output sha256 {}", adapter::sha256_hex(&r.body)), + Expect::Output { .. } => format!("refused ({}): {}", r.status, adapter::first_line(&r.body)), + _ => match serde_json::from_slice::(&r.body).ok().and_then(|v| v.get("verdict").and_then(Value::as_str).map(String::from)) { + Some(v) => v, + None => adapter::first_line(&r.body), + }, + }), + } + } + (Impl::Command(cmd), input, expect) => { + let case_file = self.scratch.join("case.json"); + let out_file = self.scratch.join("out"); + let _ = std::fs::remove_file(&out_file); + let mut with: Vec<(&str, &Path)> = vec![("out", out_file.as_path())]; + let mut stdin: Option> = None; + match input { + Input::Json(v) => { + if cmd.contains("{case}") { + std::fs::write(&case_file, v.to_string()).map_err(|e| internal(e.to_string()))?; + with.push(("case", case_file.as_path())); + } else { + stdin = Some(v.to_string().into_bytes()); + } + } + Input::File(p) => { + if cmd.contains("{in}") { + with.push(("in", p.as_path())); + } else { + stdin = Some(std::fs::read(p).map_err(|e| internal(format!("{}: {e}", p.display())))?); + } + } + Input::Served => {} + } + let line = adapter::fill(cmd, &with); + let ran = adapter::run(&line, stdin.as_deref(), self.timeout).map_err(internal)?; + if ran.timed_out { + return Ok(format!("timed out after {:?}", self.timeout)); + } + Ok(match expect { + Expect::Output { .. } => { + if ran.code != Some(0) { + format!("refused (exit {:?}): {}", ran.code, adapter::first_line(&ran.stdout)) + } else if cmd.contains("{out}") { + match std::fs::read(&out_file) { + Ok(b) => format!("output sha256 {}", adapter::sha256_hex(&b)), + Err(_) => "no output written to {out}".into(), + } + } else { + format!("output sha256 {}", adapter::sha256_hex(&ran.stdout)) + } + } + _ => { + let l = adapter::first_line(&ran.stdout); + if l.is_empty() && ran.code != Some(0) { + format!("no verdict (exit {:?}): {}", ran.code, adapter::first_line(&ran.stderr)) + } else { + l + } + } + }) + } + } + } + + /// Every case, the whole corpus given to one run of the command (`{corpus}`), which + /// answers `nameverdict` lines. + fn whole_corpus(&self, cmd: &str, corpus: &Path, cases: &[Case]) -> Result, OpError> { + let line = adapter::fill(cmd, &[("corpus", corpus)]); + let ran = adapter::run(&line, None, self.timeout * 20).map_err(|e| OpError::new(EXIT_INTERNAL, "internal", e, ""))?; + let mut got: HashMap = HashMap::new(); + for l in String::from_utf8_lossy(&ran.stdout).lines() { + if let Some((n, v)) = l.split_once('\t') { + got.entry(n.to_string()).or_insert_with(|| v.trim().to_string()); + } + } + let missing = if ran.timed_out { "timed out".to_string() } else { format!("no verdict (exit {:?}): {}", ran.code, adapter::first_line(&ran.stderr)) }; + Ok(cases + .iter() + .map(|c| { + let g = got.get(&c.name).cloned().unwrap_or_else(|| missing.clone()); + Outcome { name: c.name.clone(), pass: judge(&c.expect, &g, None), expected: cut(&expected_text(&c.expect)), got: cut(&g) } + }) + .collect()) + } +} + +fn list(json_mode: bool, root: Option<&Path>) { + let rows: Vec = SUITES + .iter() + .map(|s| { + let n = root.and_then(|r| (s.load)(&r.join(s.dir)).ok()).map(|l| l.cases.len()); + let adapters: Vec<&str> = [s.command.then_some("command"), s.http.then_some("http")].into_iter().flatten().collect(); + json!({"kind": s.kind, "suite": s.name, "corpus": s.dir, "cases": n, "adapters": adapters, "about": s.about}) + }) + .collect(); + if json_mode { + let mut m = Map::new(); + m.insert("suites".into(), Value::Array(rows)); + println!("{}", envelope::envelope("conformance.list", true, m)); + return; + } + for r in rows { + println!( + "{:<9} {:<7} {:<20} {:>5} cases {:<13} {}", + r["kind"].as_str().unwrap_or(""), + r["suite"].as_str().unwrap_or(""), + r["corpus"].as_str().unwrap_or(""), + r["cases"].as_u64().map(|n| n.to_string()).unwrap_or_else(|| "?".into()), + r["adapters"].as_array().map(|a| a.iter().filter_map(Value::as_str).collect::>().join("+")).unwrap_or_default(), + r["about"].as_str().unwrap_or("") + ); + } +} + +pub fn main(args: &[String]) -> Result { + let json_mode = envelope::json_wanted(args); + match run(args, json_mode) { + Ok(true) => Ok(true), + Ok(false) => envelope::exit(EXIT_FAILED), + Err(e) => envelope::fail(OP, json_mode, &e), + } +} + +fn run(args: &[String], json_mode: bool) -> Result { + let flags = ["--suite", "--impl", "--key", "--out", "--confirm", "--root", "--timeout", "--name", "--version", "--dry-run", "--all", "--json", "--text", "--list"]; + let with_value = ["--suite", "--impl", "--key", "--out", "--root", "--timeout", "--name", "--version"]; + let pos = envelope::check_args(args, &flags, &with_value, 1, USAGE)?; + let v = |f: &str| envelope::value(args, f, USAGE); + if args.iter().any(|a| a == "--list") { + let root = wai_root(v("--root")?).ok(); + list(json_mode, root.as_deref()); + return Ok(true); + } + let kind = *pos.first().ok_or_else(|| OpError::usage(format!("name a kind: {}", suites::KINDS.join(", ")), USAGE))?; + if !suites::KINDS.contains(&kind) { + return Err(OpError::new(EXIT_NOT_FOUND, "kind-not-found", format!("{kind}: no suite of that kind; the kinds are {}", suites::KINDS.join(", ")), "wai conform --list names every kind and suite")); + } + let of_kind: Vec<&Suite> = SUITES.iter().filter(|s| s.kind == kind).collect(); + let suite = match v("--suite")? { + Some(n) => *of_kind.iter().find(|s| s.name == n).ok_or_else(|| { + OpError::new(EXIT_NOT_FOUND, "suite-not-found", format!("{kind} has no suite {n}; it has {}", of_kind.iter().map(|s| s.name).collect::>().join(", ")), "wai conform --list names every kind and suite") + })?, + None if of_kind.len() == 1 => of_kind[0], + None => return Err(OpError::usage(format!("--suite: {kind} has {}", of_kind.iter().map(|s| s.name).collect::>().join(", ")), USAGE)), + }; + let imp_text = v("--impl")?.ok_or_else(|| OpError::usage("--impl is required: the implementation to run the corpus against", USAGE))?.to_string(); + let imp = if imp_text.starts_with("http://") || imp_text.starts_with("https://") { + if !suite.http { + return Err(OpError::usage(format!("{kind} {}: no HTTP adapter", suite.name), USAGE)); + } + Impl::Http(http::Url::parse(&imp_text).map_err(|e| OpError::usage(e, USAGE))?) + } else { + if !suite.command { + return Err(OpError::usage(format!("{kind} {}: an origin is reached at an http:// URL", suite.name), USAGE)); + } + if !adapter::program_found(&imp_text) { + return Err(OpError::new(EXIT_NOT_FOUND, "program-not-found", format!("{imp_text}: its program is not found"), "name a program on PATH, or a path to one")); + } + Impl::Command(imp_text.clone()) + }; + let timeout = Duration::from_secs(v("--timeout")?.map(|t| t.parse::().map_err(|_| OpError::usage("--timeout takes whole seconds", USAGE))).transpose()?.unwrap_or(60)); + let root = wai_root(v("--root")?)?; + let loaded = (suite.load)(&root.join(suite.dir)).map_err(|e| OpError::new(EXIT_NOT_FOUND, "corpus-not-read", e, "the corpus is incomplete; check out the standard again"))?; + if loaded.cases.is_empty() { + return Err(OpError::new(EXIT_NOT_FOUND, "no-cases", format!("{}: no cases", suite.dir), "")); + } + let whole = matches!(&imp, Impl::Command(c) if c.contains("{corpus}")); + if whole && loaded.corpus_file.is_none() { + return Err(OpError::usage(format!("{kind} {}: {{corpus}} is for a verifier suite; this suite gives its cases one at a time", suite.name), USAGE)); + } + let out = PathBuf::from(v("--out")?.map(String::from).unwrap_or_else(|| format!("{}{}", suite.name, record::SUFFIX))); + if args.iter().any(|a| a == "--dry-run") { + let first = &loaded.cases[0]; + let plan = match &imp { + Impl::Command(c) if whole => adapter::fill(c, &[("corpus", loaded.corpus_file.as_deref().unwrap_or(Path::new("")))]), + Impl::Command(c) => adapter::fill(c, &[("case", Path::new("")), ("in", match &first.input { + Input::File(p) => p.as_path(), + _ => Path::new(""), + }), ("out", Path::new(""))]), + Impl::Http(u) => format!("{} {}", if matches!(first.expect, Expect::Object { .. }) { "GET, HEAD and OPTIONS below" } else { "POST each case to" }, u.join("")), + }; + if json_mode { + let mut m = Map::new(); + m.insert("kind".into(), json!(kind)); + m.insert("suite".into(), json!(suite.name)); + m.insert("dry_run".into(), json!(true)); + m.insert("cases".into(), json!(loaded.cases.iter().map(|c| c.name.clone()).collect::>())); + m.insert("runs".into(), json!(plan)); + m.insert("record".into(), json!(out.display().to_string())); + println!("{}", envelope::envelope(OP, true, m)); + } else { + println!("wai conform {kind} {}: {} cases from {}; would run: {plan}", suite.name, loaded.cases.len(), suite.dir); + println!("the record would be written to {}", out.display()); + } + return Ok(true); + } + if out.exists() && !args.iter().any(|a| a == "--confirm") { + return Err(OpError::new(EXIT_CONFIRM, "confirmation-required", format!("{} exists; the record would replace it", out.display()), format!("re-run with --confirm, or --out : wai conform {} --confirm", args.join(" ")))); + } + let key = match v("--key")? { + Some(p) => (record::key_from_file(Path::new(p)).map_err(|e| OpError::new(EXIT_NOT_FOUND, "key-not-read", e, "--key names a file holding a 32-byte Ed25519 seed in hexadecimal"))?, "supplied"), + None => (record::ephemeral_key().map_err(|e| OpError::new(EXIT_INTERNAL, "internal", e, ""))?, "ephemeral"), + }; + let digest = corpus_digest(&root, &loaded.files)?; + let started = now_rfc3339(); + let scratch = std::env::temp_dir().join(format!("wai-conform-{}-{}", std::process::id(), std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0))); + std::fs::create_dir_all(&scratch).map_err(|e| OpError::new(EXIT_INTERNAL, "internal", e.to_string(), ""))?; + let r = Run { imp: &imp, timeout, scratch: scratch.clone() }; + eprintln!("wai conform {kind} {}: {} cases from {}", suite.name, loaded.cases.len(), suite.dir); + let outcomes: Result, OpError> = match (&imp, &loaded.corpus_file) { + (Impl::Command(c), Some(f)) if whole => r.whole_corpus(c, f, &loaded.cases), + _ => loaded + .cases + .iter() + .map(|c| { + let got = r.answer(c)?; + let input = match &c.input { + Input::File(p) => Some(p.as_path()), + _ => None, + }; + Ok(Outcome { name: c.name.clone(), pass: judge(&c.expect, &got, input), expected: cut(&expected_text(&c.expect)), got: cut(if matches!(c.expect, Expect::Object { .. }) && got.is_empty() { "served as required" } else { &got }) }) + }) + .collect(), + }; + let _ = std::fs::remove_dir_all(&scratch); + let outcomes = outcomes?; + let passed = outcomes.iter().filter(|o| o.pass).count(); + let failed = outcomes.len() - passed; + let conformant = failed == 0; + let (adapter_name, files) = match &imp { + Impl::Command(c) => ("command", adapter::identity(c).into_iter().map(|(p, h)| json!({"path": p, "sha256": h})).collect::>()), + Impl::Http(_) => ("http", Vec::new()), + }; + let mut rec = Map::new(); + rec.insert("wai".into(), json!(record::FORMAT)); + rec.insert("kind".into(), json!(kind)); + rec.insert("suite".into(), json!(suite.name)); + rec.insert("standard".into(), json!(standard_version(&root))); + rec.insert("toolchain".into(), json!({"name": "wai", "version": env!("CARGO_PKG_VERSION")})); + rec.insert("corpus".into(), json!({"dir": suite.dir, "files": loaded.files.len(), "digest": digest})); + rec.insert("implementation".into(), json!({"adapter": adapter_name, "impl": imp_text, "name": v("--name")?, "version": v("--version")?, "files": files})); + rec.insert("started_at".into(), json!(started)); + rec.insert("cases".into(), Value::Array(outcomes.iter().map(|o| json!({"name": o.name, "pass": o.pass, "expected": o.expected, "got": o.got})).collect())); + rec.insert("summary".into(), json!({"cases": outcomes.len(), "passed": passed, "failed": failed})); + rec.insert("conformant".into(), json!(conformant)); + rec.insert("key".into(), json!(key.1)); + let rec = record::sign(rec, &key.0).map_err(|e| OpError::new(EXIT_INTERNAL, "internal", e, ""))?; + let text = serde_json::to_string_pretty(&Value::Object(rec.clone())).map_err(|e| OpError::new(EXIT_INTERNAL, "internal", e.to_string(), ""))?; + std::fs::write(&out, format!("{text}\n")).map_err(|e| OpError::new(EXIT_INTERNAL, "internal", format!("{}: {e}", out.display()), ""))?; + let pubkey = rec.get("signer_pubkey").and_then(Value::as_str).unwrap_or("").to_string(); + if json_mode { + let mut m = Map::new(); + m.insert("kind".into(), json!(kind)); + m.insert("suite".into(), json!(suite.name)); + m.insert("cases".into(), json!(outcomes.len())); + m.insert("passed".into(), json!(passed)); + m.insert("failed".into(), json!(failed)); + m.insert("conformant".into(), json!(conformant)); + m.insert("failures".into(), Value::Array(outcomes.iter().filter(|o| !o.pass).take(200).map(|o| json!({"name": o.name, "expected": o.expected, "got": o.got})).collect())); + m.insert("record".into(), json!(out.display().to_string())); + m.insert("corpus_digest".into(), json!(rec["corpus"]["digest"])); + m.insert("signer_pubkey".into(), json!(pubkey)); + println!("{}", envelope::envelope(OP, conformant, m)); + } else { + let all = args.iter().any(|a| a == "--all"); + let mut shown = 0; + for o in &outcomes { + if o.pass && !all { + continue; + } + if !o.pass { + shown += 1; + if shown > 40 && !all { + continue; + } + } + println!("{} {}", if o.pass { "PASS" } else { "FAIL" }, o.name); + if !o.pass { + println!(" expected: {}", o.expected); + println!(" got: {}", o.got); + } + } + if shown > 40 && !all { + println!("… and {} more failures (--all lists every case)", shown - 40); + } + println!( + "wai conform {kind} {}: {passed} of {} cases passed: {}", + suite.name, + outcomes.len(), + if conformant { "conformant" } else { "not conformant" } + ); + println!("record: {} (signed by {}…, {} key); `wai verify {}` checks it", out.display(), pubkey.get(..16).unwrap_or(""), key.1, out.display()); + } + Ok(conformant) +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/conform/record.rs b/wai/wai-rs/src/bin/wai_cli/dx/conform/record.rs new file mode 100644 index 000000000..3c77c74f3 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/conform/record.rs @@ -0,0 +1,77 @@ +//! The result record of a `wai conform` run, `conform-result/1` +//! (`extensions/conformance-results.md`): its members (§2), its signature (§3) and its +//! verification (§4) are the library's +//! (`wai::conform_result`); this module adds the run's key and how `wai verify` reports a +//! record. + +use ed25519_dalek::SigningKey; +use serde_json::{Map, Value}; +use wai::conform_result; + +pub use wai::conform_result::FORMAT; + +/// The suffix of a result record's file name, which `wai verify` looks for. +pub const SUFFIX: &str = ".wai-conform.json"; + +fn unhex(s: &str) -> Option> { + if !s.len().is_multiple_of(2) { + return None; + } + (0..s.len() / 2).map(|i| u8::from_str_radix(s.get(2 * i..2 * i + 2)?, 16).ok()).collect() +} + +/// A signing key from `--key`'s file: 64 hexadecimal digits (a 32-byte seed), or the 32 +/// bytes themselves. +pub fn key_from_file(path: &std::path::Path) -> Result { + let b = std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?; + let t = String::from_utf8_lossy(&b); + let seed: Vec = match unhex(t.trim()) { + Some(s) if s.len() == 32 => s, + _ if b.len() == 32 => b, + _ => return Err(format!("{}: a key file holds a 32-byte Ed25519 seed, as 64 hexadecimal digits or 32 bytes", path.display())), + }; + Ok(SigningKey::from_bytes(&seed.try_into().expect("32 bytes"))) +} + +/// A key made for one run. +pub fn ephemeral_key() -> Result { + let mut seed = [0u8; 32]; + getrandom::fill(&mut seed).map_err(|e| format!("no randomness for a run key: {e}"))?; + Ok(SigningKey::from_bytes(&seed)) +} + +/// `record` signed with `key` (§3). +pub fn sign(record: Map, key: &SigningKey) -> Result, String> { + conform_result::sign(record, key) +} + +/// A result record checked (§4): whether it holds, and a line for each thing checked. +pub fn check(bytes: &[u8]) -> (bool, Vec) { + match conform_result::check(bytes) { + Err(r) => { + let what = match r.reason { + conform_result::Reason::SignatureInvalid => "the signature does not verify: the record is not the one that was signed".to_string(), + conform_result::Reason::SummaryMismatch => format!("the summary does not count the cases ({})", r.detail), + _ => r.detail.clone(), + }; + (false, vec![format!("FAIL step {} {}: {what}", r.step, r.reason.code())]) + } + Ok(a) => ( + true, + vec![ + format!("ok signed by {}… ({} key)", &a.signer_pubkey[..16], a.key), + format!( + "ok {} {} against {}: {} of {} cases passed, {}; corpus {}; WAI {}", + a.kind, + a.suite, + a.implementation, + a.passed, + a.cases, + if a.conformant { "conformant" } else { "not conformant" }, + a.corpus_digest, + a.standard + ), + ], + ), + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/conform/suites.rs b/wai/wai-rs/src/bin/wai_cli/dx/conform/suites.rs new file mode 100644 index 000000000..818f099d1 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/conform/suites.rs @@ -0,0 +1,259 @@ +//! The suites `wai conform` runs: each a kind of implementation, a corpus of WAI's, and the +//! cases read from it with what a conforming implementation answers. Every expectation is +//! the corpus's own (its declared verdicts, digests and reasons), never a verifier's output. +//! +//! | kind | suite | corpus | a case is | it passes when | +//! |---|---|---|---|---| +//! | verifier | `claim` | `claim-conformance/vectors.json` | one vector | the verdict is the one declared (jwp-receipts §3.1) | +//! | verifier | `jwp` | `jwp-conformance/vectors.json` | one vector | the verdict is the one declared (jwp-receipts §2–§6) | +//! | verifier | `signer` | `signer-conformance/vectors.json` | one vector | the verdict is the one declared (jwp-receipts §7) | +//! | verifier | `conform-result` | `conform-result-conformance/vectors.json` | one record | the outcome is the one declared (conformance-results §4) | +//! | decoder | `score` | `score-conformance/` | a `WSCR` score | the rendered `i16le` PCM has the declared SHA-256 | +//! | packager | `policy` | `pack-conformance/` | a policy file | it is admitted with the declared digest, or refused with the declared reason (packaging §3, §12) | +//! | origin | `http` | `http-conformance/objects/` | an object | it is served as http-delivery §5.3 and §6.2 require: bytes, headers, ranges, head proof, preflight | + +use serde_json::Value; +use std::path::{Path, PathBuf}; + +/// What a case expects. +#[derive(Clone, Debug)] +pub enum Expect { + /// A verdict line: exactly this, or, with `prefix`, any line that starts with it. + Verdict { line: String, prefix: bool }, + /// An output whose SHA-256 is this (hexadecimal). + Output { sha256: String }, + /// A packager's verdict: admitted with this digest, or refused with this reason. + Admit { digest: String }, + Refuse { reason: String }, + /// An origin serving this object. + Object { cid: String, bytes_file: PathBuf, head_proof: Option }, +} + +/// What a case gives the implementation. +#[derive(Clone, Debug)] +pub enum Input { + /// A vector, as the corpus writes it. + Json(Value), + /// A file. + File(PathBuf), + /// Nothing: the implementation already serves what is asked of it. + Served, +} + +#[derive(Clone, Debug)] +pub struct Case { + pub name: String, + pub input: Input, + pub expect: Expect, +} + +/// A suite read from a corpus: its cases, the files read (for the corpus digest), and the +/// corpus file a whole-corpus run is given (`{corpus}`), where the suite has one. +pub struct Loaded { + pub cases: Vec, + pub files: Vec, + pub corpus_file: Option, +} + +pub struct Suite { + pub kind: &'static str, + pub name: &'static str, + /// The corpus directory, under `wai/`. + pub dir: &'static str, + pub about: &'static str, + pub http: bool, + pub command: bool, + pub load: fn(&Path) -> Result, +} + +pub const KINDS: &[&str] = &["verifier", "decoder", "packager", "origin"]; + +pub const SUITES: &[Suite] = &[ + Suite { kind: "verifier", name: "claim", dir: "claim-conformance", about: "typed claim receipts (jwp-receipts §3.1): accept with the link hash, or reject at a step with a reason", http: true, command: true, load: claim }, + Suite { kind: "verifier", name: "jwp", dir: "jwp-conformance", about: "labelled leaves, group receipts, totals, chains and the binary form (jwp-receipts §2–§6)", http: true, command: true, load: jwp }, + Suite { kind: "verifier", name: "signer", dir: "signer-conformance", about: "key discovery, signing authority and module authorisation (jwp-receipts §7)", http: true, command: true, load: stated_lines }, + Suite { kind: "verifier", name: "conform-result", dir: "conform-result-conformance", about: "conformance results, the signed record of a run (conformance-results §4)", http: true, command: true, load: stated_lines }, + Suite { kind: "decoder", name: "score", dir: "score-conformance", about: "a WSCR score rendered to its canonical i16le PCM (the score extension's mixdown-equivalence)", http: true, command: true, load: score }, + Suite { kind: "packager", name: "policy", dir: "pack-conformance", about: "packaging policies admitted with their digest, or refused with their reason (packaging §3, §12)", http: true, command: true, load: policy }, + Suite { kind: "origin", name: "http", dir: "http-conformance", about: "objects, head proofs, ranges and headers as an origin serves them (http-delivery §5.3, §6.2)", http: true, command: false, load: http_origin }, +]; + +fn read_json(p: &Path) -> Result { + let b = std::fs::read(p).map_err(|e| format!("{}: {e}", p.display()))?; + serde_json::from_slice(&b).map_err(|e| format!("{}: {e}", p.display())) +} + +fn vectors(v: &Value, p: &Path) -> Result, String> { + v.get("vectors").and_then(Value::as_array).cloned().ok_or_else(|| format!("{}: no `vectors` list", p.display())) +} + +fn name_of(v: &Value) -> String { + v.get("name").and_then(Value::as_str).unwrap_or("?").to_string() +} + +fn verdict_cases(dir: &Path, expected_of: fn(&Value) -> (String, bool)) -> Result { + let p = dir.join("vectors.json"); + let doc = read_json(&p)?; + let cases = vectors(&doc, &p)? + .into_iter() + .map(|v| { + let (line, prefix) = expected_of(v.get("expected").unwrap_or(&Value::Null)); + Case { name: name_of(&v), input: Input::Json(v), expect: Expect::Verdict { line, prefix } } + }) + .collect(); + Ok(Loaded { cases, files: vec![p.clone()], corpus_file: Some(p) }) +} + +/// The claim corpus's verdicts (its README): `accept link_hash=` for an accepted +/// claim (any link hash where none is pinned), `accept ancestor=` for a recovery, +/// `reject step= reason=