From 47b7c30c96df6870e047b9bdaa124479ea14e1e6 Mon Sep 17 00:00:00 2001 From: Transaction Science Date: Tue, 27 Oct 2026 05:00:48 -0400 Subject: [PATCH 1/8] wai breaking and wai conform: the compatibility gate, and conformance anyone can run `wai breaking --against ` (operation compat.check) reports what the working tree breaks against the merge base of a ref, in three cumulative categories: wire (the registry export, the media and delivery corpora), record (the signing domains, magics and versions in the WAI crates' sources, the record corpora, the published JSON Schemas) and API (the Rust public API of wai and wai-quantum from rustdoc's JSON, the C header, the room service's OpenAPI document). Each finding names its item, category, what changed and why it breaks; additions are compatible. A break passes only when a line the change adds to wai/CHANGELOG.md, in an entry that says Breaking, names its id. `wai conform --suite --impl ` (operation conformance.run) runs a corpus against another implementation through a command-line or an HTTP adapter: verifiers (claim, jwp, signer), a decoder (score), a packager (policy) and an origin (http). The result is a signed conform-result/1 record that `wai verify` checks. Both follow the agent rules: the envelope/1 JSON output, the exit statuses, --dry-run. .github/workflows/wai-breaking.yml runs the gate on every pull request that touches WAI, the tests of both commands, and the corpora's independent Python verifiers and the reference packager through wai conform. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wai-breaking.yml | 153 ++++ wai/CHANGELOG.md | 34 + wai/README.md | 14 + wai/wai-rs/Cargo.toml | 11 + .../src/bin/wai_cli/dx/breaking/c_header.rs | 368 ++++++++++ .../src/bin/wai_cli/dx/breaking/corpora.rs | 110 +++ .../src/bin/wai_cli/dx/breaking/domains.rs | 165 +++++ wai/wai-rs/src/bin/wai_cli/dx/breaking/git.rs | 332 +++++++++ .../src/bin/wai_cli/dx/breaking/jsondiff.rs | 158 ++++ wai/wai-rs/src/bin/wai_cli/dx/breaking/mod.rs | 352 +++++++++ .../src/bin/wai_cli/dx/breaking/openapi.rs | 175 +++++ .../src/bin/wai_cli/dx/breaking/registry.rs | 165 +++++ .../src/bin/wai_cli/dx/breaking/rust_api.rs | 682 ++++++++++++++++++ .../src/bin/wai_cli/dx/breaking/schema.rs | 112 +++ .../src/bin/wai_cli/dx/conform/adapter.rs | 196 +++++ wai/wai-rs/src/bin/wai_cli/dx/conform/http.rs | 169 +++++ wai/wai-rs/src/bin/wai_cli/dx/conform/mod.rs | 578 +++++++++++++++ .../src/bin/wai_cli/dx/conform/record.rs | 145 ++++ .../src/bin/wai_cli/dx/conform/suites.rs | 257 +++++++ wai/wai-rs/src/bin/wai_cli/dx/envelope.rs | 135 ++++ wai/wai-rs/src/bin/wai_cli/dx/mod.rs | 30 +- wai/wai-rs/src/bin/wai_cli/dx/verify.rs | 31 +- wai/wai-rs/tests/cli_breaking.rs | 274 +++++++ wai/wai-rs/tests/cli_conform.rs | 252 +++++++ 24 files changed, 4890 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/wai-breaking.yml create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/breaking/c_header.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/breaking/corpora.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/breaking/domains.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/breaking/git.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/breaking/jsondiff.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/breaking/mod.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/breaking/openapi.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/breaking/registry.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/breaking/rust_api.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/breaking/schema.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/conform/adapter.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/conform/http.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/conform/mod.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/conform/record.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/conform/suites.rs create mode 100644 wai/wai-rs/src/bin/wai_cli/dx/envelope.rs create mode 100644 wai/wai-rs/tests/cli_breaking.rs create mode 100644 wai/wai-rs/tests/cli_conform.rs diff --git a/.github/workflows/wai-breaking.yml b/.github/workflows/wai-breaking.yml new file mode 100644 index 000000000..3782365b0 --- /dev/null +++ b/.github/workflows/wai-breaking.yml @@ -0,0 +1,153 @@ +name: WAI breaking changes and the conformance runner + +# The gate WAI's libraries are published behind (toolchain v1, decision 7): every pull request +# that touches WAI runs `wai breaking --against origin/main`, and fails on a breaking change that +# wai/CHANGELOG.md does not name. Two jobs: +# +# - breaking: the check itself, against the merge base of the pull request and main, in all +# three categories: wire (the registry export, the media and delivery corpora), record (the +# signing domains, magics and versions, the record corpora, the published JSON Schemas) and +# API (the Rust public API of the WAI library crates through rustdoc's JSON, the C header, +# the room service's OpenAPI document). +# - runner: the tests of `wai breaking` (each kind of break reported and failing until a +# changelog entry names it) and of `wai conform` (the reference implementations conform, a +# changed verdict or output byte fails, a changed record is refused by `wai verify`); then +# `wai conform` runs the corpora's independent Python verifiers and the reference packager, +# and `wai verify` checks every record those runs wrote. +# +# Push filters match the push's delta, and the `relevant` job gates the work from the pull +# request's own diff (scripts/pr-touches-paths.py, reading the list below), as wai-pack.yml +# explains. + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + pull_request: + paths: + - 'wai/**' + # The room service's API is part of the API category. + - 'joule-code/joule-code-rs/crates/joule-moq-room/openapi.json' + # The toolchain pin decides which compiler documents both sides of the Rust API. + - 'rust-toolchain.toml' + - 'scripts/pr-touches-paths.py' + - '.github/workflows/wai-breaking.yml' + +jobs: + relevant: + name: WAI breaking · does this change touch WAI + runs-on: linux-arm64 + outputs: + run: ${{ steps.gate.outputs.run }} + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - id: gate + run: | + set -euo pipefail + r=$(python3 scripts/pr-touches-paths.py .github/workflows/wai-breaking.yml \ + "${{ github.event.pull_request.base.sha }}" \ + "${{ github.event.pull_request.head.sha }}") + echo "this change touches WAI: $r" + echo "run=$r" >> "$GITHUB_OUTPUT" + + breaking: + name: WAI breaking · wire, record and API against main · linux + needs: relevant + if: needs.relevant.outputs.run == 'true' + runs-on: linux + timeout-minutes: 60 + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + - name: Install Rust 1.98.0 + shell: bash + run: | + export PATH="$HOME/.cargo/bin:$PATH" + command -v rustup >/dev/null 2>&1 || \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.98.0 + rustup toolchain install 1.98.0 --profile minimal + rustup default 1.98.0 + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo" + command -v python3 >/dev/null || { $SUDO apt-get update -qq; $SUDO apt-get install -y -qq python3; } + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - name: Build wai + working-directory: wai/wai-rs + shell: bash + run: | + set -euo pipefail + cargo build --locked --release --no-default-features --bin wai + out=$(cargo metadata --format-version 1 --no-deps --locked | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])') + mkdir -p "$RUNNER_TEMP/bin" && cp "$out/release/wai" "$RUNNER_TEMP/bin/" + # The base is main's tip as fetched now; `wai breaking` compares against its merge base + # with this pull request, so what main gained since the branch left it is not read as + # this change removing it. Every finding is printed: each break with its category, what + # changed and why it breaks, and the line that acknowledges it if it is intended. + - name: wai breaking --against origin/main + shell: bash + run: | + set -euo pipefail + git fetch -q --no-tags origin main + "$RUNNER_TEMP/bin/wai" breaking --against origin/main --text --all + + runner: + name: WAI breaking and conform · their tests, and the independent verifiers through wai conform · linux + needs: relevant + if: needs.relevant.outputs.run == 'true' + runs-on: linux + timeout-minutes: 90 + steps: + - uses: actions/checkout@v5 + - name: Install Rust 1.98.0 + shell: bash + run: | + export PATH="$HOME/.cargo/bin:$PATH" + command -v rustup >/dev/null 2>&1 || \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.98.0 + rustup toolchain install 1.98.0 --profile minimal + rustup default 1.98.0 + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + # claim_verify.py, jwp_verify.py and signer_verify.py check Ed25519's equation with it. + - name: Install the Ed25519 library + shell: bash + run: python -m pip install cryptography + # The tests build fixture repositories with git and document a fixture crate with rustdoc. + - name: The tests of wai breaking and wai conform + working-directory: wai/wai-rs + shell: bash + run: | + set -euo pipefail + git config --global user.email ci@example.invalid + git config --global user.name ci + cargo test --locked --no-default-features --features quickstart,score,key_discovery \ + --bin wai --test cli_breaking --test cli_conform + - name: Build wai and the reference packager + shell: bash + run: | + set -euo pipefail + out() { (cd "$1" && cargo metadata --format-version 1 --no-deps --locked | python3 -c 'import json,sys; print(json.load(sys.stdin)["target_directory"])'); } + mkdir -p "$RUNNER_TEMP/bin" + (cd wai/wai-rs && cargo build --locked --release --no-default-features --features quickstart --bin wai) && cp "$(out wai/wai-rs)/release/wai" "$RUNNER_TEMP/bin/" + (cd wai/wai-pack && cargo build --locked --release --bin wai-pack) && cp "$(out wai/wai-pack)/release/wai-pack" "$RUNNER_TEMP/bin/" + # The second implementations of the claim, JWP and key-discovery corpora, written from the + # extension texts and sharing no code with wai-rs, run through the conformance runner as + # any other implementation would be; then the reference packager's policy reader. Each + # run must be conformant, and every record it signs must verify. + - name: wai conform with the independent verifiers and the reference packager + shell: bash + run: | + set -euo pipefail + w="$RUNNER_TEMP/bin/wai"; d="$RUNNER_TEMP/conform"; mkdir -p "$d" + for s in claim jwp signer; do + "$w" conform verifier --suite "$s" --text --root wai \ + --impl "python3 -I wai/$s-conformance/${s}_verify.py verdicts {corpus}" --out "$d/$s.wai-conform.json" + done + "$w" conform packager --suite policy --text --root wai \ + --impl "$RUNNER_TEMP/bin/wai-pack policy-verdict {in}" --out "$d/policy.wai-conform.json" + "$w" verify "$d" diff --git a/wai/CHANGELOG.md b/wai/CHANGELOG.md index f995e7cd8..808c25339 100644 --- a/wai/CHANGELOG.md +++ b/wai/CHANGELOG.md @@ -6,6 +6,40 @@ wire format may change and such changes are recorded here. ## v0.4.0 — current +### `wai breaking` and `wai conform`: the compatibility gate, and conformance anyone can run + +- **`wai breaking --against `** (operation `compat.check`) reports what the working + tree breaks against a ref, in three cumulative categories, strictest by default: **wire** (the + registry export: capabilities and their classes, deterministic identifiers, wire identifiers; + the media and delivery corpora), **record** (the domain-separation prefixes, magics and + versions records are built from; the corpora of signed records and their verdicts; the + published JSON Schemas) and **API** (the Rust public API of `wai` and `wai-quantum`, read from + the compiler's rustdoc JSON; the C header `wai-rs/include/wai.h`; the room service's OpenAPI + document). It compares against the merge base of the ref and `HEAD`. Each finding names its + item, its category, what changed and why that breaks; additions are listed as compatible. +- **An intended break passes only when this file names it**: a line the change adds here, in + an entry that says **Breaking**, with the finding's id (as `wai breaking` prints it, such as + api:c:wai_envelope_pack) in backticks. Otherwise the run exits 1. +- **`.github/workflows/wai-breaking.yml`** runs it on every pull request that touches WAI, + against main. WAI's libraries go to public registries behind this gate. +- **`wai conform --suite --impl `** (operation + `conformance.run`) runs one of WAI's corpora against another implementation: verifiers + (`claim`, `jwp`, `signer`), a decoder (`score`), a packager (`policy`) and an origin + (`http`). A command-line adapter gives each case as a file (`{case}`, `{in}`), the corpus + whole (`{corpus}`), or standard input, and reads the answer from standard output or `{out}`; + an HTTP adapter posts each case to a URL, or, for an origin, fetches from it. Every + expectation is the corpus's own. +- **The result is a signed record**, `conform-result/1` (`*.wai-conform.json`): the corpus + digest, the implementation (its command and the SHA-256 of each file it names, or its URL), + each case's outcome, the summary and the standard's version, signed with Ed25519 by a key + made for the run or one given with `--key`. **`wai verify`** checks its signature and that its + summary and verdict are what its cases say. CI runs the corpora's independent Python + verifiers and the reference packager through it. +- Both follow the toolchain's agent rules: the JSON envelope `envelope/1` with `--json` or off a + terminal, one exit status for each outcome (0 held, 1 a check failed, 2 usage, 4 a record + would be replaced without `--confirm`, 5 not found, 6 the implementation does not answer, 7 + internal), and `wai conform --dry-run`. + ### Create: a WAI app described, built as a model writes it, and made a project by `wai create --from app.oui` - **The widgets** ([`wai-app`](wai-app/), a library with no binary): a joule-ui registry of WAI diff --git a/wai/README.md b/wai/README.md index 2dbe8f5b3..49d4e7bee 100644 --- a/wai/README.md +++ b/wai/README.md @@ -86,6 +86,20 @@ cargo run --no-default-features --bin wai -- capabilities # the registry, one r cargo build --release --lib # with the codecs feature: libwai.dylib/.so/.dll + libwai.a ``` +Two commands hold what is published to what was published before, and other implementations +to WAI's corpora: + +```bash +wai breaking --against origin/main # what this tree breaks: wire, record and API (CI runs it on every pull request) +wai conform verifier --suite claim --impl "my-verifier verdicts {corpus}" # a corpus run against your implementation +wai verify claim.wai-conform.json # the signed result, checked +``` + +A break passes only when [`CHANGELOG.md`](CHANGELOG.md) names it: an entry this change adds that +says **Breaking** and gives the finding's id in backticks. `wai conform --list` names every kind +(verifier, decoder, packager, origin) and suite, with its adapters: a command line, or an +`http://` URL. + With the `codecs` feature, `wai-rs/build.rs` finds libFLAC through pkg-config and, on macOS without pkg-config, in the Homebrew "flac" formula prefix (`/opt/homebrew/opt/flac/lib` or `/usr/local/opt/flac/lib`). Where neither finds diff --git a/wai/wai-rs/Cargo.toml b/wai/wai-rs/Cargo.toml index 866b29228..6ad1ef6b5 100644 --- a/wai/wai-rs/Cargo.toml +++ b/wai/wai-rs/Cargo.toml @@ -1483,6 +1483,17 @@ name = "cli_dx" path = "tests/cli_dx.rs" required-features = ["quickstart"] +# `wai breaking` needs no feature; `wai conform` runs the reference verifiers, the score +# renderer and the origin of this crate as the implementations under test. +[[test]] +name = "cli_breaking" +path = "tests/cli_breaking.rs" + +[[test]] +name = "cli_conform" +path = "tests/cli_conform.rs" +required-features = ["quickstart", "score", "key_discovery"] + [[test]] name = "jpegai_decode_symbols" path = "tests/jpegai_decode_symbols.rs" diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/c_header.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/c_header.rs new file mode 100644 index 000000000..0669138f1 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/c_header.rs @@ -0,0 +1,368 @@ +//! API: the C header (`wai/wai-rs/include/wai.h`, generated by cbindgen from the FFI and +//! committed). A program compiled and linked against the old header must still build and +//! run against the new library. So a function that is gone, or whose return type or +//! parameter types changed (their names are not compared), breaks; so does a struct whose +//! fields changed in any way (its size or layout differs), an opaque type or typedef that +//! is gone or changed, an enumerator or macro constant that is gone or holds another +//! value, and a declaration moved under a preprocessor condition. New declarations are +//! compatible. + +use super::{Category, Finding}; +use std::collections::BTreeMap; + +pub const PATH: &str = "wai/wai-rs/include/wai.h"; + +/// One declaration of the header. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Decl { + /// A function: its return type and its parameters' types, in order. + Function { ret: String, params: Vec }, + /// A struct or union with its fields (`type name`, in order), or none for an opaque one. + Struct { fields: Option> }, + /// An enumeration: each enumerator and its value. + Enum { values: Vec<(String, String)> }, + Typedef { ty: String }, + /// A macro constant, with every value it is defined to (cbindgen repeats some). + Macro { values: Vec }, + Variable { ty: String }, +} + +/// A declaration and the preprocessor conditions it sits under (empty for none). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Item { + pub decl: Decl, + pub cond: String, +} + +fn strip_comments(src: &str) -> String { + let mut out = String::with_capacity(src.len()); + let b = src.as_bytes(); + let mut i = 0; + while i < b.len() { + if b[i] == b'/' && b.get(i + 1) == Some(&b'*') { + i += 2; + while i + 1 < b.len() && !(b[i] == b'*' && b[i + 1] == b'/') { + if b[i] == b'\n' { + out.push('\n'); + } + i += 1; + } + i += 2; + } else if b[i] == b'/' && b.get(i + 1) == Some(&b'/') { + while i < b.len() && b[i] != b'\n' { + i += 1; + } + } else { + let ch = src[i..].chars().next().unwrap_or(' '); + out.push(ch); + i += ch.len_utf8(); + } + } + out +} + +/// Whitespace collapsed, with `*`, `(`, `)`, `[`, `]` and `,` set apart so spellings +/// that differ only in spacing compare equal. +fn norm(s: &str) -> String { + let spaced: String = s.chars().flat_map(|c| if "*()[],".contains(c) { vec![' ', c, ' '] } else { vec![c] }).collect(); + spaced.split_whitespace().collect::>().join(" ") +} + +fn ident(s: &str) -> bool { + !s.is_empty() && s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') && !s.starts_with(|c: char| c.is_ascii_digit()) +} + +/// A parameter's type: its text without the trailing name. +fn param_type(p: &str) -> String { + let n = norm(p); + let toks: Vec<&str> = n.split(' ').collect(); + let last = toks.last().copied().unwrap_or(""); + let basic = matches!(last, "void" | "int" | "char" | "long" | "short" | "unsigned" | "signed" | "float" | "double" | "bool"); + if toks.len() > 1 && ident(last) && !basic { toks[..toks.len() - 1].join(" ") } else { n.clone() } +} + +fn split_top(s: &str, sep: char) -> Vec { + let mut out = Vec::new(); + let (mut depth, mut cur) = (0i32, String::new()); + for c in s.chars() { + match c { + '(' | '{' | '[' => depth += 1, + ')' | '}' | ']' => depth -= 1, + _ => {} + } + if c == sep && depth == 0 { + out.push(std::mem::take(&mut cur)); + } else { + cur.push(c); + } + } + if !cur.trim().is_empty() { + out.push(cur); + } + out +} + +fn declaration(text: &str, cond: &str, out: &mut BTreeMap) { + let t = norm(text); + if t.is_empty() { + return; + } + let item = |decl| Item { decl, cond: cond.to_string() }; + let body = |t: &str| -> Option<(String, String, String)> { + let open = t.find('{')?; + let close = t.rfind('}')?; + Some((t[..open].trim().to_string(), t[open + 1..close].trim().to_string(), t[close + 1..].trim().to_string())) + }; + if let Some((head, inner, tail)) = body(&t) { + let words: Vec<&str> = head.split(' ').collect(); + let is_enum = words.contains(&"enum"); + let tag = words.last().filter(|w| ident(w) && !matches!(**w, "struct" | "union" | "enum" | "typedef")).map(|w| w.to_string()); + let name = if tail.is_empty() { tag } else { Some(tail.trim_end_matches(';').trim().to_string()) }; + let Some(name) = name else { return }; + if is_enum { + let mut values = Vec::new(); + let mut next: i128 = 0; + for e in split_top(&inner, ',') { + let e = e.trim(); + if e.is_empty() { + continue; + } + let (n, v) = match e.split_once('=') { + Some((n, v)) => (n.trim().to_string(), norm(v)), + None => (e.to_string(), next.to_string()), + }; + next = v.parse::().map(|x| x + 1).unwrap_or(next + 1); + values.push((n, v)); + } + out.insert(name, item(Decl::Enum { values })); + } else { + let fields = split_top(&inner, ';').into_iter().map(|f| norm(&f)).filter(|f| !f.is_empty()).collect(); + out.insert(name, item(Decl::Struct { fields: Some(fields) })); + } + return; + } + if let Some(rest) = t.strip_prefix("typedef ") { + let words: Vec<&str> = rest.split(' ').collect(); + if matches!(words.first(), Some(&"struct") | Some(&"union")) && words.len() == 3 && !rest.contains('(') { + out.entry(words[2].to_string()).or_insert_with(|| item(Decl::Struct { fields: None })); + return; + } + // `typedef `, or a function pointer `typedef R ( * name ) ( … )`. + let name = if let Some(i) = rest.find("( * ") { + rest[i + 4..].split(' ').next().unwrap_or("").to_string() + } else { + words.last().unwrap_or(&"").to_string() + }; + if ident(&name) { + out.insert(name, item(Decl::Typedef { ty: rest.to_string() })); + } + return; + } + if let Some(open) = t.find(" ( ") { + let head = &t[..open]; + let name = head.split(' ').next_back().unwrap_or("").to_string(); + if !ident(&name) { + return; + } + let ret = head[..head.len() - name.len()].trim().trim_start_matches("extern ").to_string(); + let close = t.rfind(')').unwrap_or(t.len()); + let inner = &t[open + 3..close]; + let params: Vec = split_top(inner, ',').iter().map(|p| param_type(p)).filter(|p| !p.is_empty()).collect(); + let params = if params == ["void"] { Vec::new() } else { params }; + out.insert(name, item(Decl::Function { ret, params })); + return; + } + let words: Vec<&str> = t.split(' ').collect(); + if words.len() >= 2 && ident(words[words.len() - 1]) { + let name = words[words.len() - 1].to_string(); + let ty = words[..words.len() - 1].join(" ").trim_start_matches("extern ").to_string(); + out.insert(name, item(Decl::Variable { ty })); + } +} + +/// Every declaration of a header, by name. +pub fn parse(src: &str) -> BTreeMap { + let text = strip_comments(src); + let mut out: BTreeMap = BTreeMap::new(); + let mut conds: Vec = Vec::new(); + let mut guard: Option = None; + let mut pending = String::new(); + let mut lines = text.lines().peekable(); + while let Some(line) = lines.next() { + let mut l = line.trim().to_string(); + while l.ends_with('\\') { + l.pop(); + l.push(' '); + l.push_str(lines.next().unwrap_or("").trim()); + } + if let Some(d) = l.strip_prefix('#') { + let d = d.trim(); + let (word, rest) = d.split_once(char::is_whitespace).map(|(a, b)| (a, b.trim())).unwrap_or((d, "")); + match word { + "ifndef" if guard.is_none() && conds.is_empty() => { + guard = Some(rest.to_string()); + conds.push(String::new()); + } + "if" | "ifdef" | "ifndef" => conds.push(format!("{word} {rest}")), + "else" => { + if let Some(c) = conds.last_mut() { + *c = format!("not ({c})"); + } + } + "elif" => { + if let Some(c) = conds.last_mut() { + *c = format!("elif {rest}"); + } + } + "endif" => { + conds.pop(); + } + "define" => { + let (name, value) = rest.split_once(char::is_whitespace).map(|(a, b)| (a, norm(b))).unwrap_or((rest, String::new())); + if Some(name) != guard.as_deref() && ident(name) { + let cond = conds.iter().filter(|c| !c.is_empty()).cloned().collect::>().join(" && "); + match out.get_mut(name) { + Some(Item { decl: Decl::Macro { values }, .. }) => { + if !values.contains(&value) { + values.push(value); + values.sort(); + } + } + _ => { + out.insert(name.to_string(), Item { decl: Decl::Macro { values: vec![value] }, cond }); + } + } + } + } + _ => {} + } + continue; + } + // The `extern "C" {` wrapper sits under `__cplusplus`; nothing there is a declaration. + if conds.iter().any(|c| c.contains("__cplusplus")) { + continue; + } + let cond = conds.iter().filter(|c| !c.is_empty()).cloned().collect::>().join(" && "); + for ch in l.chars().chain(std::iter::once(' ')) { + pending.push(ch); + if ch == ';' { + let depth: i32 = pending.chars().map(|c| match c { + '{' => 1, + '}' => -1, + _ => 0, + }).sum(); + if depth == 0 { + let decl = std::mem::take(&mut pending); + declaration(decl.trim().trim_end_matches(';'), &cond, &mut out); + } + } + } + } + out +} + +fn show(d: &Decl, name: &str) -> String { + match d { + Decl::Function { ret, params } => format!("{ret} {name}({})", params.join(", ")), + Decl::Struct { fields: Some(f) } => format!("struct {name} {{ {} }}", f.join("; ")), + Decl::Struct { fields: None } => format!("struct {name} (opaque)"), + Decl::Enum { values } => format!("enum {name} {{ {} }}", values.iter().map(|(n, v)| format!("{n} = {v}")).collect::>().join(", ")), + Decl::Typedef { ty } => format!("typedef {ty}"), + Decl::Macro { values } => format!("#define {name} {}", values.join(" | ")), + Decl::Variable { ty } => format!("{ty} {name}"), + } +} + +fn kind(d: &Decl) -> &'static str { + match d { + Decl::Function { .. } => "function", + Decl::Struct { .. } => "struct", + Decl::Enum { .. } => "enum", + Decl::Typedef { .. } => "typedef", + Decl::Macro { .. } => "constant", + Decl::Variable { .. } => "variable", + } +} + +/// The header's findings, from the two headers (`None` where a tree has none). +pub fn compare(old: Option<&str>, new: Option<&str>) -> Vec { + let a = old.map(parse).unwrap_or_default(); + let b = new.map(parse).unwrap_or_default(); + let mut out = Vec::new(); + for (name, x) in &a { + let k = kind(&x.decl); + let Some(y) = b.get(name) else { + out.push(Finding::broke( + Category::Api, + "c", + name.clone(), + "removed".into(), + format!("the {k} {} is gone", show(&x.decl, name)), + format!("a program that uses {name} no longer compiles against the header, or no longer links against the library"), + )); + continue; + }; + let why = match (&x.decl, &y.decl) { + _ if x.cond != y.cond => Some(format!("now declared under `{}` (was `{}`)", y.cond, x.cond)), + (Decl::Struct { fields: Some(_) }, Decl::Struct { fields: None }) => Some("its fields are no longer declared".into()), + (Decl::Struct { fields: None }, Decl::Struct { fields: Some(_) }) => None, + (Decl::Enum { values: va }, Decl::Enum { values: vb }) => { + let gone: Vec = va.iter().filter(|e| !vb.contains(e)).map(|(n, v)| format!("{n} = {v}")).collect(); + (!gone.is_empty()).then(|| format!("{} gone or renumbered", gone.join(", "))) + } + (Decl::Macro { values: va }, Decl::Macro { values: vb }) => { + (va.iter().any(|v| !vb.contains(v))).then(|| format!("value {} → {}", va.join(" | "), vb.join(" | "))) + } + (p, q) if p != q => Some(format!("{} → {}", show(p, name), show(q, name))), + _ => None, + }; + if let Some(what) = why { + let reason = match &x.decl { + Decl::Function { .. } => "a caller compiled against the old prototype passes or reads the wrong types: undefined behaviour, or a build that fails", + Decl::Struct { .. } => "the struct's size or layout differs, so a caller compiled against the old header reads and writes the wrong bytes", + Decl::Macro { .. } | Decl::Enum { .. } => "a program compiled with the old value passes or compares a value the library now reads differently", + _ => "a program compiled against the old declaration no longer matches the library", + }; + out.push(Finding::broke(Category::Api, "c", name.clone(), "changed".into(), format!("the {k} {name}: {what}"), reason.into())); + } else if x.decl != y.decl { + out.push(Finding::compatible(Category::Api, "c", name.clone(), format!("{} → {}", show(&x.decl, name), show(&y.decl, name)))); + } + } + for (name, y) in &b { + if !a.contains_key(name) { + out.push(Finding::compatible(Category::Api, "c", name.clone(), format!("the {} {} is new", kind(&y.decl), show(&y.decl, name)))); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + const H: &str = "#ifndef WAI_H\n#define WAI_H\n#include \n/** doc */\n#define WAI_OK 0\n#define CREATE 1\n#define CREATE 1\n\ + typedef struct wai_sink_t wai_sink_t;\ntypedef struct wai_buffer_t {\n uint8_t *data; // the bytes\n uintptr_t len;\n} wai_buffer_t;\n\ + #ifdef __cplusplus\nextern \"C\" {\n#endif\n int wai_pack(const char *manifest,\n const uint8_t *payload, uintptr_t len,\n struct wai_buffer_t *out);\n\ + uint32_t wai_abi(void);\n#ifdef __cplusplus\n}\n#endif\n#endif\n"; + + #[test] + fn parses_what_cbindgen_writes() { + let m = parse(H); + assert_eq!(m["wai_pack"].decl, Decl::Function { ret: "int".into(), params: vec!["const char *".into(), "const uint8_t *".into(), "uintptr_t".into(), "struct wai_buffer_t *".into()] }); + assert_eq!(m["wai_abi"].decl, Decl::Function { ret: "uint32_t".into(), params: vec![] }); + assert_eq!(m["wai_buffer_t"].decl, Decl::Struct { fields: Some(vec!["uint8_t * data".into(), "uintptr_t len".into()]) }); + assert_eq!(m["wai_sink_t"].decl, Decl::Struct { fields: None }); + assert_eq!(m["CREATE"].decl, Decl::Macro { values: vec!["1".into()] }); + assert!(!m.contains_key("WAI_H")); + assert_eq!(m["wai_pack"].cond, ""); + } + + #[test] + fn a_changed_parameter_type_breaks_and_a_renamed_one_does_not() { + let renamed = H.replace("uintptr_t len,\n", "uintptr_t payload_len,\n"); + assert!(compare(Some(H), Some(&renamed)).iter().all(|f| !f.breaking)); + let changed = H.replace("uintptr_t len,\n", "uint32_t len,\n"); + let f = compare(Some(H), Some(&changed)); + assert!(f.iter().any(|f| f.breaking && f.id == "api:c:wai_pack"), "{f:?}"); + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/corpora.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/corpora.rs new file mode 100644 index 000000000..d30828180 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/corpora.rs @@ -0,0 +1,110 @@ +//! Wire and record: the conformance corpora (`wai/*-conformance/`) and the pinned vectors +//! of the reference (`wai/wai-rs/tests/vectors/`). Each vector states bytes a conforming +//! reader must accept or refuse, and what it must make of them: a decoded output's digest, +//! a verdict, a reason. The conformance workflow holds every corpus to the code that +//! generates it, byte for byte, so a change in what the code writes or signs (a field, a +//! signing preimage, a magic) reaches these files, or that workflow fails. +//! +//! So a vector file that is gone, or whose bytes changed, breaks; a JSON file breaks where +//! an entry or member is gone or a value changed, and is compatible where it only gained +//! entries; a JSON Lines file breaks where a line is gone. A new file is compatible. +//! Verifier code, documentation and scripts beside a corpus are not vectors and are not +//! compared. The record corpora (signed records and their verdicts) report under +//! `record`; the others, under `wire`. + +use super::git::{Repo, Status}; +use super::jsondiff; +use super::super::envelope::OpError; +use super::{Category, Finding}; + +pub const PATHSPECS: &[&str] = &[":(glob)wai/*-conformance/**", ":(glob)wai/wai-rs/tests/vectors/**"]; + +/// The corpora of signed records and their verdicts. +const RECORD_CORPORA: &[&str] = &[ + "claim-conformance", + "signer-conformance", + "jwp-conformance", + "pq-conformance", + "oer-conformance", + "pin-conformance", + "edge-conformance", + "live-conformance", +]; + +/// Files beside a corpus that are not vectors: verifiers, generators, scripts, notes. +const NOT_VECTORS: &[&str] = &[ + ".py", ".pyc", ".mjs", ".js", ".ts", ".md", ".sh", ".rs", ".toml", ".lock", ".patch", ".txt", ".gitattributes", ".gitignore", +]; + +fn is_vector(path: &str) -> bool { + !path.split('/').any(|c| c == "__pycache__" || (c == "src" && path.starts_with("wai/byte-exact-conformance/"))) + && !NOT_VECTORS.iter().any(|x| path.ends_with(x)) +} + +fn category(path: &str) -> Category { + let dir = path.split('/').nth(1).unwrap_or(""); + if RECORD_CORPORA.contains(&dir) { Category::Record } else { Category::Wire } +} + +fn list(xs: &[String]) -> String { + let n = xs.len(); + let shown: Vec<&str> = xs.iter().take(4).map(String::as_str).collect(); + if n > 4 { format!("{} (and {} more)", shown.join("; "), n - 4) } else { shown.join("; ") } +} + +/// The corpora's findings between `base` and the working tree. +pub fn compare(repo: &Repo, base: &str) -> Result, OpError> { + let changed: Vec<(Status, String)> = repo.changed(base, PATHSPECS)?.into_iter().filter(|(_, p)| is_vector(p)).collect(); + let modified: Vec = changed.iter().filter(|(s, p)| *s == Status::Modified && (p.ends_with(".json") || p.ends_with(".jsonl"))).map(|x| x.1.clone()).collect(); + let olds = repo.base_files(base, &modified)?; + let mut out = Vec::new(); + for (status, path) in changed { + let cat = category(&path); + let why_bytes = if cat == Category::Record { + "a verifier conforms by reaching each vector's stated verdict on these bytes; records signed or read as before no longer match it" + } else { + "a reader conforms by decoding these bytes to the stated output; readers built against the old vector disagree with the new one" + }; + match status { + Status::Added => out.push(Finding::compatible(cat, "corpus", path, "a new vector file".into())), + Status::Removed => out.push(Finding::broke(cat, "corpus", path, "removed".into(), "the vector file is gone".into(), why_bytes.into())), + Status::Modified => { + let json = path.ends_with(".json"); + let lines = path.ends_with(".jsonl"); + let parsed = if json || lines { + let old = olds.get(&path).cloned().unwrap_or_default(); + let new = repo.head_file(&path).unwrap_or_default(); + if lines { + Some(jsondiff::diff_lines(&String::from_utf8_lossy(&old), &String::from_utf8_lossy(&new))) + } else { + match (serde_json::from_slice(&old), serde_json::from_slice(&new)) { + (Ok(a), Ok(b)) => Some(jsondiff::diff(&a, &b)), + _ => None, + } + } + } else { + None + }; + match parsed { + None => out.push(Finding::broke(cat, "corpus", path, "changed".into(), "the vector's bytes changed".into(), why_bytes.into())), + Some(d) if d.is_empty() => {} + Some(d) if !d.breaks() => out.push(Finding::compatible(cat, "corpus", path, format!("added {}", list(&d.added)))), + Some(d) => { + let mut what = Vec::new(); + if !d.removed.is_empty() { + what.push(format!("removed {}", list(&d.removed))); + } + if !d.changed.is_empty() { + what.push(format!("changed {}", list(&d.changed))); + } + if !d.added.is_empty() { + what.push(format!("added {}", d.added.len())); + } + out.push(Finding::broke(cat, "corpus", path, "changed".into(), what.join("; "), why_bytes.into())); + } + } + } + } + } + Ok(out) +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/domains.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/domains.rs new file mode 100644 index 000000000..679dcde48 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/domains.rs @@ -0,0 +1,165 @@ +//! Record: the constants a signed or framed record is built from, in the WAI crates' library +//! sources (`wai/*/src/`, binaries and tests left out). A constant whose name has the word +//! `DOMAIN`, `MAGIC` or `VERSION` and whose value is a literal is one of: +//! +//! - a domain-separation prefix of a signing preimage or a hash (`SIGN_DOMAIN`, +//! `LINK_DOMAIN`): another value changes every signature and link over the record, and a +//! verifier of the old records refuses the new ones; +//! - a record's magic (`MAGIC`, `GRANT_MAGIC`): a reader that matches the old magic refuses +//! the new records; +//! - a record's version (`RECORD_VERSION`): a reader that knows the old version refuses +//! the new one. +//! +//! So such a constant breaks when its value changes or it is gone (unless the same name +//! and value now sit in another file: a move). A new one is compatible. A record's other +//! fields and its preimage's shape reach the corpora (see `corpora`), which hold the bytes +//! the code writes and signs. + +use super::git::Repo; +use super::super::envelope::OpError; +use super::{Category, Finding}; +use std::collections::BTreeMap; + +pub const PATHSPECS: &[&str] = &[":(glob)wai/*/src/**/*.rs", ":(exclude,glob)wai/*/src/bin/**"]; + +/// The prefilter `git grep` runs (POSIX extended syntax). +const GREP: &str = "const[[:space:]]+[A-Z0-9_]*(DOMAIN|MAGIC|VERSION)"; + +/// One constant: its name, its type and its value as written (spaces dropped). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Const { + pub name: String, + pub ty: String, + pub value: String, +} + +fn named(name: &str) -> bool { + let words: Vec<&str> = name.split('_').collect(); + words.iter().any(|w| matches!(*w, "DOMAIN" | "MAGIC" | "VERSION")) + && !words.iter().any(|w| matches!(*w, "MAX" | "MIN" | "LEN" | "UNKNOWN" | "API" | "HEADER" | "VERSIONS")) +} + +fn literal(v: &str) -> bool { + v.starts_with("b\"") || v.starts_with("*b\"") || v.starts_with('"') || v.starts_with("b'") || v.starts_with('[') || v.starts_with("&[") || v.starts_with(|c: char| c.is_ascii_digit()) +} + +/// The constants of one source file, keyed by name (`NAME`, then `NAME#2`… for a name +/// declared again in the same file). Scanning stops at the file's test module. +pub fn scan(src: &str) -> BTreeMap { + let mut out = BTreeMap::new(); + let lines: Vec<&str> = src.lines().collect(); + let mut i = 0; + while i < lines.len() { + let t = lines[i].trim(); + if t == "#[cfg(test)]" && lines.get(i + 1).is_some_and(|n| n.trim_start().starts_with("mod ")) { + break; + } + let rest = t.strip_prefix("pub ").or_else(|| t.strip_prefix("pub(crate) ")).or_else(|| t.strip_prefix("pub(super) ")).unwrap_or(t); + if let Some(decl) = rest.strip_prefix("const ") { + // The declaration, joined across lines until its `;`. + let mut text = decl.to_string(); + let mut j = i; + while !text.contains(';') && j + 1 < lines.len() && j < i + 6 { + j += 1; + text.push(' '); + text.push_str(lines[j].trim()); + } + if let Some((name, after)) = text.split_once(':') { + let name = name.trim(); + if name.chars().all(|c| c.is_ascii_uppercase() || c.is_ascii_digit() || c == '_') + && named(name) + && let Some((ty, value)) = after.split_once('=') + { + let value: String = value.split(';').next().unwrap_or("").split_whitespace().collect::>().join(""); + if literal(&value) { + let ty: String = ty.split_whitespace().collect(); + let mut key = name.to_string(); + let mut n = 1; + while out.contains_key(&key) { + n += 1; + key = format!("{name}#{n}"); + } + out.insert(key, Const { name: name.to_string(), ty, value }); + } + } + } + i = j; + } + i += 1; + } + out +} + +fn role(name: &str) -> (&'static str, &'static str) { + if name.split('_').any(|w| w == "DOMAIN") { + ("signing or hashing domain", "every signature and hash made under it changes; a verifier of the records made before refuses those made after, and the other way round") + } else if name.split('_').any(|w| w == "MAGIC") { + ("magic", "a reader that matches the old magic refuses the records written with the new one") + } else { + ("version", "a reader that knows the old version refuses records of the new one, and a writer of the old version is refused by the new reader") + } +} + +fn all(repo: &Repo, base: Option<&str>) -> Result, OpError> { + let files = repo.grep_files(base, GREP, PATHSPECS)?; + let texts: BTreeMap> = match base { + Some(b) => repo.base_files(b, &files)?, + None => files.iter().filter_map(|f| Some((f.clone(), repo.head_file(f)?))).collect(), + }; + let mut out = BTreeMap::new(); + for (f, t) in texts { + for (k, c) in scan(&String::from_utf8_lossy(&t)) { + out.insert((f.clone(), k), c); + } + } + Ok(out) +} + +/// The constants' findings between `base` and the working tree. +pub fn compare(repo: &Repo, base: &str) -> Result, OpError> { + let old = all(repo, Some(base))?; + let new = all(repo, None)?; + let mut out = Vec::new(); + let moved = |c: &Const| new.values().any(|n| n.name == c.name && n.value == c.value); + for ((file, key), c) in &old { + let item = format!("{file}::{key}"); + let (what_it_is, why) = role(&c.name); + match new.get(&(file.clone(), key.clone())) { + None if moved(c) => {} + None => out.push(Finding::broke(Category::Record, "domain", item, "removed".into(), format!("the {what_it_is} {} = {} is gone", c.name, c.value), why.into())), + Some(n) if n.value != c.value || n.ty != c.ty => out.push(Finding::broke( + Category::Record, + "domain", + item, + "changed".into(), + format!("the {what_it_is} {} changed: {} → {}", c.name, c.value, n.value), + why.into(), + )), + Some(_) => {} + } + } + for ((file, key), c) in &new { + if !old.contains_key(&(file.clone(), key.clone())) && !old.values().any(|o| o.name == c.name && o.value == c.value) { + out.push(Finding::compatible(Category::Record, "domain", format!("{file}::{key}"), format!("{} = {} is new", c.name, c.value))); + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::scan; + + #[test] + fn scans_domains_magics_and_versions() { + let src = "pub const SIGN_DOMAIN: &[u8] = b\"wai:x\\x01\";\nconst MAGIC: [u8; 4] = *b\"WEV1\";\npub const RECORD_VERSION: u8 = 1;\n\ + pub(crate) const DOMAIN_LEN: u64 = X.len() as u64;\nconst MAX_VERSION: u8 = 3;\nconst OTHER: u8 = 1;\n\ + pub const LONG_DOMAIN: &[u8] =\n b\"wai:long\\x01\";\n#[cfg(test)]\nmod tests {\n const TEST_DOMAIN: &[u8] = b\"t\";\n}\n"; + let m = scan(src); + let names: Vec<&str> = m.keys().map(String::as_str).collect(); + assert_eq!(names, vec!["LONG_DOMAIN", "MAGIC", "RECORD_VERSION", "SIGN_DOMAIN"]); + assert_eq!(m["SIGN_DOMAIN"].value, "b\"wai:x\\x01\""); + assert_eq!(m["LONG_DOMAIN"].value, "b\"wai:long\\x01\""); + assert_eq!(m["MAGIC"].ty, "[u8;4]"); + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/git.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/git.rs new file mode 100644 index 000000000..e1e404d5a --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/git.rs @@ -0,0 +1,332 @@ +//! The two trees `wai breaking` compares, read through `git`: the base (a commit) and the +//! working tree (what is on disk, committed or not, untracked files included). + +use super::super::envelope::{EXIT_INTERNAL, EXIT_NOT_FOUND, OpError}; +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +/// A checkout: its top-level directory. +pub struct Repo { + pub root: PathBuf, +} + +/// How a path differs between the base and the working tree. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Status { + Added, + Modified, + Removed, +} + +fn internal(e: impl std::fmt::Display) -> OpError { + OpError::new(EXIT_INTERNAL, "git-failed", e.to_string(), "run it in a git checkout with git on PATH") +} + +impl Repo { + /// The checkout holding `dir`. + pub fn open(dir: &Path) -> Result { + let out = Command::new("git").arg("-C").arg(dir).args(["rev-parse", "--show-toplevel"]).output().map_err(|e| { + OpError::new(EXIT_NOT_FOUND, "git-not-found", format!("git: {e}"), "install git; `wai breaking` reads both trees through it") + })?; + if !out.status.success() { + return Err(OpError::new( + EXIT_NOT_FOUND, + "not-a-checkout", + format!("{}: not in a git checkout", dir.display()), + "run it inside the repository, or pass --root ", + )); + } + let root = String::from_utf8_lossy(&out.stdout).trim().to_string(); + Ok(Repo { root: PathBuf::from(root) }) + } + + fn git(&self, args: &[&str]) -> Result, OpError> { + let out = Command::new("git").arg("-C").arg(&self.root).args(args).output().map_err(internal)?; + if !out.status.success() { + return Err(internal(format!("git {}: {}", args.join(" "), String::from_utf8_lossy(&out.stderr).trim()))); + } + Ok(out.stdout) + } + + /// The commit `rev` names, or `None` when it names none. + pub fn commit(&self, rev: &str) -> Option { + let out = Command::new("git").arg("-C").arg(&self.root).args(["rev-parse", "--verify", "--quiet", &format!("{rev}^{{commit}}")]).output().ok()?; + out.status.success().then(|| String::from_utf8_lossy(&out.stdout).trim().to_string()) + } + + /// The best common ancestor of two commits. + pub fn merge_base(&self, a: &str, b: &str) -> Option { + let out = Command::new("git").arg("-C").arg(&self.root).args(["merge-base", a, b]).output().ok()?; + out.status.success().then(|| String::from_utf8_lossy(&out.stdout).trim().to_string()) + } + + /// Every path under `pathspecs` that differs between `base` and the working tree. + pub fn changed(&self, base: &str, pathspecs: &[&str]) -> Result, OpError> { + let mut args = vec!["diff", "--name-status", "--no-renames", "-z", base, "--"]; + args.extend_from_slice(pathspecs); + let out = self.git(&args)?; + let mut v = Vec::new(); + let mut it = out.split(|b| *b == 0).filter(|s| !s.is_empty()); + while let (Some(st), Some(p)) = (it.next(), it.next()) { + let s = match st.first() { + Some(b'A') => Status::Added, + Some(b'D') => Status::Removed, + _ => Status::Modified, + }; + v.push((s, String::from_utf8_lossy(p).into_owned())); + } + let mut args = vec!["ls-files", "--others", "--exclude-standard", "-z", "--"]; + args.extend_from_slice(pathspecs); + for p in self.git(&args)?.split(|b| *b == 0).filter(|s| !s.is_empty()) { + v.push((Status::Added, String::from_utf8_lossy(p).into_owned())); + } + v.sort_by(|a, b| a.1.cmp(&b.1)); + v.dedup_by(|a, b| a.1 == b.1); + Ok(v) + } + + /// The paths under `pathspecs` in `base`. (`git ls-tree` takes no pathspec magic, so + /// the listing is filtered here, by [`matches`].) + pub fn ls_base(&self, base: &str, pathspecs: &[&str]) -> Result, OpError> { + let all = split0(&self.git(&["ls-tree", "-r", "--name-only", "-z", base])?); + Ok(all.into_iter().filter(|p| matches(pathspecs, p)).collect()) + } + + /// The paths under `pathspecs` in the working tree: tracked files still on disk, and + /// untracked files that are not ignored. + pub fn ls_head(&self, pathspecs: &[&str]) -> Result, OpError> { + let mut args = vec!["ls-files", "--cached", "--others", "--exclude-standard", "-z", "--"]; + args.extend_from_slice(pathspecs); + let mut v: Vec = split0(&self.git(&args)?).into_iter().filter(|p| self.root.join(p).is_file()).collect(); + v.sort(); + v.dedup(); + Ok(v) + } + + /// The files under `pathspecs` with a line matching the extended regular expression + /// `re`: in `base` when it is given, else in the working tree (untracked files too). + pub fn grep_files(&self, base: Option<&str>, re: &str, pathspecs: &[&str]) -> Result, OpError> { + let mut args = vec!["grep", "-l", "-I", "-z", "-E", "-e", re]; + match base { + Some(b) => args.push(b), + None => args.push("--untracked"), + } + args.push("--"); + args.extend_from_slice(pathspecs); + let out = Command::new("git").arg("-C").arg(&self.root).args(&args).output().map_err(internal)?; + // git grep exits 1 when nothing matched. + if !out.status.success() && out.status.code() != Some(1) { + return Err(internal(format!("git grep: {}", String::from_utf8_lossy(&out.stderr).trim()))); + } + let prefix = base.map(|b| format!("{b}:")).unwrap_or_default(); + Ok(split0(&out.stdout).into_iter().map(|p| p.strip_prefix(&prefix).map(String::from).unwrap_or(p)).collect()) + } + + /// A file of the working tree, or `None` when it is not there. + pub fn head_file(&self, path: &str) -> Option> { + std::fs::read(self.root.join(path)).ok() + } + + /// Files of `base`, read in one `git cat-file --batch`: each path's bytes, or no entry + /// when `base` does not hold it. + pub fn base_files(&self, base: &str, paths: &[String]) -> Result>, OpError> { + let mut out = BTreeMap::new(); + if paths.is_empty() { + return Ok(out); + } + let mut child = Command::new("git") + .arg("-C") + .arg(&self.root) + .args(["cat-file", "--batch"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(internal)?; + let mut stdin = child.stdin.take().ok_or_else(|| internal("git cat-file: no stdin"))?; + let req: String = paths.iter().map(|p| format!("{base}:{p}\n")).collect(); + let writer = std::thread::spawn(move || { + let _ = stdin.write_all(req.as_bytes()); + }); + let mut all = Vec::new(); + child.stdout.take().ok_or_else(|| internal("git cat-file: no stdout"))?.read_to_end(&mut all).map_err(internal)?; + let _ = writer.join(); + let _ = child.wait(); + let mut rest: &[u8] = &all; + for p in paths { + let nl = rest.iter().position(|b| *b == b'\n').ok_or_else(|| internal("git cat-file: short answer"))?; + let header = String::from_utf8_lossy(&rest[..nl]).into_owned(); + rest = &rest[nl + 1..]; + if header.ends_with(" missing") || header.ends_with(" ambiguous") { + continue; + } + let size: usize = header.rsplit(' ').next().and_then(|s| s.parse().ok()).ok_or_else(|| internal(format!("git cat-file: {header}")))?; + if rest.len() < size + 1 { + return Err(internal("git cat-file: short object")); + } + if header.split(' ').nth(1) == Some("blob") { + out.insert(p.clone(), rest[..size].to_vec()); + } + rest = &rest[size + 1..]; + } + Ok(out) + } + + /// One file of `base`. + pub fn base_file(&self, base: &str, path: &str) -> Result>, OpError> { + Ok(self.base_files(base, &[path.to_string()])?.remove(path)) + } + + /// `base`'s tree under `path`, written below `dest` at the same relative paths. + pub fn extract(&self, base: &str, path: &str, dest: &Path) -> Result<(), OpError> { + std::fs::create_dir_all(dest).map_err(internal)?; + let mut archive = Command::new("git") + .arg("-C") + .arg(&self.root) + .args(["archive", "--format=tar", base, "--", path]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(internal)?; + let tar_in = archive.stdout.take().ok_or_else(|| internal("git archive: no stdout"))?; + let tar = Command::new("tar").arg("-x").arg("-C").arg(dest).stdin(tar_in).output().map_err(internal)?; + let a = archive.wait_with_output().map_err(internal)?; + if !a.status.success() || !tar.status.success() { + return Err(internal(format!( + "extracting {path} of {base}: {}{}", + String::from_utf8_lossy(&a.stderr).trim(), + String::from_utf8_lossy(&tar.stderr).trim() + ))); + } + Ok(()) + } + + /// The lines `base`'s CHANGELOG at `path` does not have and the working tree's has. + pub fn added_lines(&self, base: &str, path: &str) -> Result, OpError> { + let old = self.base_file(base, path)?.unwrap_or_default(); + let new = self.head_file(path).unwrap_or_default(); + Ok(added_lines(&String::from_utf8_lossy(&old), &String::from_utf8_lossy(&new))) + } +} + +/// Whether `path` is under `pathspecs`, as git reads the forms used here: a plain path (the +/// file, or everything below the directory), `:(glob)` and `:(exclude,glob)`, +/// where `*` matches within one path segment and `**` matches any number of segments. +pub fn matches(pathspecs: &[&str], path: &str) -> bool { + let mut included = false; + for spec in pathspecs { + let (exclude, pattern, glob) = if let Some(p) = spec.strip_prefix(":(exclude,glob)") { + (true, p, true) + } else if let Some(p) = spec.strip_prefix(":(glob)") { + (false, p, true) + } else { + (false, *spec, false) + }; + let hit = if glob { glob_match(&pattern.split('/').collect::>(), &path.split('/').collect::>()) } else { path == pattern || path.starts_with(&format!("{}/", pattern.trim_end_matches('/'))) }; + if hit && exclude { + return false; + } + included |= hit; + } + included +} + +fn glob_match(pat: &[&str], path: &[&str]) -> bool { + match (pat.first(), path.first()) { + (None, None) => true, + (Some(&"**"), _) => glob_match(&pat[1..], path) || (!path.is_empty() && glob_match(pat, &path[1..])), + (Some(p), Some(s)) => segment_match(p.as_bytes(), s.as_bytes()) && glob_match(&pat[1..], &path[1..]), + _ => false, + } +} + +fn segment_match(p: &[u8], s: &[u8]) -> bool { + match (p.first(), s.first()) { + (None, None) => true, + (Some(b'*'), _) => segment_match(&p[1..], s) || (!s.is_empty() && segment_match(p, &s[1..])), + (Some(b'?'), Some(_)) => segment_match(&p[1..], &s[1..]), + (Some(a), Some(b)) if a == b => segment_match(&p[1..], &s[1..]), + _ => false, + } +} + +fn split0(b: &[u8]) -> Vec { + b.split(|x| *x == 0).filter(|s| !s.is_empty()).map(|s| String::from_utf8_lossy(s).into_owned()).collect() +} + +/// The lines of `new` that a longest common subsequence with `old` leaves over: what an +/// edit added, in order. A line moved within the file counts as added where it now is. +pub fn added_lines(old: &str, new: &str) -> Vec { + let a: Vec<&str> = old.lines().collect(); + let b: Vec<&str> = new.lines().collect(); + // Trim the common head and tail first: a changelog edit is a few lines near the top of a + // long file, and what is left between is small. + let mut lo = 0; + while lo < a.len() && lo < b.len() && a[lo] == b[lo] { + lo += 1; + } + let (mut ea, mut eb) = (a.len(), b.len()); + while ea > lo && eb > lo && a[ea - 1] == b[eb - 1] { + ea -= 1; + eb -= 1; + } + let (a, b) = (&a[lo..ea], &b[lo..eb]); + if a.is_empty() { + return b.iter().map(|s| s.to_string()).collect(); + } + if (a.len() as u64) * (b.len() as u64) > 40_000_000 { + // Too far apart for the table: every line not in the old file counts as added. + let old: std::collections::HashSet<&str> = a.iter().copied().collect(); + return b.iter().filter(|l| !old.contains(*l)).map(|s| s.to_string()).collect(); + } + let (n, m) = (a.len(), b.len()); + let mut t = vec![0u32; (n + 1) * (m + 1)]; + for i in (0..n).rev() { + for j in (0..m).rev() { + t[i * (m + 1) + j] = if a[i] == b[j] { t[(i + 1) * (m + 1) + j + 1] + 1 } else { t[(i + 1) * (m + 1) + j].max(t[i * (m + 1) + j + 1]) }; + } + } + let (mut i, mut j) = (0, 0); + let mut out = Vec::new(); + while j < m { + if i < n && a[i] == b[j] { + i += 1; + j += 1; + } else if i < n && t[(i + 1) * (m + 1) + j] >= t[i * (m + 1) + j + 1] { + i += 1; + } else { + out.push(b[j].to_string()); + j += 1; + } + } + out +} + +#[cfg(test)] +mod tests { + use super::added_lines; + + #[test] + fn pathspecs_match_as_git_reads_them() { + use super::matches; + let specs = [":(glob)wai/*/src/**/*.rs", ":(exclude,glob)wai/*/src/bin/**"]; + assert!(matches(&specs, "wai/wai-rs/src/live_events.rs")); + assert!(matches(&specs, "wai/wai-rs/src/a/b/c.rs")); + assert!(!matches(&specs, "wai/wai-rs/src/bin/wai.rs")); + assert!(!matches(&specs, "wai/wai-rs/tests/x.rs")); + assert!(matches(&[":(glob)wai/**/openapi*.json"], "wai/openapi.json")); + assert!(matches(&["joule-code/x/openapi.json"], "joule-code/x/openapi.json")); + assert!(matches(&[":(glob)wai/*-conformance/**"], "wai/claim-conformance/vectors.json")); + assert!(!matches(&[":(glob)wai/*-conformance/**"], "wai/wai-rs/x")); + } + + #[test] + fn added_lines_are_the_new_ones() { + assert_eq!(added_lines("a\nb\nc\n", "a\nx\nb\nc\ny\n"), vec!["x", "y"]); + assert_eq!(added_lines("", "a\n"), vec!["a"]); + assert!(added_lines("a\nb\n", "a\n").is_empty()); + assert_eq!(added_lines("a\nb\n", "b\na\n"), vec!["a"]); + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/jsondiff.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/jsondiff.rs new file mode 100644 index 000000000..521652ea5 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/jsondiff.rs @@ -0,0 +1,158 @@ +//! What changed between two JSON documents, as a reader of the old one sees it: members +//! and entries that went away, values that changed, and members and entries that are new. +//! +//! An array whose entries are objects named by one member (`name`, `file`, `id`, …) is +//! compared entry by entry under that name, so a corpus that gains a vector shows one added +//! entry, not a shifted list. Any other array is compared position by position, and one +//! that only grew at its end shows the new positions as added. Prose members (`about`, +//! `description`, `note`, `notes`, `comment`) are not compared: rewording them changes no +//! byte a reader acts on. + +use serde_json::Value; + +/// Members never compared: prose about the document, not data a reader acts on. +pub const PROSE: &[&str] = &["about", "description", "note", "notes", "comment"]; + +/// The members an array's entries may be named by, in the order they are tried. +const ENTRY_KEYS: &[&str] = &["name", "file", "id", "capability", "identifier", "label", "case"]; + +/// Paths (`$.vectors[name=x].expected`) of what was removed, changed and added. +#[derive(Default, Debug)] +pub struct Diff { + pub removed: Vec, + pub changed: Vec, + pub added: Vec, +} + +impl Diff { + pub fn breaks(&self) -> bool { + !self.removed.is_empty() || !self.changed.is_empty() + } + pub fn is_empty(&self) -> bool { + !self.breaks() && self.added.is_empty() + } +} + +/// The difference from `old` to `new`. +pub fn diff(old: &Value, new: &Value) -> Diff { + let mut d = Diff::default(); + walk(old, new, "$", &mut d); + d +} + +/// A value shortened for a message. +pub fn short(v: &Value) -> String { + let s = v.to_string(); + if s.chars().count() > 72 { format!("{}…", s.chars().take(72).collect::()) } else { s } +} + +fn entry_key(a: &[Value], b: &[Value]) -> Option<&'static str> { + ENTRY_KEYS.iter().copied().find(|k| { + let named = |xs: &[Value]| { + let mut seen = std::collections::HashSet::new(); + xs.iter().all(|x| x.get(k).and_then(Value::as_str).is_some_and(|n| seen.insert(n.to_string()))) + }; + !(a.is_empty() && b.is_empty()) && named(a) && named(b) + }) +} + +fn walk(old: &Value, new: &Value, path: &str, d: &mut Diff) { + match (old, new) { + (Value::Object(a), Value::Object(b)) => { + for (k, av) in a { + if PROSE.contains(&k.as_str()) { + continue; + } + let p = format!("{path}.{k}"); + match b.get(k) { + None => d.removed.push(p), + Some(bv) => walk(av, bv, &p, d), + } + } + for k in b.keys() { + if !a.contains_key(k) && !PROSE.contains(&k.as_str()) { + d.added.push(format!("{path}.{k}")); + } + } + } + (Value::Array(a), Value::Array(b)) => { + if let Some(key) = entry_key(a, b) { + let name = |x: &Value| x.get(key).and_then(Value::as_str).unwrap_or_default().to_string(); + let bm: std::collections::HashMap = b.iter().map(|x| (name(x), x)).collect(); + let am: std::collections::HashSet = a.iter().map(name).collect(); + for x in a { + let n = name(x); + let p = format!("{path}[{key}={n}]"); + match bm.get(&n) { + None => d.removed.push(p), + Some(y) => walk(x, y, &p, d), + } + } + for y in b { + let n = name(y); + if !am.contains(&n) { + d.added.push(format!("{path}[{key}={n}]")); + } + } + } else { + for (i, (x, y)) in a.iter().zip(b.iter()).enumerate() { + walk(x, y, &format!("{path}[{i}]"), d); + } + for i in b.len()..a.len() { + d.removed.push(format!("{path}[{i}]")); + } + for i in a.len()..b.len() { + d.added.push(format!("{path}[{i}]")); + } + } + } + (x, y) if x == y => {} + (x, y) => d.changed.push(format!("{path}: {} → {}", short(x), short(y))), + } +} + +/// The difference between two JSON Lines documents: lines that are gone, and lines that +/// are new. Lines are compared as text. +pub fn diff_lines(old: &str, new: &str) -> Diff { + let a: std::collections::HashSet<&str> = old.lines().filter(|l| !l.trim().is_empty()).collect(); + let b: std::collections::HashSet<&str> = new.lines().filter(|l| !l.trim().is_empty()).collect(); + let mut d = Diff::default(); + for (i, l) in old.lines().enumerate() { + if !l.trim().is_empty() && !b.contains(l) { + d.removed.push(format!("line {}: {}", i + 1, short(&Value::String(l.to_string())))); + } + } + for (i, l) in new.lines().enumerate() { + if !l.trim().is_empty() && !a.contains(l) { + d.added.push(format!("line {}", i + 1)); + } + } + d +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn named_entries_are_matched_by_name() { + let a = json!({"about": "x", "vectors": [{"name": "a", "expected": "accept"}, {"name": "b", "expected": "reject"}]}); + let b = json!({"about": "y", "vectors": [{"name": "c", "expected": "accept"}, {"name": "a", "expected": "accept"}, {"name": "b", "expected": "reject"}]}); + let d = diff(&a, &b); + assert!(!d.breaks(), "{d:?}"); + assert_eq!(d.added, vec!["$.vectors[name=c]"]); + let c = json!({"about": "x", "vectors": [{"name": "a", "expected": "reject"}]}); + let d = diff(&a, &c); + assert_eq!(d.removed, vec!["$.vectors[name=b]"]); + assert_eq!(d.changed, vec![r#"$.vectors[name=a].expected: "accept" → "reject""#]); + } + + #[test] + fn a_list_that_grew_at_its_end_only_added() { + let d = diff(&json!([1, 2]), &json!([1, 2, 3])); + assert!(!d.breaks()); + let d = diff(&json!([1, 2]), &json!([2, 1])); + assert!(d.breaks()); + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/mod.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/mod.rs new file mode 100644 index 000000000..2be15de64 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/mod.rs @@ -0,0 +1,352 @@ +//! `wai breaking --against ` (operation `compat.check`): what the working tree +//! breaks, by category, for a reader, a verifier or a program written against the ref. +//! +//! - **wire**: the bytes a conforming reader must accept. The registry export (capability +//! names and their classes, deterministic identifiers, wire identifiers: magics, +//! document kinds, headers, schemes, tags) and the conformance corpora of media and +//! delivery (`registry`, `corpora`). +//! - **record**: signed record formats. The domain-separation prefixes, magics and +//! versions records are built from, the corpora of signed records and their verdicts, +//! and the published JSON Schemas (`domains`, `corpora`, `schemas`). +//! - **api**: the Rust public API of the WAI library crates, the C header and the HTTP +//! services' OpenAPI documents (`rust`, `c`, `openapi`). +//! +//! The categories are cumulative, strictest by default: `--category api` (the default) +//! reports all three, `record` reports wire and record, `wire` reports wire alone. The +//! base is the merge base of the ref and `HEAD`, so a ref that moved on since the branch +//! left it does not read as the branch removing what the ref gained. +//! +//! Each finding names its item, its category, what changed and why that breaks; additions +//! are listed as compatible. A break is intended only when `wai/CHANGELOG.md` says so: a +//! line this change adds to it, in an entry that has the word "breaking", naming the +//! finding's id in backticks. Then it is reported as acknowledged and passes; every other +//! break fails the run (exit 1). + +pub mod c_header; +pub mod corpora; +pub mod domains; +pub mod git; +pub mod jsondiff; +pub mod openapi; +pub mod registry; +pub mod rust_api; +pub mod schema; + +use super::envelope::{self, EXIT_FAILED, EXIT_NOT_FOUND, OpError}; +use serde_json::{Map, Value, json}; +use std::collections::BTreeSet; +use std::path::PathBuf; + +pub const OP: &str = "compat.check"; + +pub const USAGE: &str = "wai breaking --against [--category wire|record|api] [--only ,...] [--skip ,...]\n\ + [--root ] [--target-dir ] [--json|--text] what the working tree breaks against :\n\ + wire, record and API; a break passes only where wai/CHANGELOG.md names it (operation compat.check)"; + +/// The changelog whose added lines acknowledge a break. +pub const CHANGELOG: &str = "wai/CHANGELOG.md"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Category { + Wire, + Record, + Api, +} + +impl Category { + pub fn name(self) -> &'static str { + match self { + Category::Wire => "wire", + Category::Record => "record", + Category::Api => "api", + } + } + fn parse(s: &str) -> Option { + match s { + "wire" => Some(Category::Wire), + "record" => Some(Category::Record), + "api" => Some(Category::Api), + _ => None, + } + } +} + +/// One change between the base and the working tree. +#[derive(Clone, Debug)] +pub struct Finding { + /// `::`, the name a changelog entry acknowledges it by. + pub id: String, + pub category: Category, + pub surface: &'static str, + pub item: String, + /// `removed`, `changed` or `added`. + pub change: String, + pub what: String, + /// Why it breaks; empty for a compatible change. + pub why: String, + pub breaking: bool, + pub acknowledged: bool, +} + +impl Finding { + pub fn broke(category: Category, surface: &'static str, item: String, change: String, what: String, why: String) -> Finding { + Finding { id: format!("{}:{surface}:{item}", category.name()), category, surface, item, change, what, why, breaking: true, acknowledged: false } + } + pub fn compatible(category: Category, surface: &'static str, item: String, what: String) -> Finding { + Finding { id: format!("{}:{surface}:{item}", category.name()), category, surface, item, change: "added".into(), what, why: String::new(), breaking: false, acknowledged: false } + } + fn to_json(&self) -> Value { + json!({ + "id": self.id, "category": self.category.name(), "surface": self.surface, "item": self.item, + "change": self.change, "what": self.what, "why": self.why, "breaking": self.breaking, "acknowledged": self.acknowledged, + }) + } +} + +/// The surfaces, each with the most permissive category it reports under. +const SURFACES: &[(&str, Category)] = &[ + ("registry", Category::Wire), + ("corpora", Category::Wire), + ("domains", Category::Record), + ("schemas", Category::Record), + ("c", Category::Api), + ("openapi", Category::Api), + ("rust", Category::Api), +]; + +/// What to check. +pub struct Options { + pub against: String, + pub category: Category, + pub surfaces: BTreeSet<&'static str>, + pub root: PathBuf, + pub target_dir: Option, +} + +/// The report of one run. +pub struct Report { + pub against: String, + pub base: String, + pub category: Category, + pub surfaces: Vec<&'static str>, + pub findings: Vec, + pub notes: Vec, +} + +impl Report { + pub fn unacknowledged(&self) -> usize { + self.findings.iter().filter(|f| f.breaking && !f.acknowledged).count() + } + pub fn to_json(&self) -> Map { + let mut m = Map::new(); + m.insert("against".into(), json!(self.against)); + m.insert("base".into(), json!(self.base)); + m.insert("category".into(), json!(self.category.name())); + m.insert("surfaces".into(), json!(self.surfaces)); + m.insert("breaking".into(), json!(self.unacknowledged())); + m.insert("acknowledged".into(), json!(self.findings.iter().filter(|f| f.breaking && f.acknowledged).count())); + m.insert("compatible".into(), json!(self.findings.iter().filter(|f| !f.breaking).count())); + m.insert("findings".into(), Value::Array(self.findings.iter().map(Finding::to_json).collect())); + m.insert("notes".into(), json!(self.notes)); + m + } +} + +/// The ids a changelog's added lines acknowledge: every backticked span of an entry (a +/// bullet and its continuation lines, or a paragraph) that has the word "breaking". +pub fn acknowledgements(added: &[String]) -> BTreeSet { + let mut entries: Vec = Vec::new(); + for l in added { + let t = l.trim_start(); + let starts = t.starts_with("- ") || t.starts_with("* ") || t.starts_with('#') || (!l.starts_with(' ') && !t.is_empty()); + if starts || entries.is_empty() { + entries.push(String::new()); + } + if t.is_empty() { + entries.push(String::new()); + continue; + } + if let Some(e) = entries.last_mut() { + e.push_str(t); + e.push(' '); + } + } + let mut ids = BTreeSet::new(); + for e in entries { + if !e.to_lowercase().contains("breaking") { + continue; + } + let mut parts = e.split('`'); + parts.next(); + while let Some(span) = parts.next() { + ids.insert(span.trim().to_string()); + parts.next(); + } + } + ids +} + +/// Run the check. +pub fn run(o: &Options) -> Result { + let repo = git::Repo::open(&o.root)?; + let target = repo.commit(&o.against).ok_or_else(|| { + OpError::new(EXIT_NOT_FOUND, "ref-not-found", format!("{}: no such commit", o.against), "fetch it first (git fetch origin main), or name a commit, tag or branch this checkout holds") + })?; + let base = repo.commit("HEAD").and_then(|h| repo.merge_base(&target, &h)).unwrap_or(target); + let mut findings = Vec::new(); + let mut notes = Vec::new(); + let checked: Vec<&'static str> = SURFACES.iter().filter(|(s, c)| *c <= o.category && o.surfaces.contains(s)).map(|x| x.0).collect(); + for s in &checked { + eprintln!("wai breaking: {s}"); + match *s { + "registry" => { + let old = repo.base_file(&base, registry::PATH)?.and_then(|b| serde_json::from_slice::(&b).ok()); + let new = repo.head_file(registry::PATH).and_then(|b| serde_json::from_slice::(&b).ok()); + findings.extend(registry::compare(old.as_ref(), new.as_ref())); + } + "corpora" => findings.extend(corpora::compare(&repo, &base)?), + "domains" => findings.extend(domains::compare(&repo, &base)?), + "schemas" => findings.extend(schema::compare(&repo, &base)?), + "c" => { + let old = repo.base_file(&base, c_header::PATH)?.map(|b| String::from_utf8_lossy(&b).into_owned()); + let new = repo.head_file(c_header::PATH).map(|b| String::from_utf8_lossy(&b).into_owned()); + findings.extend(c_header::compare(old.as_deref(), new.as_deref())); + } + "openapi" => findings.extend(openapi::compare(&repo, &base)?), + "rust" => { + let scratch = std::env::temp_dir().join(format!("wai-breaking-{}-{}", std::process::id(), std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0))); + let toolchain = std::fs::read_to_string(repo.root.join("rust-toolchain.toml")).ok().and_then(|t| { + t.lines().find_map(|l| l.trim().strip_prefix("channel").and_then(|r| r.trim_start().strip_prefix('=')).map(|v| v.trim().trim_matches('"').to_string())) + }); + let builder = rust_api::Builder { target_dir: o.target_dir.clone().unwrap_or_else(|| scratch.join("target")), toolchain }; + let r = rust_api::compare(&repo, &base, &builder, &scratch, &mut notes); + let _ = std::fs::remove_dir_all(&scratch); + findings.extend(r?); + } + _ => {} + } + } + findings.retain(|f| f.category <= o.category); + let acks = acknowledgements(&repo.added_lines(&base, CHANGELOG)?); + for f in findings.iter_mut().filter(|f| f.breaking) { + f.acknowledged = acks.contains(&f.id); + } + findings.sort_by(|a, b| (!a.breaking, a.acknowledged, a.category, &a.id).cmp(&(!b.breaking, b.acknowledged, b.category, &b.id))); + Ok(Report { against: o.against.clone(), base, category: o.category, surfaces: checked, findings, notes }) +} + +fn options(args: &[String]) -> Result { + let flags = ["--against", "--category", "--only", "--skip", "--root", "--target-dir", "--json", "--text", "--all"]; + let with_value = ["--against", "--category", "--only", "--skip", "--root", "--target-dir"]; + envelope::check_args(args, &flags, &with_value, 0, USAGE)?; + let against = envelope::value(args, "--against", USAGE)?.ok_or_else(|| OpError::usage("--against is required", USAGE))?.to_string(); + let category = match envelope::value(args, "--category", USAGE)? { + None => Category::Api, + Some(c) => Category::parse(c).ok_or_else(|| OpError::usage(format!("--category {c}: wire, record or api"), USAGE))?, + }; + let names = |v: Option<&str>| -> Result>, OpError> { + let Some(v) = v else { return Ok(None) }; + v.split(',') + .map(|n| SURFACES.iter().find(|s| s.0 == n.trim()).map(|s| s.0).ok_or_else(|| OpError::usage(format!("{n}: not a surface ({})", SURFACES.iter().map(|s| s.0).collect::>().join(", ")), USAGE))) + .collect::, _>>() + .map(Some) + }; + let mut surfaces: BTreeSet<&'static str> = names(envelope::value(args, "--only", USAGE)?)?.unwrap_or_else(|| SURFACES.iter().map(|s| s.0).collect()); + if let Some(skip) = names(envelope::value(args, "--skip", USAGE)?)? { + surfaces.retain(|s| !skip.contains(s)); + } + Ok(Options { + against, + category, + surfaces, + root: envelope::value(args, "--root", USAGE)?.map(PathBuf::from).unwrap_or_else(|| PathBuf::from(".")), + target_dir: envelope::value(args, "--target-dir", USAGE)?.map(PathBuf::from), + }) +} + +fn text(r: &Report, all: bool) { + let short = r.base.get(..12).unwrap_or(&r.base); + println!("wai breaking: against {} (merge base {short}); {} and every category below it; checked {}", r.against, r.category.name(), r.surfaces.join(", ")); + let breaks: Vec<&Finding> = r.findings.iter().filter(|f| f.breaking).collect(); + let compat: Vec<&Finding> = r.findings.iter().filter(|f| !f.breaking).collect(); + if !breaks.is_empty() { + println!(); + println!("breaking changes ({}, {} acknowledged in {CHANGELOG}):", breaks.len(), breaks.iter().filter(|f| f.acknowledged).count()); + for f in &breaks { + println!(" {} {}", if f.acknowledged { "ACK " } else { "BREAK" }, f.id); + println!(" {} ({} · {}): {}", f.change, f.category.name(), f.surface, f.what); + println!(" why it breaks: {}", f.why); + } + } + if !compat.is_empty() { + println!(); + println!("compatible changes ({}):", compat.len()); + let shown = if all { compat.len() } else { compat.len().min(40) }; + for f in &compat[..shown] { + println!(" + {} {}", f.id, f.what); + } + if shown < compat.len() { + println!(" … and {} more (--all lists them)", compat.len() - shown); + } + } + for n in &r.notes { + println!("note: {n}"); + } + println!(); + let n = r.unacknowledged(); + if n == 0 { + println!("wai breaking: no unacknowledged breaking change"); + } else { + println!("wai breaking: {n} breaking change{} not acknowledged. If {} intended, add to {CHANGELOG}, under the current version:", if n == 1 { "" } else { "s" }, if n == 1 { "it is" } else { "they are" }); + for f in breaks.iter().filter(|f| !f.acknowledged).take(8) { + println!(" - **Breaking:** `{}`: ", f.id); + } + } +} + +pub fn main(args: &[String]) -> Result { + let json_mode = envelope::json_wanted(args); + let o = match options(args) { + Ok(o) => o, + Err(e) => envelope::fail(OP, json_mode, &e), + }; + let r = match run(&o) { + Ok(r) => r, + Err(e) => envelope::fail(OP, json_mode, &e), + }; + let ok = r.unacknowledged() == 0; + if json_mode { + println!("{}", envelope::envelope(OP, ok, r.to_json())); + } else { + text(&r, args.iter().any(|a| a == "--all")); + } + if !ok { + envelope::exit(EXIT_FAILED); + } + Ok(true) +} + +#[cfg(test)] +mod tests { + use super::acknowledgements; + + #[test] + fn an_entry_acknowledges_only_when_it_says_breaking() { + let added: Vec = [ + "- **Breaking:** `api:c:wai_pack` takes a", + " `uint64_t` now, and api:rust:x is not in backticks.", + "- `wire:capability:wai.x` is new (no such word here).", + "", + "Breaking, as a paragraph: `record:domain:a.rs::SIGN_DOMAIN`.", + ] + .iter() + .map(|s| s.to_string()) + .collect(); + let a = acknowledgements(&added); + assert!(a.contains("api:c:wai_pack")); + assert!(a.contains("uint64_t")); + assert!(a.contains("record:domain:a.rs::SIGN_DOMAIN")); + assert!(!a.contains("wire:capability:wai.x")); + assert!(!a.contains("api:rust:x")); + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/openapi.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/openapi.rs new file mode 100644 index 000000000..4edc469ac --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/openapi.rs @@ -0,0 +1,175 @@ +//! API: the HTTP services' OpenAPI documents (the room service's +//! `joule-moq-room/openapi.json`, and any `openapi*.json` under `wai/`). A client written +//! against the old document must still work. So a route (a path and method) that is gone +//! breaks; so does an operation whose `operationId` changed (the generated clients and +//! tools are named by it), a parameter that is gone, moved or newly required, a request +//! body that becomes required, a success response that is gone, and a schema a request +//! carries that gains a required property, or one a response carries that loses a +//! property or changes a property's type. New routes, optional parameters and response +//! properties are compatible. + +use super::git::Repo; +use super::super::envelope::OpError; +use super::{Category, Finding}; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; + +/// The documents read, by their path in the repository. +pub const PATHSPECS: &[&str] = &["joule-code/joule-code-rs/crates/joule-moq-room/openapi.json", ":(glob)wai/**/openapi*.json"]; + +const METHODS: &[&str] = &["get", "put", "post", "delete", "options", "head", "patch", "trace"]; + +/// A short name for a document: its crate's directory, else its path. +fn service(path: &str) -> String { + let parts: Vec<&str> = path.split('/').collect(); + if parts.len() >= 2 { parts[parts.len() - 2].to_string() } else { path.to_string() } +} + +fn resolve<'a>(doc: &'a Value, v: &'a Value) -> &'a Value { + let mut v = v; + for _ in 0..16 { + let Some(r) = v.get("$ref").and_then(Value::as_str) else { break }; + let Some(p) = r.strip_prefix("#/") else { break }; + let mut t = doc; + for seg in p.split('/') { + match t.get(seg.replace("~1", "/").replace("~0", "~")) { + Some(x) => t = x, + None => return v, + } + } + v = t; + } + v +} + +fn params(doc: &Value, path_item: &Value, op: &Value) -> BTreeMap<(String, String), bool> { + let mut m = BTreeMap::new(); + for list in [path_item.get("parameters"), op.get("parameters")].into_iter().flatten() { + for p in list.as_array().into_iter().flatten() { + let p = resolve(doc, p); + if let (Some(n), Some(i)) = (p.get("name").and_then(Value::as_str), p.get("in").and_then(Value::as_str)) { + m.insert((n.to_string(), i.to_string()), p.get("required").and_then(Value::as_bool).unwrap_or(i == "path")); + } + } + } + m +} + +/// The properties of a schema, resolved: each name with its type (or `$ref` name) and +/// whether it is required. +fn props(doc: &Value, schema: &Value) -> BTreeMap { + let s = resolve(doc, schema); + let required: BTreeSet = s.get("required").and_then(Value::as_array).into_iter().flatten().filter_map(|x| x.as_str().map(String::from)).collect(); + let mut m = BTreeMap::new(); + if let Some(p) = s.get("properties").and_then(Value::as_object) { + for (k, v) in p { + let ty = v.get("$ref").and_then(Value::as_str).map(String::from).or_else(|| v.get("type").map(|t| t.to_string())).unwrap_or_default(); + m.insert(k.clone(), (ty, required.contains(k))); + } + } + m +} + +fn json_schema<'a>(doc: &'a Value, body: &'a Value) -> Option<&'a Value> { + let body = resolve(doc, body); + let content = body.get("content")?.as_object()?; + content.get("application/json").or_else(|| content.values().next())?.get("schema") +} + +fn ops(doc: &Value) -> BTreeMap { + let mut m = BTreeMap::new(); + for (path, item) in doc.get("paths").and_then(Value::as_object).into_iter().flatten() { + for meth in METHODS { + if let Some(op) = item.get(*meth) { + m.insert(format!("{} {path}", meth.to_uppercase()), (item, op)); + } + } + } + m +} + +/// One document's findings (`None` where a tree has none). +pub fn compare_doc(path: &str, old: Option<&Value>, new: Option<&Value>) -> Vec { + let svc = service(path); + let mut out = Vec::new(); + let empty = Value::Null; + let (a, b) = (old.unwrap_or(&empty), new.unwrap_or(&empty)); + let (oa, ob) = (ops(a), ops(b)); + for (route, (ia, xa)) in &oa { + let item = format!("{svc}:{route}"); + let Some((ib, xb)) = ob.get(route) else { + out.push(Finding::broke(Category::Api, "openapi", item, "removed".into(), format!("the route {route} is gone from {path}"), "a client that calls it gets no such route".into())); + continue; + }; + let mut what = Vec::new(); + if xa.get("operationId") != xb.get("operationId") && xa.get("operationId").is_some() { + what.push(format!("operationId {} → {}", xa.get("operationId").unwrap_or(&empty), xb.get("operationId").unwrap_or(&empty))); + } + let (pa, pb) = (params(a, ia, xa), params(b, ib, xb)); + for ((n, i), req) in &pa { + match pb.get(&(n.clone(), i.clone())) { + None => what.push(format!("the {i} parameter {n} is gone")), + Some(r) if *r && !req => what.push(format!("the {i} parameter {n} is now required")), + _ => {} + } + } + for ((n, i), req) in &pb { + if *req && !pa.contains_key(&(n.clone(), i.clone())) { + what.push(format!("a required {i} parameter {n} is new")); + } + } + let body_required = |x: &Value, d: &Value| x.get("requestBody").map(|r| resolve(d, r).get("required").and_then(Value::as_bool).unwrap_or(false)).unwrap_or(false); + if body_required(xb, b) && !body_required(xa, a) { + what.push("the request body is now required".into()); + } + if let (Some(sa), Some(sb)) = (xa.get("requestBody").and_then(|r| json_schema(a, r)), xb.get("requestBody").and_then(|r| json_schema(b, r))) { + let (qa, qb) = (props(a, sa), props(b, sb)); + for (k, (_, req)) in &qb { + if *req && !qa.get(k).is_some_and(|x| x.1) { + what.push(format!("the request property {k} is now required")); + } + } + } + let codes = |x: &Value| -> Vec { x.get("responses").and_then(Value::as_object).map(|r| r.keys().cloned().collect()).unwrap_or_default() }; + let (ca, cb) = (codes(xa), codes(xb)); + for c in ca.iter().filter(|c| c.starts_with('2') && !cb.contains(c)) { + what.push(format!("the {c} response is gone")); + } + for c in ca.iter().filter(|c| c.starts_with('2') && cb.contains(c)) { + let schema = |x: &Value, d: &Value| x.get("responses").and_then(|r| r.get(c.as_str())).and_then(|r| json_schema(d, r)).map(|s| props(d, s)); + if let (Some(ra), Some(rb)) = (schema(xa, a), schema(xb, b)) { + for (k, (ty, _)) in &ra { + match rb.get(k) { + None => what.push(format!("the {c} response's property {k} is gone")), + Some((t, _)) if t != ty => what.push(format!("the {c} response's property {k}: {ty} → {t}")), + _ => {} + } + } + } + } + if !what.is_empty() { + out.push(Finding::broke(Category::Api, "openapi", item, "changed".into(), format!("{route}: {}", what.join("; ")), "a client written against the old document sends what is now refused, or reads what is no longer sent".into())); + } + } + for route in ob.keys() { + if !oa.contains_key(route) { + out.push(Finding::compatible(Category::Api, "openapi", format!("{svc}:{route}"), format!("the route {route} is new in {path}"))); + } + } + out +} + +/// Every document's findings between `base` and the working tree. +pub fn compare(repo: &Repo, base: &str) -> Result, OpError> { + let mut paths: BTreeSet = repo.ls_base(base, PATHSPECS)?.into_iter().collect(); + paths.extend(repo.ls_head(PATHSPECS)?); + let list: Vec = paths.into_iter().collect(); + let olds = repo.base_files(base, &list)?; + let mut out = Vec::new(); + for p in &list { + let old: Option = olds.get(p).and_then(|b| serde_json::from_slice(b).ok()); + let new: Option = repo.head_file(p).and_then(|b| serde_json::from_slice(&b).ok()); + out.extend(compare_doc(p, old.as_ref(), new.as_ref())); + } + Ok(out) +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/registry.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/registry.rs new file mode 100644 index 000000000..85780c84e --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/registry.rs @@ -0,0 +1,165 @@ +//! Wire: the registry export (`wai/registry/capabilities.json`, written by `wai registry +//! export` from REGISTRY.md and held to it by `registry export --check`). It names every +//! capability a reader dispatches on, the deterministic identifiers, and the wire +//! identifiers: magics, document kinds, headers, paths, schemes and tags. +//! +//! A capability, identifier or wire identifier that is gone breaks: a reader that +//! dispatches on it, or a writer that emits it, no longer conforms. So does a capability +//! whose kind, media, role, tier, licence, replicate class, edition or operator set +//! changed (one the old export did not state at all is new, and compatible), one that left a set or joined the mandatory floor (every old reader lacks +//! it), a registered capability demoted to candidate, a wire identifier (named by its kind +//! and name together) whose status fell back from `defined`, and an identifier whose +//! meaning changed. A section moving between documents is editorial and is not compared. + +use super::jsondiff; +use super::{Category, Finding}; +use serde_json::{Map, Value}; + +pub const PATH: &str = "wai/registry/capabilities.json"; + +const COMPARED: &[&str] = &["kind", "media", "role", "tier", "licence", "replicate", "edition", "ops"]; + +fn by<'a>(doc: &'a Value, list: &str, key: &str) -> Vec<(String, &'a Map)> { + doc.get(list) + .and_then(Value::as_array) + .map(|a| a.iter().filter_map(|x| Some((x.get(key)?.as_str()?.to_string(), x.as_object()?))).collect()) + .unwrap_or_default() +} + +/// The wire identifiers, each named by its kind and its name together (`fourcc/wai1`): one +/// name can be, say, a `codecs` prefix and a sample-entry 4CC at once. +fn wire(doc: &Value) -> Vec<(String, &Map)> { + by(doc, "wire_identifiers", "name").into_iter().map(|(n, m)| (format!("{}/{n}", m.get("kind").and_then(Value::as_str).unwrap_or("identifier")), m)).collect() +} + +fn s(m: &Map, k: &str) -> String { + m.get(k).map(jsondiff::short).unwrap_or_else(|| "absent".into()) +} + +/// The registry's findings, from the two exports (`None` where a tree has none). +pub fn compare(old: Option<&Value>, new: Option<&Value>) -> Vec { + let mut out = Vec::new(); + let (old, new) = match (old, new) { + (None, None) => return out, + (None, Some(_)) => { + out.push(Finding::compatible(Category::Wire, "registry", PATH.into(), "the registry export is new".into())); + return out; + } + (Some(_), None) => { + out.push(Finding::broke( + Category::Wire, + "registry", + PATH.into(), + "removed".into(), + "the registry export is gone".into(), + "every capability and wire identifier a reader dispatches on loses its registration".into(), + )); + return out; + } + (Some(a), Some(b)) => (a, b), + }; + + let caps_new: std::collections::HashMap> = by(new, "capabilities", "capability").into_iter().collect(); + let caps_old: std::collections::HashSet = by(old, "capabilities", "capability").into_iter().map(|x| x.0).collect(); + for (name, a) in by(old, "capabilities", "capability") { + let id = name.clone(); + let Some(b) = caps_new.get(&name) else { + out.push(Finding::broke(Category::Wire, "capability", id, "removed".into(), format!("capability {name} is no longer registered"), "a reader that dispatches on it, or a writer that names it, no longer conforms".into())); + continue; + }; + let mut what = Vec::new(); + let mut gained = Vec::new(); + for k in COMPARED { + match (a.get(*k), b.get(*k)) { + (x, y) if x == y => {} + // A member the old export did not have is new information, not a change. + (None, Some(_)) => gained.push(format!("{k} {}", s(b, k))), + _ => what.push(format!("{k} {} → {}", s(a, k), s(b, k))), + } + } + if a.get("status").and_then(Value::as_str) == Some("registered") && b.get("status").and_then(Value::as_str) != Some("registered") { + what.push(format!("status registered → {}", s(b, "status"))); + } + let sets = |m: &Map| -> Vec { m.get("sets").and_then(Value::as_array).map(|a| a.iter().filter_map(|x| x.as_str().map(String::from)).collect()).unwrap_or_default() }; + let (sa, sb) = (sets(a), sets(b)); + for x in sa.iter().filter(|x| !sb.contains(x)) { + what.push(format!("left the set {x}")); + } + if sb.iter().any(|x| x == "floor") && !sa.iter().any(|x| x == "floor") { + what.push("joined the mandatory floor, which every older reader lacks".into()); + } + if !what.is_empty() { + out.push(Finding::broke(Category::Wire, "capability", id, "changed".into(), what.join("; "), "readers and writers dispatch, select and verify by these fields".into())); + } else if a.get("status") != b.get("status") || sb.iter().any(|x| !sa.contains(x)) || !gained.is_empty() { + gained.insert(0, format!("status {} → {}, sets {:?} → {:?}", s(a, "status"), s(b, "status"), sa, sb)); + out.push(Finding::compatible(Category::Wire, "capability", name.clone(), gained.join("; "))); + } + } + for (name, _) in by(new, "capabilities", "capability") { + if !caps_old.contains(&name) { + out.push(Finding::compatible(Category::Wire, "capability", name.clone(), format!("capability {name} is registered"))); + } + } + + let ids_new: std::collections::HashMap> = by(new, "identifiers", "identifier").into_iter().collect(); + let ids_old: std::collections::HashSet = by(old, "identifiers", "identifier").into_iter().map(|x| x.0).collect(); + for (name, a) in by(old, "identifiers", "identifier") { + let id = name.clone(); + match ids_new.get(&name) { + None => out.push(Finding::broke(Category::Wire, "identifier", id, "removed".into(), format!("identifier {name} is gone"), "a manifest that names it is no longer read as it was".into())), + Some(b) if a.get("meaning") != b.get("meaning") => out.push(Finding::broke( + Category::Wire, + "identifier", + id, + "changed".into(), + format!("its meaning changed: {} → {}", s(a, "meaning"), s(b, "meaning")), + "a deterministic identifier names one arithmetic; another meaning gives other bytes under the same name".into(), + )), + Some(_) => {} + } + } + for (name, _) in by(new, "identifiers", "identifier") { + if !ids_old.contains(&name) { + out.push(Finding::compatible(Category::Wire, "identifier", name.clone(), format!("identifier {name} is new"))); + } + } + + let rank = |m: &Map| match m.get("status").and_then(Value::as_str) { + Some("defined") => 2, + Some("provisional") => 1, + _ => 0, + }; + let wire_new: std::collections::HashMap> = wire(new).into_iter().collect(); + let wire_old: std::collections::HashSet = wire(old).into_iter().map(|x| x.0).collect(); + for (id, a) in wire(old) { + let kind = a.get("kind").and_then(Value::as_str).unwrap_or("identifier"); + let name = a.get("name").and_then(Value::as_str).unwrap_or("?"); + match wire_new.get(&id) { + None => out.push(Finding::broke( + Category::Wire, + "wire-identifier", + id.clone(), + "removed".into(), + format!("the {kind} {name} ({}) is gone", s(a, "document")), + format!("a reader that matches this {kind} refuses what is written without it, and what was written with it is no longer defined"), + )), + Some(b) if rank(b) < rank(a) => out.push(Finding::broke( + Category::Wire, + "wire-identifier", + id.clone(), + "changed".into(), + format!("the {kind} {name}: status {} → {}", s(a, "status"), s(b, "status")), + format!("readers and writers rely on this {kind} as it was defined"), + )), + Some(_) => {} + } + } + for (id, b) in wire(new) { + if !wire_old.contains(&id) { + let kind = b.get("kind").and_then(Value::as_str).unwrap_or("identifier"); + let name = b.get("name").and_then(Value::as_str).unwrap_or("?"); + out.push(Finding::compatible(Category::Wire, "wire-identifier", id.clone(), format!("the {kind} {name} is new"))); + } + } + out +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/rust_api.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/rust_api.rs new file mode 100644 index 000000000..e2833108e --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/rust_api.rs @@ -0,0 +1,682 @@ +//! API: the Rust public API of the WAI library crates, read from the compiler's own +//! description of it. Each crate is documented twice by `cargo rustdoc` in rustdoc's JSON +//! output, once from the base (its tree extracted to a scratch directory) and once from the +//! working tree, with the same toolchain and the same features: every feature but those +//! that need system libraries or downloads (`DENY`, and any feature that enables one). +//! +//! The public surface is walked from the crate root through public modules and re-exports: +//! every public path to a module, function, struct and its public fields, enum and its +//! variants, union, trait and its items, inherent method and associated item, constant, +//! static, type alias and macro, and every trait a public type implements (auto traits +//! included, blanket implementations left out). A path that is gone breaks. So does a +//! signature that changed (parameter and return types, generics and bounds, `unsafe`, +//! `const`, `async`, ABI; parameter names are not part of it), a field's or alias's type +//! that changed, a struct that can no longer be built with a literal, an enum or struct +//! that became `#[non_exhaustive]`, a new public field of a struct a literal could build, +//! a new variant of an exhaustive enum, and a new required method of a trait. Other new +//! paths are compatible. +//! +//! Rustdoc's JSON is an unstable output of a stable toolchain, read here with +//! `RUSTC_BOOTSTRAP=1`; both sides are written by one toolchain, so they share one format +//! version, and a run that finds two refuses to compare them. + +use super::git::Repo; +use super::super::envelope::{EXIT_INTERNAL, EXIT_NOT_FOUND, OpError}; +use super::{Category, Finding}; +use serde_json::{Map, Value}; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// A library crate whose public API is held. +pub struct Crate { + pub name: &'static str, + pub dir: &'static str, + /// Features left out of the documented set: they need system libraries, a download, or + /// a GPU toolchain, which a check of the API does not. + pub deny: &'static [&'static str], +} + +pub const CRATES: &[Crate] = &[ + Crate { name: "wai", dir: "wai/wai-rs", deny: &["default", "codecs", "semantic", "neural", "c2pa_emit", "ops_gpu"] }, + Crate { name: "wai-quantum", dir: "wai/wai-quantum", deny: &["default"] }, +]; + +/// One public item: what it is, its comparable form, how it reads, and the facts the +/// semver rules above need. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ApiItem { + pub kind: &'static str, + pub sig: String, + pub show: String, + /// A struct every field of which is public and which is not `#[non_exhaustive]`, or an + /// enum with every variant public and not `#[non_exhaustive]`. + pub exhaustive: bool, + /// A trait method, type or constant with no default. + pub required: bool, +} + +/// The public surface of one crate: each public path, and its item. +pub type Surface = BTreeMap; + +struct Doc<'a> { + index: &'a Map, + paths: &'a Map, + crate_name: String, +} + +impl Doc<'_> { + fn item(&self, id: &Value) -> Option<&Value> { + self.index.get(&id_key(id)) + } + fn path_of(&self, id: &Value) -> Option { + let p = self.paths.get(&id_key(id))?.get("path")?.as_array()?; + Some(p.iter().filter_map(Value::as_str).collect::>().join("::")) + } +} + +fn id_key(id: &Value) -> String { + match id { + Value::String(s) => s.clone(), + other => other.to_string(), + } +} + +fn inner<'a>(item: &'a Value, kind: &str) -> Option<&'a Value> { + item.get("inner")?.get(kind) +} + +fn attrs_have(item: &Value, word: &str) -> bool { + item.get("attrs").and_then(Value::as_array).is_some_and(|a| a.iter().any(|x| x.as_str() == Some(word) || x.to_string().contains(word))) +} + +/// `v` with every `id` dropped and every resolved path written as its full path, keys +/// sorted: equal for two builds of one API. +fn canon(doc: &Doc, v: &Value) -> Value { + match v { + Value::Object(m) => { + let mut keys: Vec<&String> = m.keys().collect(); + keys.sort(); + let mut out = Map::new(); + for k in keys { + if k == "id" { + continue; + } + let mut x = canon(doc, &m[k]); + if k == "path" && x.is_string() && let Some(full) = m.get("id").and_then(|id| doc.path_of(id)) { + x = Value::String(full); + } + out.insert(k.clone(), x); + } + Value::Object(out) + } + Value::Array(a) => Value::Array(a.iter().map(|x| canon(doc, x)).collect()), + other => other.clone(), + } +} + +/// A type as Rust spells it, as far as these forms go; anything else as JSON. +fn ty(doc: &Doc, t: &Value) -> String { + let Some((k, v)) = t.as_object().and_then(|m| m.iter().next()) else { + return t.as_str().map(String::from).unwrap_or_else(|| t.to_string()); + }; + match k.as_str() { + "resolved_path" => path_ty(doc, v), + "generic" | "primitive" => v.as_str().unwrap_or("?").to_string(), + "borrowed_ref" => format!( + "&{}{}{}", + v.get("lifetime").and_then(Value::as_str).map(|l| format!("{l} ")).unwrap_or_default(), + if v.get("is_mutable").and_then(Value::as_bool) == Some(true) { "mut " } else { "" }, + ty(doc, &v["type"]) + ), + "raw_pointer" => format!("*{} {}", if v.get("is_mutable").and_then(Value::as_bool) == Some(true) { "mut" } else { "const" }, ty(doc, &v["type"])), + "slice" => format!("[{}]", ty(doc, v)), + "array" => format!("[{}; {}]", ty(doc, &v["type"]), v.get("len").and_then(Value::as_str).unwrap_or("?")), + "tuple" => format!("({})", v.as_array().map(|a| a.iter().map(|x| ty(doc, x)).collect::>().join(", ")).unwrap_or_default()), + "impl_trait" => format!("impl {}", bounds(doc, v)), + "dyn_trait" => format!( + "dyn {}", + v.get("traits").and_then(Value::as_array).map(|a| a.iter().map(|x| path_ty(doc, &x["trait"])).collect::>().join(" + ")).unwrap_or_default() + ), + "qualified_path" => format!( + "<{} as {}>::{}", + ty(doc, &v["self_type"]), + v.get("trait").filter(|t| !t.is_null()).map(|t| path_ty(doc, t)).unwrap_or_default(), + v.get("name").and_then(Value::as_str).unwrap_or("?") + ), + "function_pointer" => { + let sig = &v["sig"]; + format!("fn({}){}", inputs(doc, sig), output(doc, sig)) + } + "infer" => "_".into(), + _ => t.to_string(), + } +} + +fn path_ty(doc: &Doc, v: &Value) -> String { + let name = v.get("id").and_then(|id| doc.path_of(id)).or_else(|| v.get("path").and_then(Value::as_str).map(String::from)).unwrap_or_default(); + format!("{name}{}", args(doc, v.get("args"))) +} + +fn args(doc: &Doc, a: Option<&Value>) -> String { + let Some(a) = a.filter(|a| !a.is_null()) else { return String::new() }; + if let Some(ab) = a.get("angle_bracketed") { + let mut parts: Vec = ab + .get("args") + .and_then(Value::as_array) + .into_iter() + .flatten() + .map(|x| match x { + Value::Object(m) if m.contains_key("type") => ty(doc, &m["type"]), + Value::Object(m) if m.contains_key("lifetime") => m["lifetime"].as_str().unwrap_or("'_").to_string(), + other => other.to_string(), + }) + .collect(); + for c in ab.get("constraints").and_then(Value::as_array).into_iter().flatten() { + let n = c.get("name").and_then(Value::as_str).unwrap_or("?"); + match c.get("binding").and_then(|b| b.get("equality")) { + Some(e) => parts.push(format!("{n} = {}", e.get("type").map(|t| ty(doc, t)).unwrap_or_else(|| e.to_string()))), + None => parts.push(format!("{n}: …")), + } + } + return if parts.is_empty() { String::new() } else { format!("<{}>", parts.join(", ")) }; + } + if let Some(p) = a.get("parenthesized") { + let ins = p.get("inputs").and_then(Value::as_array).map(|i| i.iter().map(|x| ty(doc, x)).collect::>().join(", ")).unwrap_or_default(); + let out = p.get("output").filter(|o| !o.is_null()).map(|o| format!(" -> {}", ty(doc, o))).unwrap_or_default(); + return format!("({ins}){out}"); + } + a.to_string() +} + +fn bounds(doc: &Doc, b: &Value) -> String { + b.as_array() + .map(|a| { + a.iter() + .map(|x| match x.get("trait_bound") { + Some(tb) => path_ty(doc, &tb["trait"]), + None => x.get("outlives").and_then(Value::as_str).map(String::from).unwrap_or_else(|| x.to_string()), + }) + .collect::>() + .join(" + ") + }) + .unwrap_or_default() +} + +fn inputs(doc: &Doc, sig: &Value) -> String { + sig.get("inputs").and_then(Value::as_array).map(|a| a.iter().map(|p| ty(doc, &p[1])).collect::>().join(", ")).unwrap_or_default() +} + +fn output(doc: &Doc, sig: &Value) -> String { + sig.get("output").filter(|o| !o.is_null()).map(|o| format!(" -> {}", ty(doc, o))).unwrap_or_default() +} + +fn generics_show(doc: &Doc, g: &Value) -> String { + let params: Vec = g + .get("params") + .and_then(Value::as_array) + .into_iter() + .flatten() + .map(|p| { + let n = p.get("name").and_then(Value::as_str).unwrap_or("?").to_string(); + match p.get("kind").and_then(|k| k.get("type")).and_then(|t| t.get("bounds")) { + Some(b) if b.as_array().is_some_and(|a| !a.is_empty()) => format!("{n}: {}", bounds(doc, b)), + _ => n, + } + }) + .filter(|n| !n.starts_with("impl ")) + .collect(); + if params.is_empty() { String::new() } else { format!("<{}>", params.join(", ")) } +} + +/// A function's comparable form (types only, parameter names dropped) and how it reads. +fn function(doc: &Doc, name: &str, f: &Value) -> (String, String) { + let sig = &f["sig"]; + let types: Vec = sig.get("inputs").and_then(Value::as_array).into_iter().flatten().map(|p| canon(doc, &p[1])).collect(); + let cmp = serde_json::json!({"inputs": types, "output": canon(doc, &sig["output"]), "c_variadic": sig.get("is_c_variadic"), "generics": canon(doc, &f["generics"]), "header": canon(doc, &f["header"])}); + let h = &f["header"]; + let mut q = String::new(); + for (k, w) in [("is_const", "const "), ("is_async", "async "), ("is_unsafe", "unsafe ")] { + if h.get(k).and_then(Value::as_bool) == Some(true) { + q.push_str(w); + } + } + let show = format!("{q}fn {name}{}({}){}", generics_show(doc, &f["generics"]), inputs(doc, sig), output(doc, sig)); + (cmp.to_string(), show) +} + +struct Walk<'a, 'b> { + doc: &'b Doc<'a>, + out: Surface, + stack: Vec, +} + +impl Walk<'_, '_> { + fn put(&mut self, path: String, kind: &'static str, sig: String, show: String, exhaustive: bool, required: bool) { + self.out.insert(path, ApiItem { kind, sig, show, exhaustive, required }); + } + + fn module(&mut self, id: &Value, path: &str) { + let key = id_key(id); + if self.stack.contains(&key) || self.stack.len() > 32 { + return; + } + let Some(m) = self.doc.item(id).and_then(|i| inner(i, "module")) else { return }; + self.stack.push(key); + for child in m.get("items").and_then(Value::as_array).into_iter().flatten() { + self.child(child, path); + } + self.stack.pop(); + } + + fn child(&mut self, id: &Value, parent: &str) { + let doc = self.doc; + let Some(item) = doc.item(id) else { return }; + if item.get("visibility").and_then(Value::as_str) != Some("public") { + return; + } + if let Some(u) = inner(item, "use") { + let name = u.get("name").and_then(Value::as_str).unwrap_or("?"); + let source = u.get("source").and_then(Value::as_str).unwrap_or("?").to_string(); + let target = u.get("id").filter(|t| !t.is_null()).and_then(|t| doc.item(t).map(|i| (t, i))); + let glob = u.get("is_glob").and_then(Value::as_bool) == Some(true); + match (glob, target) { + (true, Some((t, i))) if inner(i, "module").is_some() => self.module(t, parent), + (false, Some((t, _))) => self.named(t, &format!("{parent}::{name}")), + (true, _) => self.put(format!("{parent}::*{source}"), "re-export", source.clone(), format!("pub use {source}::*"), false, false), + (false, None) => self.put(format!("{parent}::{name}"), "re-export", source.clone(), format!("pub use {source}"), false, false), + } + return; + } + let Some(name) = item.get("name").and_then(Value::as_str) else { return }; + self.named(id, &format!("{parent}::{name}")); + } + + /// The item `id` at the public path `path`, and its members below it. + fn named(&mut self, id: &Value, path: &str) { + let doc = self.doc; + let Some(item) = doc.item(id) else { return }; + let Some((kind, v)) = item.get("inner").and_then(Value::as_object).and_then(|m| m.iter().next()) else { return }; + let name = path.rsplit("::").next().unwrap_or(path).to_string(); + let non_exhaustive = attrs_have(item, "non_exhaustive"); + match kind.as_str() { + "module" => { + self.put(path.into(), "module", String::new(), format!("mod {name}"), false, false); + self.module(id, path); + } + "function" => { + let (sig, show) = function(doc, &name, v); + self.put(path.into(), "function", sig, show, false, false); + } + "struct" | "union" => { + let k = &v["kind"]; + let mut exhaustive = !non_exhaustive; + let fields: Vec<&Value> = if let Some(p) = k.get("plain") { + exhaustive &= p.get("has_stripped_fields").and_then(Value::as_bool) != Some(true); + p.get("fields").and_then(Value::as_array).map(|a| a.iter().collect()).unwrap_or_default() + } else if let Some(t) = k.get("tuple").and_then(Value::as_array) { + exhaustive &= t.iter().all(|f| !f.is_null()); + t.iter().filter(|f| !f.is_null()).collect() + } else if let Some(f) = v.get("fields").and_then(Value::as_array) { + exhaustive &= v.get("has_stripped_fields").and_then(Value::as_bool) != Some(true); + f.iter().collect() + } else { + Vec::new() + }; + let shape = if k.get("tuple").is_some() { "tuple" } else if k.as_str() == Some("unit") { "unit" } else { "plain" }; + let kind_name: &'static str = if kind == "union" { "union" } else { "struct" }; + self.put(path.into(), kind_name, format!("{shape} {}", canon(doc, &v["generics"])), format!("{kind_name} {name}{}", generics_show(doc, &v["generics"])), exhaustive, false); + for f in fields { + if let Some(fi) = doc.item(f) + && fi.get("visibility").and_then(Value::as_str) == Some("public") + { + let fname = fi.get("name").and_then(Value::as_str).unwrap_or("?"); + let t = inner(fi, "struct_field").cloned().unwrap_or(Value::Null); + self.put(format!("{path}::{fname}"), "field", canon(doc, &t).to_string(), format!("{fname}: {}", ty(doc, &t)), false, false); + } + } + self.impls(v, path); + } + "enum" => { + let exhaustive = !non_exhaustive && v.get("has_stripped_variants").and_then(Value::as_bool) != Some(true); + self.put(path.into(), "enum", canon(doc, &v["generics"]).to_string(), format!("enum {name}{}", generics_show(doc, &v["generics"])), exhaustive, false); + for var in v.get("variants").and_then(Value::as_array).into_iter().flatten() { + if let Some(vi) = doc.item(var) { + let vname = vi.get("name").and_then(Value::as_str).unwrap_or("?"); + let vk = inner(vi, "variant").cloned().unwrap_or(Value::Null); + let mut fields = Vec::new(); + let field_ids: Vec<&Value> = match vk.get("kind") { + Some(k) if k.get("tuple").is_some() => k["tuple"].as_array().map(|a| a.iter().collect()).unwrap_or_default(), + Some(k) if k.get("struct").is_some() => k["struct"].get("fields").and_then(Value::as_array).map(|a| a.iter().collect()).unwrap_or_default(), + _ => Vec::new(), + }; + for fid in field_ids { + let t = doc.item(fid).and_then(|fi| inner(fi, "struct_field").map(|t| (fi.get("name").and_then(Value::as_str).unwrap_or("?").to_string(), t.clone()))); + if let Some((fname, t)) = t { + fields.push(serde_json::json!([fname, canon(doc, &t)])); + } + } + let sig = serde_json::json!({"shape": vk.get("kind").map(|k| if k.is_string() { "plain" } else if k.get("tuple").is_some() { "tuple" } else { "struct" }), "fields": fields, "discriminant": vk.get("discriminant")}); + self.put(format!("{path}::{vname}"), "variant", sig.to_string(), format!("{name}::{vname}"), false, false); + } + } + self.impls(v, path); + } + "trait" => { + let sig = serde_json::json!({"generics": canon(doc, &v["generics"]), "bounds": canon(doc, &v["bounds"]), "unsafe": v.get("is_unsafe"), "auto": v.get("is_auto")}); + self.put(path.into(), "trait", sig.to_string(), format!("trait {name}{}", generics_show(doc, &v["generics"])), false, false); + for t in v.get("items").and_then(Value::as_array).into_iter().flatten() { + let Some(ti) = doc.item(t) else { continue }; + let tname = ti.get("name").and_then(Value::as_str).unwrap_or("?"); + let Some((tk, tv)) = ti.get("inner").and_then(Value::as_object).and_then(|m| m.iter().next()) else { continue }; + let p = format!("{path}::{tname}"); + match tk.as_str() { + "function" => { + let (sig, show) = function(doc, tname, tv); + let required = tv.get("has_body").and_then(Value::as_bool) != Some(true); + self.put(p, "trait method", sig, show, false, required); + } + "assoc_type" => { + let required = tv.get("type").is_none_or(Value::is_null); + self.put(p, "associated type", canon(doc, &tv["bounds"]).to_string(), format!("type {tname}"), false, required); + } + "assoc_const" => { + let required = tv.get("value").is_none_or(Value::is_null); + self.put(p, "associated constant", canon(doc, &tv["type"]).to_string(), format!("const {tname}: {}", ty(doc, &tv["type"])), false, required); + } + _ => {} + } + } + } + "constant" => { + let t = v.get("type").cloned().unwrap_or(Value::Null); + self.put(path.into(), "constant", canon(doc, &t).to_string(), format!("const {name}: {}", ty(doc, &t)), false, false); + } + "static" => { + let t = v.get("type").cloned().unwrap_or(Value::Null); + let sig = serde_json::json!({"type": canon(doc, &t), "mutable": v.get("is_mutable")}); + self.put(path.into(), "static", sig.to_string(), format!("static {name}: {}", ty(doc, &t)), false, false); + } + "type_alias" => { + let t = v.get("type").cloned().unwrap_or(Value::Null); + let sig = serde_json::json!({"type": canon(doc, &t), "generics": canon(doc, &v["generics"])}); + self.put(path.into(), "type alias", sig.to_string(), format!("type {name} = {}", ty(doc, &t)), false, false); + } + "macro" | "proc_macro" => self.put(path.into(), "macro", String::new(), format!("macro {name}!"), false, false), + _ => {} + } + } + + /// The inherent methods and associated items, and the traits implemented, of the type + /// at `path`. + fn impls(&mut self, v: &Value, path: &str) { + let doc = self.doc; + for imp in v.get("impls").and_then(Value::as_array).into_iter().flatten() { + let Some(ii) = doc.item(imp).and_then(|i| inner(i, "impl")) else { continue }; + if ii.get("blanket_impl").is_some_and(|b| !b.is_null()) || ii.get("is_negative").and_then(Value::as_bool) == Some(true) { + continue; + } + match ii.get("trait").filter(|t| !t.is_null()) { + Some(t) => { + let tr = path_ty(doc, t); + let sig = serde_json::json!({"generics": canon(doc, &ii["generics"]), "unsafe": ii.get("is_unsafe")}); + self.put(format!("impl {tr} for {path}"), "trait implementation", sig.to_string(), format!("impl {tr} for {path}"), false, false); + } + None => { + for m in ii.get("items").and_then(Value::as_array).into_iter().flatten() { + let Some(mi) = doc.item(m) else { continue }; + if mi.get("visibility").and_then(Value::as_str) != Some("public") { + continue; + } + let mname = mi.get("name").and_then(Value::as_str).unwrap_or("?"); + let Some((mk, mv)) = mi.get("inner").and_then(Value::as_object).and_then(|m| m.iter().next()) else { continue }; + let p = format!("{path}::{mname}"); + match mk.as_str() { + "function" => { + let (sig, show) = function(doc, mname, mv); + self.put(p, "method", sig, show, false, false); + } + "assoc_const" => self.put(p, "associated constant", canon(doc, &mv["type"]).to_string(), format!("const {mname}: {}", ty(doc, &mv["type"])), false, false), + "assoc_type" => self.put(p, "associated type", canon(doc, &mv["type"]).to_string(), format!("type {mname}"), false, false), + _ => {} + } + } + } + } + } + } +} + +/// The public surface a rustdoc JSON document describes, and its format version. +pub fn surface(json: &Value) -> Result<(Surface, u64), String> { + let index = json.get("index").and_then(Value::as_object).ok_or("rustdoc JSON without an index")?; + let paths = json.get("paths").and_then(Value::as_object).ok_or("rustdoc JSON without paths")?; + let root = json.get("root").ok_or("rustdoc JSON without a root")?; + let crate_name = index.get(&id_key(root)).and_then(|r| r.get("name")).and_then(Value::as_str).unwrap_or("crate").to_string(); + let doc = Doc { index, paths, crate_name }; + let mut w = Walk { doc: &doc, out: Surface::new(), stack: Vec::new() }; + let name = doc.crate_name.clone(); + w.module(root, &name); + Ok((w.out, json.get("format_version").and_then(Value::as_u64).unwrap_or(0))) +} + +fn parent(path: &str) -> &str { + path.rsplit_once("::").map(|x| x.0).unwrap_or("") +} + +/// The findings between two surfaces of the crate `name`. +pub fn compare_surfaces(old: &Surface, new: &Surface) -> Vec { + let mut out = Vec::new(); + for (path, a) in old { + match new.get(path) { + None => out.push(Finding::broke( + Category::Api, + "rust", + path.clone(), + "removed".into(), + format!("the {} {} is gone", a.kind, a.show), + format!("code that names {path} no longer compiles"), + )), + Some(b) if a.kind != b.kind => out.push(Finding::broke(Category::Api, "rust", path.clone(), "changed".into(), format!("a {} is now a {}", a.kind, b.kind), "code that uses it as it was no longer compiles".into())), + Some(b) if a.sig != b.sig => { + let what = if a.show != b.show { format!("{} → {}", a.show, b.show) } else { format!("the {} {}: its generics, bounds or qualifiers changed", a.kind, a.show) }; + out.push(Finding::broke(Category::Api, "rust", path.clone(), "changed".into(), what, "code written against the old signature no longer compiles, or means something else".into())); + } + Some(b) if a.exhaustive && !b.exhaustive => out.push(Finding::broke( + Category::Api, + "rust", + path.clone(), + "changed".into(), + format!("the {} {} is no longer exhaustive (a private field or variant, or #[non_exhaustive])", a.kind, a.show), + if a.kind == "enum" { "a match over every variant no longer compiles outside the crate".into() } else { "a struct literal or pattern that lists every field no longer compiles outside the crate".into() }, + )), + Some(b) if !a.required && b.required => out.push(Finding::broke(Category::Api, "rust", path.clone(), "changed".into(), format!("{} lost its default", b.show), "every implementation outside the crate must now provide it".into())), + Some(_) => {} + } + } + for (path, b) in new { + if old.contains_key(path) { + continue; + } + let up = old.get(parent(path)); + let (breaks, why) = match (b.kind, up) { + ("field", Some(s)) if s.exhaustive && new.get(parent(path)).is_some_and(|n| n.kind == s.kind) => (true, "a struct literal or pattern that lists every field no longer compiles outside the crate"), + ("variant", Some(e)) if e.exhaustive && e.kind == "enum" => (true, "a match over every variant no longer compiles outside the crate"), + ("trait method" | "associated type" | "associated constant", Some(t)) if b.required && t.kind == "trait" => (true, "every implementation outside the crate must now provide it"), + _ => (false, ""), + }; + if breaks { + out.push(Finding::broke(Category::Api, "rust", path.clone(), "added".into(), format!("the {} {} is new", b.kind, b.show), why.into())); + } else { + out.push(Finding::compatible(Category::Api, "rust", path.clone(), format!("the {} {} is new", b.kind, b.show))); + } + } + out +} + +/// The scratch space and toolchain the two documentation builds share. +pub struct Builder { + pub target_dir: PathBuf, + pub toolchain: Option, +} + +fn internal(e: impl std::fmt::Display) -> OpError { + OpError::new(EXIT_INTERNAL, "rustdoc-failed", e.to_string(), "the API check builds each crate's documentation with cargo; run `cargo rustdoc` in the crate to see why it fails") +} + +impl Builder { + fn cargo(&self, dir: &Path) -> Command { + let mut c = Command::new("cargo"); + c.current_dir(dir).env("RUSTC_BOOTSTRAP", "1").env_remove("RUSTDOCFLAGS"); + if let Some(t) = &self.toolchain { + c.env("RUSTUP_TOOLCHAIN", t); + } + c + } + + /// The features documented for `krate` in its directory `dir`: all, less the denied + /// ones and every feature that enables a denied one. + fn features(&self, krate: &Crate, dir: &Path) -> Result, OpError> { + let out = self.cargo(dir).args(["metadata", "--no-deps", "--format-version", "1", "--offline"]).output().map_err(|e| { + OpError::new(EXIT_NOT_FOUND, "cargo-not-found", format!("cargo: {e}"), "the API check needs a Rust toolchain on PATH, or `--skip rust`") + })?; + if !out.status.success() { + return Err(internal(format!("cargo metadata in {}: {}", dir.display(), String::from_utf8_lossy(&out.stderr).trim()))); + } + let meta: Value = serde_json::from_slice(&out.stdout).map_err(internal)?; + let pkg = meta["packages"].as_array().and_then(|a| a.iter().find(|p| p["name"] == krate.name)).ok_or_else(|| internal(format!("no package {} in {}", krate.name, dir.display())))?; + let feats: BTreeMap> = pkg["features"] + .as_object() + .map(|m| m.iter().map(|(k, v)| (k.clone(), v.as_array().map(|a| a.iter().filter_map(|x| x.as_str().map(String::from)).collect()).unwrap_or_default())).collect()) + .unwrap_or_default(); + let mut denied: BTreeSet = krate.deny.iter().map(|s| s.to_string()).collect(); + loop { + let more: Vec = feats.iter().filter(|(k, v)| !denied.contains(*k) && v.iter().any(|x| denied.contains(x))).map(|x| x.0.clone()).collect(); + if more.is_empty() { + break; + } + denied.extend(more); + } + Ok(feats.keys().filter(|k| !denied.contains(*k)).cloned().collect()) + } + + /// `krate`'s public surface in the tree whose top is `top`. + pub fn surface(&self, krate: &Crate, top: &Path) -> Result<(Surface, u64), OpError> { + let dir = top.join(krate.dir); + let feats = self.features(krate, &dir)?; + let mut c = self.cargo(&dir); + c.args(["rustdoc", "--lib", "--no-default-features", "--quiet"]); + if !feats.is_empty() { + c.arg("--features").arg(feats.join(",")); + } + if dir.join("Cargo.lock").exists() { + c.arg("--locked"); + } + c.arg("--target-dir").arg(&self.target_dir); + c.args(["--", "-Z", "unstable-options", "--output-format", "json", "--cap-lints", "allow"]); + eprintln!("wai breaking: documenting {} in {}", krate.name, dir.display()); + let out = c.output().map_err(internal)?; + if !out.status.success() { + let err = String::from_utf8_lossy(&out.stderr); + let tail: Vec<&str> = err.lines().rev().take(30).collect::>().into_iter().rev().collect(); + return Err(internal(format!("cargo rustdoc for {} in {}:\n{}", krate.name, dir.display(), tail.join("\n")))); + } + let file = self.target_dir.join("doc").join(format!("{}.json", krate.name.replace('-', "_"))); + let bytes = std::fs::read(&file).map_err(|e| internal(format!("{}: {e}", file.display())))?; + let json: Value = serde_json::from_slice(&bytes).map_err(internal)?; + let _ = std::fs::remove_file(&file); + surface(&json).map_err(internal) + } +} + +/// The directories `dir`'s crate depends on by path, `dir` included, found through each +/// manifest's `path = "…"` dependencies: what can change its API. +pub fn path_closure(read: &dyn Fn(&str) -> Option, dir: &str) -> BTreeSet { + let mut seen = BTreeSet::new(); + let mut todo = vec![dir.to_string()]; + while let Some(d) = todo.pop() { + if !seen.insert(d.clone()) { + continue; + } + let Some(toml) = read(&format!("{d}/Cargo.toml")) else { continue }; + let mut section = String::new(); + for line in toml.lines() { + let l = line.trim(); + if l.starts_with('[') { + section = l.to_string(); + continue; + } + if !section.contains("dependencies") { + continue; + } + if let Some(i) = l.find("path") { + let rest = l[i + 4..].trim_start(); + if let Some(rest) = rest.strip_prefix('=') + && let Some(p) = rest.trim_start().strip_prefix('"').and_then(|r| r.split('"').next()) + { + let mut parts: Vec<&str> = d.split('/').collect(); + for seg in p.split('/') { + match seg { + ".." => { + parts.pop(); + } + "." | "" => {} + s => parts.push(s), + } + } + todo.push(parts.join("/")); + } + } + } + } + seen +} + +/// The Rust API's findings between `base` and the working tree, for every crate in +/// [`CRATES`] whose sources (or those it depends on by path) differ. +pub fn compare(repo: &Repo, base: &str, builder: &Builder, scratch: &Path, notes: &mut Vec) -> Result, OpError> { + let mut out = Vec::new(); + let mut extracted: Option = None; + for krate in CRATES { + let read_head = |p: &str| std::fs::read_to_string(repo.root.join(p)).ok(); + let read_base = |p: &str| repo.base_file(base, p).ok().flatten().map(|b| String::from_utf8_lossy(&b).into_owned()); + let mut dirs = path_closure(&read_head, krate.dir); + dirs.extend(path_closure(&read_base, krate.dir)); + let specs: Vec = dirs.iter().cloned().collect(); + let spec_refs: Vec<&str> = specs.iter().map(String::as_str).collect(); + let in_head = repo.root.join(krate.dir).join("Cargo.toml").exists(); + let in_base = read_base(&format!("{}/Cargo.toml", krate.dir)).is_some(); + if !in_head && !in_base { + continue; + } + if repo.changed(base, &spec_refs)?.is_empty() { + notes.push(format!("rust: {} unchanged (nothing under {} differs)", krate.name, specs.join(", "))); + continue; + } + let new = if in_head { Some(builder.surface(krate, &repo.root)?) } else { None }; + let old = if in_base { + let top = match &extracted { + Some(t) => t.clone(), + None => { + let t = scratch.join("base"); + repo.extract(base, "wai", &t)?; + extracted = Some(t.clone()); + t + } + }; + Some(builder.surface(krate, &top)?) + } else { + None + }; + if let (Some((_, fa)), Some((_, fb))) = (&old, &new) + && fa != fb + { + return Err(internal(format!("rustdoc wrote format {fa} for the base and {fb} for the working tree; they are not compared"))); + } + let empty = Surface::new(); + let (a, b) = (old.as_ref().map(|x| &x.0).unwrap_or(&empty), new.as_ref().map(|x| &x.0).unwrap_or(&empty)); + notes.push(format!("rust: {} documented, {} public paths in the base and {} in the working tree", krate.name, a.len(), b.len())); + out.extend(compare_surfaces(a, b)); + } + Ok(out) +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/breaking/schema.rs b/wai/wai-rs/src/bin/wai_cli/dx/breaking/schema.rs new file mode 100644 index 000000000..b63c8f7f6 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/breaking/schema.rs @@ -0,0 +1,112 @@ +//! Record: the JSON Schemas WAI publishes for its documents (`wai/**/*.schema.json`). A +//! document that validated before must validate after. So a schema breaks where a property +//! is gone, a property becomes required, a type or enumeration narrows, a pattern or +//! constant changes, or additional properties are no longer allowed; it is compatible +//! where it only gains optional properties, types or enumerated values. + +use super::git::{Repo, Status}; +use super::super::envelope::OpError; +use super::{Category, Finding}; +use serde_json::Value; + +pub const PATHSPECS: &[&str] = &[":(glob)wai/**/*.schema.json"]; + +fn strings(v: Option<&Value>) -> Vec { + match v { + Some(Value::Array(a)) => a.iter().map(|x| x.as_str().map(String::from).unwrap_or_else(|| x.to_string())).collect(), + Some(Value::String(s)) => vec![s.clone()], + _ => Vec::new(), + } +} + +/// What a document valid under `old` may fail under `new` (`breaks`), and what `new` admits +/// that `old` did not (`adds`), each a line naming the schema path. +pub fn walk(old: &Value, new: &Value, path: &str, breaks: &mut Vec, adds: &mut Vec) { + let (Some(a), Some(b)) = (old.as_object(), new.as_object()) else { + if old != new { + breaks.push(format!("{path}: {old} → {new}")); + } + return; + }; + let (ta, tb) = (strings(a.get("type")), strings(b.get("type"))); + if (!tb.is_empty() && ta.iter().any(|t| !tb.contains(t))) || (ta.is_empty() && !tb.is_empty()) { + breaks.push(format!("{path}: type {ta:?} → {tb:?}")); + } + if let (Some(Value::Array(ea)), Some(Value::Array(eb))) = (a.get("enum"), b.get("enum")) { + for x in ea.iter().filter(|x| !eb.contains(x)) { + breaks.push(format!("{path}: the value {x} is no longer allowed")); + } + for x in eb.iter().filter(|x| !ea.contains(x)) { + adds.push(format!("{path}: the value {x} is allowed")); + } + } else if a.get("enum").is_none() && b.get("enum").is_some() { + breaks.push(format!("{path}: values are now enumerated")); + } + for k in ["pattern", "const", "format", "maxLength", "maximum", "maxItems", "minLength", "minimum", "minItems"] { + if a.get(k) != b.get(k) && b.get(k).is_some() { + breaks.push(format!("{path}: {k} {} → {}", a.get(k).map(Value::to_string).unwrap_or_else(|| "absent".into()), b.get(k).map(Value::to_string).unwrap_or_default())); + } + } + let (ra, rb) = (strings(a.get("required")), strings(b.get("required"))); + for r in rb.iter().filter(|r| !ra.contains(r)) { + breaks.push(format!("{path}: {r} is now required")); + } + if b.get("additionalProperties") == Some(&Value::Bool(false)) && a.get("additionalProperties") != Some(&Value::Bool(false)) { + breaks.push(format!("{path}: other properties are no longer allowed")); + } + for key in ["properties", "$defs", "definitions"] { + let (pa, pb) = (a.get(key).and_then(Value::as_object), b.get(key).and_then(Value::as_object)); + if let Some(pa) = pa { + for (name, sa) in pa { + match pb.and_then(|pb| pb.get(name)) { + None => breaks.push(format!("{path}/{key}/{name}: removed")), + Some(sb) => walk(sa, sb, &format!("{path}/{key}/{name}"), breaks, adds), + } + } + } + if let Some(pb) = pb { + for name in pb.keys().filter(|n| pa.is_none_or(|pa| !pa.contains_key(*n))) { + adds.push(format!("{path}/{key}/{name}: new")); + } + } + } + for key in ["items", "additionalProperties", "not"] { + if let (Some(x @ Value::Object(_)), Some(y @ Value::Object(_))) = (a.get(key), b.get(key)) { + walk(x, y, &format!("{path}/{key}"), breaks, adds); + } + } + for key in ["anyOf", "oneOf", "allOf"] { + if a.get(key) != b.get(key) { + breaks.push(format!("{path}: {key} changed")); + } + } +} + +/// The schemas' findings between `base` and the working tree. +pub fn compare(repo: &Repo, base: &str) -> Result, OpError> { + let changed = repo.changed(base, PATHSPECS)?; + let olds = repo.base_files(base, &changed.iter().map(|x| x.1.clone()).collect::>())?; + let mut out = Vec::new(); + for (status, path) in changed { + match status { + Status::Added => out.push(Finding::compatible(Category::Record, "schema", path, "a new schema".into())), + Status::Removed => out.push(Finding::broke(Category::Record, "schema", path, "removed".into(), "the schema is gone".into(), "documents written against it lose their published shape".into())), + Status::Modified => { + let old: Option = olds.get(&path).and_then(|b| serde_json::from_slice(b).ok()); + let new: Option = repo.head_file(&path).and_then(|b| serde_json::from_slice(&b).ok()); + let (Some(old), Some(new)) = (old, new) else { + out.push(Finding::broke(Category::Record, "schema", path, "changed".into(), "the schema is no longer JSON".into(), "documents cannot be checked against it".into())); + continue; + }; + let (mut breaks, mut adds) = (Vec::new(), Vec::new()); + walk(&old, &new, "#", &mut breaks, &mut adds); + if !breaks.is_empty() { + out.push(Finding::broke(Category::Record, "schema", path, "changed".into(), breaks.join("; "), "a document valid under the old schema is refused under the new one".into())); + } else if !adds.is_empty() { + out.push(Finding::compatible(Category::Record, "schema", path, adds.join("; "))); + } + } + } + } + Ok(out) +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/conform/adapter.rs b/wai/wai-rs/src/bin/wai_cli/dx/conform/adapter.rs new file mode 100644 index 000000000..f1e4e0e65 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/conform/adapter.rs @@ -0,0 +1,196 @@ +//! The two ways `wai conform` reaches an implementation. +//! +//! **Command.** `--impl` is a command line, run by the system shell once a case (or once for +//! the whole corpus, where the suite allows it), with these placeholders replaced by quoted +//! paths: `{case}` a file holding the case (a verifier's vector, as the corpus writes it), +//! `{corpus}` the corpus file whole, `{in}` the input file, `{out}` a file to write the +//! output to. Without `{case}` or `{in}`, the case or input is given on standard input; +//! without `{out}`, the output is read from standard output. What the command prints first +//! on standard output is its verdict; its exit status says whether it refused (non-zero) +//! where the suite asks. +//! +//! **HTTP.** `--impl` is an `http://` URL: each case is a `POST` of the case or input to +//! it (an origin is instead fetched from, below it), and the response's status and body are +//! the answer. + +use sha2::{Digest, Sha256}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +/// What a command did. +pub struct Ran { + pub code: Option, + pub stdout: Vec, + pub stderr: Vec, + pub timed_out: bool, +} + +/// A path quoted for the shell that runs the command. +pub fn quote(p: &Path) -> String { + let s = p.to_string_lossy(); + if cfg!(windows) { format!("\"{s}\"") } else { format!("'{}'", s.replace('\'', "'\\''")) } +} + +/// `cmd` with each `{name}` replaced by its quoted path. +pub fn fill(cmd: &str, with: &[(&str, &Path)]) -> String { + let mut s = cmd.to_string(); + for (k, p) in with { + s = s.replace(&format!("{{{k}}}"), "e(p)); + } + s +} + +/// Run `cmd` through the shell, `stdin` given to it, killed after `timeout`. +pub fn run(cmd: &str, stdin: Option<&[u8]>, timeout: Duration) -> Result { + let mut c = if cfg!(windows) { + let mut c = Command::new("cmd"); + c.arg("/C").arg(cmd); + c + } else { + let mut c = Command::new("sh"); + c.arg("-c").arg(cmd); + c + }; + let mut child = c + .stdin(if stdin.is_some() { Stdio::piped() } else { Stdio::null() }) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("{cmd}: {e}"))?; + let writer = match (stdin, child.stdin.take()) { + (Some(b), Some(mut w)) => { + let b = b.to_vec(); + Some(std::thread::spawn(move || { + let _ = w.write_all(&b); + })) + } + _ => None, + }; + let read = |r: Option>| { + std::thread::spawn(move || { + let mut v = Vec::new(); + if let Some(mut r) = r { + let _ = r.read_to_end(&mut v); + } + v + }) + }; + let out = read(child.stdout.take().map(|x| Box::new(x) as Box)); + let err = read(child.stderr.take().map(|x| Box::new(x) as Box)); + let t0 = Instant::now(); + let mut timed_out = false; + let status = loop { + match child.try_wait().map_err(|e| e.to_string())? { + Some(s) => break Some(s), + None if t0.elapsed() >= timeout => { + timed_out = true; + let _ = child.kill(); + break child.wait().ok(); + } + None => std::thread::sleep(Duration::from_millis(5)), + } + }; + if let Some(w) = writer { + let _ = w.join(); + } + let stdout = out.join().unwrap_or_default(); + let stderr = err.join().unwrap_or_default(); + Ok(Ran { code: status.and_then(|s| s.code()), stdout, stderr, timed_out }) +} + +/// The first line of `b`, trimmed. +pub fn first_line(b: &[u8]) -> String { + String::from_utf8_lossy(b).lines().map(str::trim).find(|l| !l.is_empty()).unwrap_or("").to_string() +} + +pub fn sha256_hex(b: &[u8]) -> String { + Sha256::digest(b).iter().map(|x| format!("{x:02x}")).collect() +} + +/// The words of a command line, quotes removed (the shell's rules, roughly). +fn words(cmd: &str) -> Vec { + let mut out = Vec::new(); + let mut cur = String::new(); + let mut quote: Option = None; + let mut any = false; + for c in cmd.chars() { + match (quote, c) { + (Some(q), c) if c == q => quote = None, + (Some(_), c) => cur.push(c), + (None, '\'' | '"') => { + quote = Some(c); + any = true; + } + (None, c) if c.is_whitespace() => { + if any || !cur.is_empty() { + out.push(std::mem::take(&mut cur)); + any = false; + } + } + (None, c) => cur.push(c), + } + } + if any || !cur.is_empty() { + out.push(cur); + } + out +} + +fn on_path(name: &str) -> Option { + if name.contains('/') || name.contains('\\') { + return Path::new(name).is_file().then(|| PathBuf::from(name)); + } + let path = std::env::var_os("PATH")?; + std::env::split_paths(&path).map(|d| d.join(name)).find(|p| p.is_file()) +} + +/// The files a command line names (its program found on `PATH`, and every argument that +/// is a file), each with its SHA-256: what identifies the implementation that ran. +pub fn identity(cmd: &str) -> Vec<(String, String)> { + let mut out = Vec::new(); + for (i, w) in words(cmd).iter().enumerate() { + if w.starts_with('{') && w.ends_with('}') { + continue; + } + let p = if i == 0 { on_path(w) } else { Path::new(w).is_file().then(|| PathBuf::from(w)) }; + if let Some(p) = p + && let Ok(b) = std::fs::read(&p) + { + out.push((p.to_string_lossy().into_owned(), sha256_hex(&b))); + } + } + out +} + +/// Whether the program a command line runs can be found. +pub fn program_found(cmd: &str) -> bool { + match words(cmd).first() { + None => false, + Some(p) => on_path(p).is_some() || matches!(p.as_str(), "cd" | "exec" | "env" | "echo" | "printf" | "test" | "[" | "true" | "false"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg(unix)] + #[test] + fn placeholders_are_quoted_paths() { + let s = fill("dec {in} {out}", &[("in", Path::new("/t/a b.wscr")), ("out", Path::new("/t/it's.pcm"))]); + assert_eq!(s, "dec '/t/a b.wscr' '/t/it'\\''s.pcm'"); + assert_eq!(words("python3 -I 'a b.py' verdicts {corpus}"), vec!["python3", "-I", "a b.py", "verdicts", "{corpus}"]); + } + + #[cfg(unix)] + #[test] + fn a_command_runs_and_one_that_hangs_is_stopped() { + let r = run("cat", Some(b"x\ny"), Duration::from_secs(20)).unwrap(); + assert_eq!(r.stdout, b"x\ny"); + assert_eq!(first_line(&r.stdout), "x"); + let r = run("sleep 30", None, Duration::from_millis(200)).unwrap(); + assert!(r.timed_out); + } +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/conform/http.rs b/wai/wai-rs/src/bin/wai_cli/dx/conform/http.rs new file mode 100644 index 000000000..3c4cf8a41 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/conform/http.rs @@ -0,0 +1,169 @@ +//! A small HTTP/1.1 client over plain TCP, enough to drive an implementation behind a URL: +//! one request a connection (`Connection: close`), a body by `Content-Length` or chunked, +//! and a response read whole up to a bound. + +use std::io::{Read, Write}; +use std::net::{TcpStream, ToSocketAddrs}; +use std::time::{Duration, Instant}; + +/// The most a response may hold. +const MAX_RESPONSE: usize = 512 << 20; + +/// An `http://` URL: where to connect, the `Host` header, and the path it starts with. +#[derive(Clone, Debug)] +pub struct Url { + pub authority: String, + pub prefix: String, +} + +impl Url { + pub fn parse(u: &str) -> Result { + let rest = u.strip_prefix("http://").ok_or_else(|| { + if u.starts_with("https://") { + format!("{u}: this build speaks plain HTTP to an implementation; serve it on an http:// address (a loopback port, or a tunnel to one)") + } else { + format!("{u}: not an http:// URL") + } + })?; + let (authority, path) = match rest.find('/') { + Some(i) => (&rest[..i], &rest[i..]), + None => (rest, "/"), + }; + if authority.is_empty() { + return Err(format!("{u}: no host")); + } + Ok(Url { authority: authority.to_string(), prefix: path.trim_end_matches('/').to_string() }) + } + + /// The path `p` (which starts with `/`, or is empty for the URL itself) below the URL. + pub fn join(&self, p: &str) -> String { + let s = format!("{}{p}", self.prefix); + if s.is_empty() { "/".into() } else { s } + } +} + +#[derive(Debug)] +pub struct Response { + pub status: u16, + pub headers: Vec<(String, String)>, + pub body: Vec, +} + +impl Response { + /// Every value of the header `name` (case-insensitive), joined with `, `. + pub fn header(&self, name: &str) -> Option { + let v: Vec<&str> = self.headers.iter().filter(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str()).collect(); + if v.is_empty() { None } else { Some(v.join(", ")) } + } +} + +/// Why a request got no response. +#[derive(Debug)] +pub enum Failure { + /// No connection: the implementation is not there (or not yet). + Unreachable(String), + /// A connection, and no readable response. + Broken(String), +} + +impl std::fmt::Display for Failure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Failure::Unreachable(e) => write!(f, "unreachable: {e}"), + Failure::Broken(e) => write!(f, "{e}"), + } + } +} + +fn dechunk(mut b: &[u8]) -> Option> { + let mut out = Vec::new(); + loop { + let nl = b.windows(2).position(|w| w == b"\r\n")?; + let size_text = std::str::from_utf8(&b[..nl]).ok()?; + let size = usize::from_str_radix(size_text.split(';').next()?.trim(), 16).ok()?; + b = &b[nl + 2..]; + if size == 0 { + return Some(out); + } + if b.len() < size + 2 { + return None; + } + out.extend_from_slice(&b[..size]); + b = &b[size + 2..]; + } +} + +/// One request, answered whole within `timeout`. +pub fn request(url: &Url, method: &str, path: &str, headers: &[(&str, &str)], body: Option<&[u8]>, timeout: Duration) -> Result { + let deadline = Instant::now() + timeout; + let addrs: Vec<_> = url.authority.to_socket_addrs().map_err(|e| Failure::Unreachable(format!("{}: {e}", url.authority)))?.collect(); + let mut last = String::from("no address"); + let mut stream = None; + for a in &addrs { + match TcpStream::connect_timeout(a, timeout.min(Duration::from_secs(10))) { + Ok(s) => { + stream = Some(s); + break; + } + Err(e) => last = format!("{a}: {e}"), + } + } + let mut s = stream.ok_or(Failure::Unreachable(last))?; + let left = || deadline.saturating_duration_since(Instant::now()).max(Duration::from_millis(1)); + let _ = s.set_read_timeout(Some(left())); + let _ = s.set_write_timeout(Some(left())); + let mut head = format!("{method} {} HTTP/1.1\r\nHost: {}\r\nConnection: close\r\nUser-Agent: wai-conform\r\n", url.join(path), url.authority); + for (k, v) in headers { + head.push_str(&format!("{k}: {v}\r\n")); + } + if let Some(b) = body { + head.push_str(&format!("Content-Length: {}\r\n", b.len())); + } + head.push_str("\r\n"); + let broken = |e: std::io::Error| Failure::Broken(format!("{method} {path}: {e}")); + s.write_all(head.as_bytes()).map_err(broken)?; + if let Some(b) = body { + s.write_all(b).map_err(broken)?; + } + let mut buf = Vec::new(); + let mut chunk = [0u8; 64 * 1024]; + loop { + let _ = s.set_read_timeout(Some(left())); + match s.read(&mut chunk) { + Ok(0) => break, + Ok(n) => { + buf.extend_from_slice(&chunk[..n]); + if buf.len() > MAX_RESPONSE { + return Err(Failure::Broken(format!("{method} {path}: the response is larger than {MAX_RESPONSE} bytes"))); + } + } + Err(e) if Instant::now() >= deadline => return Err(Failure::Broken(format!("{method} {path}: no complete response in {timeout:?} ({e})"))), + Err(e) if matches!(e.kind(), std::io::ErrorKind::Interrupted) => {} + Err(e) => { + if buf.is_empty() { + return Err(broken(e)); + } + break; + } + } + } + let end = buf.windows(4).position(|w| w == b"\r\n\r\n").ok_or_else(|| Failure::Broken(format!("{method} {path}: no response head")))?; + let head = String::from_utf8_lossy(&buf[..end]).into_owned(); + let mut lines = head.split("\r\n"); + let status_line = lines.next().unwrap_or(""); + let status: u16 = status_line.split(' ').nth(1).and_then(|s| s.parse().ok()).ok_or_else(|| Failure::Broken(format!("{method} {path}: status line {status_line:?}")))?; + let headers: Vec<(String, String)> = lines.filter_map(|l| l.split_once(':').map(|(k, v)| (k.trim().to_string(), v.trim().to_string()))).collect(); + let mut body = buf[end + 4..].to_vec(); + let r = Response { status, headers, body: Vec::new() }; + if method == "HEAD" || status == 204 || status == 304 { + body.clear(); + } else if r.header("Transfer-Encoding").is_some_and(|t| t.to_ascii_lowercase().contains("chunked")) { + body = dechunk(&body).ok_or_else(|| Failure::Broken(format!("{method} {path}: a chunked body that does not read")))?; + } else if let Some(n) = r.header("Content-Length").and_then(|v| v.trim().parse::().ok()) { + if body.len() < n { + return Err(Failure::Broken(format!("{method} {path}: {} of {n} body bytes", body.len()))); + } + body.truncate(n); + } + Ok(Response { body, ..r }) +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/conform/mod.rs b/wai/wai-rs/src/bin/wai_cli/dx/conform/mod.rs new file mode 100644 index 000000000..6f913c03e --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/conform/mod.rs @@ -0,0 +1,578 @@ +//! `wai conform --suite --impl ` (operation +//! `conformance.run`): one of WAI's conformance corpora run against another implementation, +//! and the outcome written as a signed result record (`record`), which `wai verify` checks. +//! +//! The kinds and their suites are in `suites`; the two adapters (a command line, or an +//! `http://` URL) in `adapter`. Every case's expectation is the corpus's own. A run reads +//! the corpus from a checkout of the standard (`--root `, or found from the +//! current directory), and records the digest of every corpus file it read, the +//! implementation it ran (the command and the SHA-256 of every file it names, or the URL), +//! each case's outcome, and the standard's version. +//! +//! Exit status: 0 when every case passed, 1 when one did not, 2 for a usage error, 4 when +//! the record would replace a file and `--confirm` was not given, 5 when the suite, corpus +//! or program is not found, 6 when the implementation's URL does not answer, 7 otherwise. + +mod adapter; +mod http; +pub mod record; +pub mod suites; + +use super::envelope::{self, EXIT_CONFIRM, EXIT_FAILED, EXIT_INTERNAL, EXIT_NOT_FOUND, EXIT_UNAVAILABLE, OpError}; +use serde_json::{Map, Value, json}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use suites::{Case, Expect, Input, SUITES, Suite}; + +pub const OP: &str = "conformance.run"; + +const USAGE: &str = "wai conform --suite --impl [--key ] [--out ] [--confirm]\n\ + [--root ] [--timeout ] [--name ] [--version ] [--dry-run] [--all] [--json|--text]\n\ + wai conform --list"; + +/// How the implementation is reached. +enum Impl { + Command(String), + Http(http::Url), +} + +/// One case's outcome. +struct Outcome { + name: String, + pass: bool, + expected: String, + got: String, +} + +fn cut(s: &str) -> String { + if s.chars().count() > 300 { format!("{}…", s.chars().take(300).collect::()) } else { s.to_string() } +} + +/// The `wai/` directory of a checkout: `--root`, else the first directory up from here +/// that is one, or holds one. +fn wai_root(given: Option<&str>) -> Result { + let is_root = |d: &Path| d.join("SPEC.md").is_file() && d.join("registry").join("capabilities.json").is_file(); + if let Some(g) = given { + let d = PathBuf::from(g); + return if is_root(&d) { + Ok(d) + } else { + Err(OpError::new(EXIT_NOT_FOUND, "no-corpus", format!("{g}: not the wai/ directory of a checkout of the standard"), "--root names the directory that holds SPEC.md and the *-conformance corpora")) + }; + } + let mut d = std::env::current_dir().map_err(|e| OpError::new(EXIT_INTERNAL, "internal", e.to_string(), ""))?; + loop { + if is_root(&d) { + return Ok(d); + } + if is_root(&d.join("wai")) { + return Ok(d.join("wai")); + } + if !d.pop() { + return Err(OpError::new( + EXIT_NOT_FOUND, + "no-corpus", + "no checkout of the WAI standard here, so no corpus to run", + "run it inside a checkout (git clone https://git.transaction.science/Transaction-Science/open-standards), or pass --root /wai", + )); + } + } +} + +/// The standard's version: the first `## v…` heading of `CHANGELOG.md`. +fn standard_version(root: &Path) -> String { + std::fs::read_to_string(root.join("CHANGELOG.md")) + .ok() + .and_then(|t| t.lines().find_map(|l| l.strip_prefix("## v").map(|r| format!("v{}", r.split_whitespace().next().unwrap_or(""))))) + .unwrap_or_else(|| "unknown".into()) +} + +/// Now, in RFC 3339 UTC to the second. +fn now_rfc3339() -> String { + let secs = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs() as i64).unwrap_or(0); + let (days, rem) = (secs.div_euclid(86_400), secs.rem_euclid(86_400)); + // Civil date from days since 1970-01-01 (Howard Hinnant's algorithm). + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + i64::from(m <= 2); + format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z", rem / 3600, rem % 3600 / 60, rem % 60) +} + +fn corpus_digest(root: &Path, files: &[PathBuf]) -> Result { + let mut lines: Vec = Vec::new(); + for f in files { + let b = std::fs::read(f).map_err(|e| OpError::new(EXIT_NOT_FOUND, "corpus-file-missing", format!("{}: {e}", f.display()), "the corpus is incomplete; check out the standard again"))?; + let rel = f.strip_prefix(root).unwrap_or(f).to_string_lossy().replace('\\', "/"); + lines.push(format!("{} {rel}\n", adapter::sha256_hex(&b))); + } + lines.sort(); + lines.dedup(); + Ok(format!("sha256:{}", adapter::sha256_hex(lines.concat().as_bytes()))) +} + +/// A packager's verdict line read: `ok `, or a refusal's reason, a leading copy of +/// the input's path ignored. +fn packager_tokens(line: &str, input: &Path) -> Vec { + let mut t: Vec = line.split_whitespace().map(String::from).collect(); + if t.first().is_some_and(|w| Path::new(w) == input || input.file_name().is_some_and(|n| w.ends_with(&*n.to_string_lossy()))) { + t.remove(0); + } + t +} + +fn expected_text(e: &Expect) -> String { + match e { + Expect::Verdict { line, prefix: true } => format!("{line}…"), + Expect::Verdict { line, .. } => line.clone(), + Expect::Output { sha256 } => format!("output sha256 {sha256}"), + Expect::Admit { digest } => format!("ok {digest}"), + Expect::Refuse { reason } => reason.clone(), + Expect::Object { cid, .. } => format!("{cid}.wai served as http-delivery §5.3 and §6.2 require"), + } +} + +fn judge(e: &Expect, got: &str, input: Option<&Path>) -> bool { + match e { + Expect::Verdict { line, prefix } => { + if *prefix { + got.starts_with(line.as_str()) + } else { + got == line + } + } + Expect::Output { sha256 } => got == format!("output sha256 {sha256}"), + Expect::Admit { digest } => { + let t = packager_tokens(got, input.unwrap_or(Path::new(""))); + t.len() >= 2 && t[0] == "ok" && t[1] == *digest + } + Expect::Refuse { reason } => packager_tokens(got, input.unwrap_or(Path::new(""))).first() == Some(reason), + Expect::Object { .. } => got.is_empty(), + } +} + +/// What an origin serving the object answers, against http-delivery §5.3 and §6.2: the +/// problems found, none when it conforms. +fn check_object(url: &http::Url, cid: &str, bytes: &[u8], head_proof: Option<&[u8]>, timeout: Duration) -> Result, http::Failure> { + let mut p = Vec::new(); + let path = format!("/{cid}.wai"); + let r = http::request(url, "GET", &path, &[], None, timeout)?; + if r.status != 200 { + p.push(format!("GET {path}: status {} (200)", r.status)); + } else if r.body != bytes { + p.push(format!("GET {path}: the body is not the object ({} bytes, sha256 {})", r.body.len(), &adapter::sha256_hex(&r.body)[..16])); + } + let want_etag = format!("\"{cid}\""); + let h = |name: &str| r.header(name).unwrap_or_default(); + if h("ETag") != want_etag { + p.push(format!("ETag {:?} ({want_etag})", h("ETag"))); + } + if !h("Content-Type").starts_with("application/wai") { + p.push(format!("Content-Type {:?} (application/wai)", h("Content-Type"))); + } + let cc: Vec = h("Cache-Control").split(',').map(|x| x.trim().to_ascii_lowercase()).collect(); + for want in ["public", "max-age=31536000", "immutable", "no-transform"] { + if !cc.iter().any(|x| x == want) { + p.push(format!("Cache-Control {:?} lacks {want}", h("Cache-Control"))); + } + } + if !h("Accept-Ranges").eq_ignore_ascii_case("bytes") { + p.push(format!("Accept-Ranges {:?} (bytes)", h("Accept-Ranges"))); + } + if h("Access-Control-Allow-Origin") != "*" { + p.push(format!("Access-Control-Allow-Origin {:?} (*)", h("Access-Control-Allow-Origin"))); + } + let expose = h("Access-Control-Expose-Headers").to_ascii_lowercase(); + for want in ["content-range", "content-length", "etag"] { + if !expose.split(',').any(|x| x.trim() == want) { + p.push(format!("Access-Control-Expose-Headers {:?} lacks {want}", h("Access-Control-Expose-Headers"))); + } + } + if h("Timing-Allow-Origin") != "*" { + p.push(format!("Timing-Allow-Origin {:?} (*)", h("Timing-Allow-Origin"))); + } + if r.header("Vary").is_some() { + p.push(format!("Vary {:?} (an object is never selected by a request header)", h("Vary"))); + } + if r.header("Content-Encoding").is_some_and(|e| !e.eq_ignore_ascii_case("identity")) { + p.push(format!("Content-Encoding {:?} (none)", h("Content-Encoding"))); + } + // A single byte range: 206 with the bytes and the whole length, or 200 with the object. + let last = bytes.len().min(32).saturating_sub(1); + let range = format!("bytes=0-{last}"); + let r = http::request(url, "GET", &path, &[("Range", &range)], None, timeout)?; + match r.status { + 206 => { + let want = format!("bytes 0-{last}/{}", bytes.len()); + if r.header("Content-Range").as_deref() != Some(want.as_str()) { + p.push(format!("Range {range}: Content-Range {:?} ({want})", r.header("Content-Range"))); + } + if r.body != bytes[..=last] { + p.push(format!("Range {range}: the body is not those bytes")); + } + } + 200 if r.body == bytes => {} + s => p.push(format!("Range {range}: status {s} (206 with the bytes, or 200 with the object)")), + } + let r = http::request(url, "HEAD", &path, &[], None, timeout)?; + if r.status != 200 || r.header("ETag").as_deref() != Some(want_etag.as_str()) { + p.push(format!("HEAD {path}: status {}, ETag {:?}", r.status, r.header("ETag"))); + } + let r = http::request(url, "OPTIONS", &path, &[("Origin", "https://conformance.example"), ("Access-Control-Request-Method", "GET"), ("Access-Control-Request-Headers", "range, priority")], None, timeout)?; + let allow = r.header("Access-Control-Allow-Headers").unwrap_or_default().to_ascii_lowercase(); + for want in ["range", "priority"] { + if !allow.split(',').any(|x| x.trim() == want) { + p.push(format!("preflight: Access-Control-Allow-Headers {allow:?} lacks {want}")); + } + } + if r.header("Access-Control-Max-Age").is_none() { + p.push("preflight: no Access-Control-Max-Age".into()); + } + if let Some(hp) = head_proof { + let hpath = format!("/{cid}.wai.hp"); + let r = http::request(url, "GET", &hpath, &[], None, timeout)?; + if r.status != 200 || r.body != hp { + p.push(format!("GET {hpath}: status {}, {} bytes (the head proof, {} bytes)", r.status, r.body.len(), hp.len())); + } else if r.header("Content-Type").as_deref() != Some("application/octet-stream") { + p.push(format!("GET {hpath}: Content-Type {:?} (application/octet-stream)", r.header("Content-Type"))); + } + } + Ok(p) +} + +struct Run<'a> { + imp: &'a Impl, + timeout: Duration, + scratch: PathBuf, +} + +impl Run<'_> { + fn http_failure(&self, f: http::Failure) -> Result { + match f { + http::Failure::Unreachable(e) => Err(OpError::new(EXIT_UNAVAILABLE, "implementation-unreachable", format!("the implementation does not answer: {e}"), "start it, then run again")), + http::Failure::Broken(e) => Ok(format!("no answer: {e}")), + } + } + + /// One case's answer, as the text `judge` reads. + fn answer(&self, case: &Case) -> Result { + let internal = |e: String| OpError::new(EXIT_INTERNAL, "internal", e, ""); + match (self.imp, &case.input, &case.expect) { + (Impl::Http(url), _, Expect::Object { cid, bytes_file, head_proof }) => { + let bytes = std::fs::read(bytes_file).map_err(|e| internal(e.to_string()))?; + let hp = head_proof.as_ref().and_then(|p| std::fs::read(p).ok()); + match check_object(url, cid, &bytes, hp.as_deref(), self.timeout) { + Ok(problems) => Ok(problems.join("; ")), + Err(f) => self.http_failure(f), + } + } + (Impl::Http(url), input, expect) => { + let (body, ctype) = match input { + Input::Json(v) => (v.to_string().into_bytes(), "application/json"), + Input::File(p) => (std::fs::read(p).map_err(|e| internal(format!("{}: {e}", p.display())))?, "application/octet-stream"), + Input::Served => (Vec::new(), "application/octet-stream"), + }; + match http::request(url, "POST", "", &[("Content-Type", ctype)], Some(&body), self.timeout) { + Err(f) => self.http_failure(f), + Ok(r) => Ok(match expect { + Expect::Output { .. } if r.status == 200 => format!("output sha256 {}", adapter::sha256_hex(&r.body)), + Expect::Output { .. } => format!("refused ({}): {}", r.status, adapter::first_line(&r.body)), + _ => match serde_json::from_slice::(&r.body).ok().and_then(|v| v.get("verdict").and_then(Value::as_str).map(String::from)) { + Some(v) => v, + None => adapter::first_line(&r.body), + }, + }), + } + } + (Impl::Command(cmd), input, expect) => { + let case_file = self.scratch.join("case.json"); + let out_file = self.scratch.join("out"); + let _ = std::fs::remove_file(&out_file); + let mut with: Vec<(&str, &Path)> = vec![("out", out_file.as_path())]; + let mut stdin: Option> = None; + match input { + Input::Json(v) => { + if cmd.contains("{case}") { + std::fs::write(&case_file, v.to_string()).map_err(|e| internal(e.to_string()))?; + with.push(("case", case_file.as_path())); + } else { + stdin = Some(v.to_string().into_bytes()); + } + } + Input::File(p) => { + if cmd.contains("{in}") { + with.push(("in", p.as_path())); + } else { + stdin = Some(std::fs::read(p).map_err(|e| internal(format!("{}: {e}", p.display())))?); + } + } + Input::Served => {} + } + let line = adapter::fill(cmd, &with); + let ran = adapter::run(&line, stdin.as_deref(), self.timeout).map_err(internal)?; + if ran.timed_out { + return Ok(format!("timed out after {:?}", self.timeout)); + } + Ok(match expect { + Expect::Output { .. } => { + if ran.code != Some(0) { + format!("refused (exit {:?}): {}", ran.code, adapter::first_line(&ran.stdout)) + } else if cmd.contains("{out}") { + match std::fs::read(&out_file) { + Ok(b) => format!("output sha256 {}", adapter::sha256_hex(&b)), + Err(_) => "no output written to {out}".into(), + } + } else { + format!("output sha256 {}", adapter::sha256_hex(&ran.stdout)) + } + } + _ => { + let l = adapter::first_line(&ran.stdout); + if l.is_empty() && ran.code != Some(0) { + format!("no verdict (exit {:?}): {}", ran.code, adapter::first_line(&ran.stderr)) + } else { + l + } + } + }) + } + } + } + + /// Every case, the whole corpus given to one run of the command (`{corpus}`), which + /// answers `nameverdict` lines. + fn whole_corpus(&self, cmd: &str, corpus: &Path, cases: &[Case]) -> Result, OpError> { + let line = adapter::fill(cmd, &[("corpus", corpus)]); + let ran = adapter::run(&line, None, self.timeout * 20).map_err(|e| OpError::new(EXIT_INTERNAL, "internal", e, ""))?; + let mut got: HashMap = HashMap::new(); + for l in String::from_utf8_lossy(&ran.stdout).lines() { + if let Some((n, v)) = l.split_once('\t') { + got.entry(n.to_string()).or_insert_with(|| v.trim().to_string()); + } + } + let missing = if ran.timed_out { "timed out".to_string() } else { format!("no verdict (exit {:?}): {}", ran.code, adapter::first_line(&ran.stderr)) }; + Ok(cases + .iter() + .map(|c| { + let g = got.get(&c.name).cloned().unwrap_or_else(|| missing.clone()); + Outcome { name: c.name.clone(), pass: judge(&c.expect, &g, None), expected: cut(&expected_text(&c.expect)), got: cut(&g) } + }) + .collect()) + } +} + +fn list(json_mode: bool, root: Option<&Path>) { + let rows: Vec = SUITES + .iter() + .map(|s| { + let n = root.and_then(|r| (s.load)(&r.join(s.dir)).ok()).map(|l| l.cases.len()); + let adapters: Vec<&str> = [s.command.then_some("command"), s.http.then_some("http")].into_iter().flatten().collect(); + json!({"kind": s.kind, "suite": s.name, "corpus": s.dir, "cases": n, "adapters": adapters, "about": s.about}) + }) + .collect(); + if json_mode { + let mut m = Map::new(); + m.insert("suites".into(), Value::Array(rows)); + println!("{}", envelope::envelope("conformance.list", true, m)); + return; + } + for r in rows { + println!( + "{:<9} {:<7} {:<20} {:>5} cases {:<13} {}", + r["kind"].as_str().unwrap_or(""), + r["suite"].as_str().unwrap_or(""), + r["corpus"].as_str().unwrap_or(""), + r["cases"].as_u64().map(|n| n.to_string()).unwrap_or_else(|| "?".into()), + r["adapters"].as_array().map(|a| a.iter().filter_map(Value::as_str).collect::>().join("+")).unwrap_or_default(), + r["about"].as_str().unwrap_or("") + ); + } +} + +pub fn main(args: &[String]) -> Result { + let json_mode = envelope::json_wanted(args); + match run(args, json_mode) { + Ok(true) => Ok(true), + Ok(false) => envelope::exit(EXIT_FAILED), + Err(e) => envelope::fail(OP, json_mode, &e), + } +} + +fn run(args: &[String], json_mode: bool) -> Result { + let flags = ["--suite", "--impl", "--key", "--out", "--confirm", "--root", "--timeout", "--name", "--version", "--dry-run", "--all", "--json", "--text", "--list"]; + let with_value = ["--suite", "--impl", "--key", "--out", "--root", "--timeout", "--name", "--version"]; + let pos = envelope::check_args(args, &flags, &with_value, 1, USAGE)?; + let v = |f: &str| envelope::value(args, f, USAGE); + if args.iter().any(|a| a == "--list") { + let root = wai_root(v("--root")?).ok(); + list(json_mode, root.as_deref()); + return Ok(true); + } + let kind = *pos.first().ok_or_else(|| OpError::usage(format!("name a kind: {}", suites::KINDS.join(", ")), USAGE))?; + if !suites::KINDS.contains(&kind) { + return Err(OpError::new(EXIT_NOT_FOUND, "kind-not-found", format!("{kind}: no suite of that kind; the kinds are {}", suites::KINDS.join(", ")), "wai conform --list names every kind and suite")); + } + let of_kind: Vec<&Suite> = SUITES.iter().filter(|s| s.kind == kind).collect(); + let suite = match v("--suite")? { + Some(n) => *of_kind.iter().find(|s| s.name == n).ok_or_else(|| { + OpError::new(EXIT_NOT_FOUND, "suite-not-found", format!("{kind} has no suite {n}; it has {}", of_kind.iter().map(|s| s.name).collect::>().join(", ")), "wai conform --list names every kind and suite") + })?, + None if of_kind.len() == 1 => of_kind[0], + None => return Err(OpError::usage(format!("--suite: {kind} has {}", of_kind.iter().map(|s| s.name).collect::>().join(", ")), USAGE)), + }; + let imp_text = v("--impl")?.ok_or_else(|| OpError::usage("--impl is required: the implementation to run the corpus against", USAGE))?.to_string(); + let imp = if imp_text.starts_with("http://") || imp_text.starts_with("https://") { + if !suite.http { + return Err(OpError::usage(format!("{kind} {}: no HTTP adapter", suite.name), USAGE)); + } + Impl::Http(http::Url::parse(&imp_text).map_err(|e| OpError::usage(e, USAGE))?) + } else { + if !suite.command { + return Err(OpError::usage(format!("{kind} {}: an origin is reached at an http:// URL", suite.name), USAGE)); + } + if !adapter::program_found(&imp_text) { + return Err(OpError::new(EXIT_NOT_FOUND, "program-not-found", format!("{imp_text}: its program is not found"), "name a program on PATH, or a path to one")); + } + Impl::Command(imp_text.clone()) + }; + let timeout = Duration::from_secs(v("--timeout")?.map(|t| t.parse::().map_err(|_| OpError::usage("--timeout takes whole seconds", USAGE))).transpose()?.unwrap_or(60)); + let root = wai_root(v("--root")?)?; + let loaded = (suite.load)(&root.join(suite.dir)).map_err(|e| OpError::new(EXIT_NOT_FOUND, "corpus-not-read", e, "the corpus is incomplete; check out the standard again"))?; + if loaded.cases.is_empty() { + return Err(OpError::new(EXIT_NOT_FOUND, "no-cases", format!("{}: no cases", suite.dir), "")); + } + let whole = matches!(&imp, Impl::Command(c) if c.contains("{corpus}")); + if whole && loaded.corpus_file.is_none() { + return Err(OpError::usage(format!("{kind} {}: {{corpus}} is for a verifier suite; this suite gives its cases one at a time", suite.name), USAGE)); + } + let out = PathBuf::from(v("--out")?.map(String::from).unwrap_or_else(|| format!("{}{}", suite.name, record::SUFFIX))); + if args.iter().any(|a| a == "--dry-run") { + let first = &loaded.cases[0]; + let plan = match &imp { + Impl::Command(c) if whole => adapter::fill(c, &[("corpus", loaded.corpus_file.as_deref().unwrap_or(Path::new("")))]), + Impl::Command(c) => adapter::fill(c, &[("case", Path::new("")), ("in", match &first.input { + Input::File(p) => p.as_path(), + _ => Path::new(""), + }), ("out", Path::new(""))]), + Impl::Http(u) => format!("{} {}", if matches!(first.expect, Expect::Object { .. }) { "GET, HEAD and OPTIONS below" } else { "POST each case to" }, u.join("")), + }; + if json_mode { + let mut m = Map::new(); + m.insert("kind".into(), json!(kind)); + m.insert("suite".into(), json!(suite.name)); + m.insert("dry_run".into(), json!(true)); + m.insert("cases".into(), json!(loaded.cases.iter().map(|c| c.name.clone()).collect::>())); + m.insert("runs".into(), json!(plan)); + m.insert("record".into(), json!(out.display().to_string())); + println!("{}", envelope::envelope(OP, true, m)); + } else { + println!("wai conform {kind} {}: {} cases from {}; would run: {plan}", suite.name, loaded.cases.len(), suite.dir); + println!("the record would be written to {}", out.display()); + } + return Ok(true); + } + if out.exists() && !args.iter().any(|a| a == "--confirm") { + return Err(OpError::new(EXIT_CONFIRM, "confirmation-required", format!("{} exists; the record would replace it", out.display()), format!("re-run with --confirm, or --out : wai conform {} --confirm", args.join(" ")))); + } + let key = match v("--key")? { + Some(p) => (record::key_from_file(Path::new(p)).map_err(|e| OpError::new(EXIT_NOT_FOUND, "key-not-read", e, "--key names a file holding a 32-byte Ed25519 seed in hexadecimal"))?, "supplied"), + None => (record::ephemeral_key().map_err(|e| OpError::new(EXIT_INTERNAL, "internal", e, ""))?, "ephemeral"), + }; + let digest = corpus_digest(&root, &loaded.files)?; + let started = now_rfc3339(); + let scratch = std::env::temp_dir().join(format!("wai-conform-{}-{}", std::process::id(), std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0))); + std::fs::create_dir_all(&scratch).map_err(|e| OpError::new(EXIT_INTERNAL, "internal", e.to_string(), ""))?; + let r = Run { imp: &imp, timeout, scratch: scratch.clone() }; + eprintln!("wai conform {kind} {}: {} cases from {}", suite.name, loaded.cases.len(), suite.dir); + let outcomes: Result, OpError> = match (&imp, &loaded.corpus_file) { + (Impl::Command(c), Some(f)) if whole => r.whole_corpus(c, f, &loaded.cases), + _ => loaded + .cases + .iter() + .map(|c| { + let got = r.answer(c)?; + let input = match &c.input { + Input::File(p) => Some(p.as_path()), + _ => None, + }; + Ok(Outcome { name: c.name.clone(), pass: judge(&c.expect, &got, input), expected: cut(&expected_text(&c.expect)), got: cut(if matches!(c.expect, Expect::Object { .. }) && got.is_empty() { "served as required" } else { &got }) }) + }) + .collect(), + }; + let _ = std::fs::remove_dir_all(&scratch); + let outcomes = outcomes?; + let passed = outcomes.iter().filter(|o| o.pass).count(); + let failed = outcomes.len() - passed; + let conformant = failed == 0; + let (adapter_name, files) = match &imp { + Impl::Command(c) => ("command", adapter::identity(c).into_iter().map(|(p, h)| json!({"path": p, "sha256": h})).collect::>()), + Impl::Http(_) => ("http", Vec::new()), + }; + let mut rec = Map::new(); + rec.insert("wai".into(), json!(record::FORMAT)); + rec.insert("kind".into(), json!(kind)); + rec.insert("suite".into(), json!(suite.name)); + rec.insert("standard".into(), json!(standard_version(&root))); + rec.insert("toolchain".into(), json!({"name": "wai", "version": env!("CARGO_PKG_VERSION")})); + rec.insert("corpus".into(), json!({"dir": suite.dir, "files": loaded.files.len(), "digest": digest})); + rec.insert("implementation".into(), json!({"adapter": adapter_name, "impl": imp_text, "name": v("--name")?, "version": v("--version")?, "files": files})); + rec.insert("started_at".into(), json!(started)); + rec.insert("cases".into(), Value::Array(outcomes.iter().map(|o| json!({"name": o.name, "pass": o.pass, "expected": o.expected, "got": o.got})).collect())); + rec.insert("summary".into(), json!({"cases": outcomes.len(), "passed": passed, "failed": failed})); + rec.insert("conformant".into(), json!(conformant)); + rec.insert("key".into(), json!(key.1)); + let rec = record::sign(rec, &key.0).map_err(|e| OpError::new(EXIT_INTERNAL, "internal", e, ""))?; + let text = serde_json::to_string_pretty(&Value::Object(rec.clone())).map_err(|e| OpError::new(EXIT_INTERNAL, "internal", e.to_string(), ""))?; + std::fs::write(&out, format!("{text}\n")).map_err(|e| OpError::new(EXIT_INTERNAL, "internal", format!("{}: {e}", out.display()), ""))?; + let pubkey = rec.get("signer_pubkey").and_then(Value::as_str).unwrap_or("").to_string(); + if json_mode { + let mut m = Map::new(); + m.insert("kind".into(), json!(kind)); + m.insert("suite".into(), json!(suite.name)); + m.insert("cases".into(), json!(outcomes.len())); + m.insert("passed".into(), json!(passed)); + m.insert("failed".into(), json!(failed)); + m.insert("conformant".into(), json!(conformant)); + m.insert("failures".into(), Value::Array(outcomes.iter().filter(|o| !o.pass).take(200).map(|o| json!({"name": o.name, "expected": o.expected, "got": o.got})).collect())); + m.insert("record".into(), json!(out.display().to_string())); + m.insert("corpus_digest".into(), json!(rec["corpus"]["digest"])); + m.insert("signer_pubkey".into(), json!(pubkey)); + println!("{}", envelope::envelope(OP, conformant, m)); + } else { + let all = args.iter().any(|a| a == "--all"); + let mut shown = 0; + for o in &outcomes { + if o.pass && !all { + continue; + } + if !o.pass { + shown += 1; + if shown > 40 && !all { + continue; + } + } + println!("{} {}", if o.pass { "PASS" } else { "FAIL" }, o.name); + if !o.pass { + println!(" expected: {}", o.expected); + println!(" got: {}", o.got); + } + } + if shown > 40 && !all { + println!("… and {} more failures (--all lists every case)", shown - 40); + } + println!( + "wai conform {kind} {}: {passed} of {} cases passed: {}", + suite.name, + outcomes.len(), + if conformant { "conformant" } else { "not conformant" } + ); + println!("record: {} (signed by {}…, {} key); `wai verify {}` checks it", out.display(), pubkey.get(..16).unwrap_or(""), key.1, out.display()); + } + Ok(conformant) +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/conform/record.rs b/wai/wai-rs/src/bin/wai_cli/dx/conform/record.rs new file mode 100644 index 000000000..52e347db7 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/conform/record.rs @@ -0,0 +1,145 @@ +//! The result record of a `wai conform` run, `conform-result/1`: what was run against what, +//! each case's outcome, and an Ed25519 signature over all of it. +//! +//! ```json +//! { "wai": "conform-result/1", "kind": "verifier", "suite": "claim", +//! "standard": "v0.4.0", "toolchain": {"name": "wai", "version": "0.3.0"}, +//! "corpus": {"dir": "claim-conformance", "files": 1, "digest": "sha256:"}, +//! "implementation": {"adapter": "command", "impl": "", "name": null, +//! "version": null, "files": [{"path": "…", "sha256": ""}]}, +//! "started_at": "2026-10-27T12:00:00Z", +//! "cases": [{"name": "…", "pass": true, "expected": "…", "got": "…"}], +//! "summary": {"cases": 636, "passed": 636, "failed": 0}, "conformant": true, +//! "key": "ephemeral", "signer_pubkey": "", "sig": "" } +//! ``` +//! +//! The corpus digest is the SHA-256 of the lines ` `, +//! one for every corpus file the run read, in path order. The signature is Ed25519 (RFC +//! 8032, verified strictly) by `signer_pubkey` over `wai:conform-result\x01` followed by the +//! record without `sig` in RFC 8785 form. `key` is `ephemeral` for a key made for the run, +//! whose signature shows the record is as the run wrote it, or `supplied` for a key the +//! runner holds (`--key`), which also says who ran it. + +use ed25519_dalek::{Signature, Signer, SigningKey, VerifyingKey}; +use serde_json::{Map, Value}; + +pub const FORMAT: &str = "conform-result/1"; +pub const SIGN_DOMAIN: &[u8] = b"wai:conform-result\x01"; +/// The suffix of a result record's file name, which `wai verify` looks for. +pub const SUFFIX: &str = ".wai-conform.json"; + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +fn unhex(s: &str) -> Option> { + if !s.len().is_multiple_of(2) { + return None; + } + (0..s.len() / 2).map(|i| u8::from_str_radix(s.get(2 * i..2 * i + 2)?, 16).ok()).collect() +} + +fn signing_bytes(record: &Map) -> Result, String> { + let mut r = record.clone(); + r.remove("sig"); + let mut m = SIGN_DOMAIN.to_vec(); + m.extend_from_slice(wai::jcs::jcs(&Value::Object(r)).map_err(|e| format!("the record is not in the signed JSON subset: {e:?}"))?.as_bytes()); + Ok(m) +} + +/// A signing key from `--key`'s file: 64 hexadecimal digits (a 32-byte seed), or the 32 +/// bytes themselves. +pub fn key_from_file(path: &std::path::Path) -> Result { + let b = std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?; + let t = String::from_utf8_lossy(&b); + let seed: Vec = match unhex(t.trim()) { + Some(s) if s.len() == 32 => s, + _ if b.len() == 32 => b, + _ => return Err(format!("{}: a key file holds a 32-byte Ed25519 seed, as 64 hexadecimal digits or 32 bytes", path.display())), + }; + Ok(SigningKey::from_bytes(&seed.try_into().expect("32 bytes"))) +} + +/// A key made for one run. +pub fn ephemeral_key() -> Result { + let mut seed = [0u8; 32]; + getrandom::fill(&mut seed).map_err(|e| format!("no randomness for a run key: {e}"))?; + Ok(SigningKey::from_bytes(&seed)) +} + +/// `record` signed with `key`: `signer_pubkey` and `sig` set. +pub fn sign(mut record: Map, key: &SigningKey) -> Result, String> { + record.insert("signer_pubkey".into(), Value::String(hex(key.verifying_key().as_bytes()))); + record.remove("sig"); + let sig = key.sign(&signing_bytes(&record)?); + record.insert("sig".into(), Value::String(hex(&sig.to_bytes()))); + Ok(record) +} + +/// A result record checked: whether it holds, and a line for each thing checked. +pub fn check(bytes: &[u8]) -> (bool, Vec) { + let mut lines = Vec::new(); + let v: Value = match serde_json::from_slice(bytes) { + Ok(v) => v, + Err(e) => return (false, vec![format!("FAIL not JSON: {e}")]), + }; + let Some(r) = v.as_object() else { return (false, vec!["FAIL not a JSON object".into()]) }; + if r.get("wai").and_then(Value::as_str) != Some(FORMAT) { + return (false, vec![format!("FAIL not a {FORMAT} record")]); + } + let mut ok = true; + let key = r.get("signer_pubkey").and_then(Value::as_str).and_then(unhex).and_then(|b| <[u8; 32]>::try_from(b).ok()).and_then(|b| VerifyingKey::from_bytes(&b).ok()); + let sig = r.get("sig").and_then(Value::as_str).and_then(unhex).and_then(|b| <[u8; 64]>::try_from(b).ok()).map(|b| Signature::from_bytes(&b)); + match (key, sig, signing_bytes(r)) { + (Some(k), Some(s), Ok(m)) if k.verify_strict(&m, &s).is_ok() => { + let who = r.get("key").and_then(Value::as_str).unwrap_or("?"); + lines.push(format!("ok signed by {}… ({who} key)", &hex(k.as_bytes())[..16])); + } + (None, _, _) => { + ok = false; + lines.push("FAIL signer_pubkey is not an Ed25519 public key".into()); + } + (_, None, _) => { + ok = false; + lines.push("FAIL sig is not an Ed25519 signature".into()); + } + (_, _, Err(e)) => { + ok = false; + lines.push(format!("FAIL {e}")); + } + _ => { + ok = false; + lines.push("FAIL the signature does not verify: the record is not the one that was signed".into()); + } + } + let cases = r.get("cases").and_then(Value::as_array).cloned().unwrap_or_default(); + let passed = cases.iter().filter(|c| c.get("pass").and_then(Value::as_bool) == Some(true)).count() as u64; + let failed = cases.len() as u64 - passed; + let s = r.get("summary"); + let stated = |k: &str| s.and_then(|s| s.get(k)).and_then(Value::as_u64); + if stated("cases") != Some(cases.len() as u64) || stated("passed") != Some(passed) || stated("failed") != Some(failed) { + ok = false; + lines.push(format!("FAIL the summary does not count the cases: {} cases, {passed} passed, {failed} failed", cases.len())); + } + let conformant = failed == 0 && !cases.is_empty(); + if r.get("conformant").and_then(Value::as_bool) != Some(conformant) { + ok = false; + lines.push(format!("FAIL `conformant` is not what the cases say ({conformant})")); + } + if ok { + let s = |k: &str| r.get(k).and_then(Value::as_str).unwrap_or("?").to_string(); + let corpus = r.get("corpus").and_then(|c| c.get("digest")).and_then(Value::as_str).unwrap_or("?"); + let imp = r.get("implementation").and_then(|i| i.get("impl")).and_then(Value::as_str).unwrap_or("?"); + lines.push(format!( + "ok {} {} against {}: {} of {} cases passed, {}; corpus {corpus}; WAI {}", + s("kind"), + s("suite"), + imp, + passed, + cases.len(), + if conformant { "conformant" } else { "not conformant" }, + s("standard") + )); + } + (ok, lines) +} diff --git a/wai/wai-rs/src/bin/wai_cli/dx/conform/suites.rs b/wai/wai-rs/src/bin/wai_cli/dx/conform/suites.rs new file mode 100644 index 000000000..2d2c6c5e9 --- /dev/null +++ b/wai/wai-rs/src/bin/wai_cli/dx/conform/suites.rs @@ -0,0 +1,257 @@ +//! The suites `wai conform` runs: each a kind of implementation, a corpus of WAI's, and the +//! cases read from it with what a conforming implementation answers. Every expectation is +//! the corpus's own (its declared verdicts, digests and reasons), never a verifier's output. +//! +//! | kind | suite | corpus | a case is | it passes when | +//! |---|---|---|---|---| +//! | verifier | `claim` | `claim-conformance/vectors.json` | one vector | the verdict is the one declared (jwp-receipts §3.1) | +//! | verifier | `jwp` | `jwp-conformance/vectors.json` | one vector | the verdict is the one declared (jwp-receipts §2–§6) | +//! | verifier | `signer` | `signer-conformance/vectors.json` | one vector | the verdict is the one declared (jwp-receipts §7) | +//! | decoder | `score` | `score-conformance/` | a `WSCR` score | the rendered `i16le` PCM has the declared SHA-256 | +//! | packager | `policy` | `pack-conformance/` | a policy file | it is admitted with the declared digest, or refused with the declared reason (packaging §3, §12) | +//! | origin | `http` | `http-conformance/objects/` | an object | it is served as http-delivery §5.3 and §6.2 require: bytes, headers, ranges, head proof, preflight | + +use serde_json::Value; +use std::path::{Path, PathBuf}; + +/// What a case expects. +#[derive(Clone, Debug)] +pub enum Expect { + /// A verdict line: exactly this, or, with `prefix`, any line that starts with it. + Verdict { line: String, prefix: bool }, + /// An output whose SHA-256 is this (hexadecimal). + Output { sha256: String }, + /// A packager's verdict: admitted with this digest, or refused with this reason. + Admit { digest: String }, + Refuse { reason: String }, + /// An origin serving this object. + Object { cid: String, bytes_file: PathBuf, head_proof: Option }, +} + +/// What a case gives the implementation. +#[derive(Clone, Debug)] +pub enum Input { + /// A vector, as the corpus writes it. + Json(Value), + /// A file. + File(PathBuf), + /// Nothing: the implementation already serves what is asked of it. + Served, +} + +#[derive(Clone, Debug)] +pub struct Case { + pub name: String, + pub input: Input, + pub expect: Expect, +} + +/// A suite read from a corpus: its cases, the files read (for the corpus digest), and the +/// corpus file a whole-corpus run is given (`{corpus}`), where the suite has one. +pub struct Loaded { + pub cases: Vec, + pub files: Vec, + pub corpus_file: Option, +} + +pub struct Suite { + pub kind: &'static str, + pub name: &'static str, + /// The corpus directory, under `wai/`. + pub dir: &'static str, + pub about: &'static str, + pub http: bool, + pub command: bool, + pub load: fn(&Path) -> Result, +} + +pub const KINDS: &[&str] = &["verifier", "decoder", "packager", "origin"]; + +pub const SUITES: &[Suite] = &[ + Suite { kind: "verifier", name: "claim", dir: "claim-conformance", about: "typed claim receipts (jwp-receipts §3.1): accept with the link hash, or reject at a step with a reason", http: true, command: true, load: claim }, + Suite { kind: "verifier", name: "jwp", dir: "jwp-conformance", about: "labelled leaves, group receipts, totals, chains and the binary form (jwp-receipts §2–§6)", http: true, command: true, load: jwp }, + Suite { kind: "verifier", name: "signer", dir: "signer-conformance", about: "key discovery, signing authority and module authorisation (jwp-receipts §7)", http: true, command: true, load: signer }, + Suite { kind: "decoder", name: "score", dir: "score-conformance", about: "a WSCR score rendered to its canonical i16le PCM (the score extension's mixdown-equivalence)", http: true, command: true, load: score }, + Suite { kind: "packager", name: "policy", dir: "pack-conformance", about: "packaging policies admitted with their digest, or refused with their reason (packaging §3, §12)", http: true, command: true, load: policy }, + Suite { kind: "origin", name: "http", dir: "http-conformance", about: "objects, head proofs, ranges and headers as an origin serves them (http-delivery §5.3, §6.2)", http: true, command: false, load: http_origin }, +]; + +fn read_json(p: &Path) -> Result { + let b = std::fs::read(p).map_err(|e| format!("{}: {e}", p.display()))?; + serde_json::from_slice(&b).map_err(|e| format!("{}: {e}", p.display())) +} + +fn vectors(v: &Value, p: &Path) -> Result, String> { + v.get("vectors").and_then(Value::as_array).cloned().ok_or_else(|| format!("{}: no `vectors` list", p.display())) +} + +fn name_of(v: &Value) -> String { + v.get("name").and_then(Value::as_str).unwrap_or("?").to_string() +} + +fn verdict_cases(dir: &Path, expected_of: fn(&Value) -> (String, bool)) -> Result { + let p = dir.join("vectors.json"); + let doc = read_json(&p)?; + let cases = vectors(&doc, &p)? + .into_iter() + .map(|v| { + let (line, prefix) = expected_of(v.get("expected").unwrap_or(&Value::Null)); + Case { name: name_of(&v), input: Input::Json(v), expect: Expect::Verdict { line, prefix } } + }) + .collect(); + Ok(Loaded { cases, files: vec![p.clone()], corpus_file: Some(p) }) +} + +/// The claim corpus's verdicts (its README): `accept link_hash=` for an accepted +/// claim (any link hash where none is pinned), `accept ancestor=` for a recovery, +/// `reject step= reason=